From 8ef367ec3107d9e0deff6a2321b8b087313a05c8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:38:39 +0000 Subject: [PATCH 01/20] feat(email): scaffold per-user Mailbox durable object Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 164 +++- .../contributing/architecture/primitives.yaml | 9 + .../worker/src/account/export.node.test.ts | 155 ++++ packages/worker/src/account/export.ts | 89 ++ .../account/user-owned-surfaces.node.test.ts | 14 + .../worker/src/account/user-owned-surfaces.ts | 9 + .../src/app/account-deletion.node.test.ts | 41 + packages/worker/src/app/account-deletion.ts | 27 + packages/worker/src/email/mailbox-client.ts | 28 + packages/worker/src/email/mailbox-do.ts | 292 ++++++ .../src/email/mailbox-do.workers.test.ts | 660 ++++++++++++++ packages/worker/src/email/mailbox-export.ts | 293 ++++++ packages/worker/src/email/mailbox-mappers.ts | 319 +++++++ .../worker/src/email/mailbox-retention.ts | 137 +++ packages/worker/src/email/mailbox-schema.ts | 328 +++++++ packages/worker/src/email/mailbox-store.ts | 840 ++++++++++++++++++ packages/worker/src/email/mailbox-types.ts | 631 +++++++++++++ packages/worker/src/index.ts | 2 + ...er-scoped-durable-object-name.node.test.ts | 5 + .../src/user-scoped-durable-object-name.ts | 5 + packages/worker/worker-configuration.d.ts | 16 +- packages/worker/wrangler.jsonc | 16 + 22 files changed, 4051 insertions(+), 29 deletions(-) create mode 100644 packages/worker/src/email/mailbox-client.ts create mode 100644 packages/worker/src/email/mailbox-do.ts create mode 100644 packages/worker/src/email/mailbox-do.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-export.ts create mode 100644 packages/worker/src/email/mailbox-mappers.ts create mode 100644 packages/worker/src/email/mailbox-retention.ts create mode 100644 packages/worker/src/email/mailbox-schema.ts create mode 100644 packages/worker/src/email/mailbox-store.ts create mode 100644 packages/worker/src/email/mailbox-types.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 7bd1099f08..c56f08bf91 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -78,22 +78,28 @@ Deletion must cover these user-owned surfaces: references a stale column. - **Durable Objects:** `JobManager`, `StorageRunner`, `RepoSession`, `RemoteConnectorSession`, `PackageRealtimeSession`, `PackageServiceInstance`, - `McpClientHub`, `RunLog`, and `UserMeter` are purged through account-deletion - RPCs after their D1 identifiers are collected (`RunLog` and `UserMeter` are - one object per user and need no D1 id scan). `MCP` objects remain SDK - session-keyed, while `mcp_agent_sessions` indexes each Durable Object id by - authenticated stable user id so account deletion can purge stored props, - conversation state, raw-fetch state, and transport storage before revoking - OAuth grants. + `McpClientHub`, `RunLog`, `UserMeter`, and `Mailbox` are purged through + account-deletion RPCs after their D1 identifiers are collected (`RunLog`, + `UserMeter`, and `Mailbox` are one object per user and need no D1 id scan). + During the Mailbox expand phase, `Mailbox.purge()` clears DO SQLite only; D1 + `email_*` rows and `EMAIL_BLOBS` deletion remain the authoritative account- + deletion path for mail content (see [Mailbox](#durable-objects-mailbox)). + `MCP` objects remain SDK session-keyed, while `mcp_agent_sessions` indexes + each Durable Object id by authenticated stable user id so account deletion can + purge stored props, conversation state, raw-fetch state, and transport storage + before revoking OAuth grants. - **Vectorize:** memory, job, and saved-package vector ids are derived from D1 rows and removed with `deleteByIds`. -- **R2:** raw email MIME blobs in `EMAIL_BLOBS` are enumerated from - `email_messages` (deterministic `emailRawMimeKey` keys) and attachment storage - keys while those rows still exist. A failed object delete aborts D1 - finalization so those inventory rows remain available for retry. Rows owned by - `system:email` keep their blobs here (they are not user data); those blobs are - removed when the system-email retention prune deletes the messages through the - shared delete-message helper. +- **R2:** raw email MIME and attachment blobs in `EMAIL_BLOBS` are deleted by + per-user prefix cleanup (`email-raw:v1:{userId}/` and + `email-attachment:v1:{userId}/`) plus any remaining D1-inventoried keys + outside those prefixes. A failed object delete aborts D1 finalization so + inventory rows remain available for retry. Rows owned by `system:email` keep + their blobs here (they are not user data); those blobs are removed when the + system-email retention prune deletes the messages through the shared + delete-message helper. Mailbox expand-phase `purge` does not delete R2 objects + — D1 email rows and this existing R2 prefix deletion stay authoritative during + expand. - **KV:** published bundle artifact keys, source/manifest snapshot keys, community listing snapshots, and per-user package retriever cache/index keys in `BUNDLE_ARTIFACTS_KV` are deleted before D1 projection rows are removed. @@ -168,8 +174,9 @@ migration-safe chunked interface: storage export capability. User meter counters use `section: "user_meter"` and the `UserMeter.exportCounters` RPC (daily counters plus additive `storageBytesShadow` on the first page only when present; explicitly - non-authoritative). R2 raw MIME, attachment, avatar, and icon objects use - `section: "r2_object"`; each response contains at most one 256 KiB base64 + non-authoritative). Mailbox metadata uses `section: "mailbox"` and the + `Mailbox.exportMailbox` RPC. R2 raw MIME, attachment, avatar, and icon objects + use `section: "r2_object"`; each response contains at most one 256 KiB base64 chunk and an opaque cursor. Each request uses bounded `LIMIT 1` ownership queries rather than reconstructing inventory. Continuation cursors bind the source row, object key, size, and ETag; ownership/key mutations and object @@ -206,6 +213,13 @@ Durable Object export behavior: (seven UTC days of counter and inbound-delivery-claim rows); shadow storage-byte state is not time-pruned. See [Entitlements](./entitlements.md#usermeter-expand-phase). +- `Mailbox` exports per-user email metadata (threads, messages, attachments, + delivery events) through the account-export `mailbox` section (`exportMailbox` + RPC; keyset pagination with prefixed cursors over those tables). Manifest + counts use `countMailbox`. Phase 1 registers this consumption without + dual-write or a change of D1 authority — D1 `email_*` rows remain the live + source of truth and are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -560,6 +574,110 @@ section / `account_export_section` (daily counters plus additive `storageBytesShadow` on the first page only when present; shadow field is non-authoritative). +## Durable Objects (`Mailbox`) + +User-owned email **metadata** moves to a per-user `Mailbox` Durable Object with +SQLite (`packages/worker/src/email/mailbox-do.ts` and siblings under +`mailbox-*.ts`; client in `packages/worker/src/email/mailbox-client.ts`). The +Worker binding is `MAILBOX` (class `Mailbox`; Wrangler SQLite migration tag +`v22` via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Raw MIME +and outbound attachment bytes stay in `EMAIL_BLOBS` R2; the DO stores object +keys, not payload bytes. + +Naming matches `RunLog`, `JobManager`, and `UserMeter`: one object per untrimmed +stable MCP `userId` via `mailboxDurableObjectName(userId)` → +`idFromName(userId)` in +`packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` +column inside the DO because the object identity is the user. + +**SQLite ownership** (schema version in `mailbox_meta`, currently +`mailboxSchemaVersion = 2`): + +- `email_threads`, `email_messages`, `email_attachments`, and + `email_delivery_events` for the owning user (same logical names as the D1 + tables they will eventually replace) +- latest per-message delivery status on `email_messages.delivery_status`, kept + separate from send-request `processing_status` + +**Retention** is self-enforced inside the DO with alarms +(`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). +Alarm-driven deletes are strict blob-before-row for `EMAIL_BLOBS` keys (failed +blob deletes skip the row for retry), matching today's D1 retention ordering. + +Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; +result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state +only and reinitializes schema — it does **not** delete R2 objects. D1 `email_*` +row deletion and existing `EMAIL_BLOBS` prefix deletion remain authoritative for +mail content. Account export pages Mailbox state through the `mailbox` section +(`exportMailbox` / `countMailbox`). + +### Expand/contract phases + +This is an expand/contract migration. **The current PR is phase 1 only:** +additive scaffold (DO class, binding, naming, client, mirror/export/purge RPCs, +and alarm retention) **plus registered account-deletion `purge` and +account-export `mailbox` section consumption**. It does **not** dual-write, does +**not** change live email read/write authority, and leaves D1 `email_*` tables +authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. + +1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze + `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / + `exportMailbox` / `countMailbox` / `purge` RPCs and alarm retention, and + register account deletion/export consumption. No dual-write; D1 remains sole + authority for live mail. +2. **D1-authoritative dual-write + parity** — every user-mail mutation that + writes D1 also writes the DO; parity counters and reconciliation prove DO + completeness against D1. D1 stays authoritative for reads. +3. **Reads cut over after production parity** — user-mail reads move to the DO + only after production parity is verified. D1 writes continue. +4. **D1 write-off / event retirement** — stop writing moved user-mail metadata + to D1; retire dual-write and event/mirror machinery used only for the + migration. +5. **Later contract migrations** — drop retired D1 user-mail tables/columns only + after verification. No premature schema deletion. + +A cron sibling track owns scheduled-lane wiring (for example parity / +reconciliation ticks). This Mailbox track owns DO storage semantics, alarm +retention, and the inbound durability boundary below. + +### What stays in D1 + +- **`system:email` operator inbox** — remains in D1 for cross-account / admin + access, fixed bounded caps, and separate system-email retention. Those rows + stay excluded from account deletion and export + (`accountUserDataExcludedOwnerIds`). They are not migrated into per-user + Mailbox objects. +- **Low-write email config** — sender identities, inboxes, inbox addresses, + sender rules, and similar low-churn configuration stay in D1. +- **Provider-message reverse lookup** — outbound `provider_message_id` → owner + resolution stays in D1 until the delivery webhook path already knows the + owning user. Contextless provider-id reverse lookups must not require + enumerating per-user Mailbox objects. + +### Inbound durability boundary (eventual DO phase) + +Today's D1-authoritative inbound commit boundary is documented under +[R2 (`EMAIL_BLOBS`)](#r2-community_assets-email_blobs): thread prework + R2 put + +- D1 message/attachment rows are pre-commit; `touchEmailThread` / `received` + delivery-event writes are post-commit best-effort; ambiguous attachment-insert + failures acknowledge rather than risk duplicates. + +When Mailbox becomes authoritative, the same shape applies with the DO as the +metadata store: + +- **Pre-commit:** thread prework + R2 raw-MIME put + atomic Mailbox + message/attachment commit. +- **Post-commit (best effort):** thread touch and delivery-event writes — log + failures without throwing (retry would duplicate mail). +- **Ambiguity:** if attachment commit fails but message cleanup (or a residual + probe) cannot prove the pre-commit state, acknowledge the already-created + message (logged, non-retry) rather than risking a duplicate on Email Routing + retry. + +Phase 1 scaffolding does not change that live D1 boundary. + ### Package state model Saved packages are the only top-level persisted primitive. Their state maps onto @@ -599,6 +717,9 @@ via `durableObjectNameFromParts`); domain helpers such as daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`), plus optional schema-v4 D1 storage-byte shadow. See [Entitlements](./entitlements.md#usermeter-expand-phase). +- `Mailbox` — `mailboxDurableObjectName(userId)` → `idFromName(userId)`. One + email-metadata DO per user (untrimmed stable id, same as `RunLog`). See + [Mailbox](#durable-objects-mailbox). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -666,6 +787,9 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` [Run records](./run-records.md)) - `USER_METER` (Durable Objects; per-user daily entitlement counters — see [Entitlements](./entitlements.md#usermeter-expand-phase)) +- `MAILBOX` (Durable Objects; per-user email metadata — see + [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export + consumption without changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) @@ -914,6 +1038,7 @@ to `durableObjectNameFromParts`). - `RunLog`: `idFromName(userId)` (no trim); one execution-history DO per user. - `UserMeter`: `idFromName(userId)` (no trim); one daily-entitlement meter DO per user, plus optional schema-v4 D1 storage-byte shadow. +- `Mailbox`: `idFromName(userId)` (no trim); one email-metadata DO per user. - `McpClientHub`: `idFromName(userId.trim())`. - `StorageRunner`: `idFromName(JSON.stringify([userId, storageId]))`. - `RepoSession`: `idFromName(repo_sessions.id)`; the key is not user-prefixed, @@ -1091,7 +1216,9 @@ Current retention policies: which prunes messages (and their `EMAIL_BLOBS` raw-MIME blobs) and delivery events older than 90 days in bounded batches within its own time budget, deletes stale `system_email_daily_counters`, and caps stored system messages - at 5000. + at 5000. After Mailbox cut-over, user-owned delivery-event retention moves to + the per-user Mailbox DO alarm (still 90 days, strict blob-before-row); D1 + policies here remain authoritative until that phase. - `email_messages` / `email_attachments` / `email_threads`: user-owned messages (excluding the `system:email` owner) keep 365 days, deleted oldest first in batches. Retention deletes the deterministic @@ -1099,6 +1226,9 @@ Current retention policies: blob delete fails, those rows are skipped and still selected on later runs so cleanup can retry. Dependent `email_attachments` rows are deleted before their messages, and threads left with no messages are pruned for the affected users. + After Mailbox cut-over, the same 365-day window and blob-before-row ordering + are self-enforced by the Mailbox DO alarm; `system:email` stays on the D1 + system-email retention job. - `entitlement_daily_counters`: expand-phase **mirror** of UserMeter daily counters (authoritative state lives in the per-user `UserMeter` DO). Rows keep 400 days by `day` key until mirror retirement is verified after diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index 5302603cd6..6a0731c858 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -522,6 +522,15 @@ primitives: - docs/contributing/architecture/data-storage.md - docs/contributing/architecture/entitlements.md + - id: mailbox + group: storage + name: Mailbox + summary: Per-user Mailbox DO SQLite for email metadata; R2 keeps raw MIME. + code: + - packages/worker/src/email/mailbox- + docs: + - docs/contributing/architecture/data-storage.md + - id: email-blobs-r2 group: storage name: Email blob storage (R2) diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index 815d16f8cb..657b5a1f42 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1677,6 +1677,161 @@ test('account export includes user_meter counters, pages them, and warns on trun ) }) +test('account export includes mailbox rows, pages them, and warns on truncation', async () => { + const { sqlite, db } = createMigratedDb() + sqlite.exec(` + INSERT INTO users ( + id, username, email, password_hash, created_at, updated_at, + email_verified_at, stable_user_id + ) + VALUES ( + 1, 'user-a', 'a@example.com', 'password-hash-a', '2026-07-05', + '2026-07-05', '2026-07-05', 'user-aaa' + ); + `) + const rows = [ + { + kind: 'thread' as const, + row: { + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: null, + lastMessageAt: '2026-07-30T00:00:00.000Z', + createdAt: '2026-07-30T00:00:00.000Z', + updatedAt: '2026-07-30T00:00:00.000Z', + }, + }, + { + kind: 'message' as const, + row: { + id: 'message-1', + threadId: 'thread-1', + inboxId: 'inbox-1', + direction: 'inbound' as const, + processingStatus: 'received' as const, + deliveryStatus: 'delivered' as const, + classification: 'personal' as const, + subject: 'hello', + fromAddress: 'a@example.com', + toAddresses: ['b@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + messageIdHeader: null, + inReplyToHeader: null, + referencesHeader: null, + sentAt: '2026-07-30T00:00:00.000Z', + receivedAt: '2026-07-30T00:00:00.000Z', + rawMimeKey: null, + rawMimeStorageKind: 'unavailable' as const, + bodyText: 'hi', + bodyHtml: null, + snippet: 'hi', + hasAttachments: false, + createdAt: '2026-07-30T00:00:00.000Z', + updatedAt: '2026-07-30T00:00:00.000Z', + }, + }, + { + kind: 'attachment' as const, + row: { + id: 'attachment-1', + messageId: 'message-1', + filename: 'file.txt', + contentType: 'text/plain', + sizeBytes: 3, + storageKey: null, + storageKind: 'unavailable' as const, + contentId: null, + isInline: false, + createdAt: '2026-07-30T00:00:00.000Z', + }, + }, + ] + const exportMailbox = vi.fn( + async (input: { pageSize?: number; startAfter?: string | null }) => { + const pageSize = input.pageSize ?? 100 + const startIndex = input.startAfter + ? rows.findIndex((row) => row.row.id === input.startAfter) + 1 + : 0 + const page = rows.slice(startIndex, startIndex + pageSize) + const truncated = startIndex + pageSize < rows.length + return { + rows: page, + nextStartAfter: truncated ? page.at(-1)!.row.id : null, + truncated, + } + }, + ) + const countMailbox = vi.fn(async () => ({ + threads: 1, + messages: 1, + attachments: 1, + deliveryEvents: 0, + })) + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const env = { + APP_DB: db, + MAILBOX: { + idFromName, + get: () => ({ exportMailbox, countMailbox }), + }, + } as unknown as Env + + const accountExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(idFromName).toHaveBeenCalledWith('user-aaa') + expect(accountExport.manifest.sections.mailbox?.count).toBe(3) + expect(accountExport.durableObjects.mailbox).toEqual({ + rows, + nextStartAfter: null, + truncated: false, + }) + + const first = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'mailbox', + pageSize: 2, + }) + expect(first.items).toEqual(rows.slice(0, 2)) + expect(first.truncated).toBe(true) + expect(first.nextStartAfter).toBe('message-1') + + const second = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'mailbox', + pageSize: 2, + startAfter: first.nextStartAfter ?? undefined, + }) + expect(second.items).toEqual(rows.slice(2)) + expect(second.truncated).toBe(false) + expect(second.nextStartAfter).toBeNull() + expect(exportMailbox).toHaveBeenCalled() + + exportMailbox.mockImplementation(async () => ({ + rows: [rows[0]!], + nextStartAfter: 'cursor-more', + truncated: true, + })) + const truncatedExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(truncatedExport.durableObjects.mailbox?.truncated).toBe(true) + expect(truncatedExport.manifest.warnings).toContain( + 'Mailbox rows were truncated in the full export; use account_export_section with section "mailbox" to retrieve additional pages.', + ) +}) + test('run_records section paging preserves exportRuns cursor across dedicated phases', async () => { const { sqlite, db } = createMigratedDb() sqlite.exec(` diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index a2ac581228..a5796c6378 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -39,6 +39,8 @@ import { userMeterRpc, } from '#worker/entitlements/user-meter-client.ts' import { type UserMeterExportResult } from '#worker/entitlements/user-meter-do.ts' +import { mailboxNamespace, mailboxRpc } from '#worker/email/mailbox-client.ts' +import { type MailboxExportResult } from '#worker/email/mailbox-types.ts' import { resolveUserStableId } from '#worker/user-id.ts' import { listAccountUserPackageServices, @@ -61,6 +63,7 @@ export const accountExportSectionNames = [ 'job_manager', 'run_records', 'user_meter', + 'mailbox', 'remote_connector_session', 'package_service', 'oauth_grants', @@ -133,6 +136,7 @@ type ManifestInventoryCounts = { storageRunners: number runRecords: number userMeterCounters: number + mailboxRows: number remoteConnectorSessions: number packageServices: number artifactRepos: number @@ -250,6 +254,7 @@ type AccountExportDurableObjects = { jobManager: unknown | null runRecords: RunRecordsExportPayload | null userMeter: UserMeterExportResult | null + mailbox: MailboxExportResult | null remoteConnectorSessions: Array<{ instanceId: string export: RemoteConnectorSessionExport | null @@ -1035,6 +1040,7 @@ async function collectManifestInventoryCounts(input: { storageRunners, runRecords, userMeterCounters, + mailboxRows, remoteConnectorSessions, packageServices, artifactRepos, @@ -1061,6 +1067,19 @@ async function collectManifestInventoryCounts(input: { }) return countUserMeterExportEntries(page) }), + safeCount('mailbox rows', async () => { + if (!mailboxNamespace(input.env)) return 0 + const counts = await mailboxRpc({ + env: input.env, + userId: input.userId, + }).countMailbox() + return ( + counts.threads + + counts.messages + + counts.attachments + + counts.deliveryEvents + ) + }), safeCount('remote connectors', async () => countScalar( input.env, @@ -1099,6 +1118,7 @@ async function collectManifestInventoryCounts(input: { storageRunners, runRecords, userMeterCounters, + mailboxRows, remoteConnectorSessions, packageServices, artifactRepos, @@ -1371,6 +1391,36 @@ async function exportUserMeterCounters(input: { } } +async function exportMailboxRows(input: { + env: AccountExportEnv + userId: string + warnings: Array +}): Promise { + try { + if (!mailboxNamespace(input.env)) { + input.warnings.push( + 'MAILBOX binding was unavailable; mailbox rows were not exported.', + ) + return null + } + const page = await mailboxRpc({ + env: input.env, + userId: input.userId, + }).exportMailbox({ + pageSize: maxExportPageSize, + }) + if (page.truncated) { + input.warnings.push( + `Mailbox rows were truncated in the full export; use account_export_section with section "mailbox" to retrieve additional pages.`, + ) + } + return page + } catch (error) { + input.warnings.push(`Mailbox export failed: ${getErrorMessage(error)}`) + return null + } +} + async function exportRemoteConnectorSessions(input: { env: AccountExportEnv userId: string @@ -1469,6 +1519,7 @@ async function exportDurableObjects(input: { packageServices, runRecords, userMeter, + mailbox, ] = await Promise.all([ exportStorageRunners({ env: input.env, @@ -1498,6 +1549,11 @@ async function exportDurableObjects(input: { userId: input.userId, warnings: input.warnings, }), + exportMailboxRows({ + env: input.env, + userId: input.userId, + warnings: input.warnings, + }), ]) let jobManager: unknown | null = null try { @@ -1512,6 +1568,7 @@ async function exportDurableObjects(input: { jobManager, runRecords, userMeter, + mailbox, remoteConnectorSessions, packageServices, storageRunners, @@ -1587,6 +1644,17 @@ function buildManifest(input: { ), discovery: { section: 'user_meter' }, } + sections.mailbox = { + count: + input.durableObjects?.mailbox == null + ? (input.inventoryCounts?.mailboxRows ?? 0) + : input.durableObjects.mailbox.rows.length, + warnings: input.warnings.filter( + (warning) => + warning.startsWith('Mailbox ') || warning.startsWith('MAILBOX '), + ), + discovery: { section: 'mailbox' }, + } sections.remote_connector_sessions = { count: input.inventoryCounts?.remoteConnectorSessions ?? @@ -1927,6 +1995,27 @@ export async function readAccountExportSection(input: { warnings, } } + if (input.section === 'mailbox') { + if (!mailboxNamespace(input.env)) { + throw new Error('MAILBOX binding was unavailable.') + } + const pageSize = normalizePageSize(input.pageSize) + const page = await mailboxRpc({ + env: input.env, + userId: input.mcpUserId, + }).exportMailbox({ + pageSize, + startAfter: input.startAfter ?? null, + }) + return { + section: input.section, + items: page.rows, + truncated: page.truncated, + nextStartAfter: page.nextStartAfter, + pageSize, + warnings, + } + } if (input.section === 'remote_connector_session') { if (!input.instanceId) { throw new Error( diff --git a/packages/worker/src/account/user-owned-surfaces.node.test.ts b/packages/worker/src/account/user-owned-surfaces.node.test.ts index 41cb3b0280..2882e4311a 100644 --- a/packages/worker/src/account/user-owned-surfaces.node.test.ts +++ b/packages/worker/src/account/user-owned-surfaces.node.test.ts @@ -60,6 +60,7 @@ test('account deletion and export consume the out-of-band surface registry', () expect(coverage.r2SurfaceIds.has('email_raw_mime')).toBe(true) expect(coverage.r2SurfaceIds.has('user_avatar')).toBe(true) expect(coverage.durableObjectIds.has('user_meter')).toBe(true) + expect(coverage.durableObjectIds.has('mailbox')).toBe(true) expect( accountUserOwnedDurableObjectSurfaces.find( (surface) => surface.id === 'user_meter', @@ -69,6 +70,15 @@ test('account deletion and export consume the out-of-band surface registry', () deletionResultKey: 'userMeters', export: 'include', }) + expect( + accountUserOwnedDurableObjectSurfaces.find( + (surface) => surface.id === 'mailbox', + ), + ).toMatchObject({ + binding: 'MAILBOX', + deletionResultKey: 'mailboxes', + export: 'include', + }) expect( accountUserOwnedKvKeySchemes.some((scheme) => scheme.prefixTemplate?.includes('source-snapshot:v1:'), @@ -76,8 +86,12 @@ test('account deletion and export consume the out-of-band surface registry', () ).toBe(true) expect(accountExportSource).toContain("'user_meter'") expect(accountExportSource).toContain('userMeterRpc') + expect(accountExportSource).toContain("'mailbox'") + expect(accountExportSource).toContain('mailboxRpc') expect(accountDeletionSource).toContain('userMeterRpc') expect(accountDeletionSource).toContain('userMeters') + expect(accountDeletionSource).toContain('mailboxRpc') + expect(accountDeletionSource).toContain('mailboxes') expect(accountDeletionSource).toContain( "from '#worker/account/user-owned-surfaces.ts'", diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 47a46d6fc3..885110edd9 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -9,6 +9,7 @@ export type UserOwnedDurableObjectSurface = { | 'package_service_instance' | 'run_log' | 'user_meter' + | 'mailbox' | 'mcp' binding: string /** Result key used in AccountDeletionResult.clearedDurableObjects when purged */ @@ -132,6 +133,14 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray ({ purge: async () => ({ ok: true as const }) }), }, + MAILBOX: { + idFromName: durableObjectId, + get: () => ({ purge: async () => ({ ok: true as const }) }), + }, JOB_MANAGER: { idFromName: durableObjectId, get: () => ({ purgeUser: async () => ({ ok: true as const }) }), @@ -1111,6 +1115,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async const clearStorageMock = vi.fn(async () => ({ ok: true as const })) const clearRunLogMock = vi.fn(async () => ({ ok: true as const })) const purgeUserMeterMock = vi.fn(async () => ({ ok: true as const })) + const purgeMailboxMock = vi.fn(async () => ({ ok: true as const })) const purgeJobManagerMock = vi.fn(async () => ({ ok: true as const })) const purgeRepoSessionMock = vi.fn(async () => ({ ok: true as const })) const purgeRemoteConnectorMock = vi.fn(async () => ({ ok: true as const })) @@ -1140,6 +1145,10 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async idFromName: (name: string) => name as unknown as DurableObjectId, get: () => ({ purge: purgeUserMeterMock }), }, + MAILBOX: { + idFromName: (name: string) => name as unknown as DurableObjectId, + get: () => ({ purge: purgeMailboxMock }), + }, JOB_MANAGER: { idFromName: (name: string) => name as unknown as DurableObjectId, get: () => ({ purgeUser: purgeJobManagerMock }), @@ -1441,6 +1450,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async storageRunners: 6, runLogs: 1, userMeters: 1, + mailboxes: 1, jobManagers: 1, repoSessions: 1, remoteConnectorSessions: 1, @@ -1454,6 +1464,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async }) expect(clearRunLogMock).toHaveBeenCalledTimes(1) expect(purgeUserMeterMock).toHaveBeenCalledTimes(1) + expect(purgeMailboxMock).toHaveBeenCalledTimes(1) expect(purgeMcpClientHubMock).toHaveBeenCalledTimes(1) expect(purgeMcpAgentSessionMock).toHaveBeenCalledWith({ userId: userAaa, @@ -2031,6 +2042,36 @@ test('account deletion reports a missing USER_METER binding and remains retryabl ]) }) +test('account deletion reports a missing MAILBOX binding and remains retryable', async () => { + const { db, rows } = createTestDb({ + users: [{ id: 1, email: 'a@example.com', stable_user_id: 'user-aaa' }], + }) + await expect( + deleteUserAccount({ + env: createSuccessfulDeletionEnv(db, { + MAILBOX: undefined, + }), + dbUserId: 1, + mcpUserId: 'user-aaa', + }), + ).rejects.toMatchObject({ + cleanupErrors: expect.arrayContaining([ + 'MAILBOX binding was unavailable; the mailbox Durable Object was not purged.', + ]), + partialResult: expect.objectContaining({ + clearedDurableObjects: expect.objectContaining({ + mailboxes: 0, + }), + }), + }) + expect(rows.users).toEqual([ + expect.objectContaining({ + id: 1, + deleting_at: expect.any(String), + }), + ]) +}) + test('account deletion purges a StorageRunner known only via user_storage_buckets', async () => { const userId = 'user-bucket-only' const clearStorage = vi.fn(async () => ({ ok: true as const })) diff --git a/packages/worker/src/app/account-deletion.ts b/packages/worker/src/app/account-deletion.ts index bb2d825d7d..c55d5d0992 100644 --- a/packages/worker/src/app/account-deletion.ts +++ b/packages/worker/src/app/account-deletion.ts @@ -21,6 +21,7 @@ import { userMeterNamespace, userMeterRpc, } from '#worker/entitlements/user-meter-client.ts' +import { mailboxNamespace, mailboxRpc } from '#worker/email/mailbox-client.ts' import { listAccountUserPackageServices, listAccountUserStorageIds, @@ -621,6 +622,27 @@ async function purgeUserMeter(input: { } } +async function purgeMailbox(input: { + env: Env + userId: string + warnings: Array +}): Promise { + try { + if (!mailboxNamespace(input.env)) { + input.warnings.push( + 'MAILBOX binding was unavailable; the mailbox Durable Object was not purged.', + ) + return 0 + } + await mailboxRpc({ env: input.env, userId: input.userId }).purge() + return 1 + } catch (error) { + const message = getErrorMessage(error) + input.warnings.push(`Mailbox purge failed: ${message}`) + return 0 + } +} + async function purgeJobManager(input: { env: Env userId: string @@ -1122,6 +1144,11 @@ export async function deleteUserAccount(input: { userId: input.mcpUserId, warnings, }) + result.clearedDurableObjects.mailboxes = await purgeMailbox({ + env: input.env, + userId: input.mcpUserId, + warnings, + }) if (input.env.BUNDLE_ARTIFACTS_KV) { const sourceSnapshotKeys = await listKvKeysByPrefix({ diff --git a/packages/worker/src/email/mailbox-client.ts b/packages/worker/src/email/mailbox-client.ts new file mode 100644 index 0000000000..a261765b0d --- /dev/null +++ b/packages/worker/src/email/mailbox-client.ts @@ -0,0 +1,28 @@ +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { type MailboxRpc } from './mailbox-do.ts' + +export type MailboxEnv = { + MAILBOX?: DurableObjectNamespace +} + +export function mailboxNamespace( + env: MailboxEnv, +): DurableObjectNamespace | null { + return env.MAILBOX ?? null +} + +/** Typed per-user Mailbox RPC stub; throws when `MAILBOX` is missing. */ +export function mailboxRpc(input: { + env: MailboxEnv + userId: string +}): MailboxRpc { + const namespace = mailboxNamespace(input.env) + if (!namespace) { + throw new Error('MAILBOX Durable Object binding is not configured.') + } + return namespace.get( + namespace.idFromName(mailboxDurableObjectName(input.userId)), + ) as unknown as MailboxRpc +} + +export type { MailboxRpc } diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts new file mode 100644 index 0000000000..cce457aca3 --- /dev/null +++ b/packages/worker/src/email/mailbox-do.ts @@ -0,0 +1,292 @@ +import * as Sentry from '@sentry/cloudflare' +import { DurableObject } from 'cloudflare:workers' +import { buildSentryOptions } from '#worker/sentry-options.ts' +import { + type EmailDeliveryEventType, + type EmailDeliveryStatus, +} from './types.ts' +import { + enforceMailboxRetention, + mailboxRetentionAlarmAtMs, + nextMailboxRetentionDueAtMs, +} from './mailbox-retention.ts' +import { MailboxStore } from './mailbox-store.ts' +import { + assertMailboxNonEmptyString, + mailboxRetentionAlarmSkewMs, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxListMessagesInput, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxSearchMessagesInput, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' + +/** + * Per-owner Mailbox Durable Object: SQLite metadata for threads, messages, + * attachments, and delivery events. Object identity is ownership — no + * `user_id` columns. Raw MIME / external attachment bytes stay in + * `EMAIL_BLOBS`; rows retain keys. `system:email` stays in D1 by design. + * + * Scaffold only: dual-write / cutover callers are not wired here. + */ + +class MailboxBase extends DurableObject { + private readonly store: MailboxStore + /** + * In-isolate cache: once an alarm is scheduled, hot writes skip + * getAlarm/setAlarm. Cleared when retention becomes idle or a write may + * need an earlier wake. + */ + private retentionAlarmArmed = false + /** + * In-isolate: no future retention work. Cleared on every data write so + * ensureRetentionAlarm re-arms for the new row's due-time. + */ + private retentionIdleConfirmed = false + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env) + this.store = new MailboxStore(ctx.storage) + this.ctx.blockConcurrencyWhile(async () => { + this.store.initializeSchema() + // Observe existing alarm only — never arm in the constructor. + this.retentionAlarmArmed = (await this.ctx.storage.getAlarm()) != null + }) + } + + /** + * Arming rule (keep this comment accurate — reviewers rely on it): + * + * - Schedule at most one alarm for the soonest retention due-time. + * - Overdue work uses an hourly retry delay (not now+1s) so persistent R2 + * failures cannot storm alarm wakes; future rows keep due-time scheduling. + * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. + * - `retentionIdleConfirmed` skips re-evaluation only while no new row has + * been written; every write clears it so the next ensure re-arms. + * - Never arm in the constructor. + */ + private async ensureRetentionAlarm() { + if (this.retentionAlarmArmed) return + if (this.retentionIdleConfirmed) return + + const dueAtMs = nextMailboxRetentionDueAtMs(this.store) + const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) + if (alarmAt == null) { + this.retentionIdleConfirmed = true + this.retentionAlarmArmed = false + return + } + + this.retentionIdleConfirmed = false + const existing = await this.ctx.storage.getAlarm() + if ( + existing != null && + Math.abs(existing - alarmAt) < mailboxRetentionAlarmSkewMs + ) { + this.retentionAlarmArmed = true + return + } + await this.ctx.storage.setAlarm(alarmAt) + this.retentionAlarmArmed = true + } + + private markRetentionDirty() { + this.retentionIdleConfirmed = false + this.retentionAlarmArmed = false + } + + async alarm(): Promise { + await enforceMailboxRetention({ + store: this.store, + blobs: this.env.EMAIL_BLOBS, + }) + const dueAtMs = nextMailboxRetentionDueAtMs(this.store) + const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) + if (alarmAt == null) { + this.retentionAlarmArmed = false + this.retentionIdleConfirmed = true + return + } + await this.ctx.storage.setAlarm(alarmAt) + this.retentionAlarmArmed = true + this.retentionIdleConfirmed = false + } + + /** Atomic mirror of thread + message + attachments for dual-write. */ + async mirrorMessage(input: { + thread?: MailboxThreadInput | null + message: MailboxMessageInput + attachments?: Array + }): Promise<{ ok: true }> { + const message = input.message + assertMailboxNonEmptyString(message.id, 'message.id') + const attachments = Array.isArray(input.attachments) + ? input.attachments + : [] + this.ctx.storage.transactionSync(() => { + if (input.thread) { + this.store.upsertThreadRow(input.thread) + } + this.store.upsertMessageRow(message) + this.store.replaceAttachmentsForMessage(message.id, attachments) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { ok: true } + } + + /** + * Upsert a delivery event (idempotent on `provider_event_id`) and optionally + * apply a monotonic latest `delivery_status` update on the message. + */ + async upsertDeliveryEvent(input: { + event: MailboxDeliveryEventInput + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null + }): Promise<{ inserted: boolean; updatedLatestStatus: boolean }> { + let inserted = false + let updatedLatestStatus = false + this.ctx.storage.transactionSync(() => { + inserted = this.store.writeDeliveryEventRow(input.event) + if (input.latestDeliveryStatus) { + const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') + const messageId = assertMailboxNonEmptyString( + input.latestDeliveryStatus.messageId, + 'latestDeliveryStatus.messageId', + ) + if (this.store.deliveryEventOwnsMessage(eventId, messageId)) { + updatedLatestStatus = this.store.updateLatestDeliveryStatus( + input.latestDeliveryStatus, + ) + } + } + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { inserted, updatedLatestStatus } + } + + async getThread(input: { + threadId: string + }): Promise { + return this.store.getThread(input.threadId) + } + + async getMessage(input: { + messageId: string + }): Promise { + return this.store.getMessage(input.messageId) + } + + async getMessageByMessageIdHeader(input: { + messageIdHeader: string + }): Promise { + return this.store.getMessageByMessageIdHeader(input.messageIdHeader) + } + + async getOutboundMessageByProviderMessageId(input: { + providerMessageId: string + }): Promise { + return this.store.getOutboundMessageByProviderMessageId( + input.providerMessageId, + ) + } + + async listMessages(input: MailboxListMessagesInput): Promise<{ + messages: Array + nextCursor: string | null + }> { + return this.store.listMessages(input) + } + + async searchMessages(input: MailboxSearchMessagesInput): Promise<{ + messages: Array + }> { + return this.store.searchMessages(input) + } + + async listAttachmentsForMessage(input: { + messageId: string + }): Promise> { + return this.store.listAttachmentsForMessage(input.messageId) + } + + async listDeliveryEvents(input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }): Promise> { + return this.store.listDeliveryEvents(input) + } + + async countMailbox(): Promise { + return this.store.countMailbox() + } + + async exportMailbox(input: { + pageSize?: number + startAfter?: string | null + }): Promise { + return this.store.exportMailbox(input) + } + + async listBlobReferences(input: { + pageSize?: number + startAfter?: string | null + }): Promise { + return this.store.listBlobReferences(input) + } + + /** + * Clear SQLite state only. During expand, D1 deletion remains authoritative + * for R2 objects. + */ + async purge(): Promise<{ ok: true }> { + await this.ctx.blockConcurrencyWhile(async () => { + await this.ctx.storage.deleteAlarm().catch(() => { + // Best effort: deleteAll below still clears persisted alarm state. + }) + await this.ctx.storage.deleteAll() + this.retentionAlarmArmed = false + this.retentionIdleConfirmed = true + this.store.initializeSchema() + }) + return { ok: true } + } +} + +export const Mailbox = Sentry.instrumentDurableObjectWithSentry( + (env: Env) => buildSentryOptions(env), + MailboxBase, +) + +export type { MailboxRpc } from './mailbox-types.ts' +export { + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionRetryDelayMs, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReference, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxExportRow, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts new file mode 100644 index 0000000000..fe6b423dd7 --- /dev/null +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -0,0 +1,660 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + Mailbox, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionRetryDelayMs, + type MailboxAttachmentInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-do.ts' + +function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { + const mailbox = (env as MailboxEnv).MAILBOX + if (!mailbox) { + throw new Error( + 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', + ) + } + return { MAILBOX: mailbox } +} + +function stubFor(userId: string) { + const { MAILBOX } = mailboxEnv() + return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) +} + +function rpcFor(userId: string) { + return mailboxRpc({ env: mailboxEnv(), userId }) +} + +function uniqueUserId(label: string) { + return `mailbox-${label}-${crypto.randomUUID()}` +} + +async function assertMailboxThrows( + pattern: RegExp, + run: () => Promise, +) { + try { + await run() + throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) + } catch (error) { + expect(String(error)).toMatch(pattern) + } +} + +function baseThread( + overrides?: Partial, +): MailboxThreadInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: ``, + lastMessageAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +function baseMessage( + overrides?: Partial, +): MailboxMessageInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + direction: 'inbound', + inboxId: 'inbox-1', + threadId: null, + fromAddress: 'sender@example.com', + envelopeFrom: 'envelope@example.com', + toAddresses: ['owner@example.com'], + subject: 'Hello mailbox', + messageIdHeader: ``, + rawMimeKey: `email-raw:v1:owner/${id}`, + rawSize: 128, + processingStatus: 'stored', + classification: 'accepted', + receivedAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +function baseAttachment( + messageId: string, + overrides?: Partial, +): MailboxAttachmentInput { + const id = overrides?.id ?? crypto.randomUUID() + return { + id, + messageId, + filename: 'note.txt', + contentType: 'text/plain', + size: 12, + storageKind: 'external', + storageKey: `email-attachment:v1:owner/${messageId}/${id}`, + createdAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { + silenceIncidentalRuntimeWarnings() + const ownerA = uniqueUserId('a') + const ownerB = uniqueUserId('b') + const mailboxA = rpcFor(ownerA) + const mailboxB = rpcFor(ownerB) + + const stubA = stubFor(ownerA) + await runInDurableObject(stubA, async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + const tables = state.storage.sql + .exec<{ name: string }>( + `SELECT name FROM sqlite_master + WHERE type = 'table' + AND name IN ( + 'email_threads', 'email_messages', + 'email_attachments', 'email_delivery_events' + ) + ORDER BY name ASC`, + ) + .toArray() + .map((row) => row.name) + expect(tables).toEqual([ + 'email_attachments', + 'email_delivery_events', + 'email_messages', + 'email_threads', + ]) + expect(await state.storage.getAlarm()).toBeNull() + }) + + const thread = baseThread({ id: 'thread-1' }) + const message = baseMessage({ + id: 'msg-1', + threadId: thread.id, + subject: 'Project update', + fromAddress: 'alice@example.com', + }) + const attachment = baseAttachment(message.id, { id: 'att-1' }) + + await mailboxA.mirrorMessage({ + thread, + message, + attachments: [attachment], + }) + await mailboxA.mirrorMessage({ + thread, + message: { ...message, subject: 'Project update (edited)' }, + attachments: [attachment], + }) + + const stored = await mailboxA.getMessage({ messageId: message.id }) + expect(stored).toMatchObject({ + id: message.id, + threadId: thread.id, + subject: 'Project update (edited)', + fromAddress: 'alice@example.com', + rawMimeKey: message.rawMimeKey, + }) + expect(await mailboxA.getThread({ threadId: thread.id })).toMatchObject({ + id: thread.id, + subjectNormalized: 'hello', + }) + expect( + await mailboxA.listAttachmentsForMessage({ messageId: message.id }), + ).toEqual([ + expect.objectContaining({ + id: attachment.id, + storageKey: attachment.storageKey, + storageKind: 'external', + }), + ]) + expect( + await mailboxA.getMessageByMessageIdHeader({ + messageIdHeader: message.messageIdHeader!, + }), + ).toMatchObject({ id: message.id }) + + const listed = await mailboxA.listMessages({ limit: 10 }) + expect(listed.messages.map((row) => row.id)).toEqual([message.id]) + const searched = await mailboxA.searchMessages({ query: 'project' }) + expect(searched.messages.map((row) => row.id)).toEqual([message.id]) + expect( + (await mailboxA.searchMessages({ query: 'nope' })).messages, + ).toHaveLength(0) + + await mailboxB.mirrorMessage({ + thread: baseThread({ id: 'thread-b' }), + message: baseMessage({ + id: 'msg-b', + threadId: 'thread-b', + subject: 'Other owner', + }), + }) + expect(await mailboxB.getMessage({ messageId: message.id })).toBeNull() + expect(await mailboxA.getMessage({ messageId: 'msg-b' })).toBeNull() + expect(await mailboxA.countMailbox()).toMatchObject({ + threads: 1, + messages: 1, + attachments: 1, + deliveryEvents: 0, + }) + expect(await mailboxB.countMailbox()).toMatchObject({ + threads: 1, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + + const outboundId = 'msg-out-1' + await mailboxA.mirrorMessage({ + message: baseMessage({ + id: outboundId, + direction: 'outbound', + subject: 'Sent mail', + providerMessageId: 'provider-1', + processingStatus: 'sent', + rawMimeKey: null, + }), + }) + expect( + await mailboxA.getOutboundMessageByProviderMessageId({ + providerMessageId: 'provider-1', + }), + ).toMatchObject({ id: outboundId }) +}) + +test('Mailbox delivery status, promoted inbound fields, export paging, and cursor rejection', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('export') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'export-thread' }) + const message = baseMessage({ + id: 'export-msg', + threadId: thread.id, + direction: 'outbound', + providerMessageId: 'prov-export', + processingStatus: 'sent', + rawMimeKey: 'email-raw:v1:owner/export-msg', + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + const attachment = baseAttachment(message.id, { + id: 'export-att', + storageKey: 'email-attachment:v1:owner/export-msg/export-att', + }) + await mailbox.mirrorMessage({ + thread, + message, + attachments: [attachment], + }) + + // Stale dual-write replay must not regress a newer delivery status. + await mailbox.mirrorMessage({ + message: { + ...message, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + }, + attachments: [attachment], + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + // Equal timestamps may update (matches D1 <= semantics). + await mailbox.mirrorMessage({ + message: { + ...message, + deliveryStatus: 'complained', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }, + attachments: [attachment], + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'complained', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + const first = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-1', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }, + }) + expect(first).toEqual({ inserted: true, updatedLatestStatus: true }) + + const inbound = await mailbox.upsertDeliveryEvent({ + event: { + id: 'inbound-delivery-1', + messageId: 'inbound-msg-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'cloudflare-email-routing', + createdAt: '2026-07-02T11:00:00.000Z', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-abc', + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + expectedAttachmentCount: 2, + finalizationToken: 'lease-token-1', + usageStartedAt: '2026-07-02T10:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + subscriptionEffectState: 'pending', + subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + detailJson: JSON.stringify({ recipient: 'owner@example.com' }), + }, + }) + expect(inbound.inserted).toBe(true) + const inboundRow = ( + await mailbox.listDeliveryEvents({ + messageId: 'inbound-msg-1', + limit: 1, + }) + )[0] + expect(inboundRow).toMatchObject({ + id: 'inbound-delivery-1', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-abc', + expectedAttachmentCount: 2, + finalizationToken: 'lease-token-1', + usageStartedAt: '2026-07-02T10:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + subscriptionEffectState: 'pending', + subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + }) + expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ + recipient: 'owner@example.com', + }) + + // Default needsEffectReconcile is false unless explicitly true. + await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-default-reconcile', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + createdAt: '2026-07-02T11:30:00.000Z', + }, + }) + const defaultReconcile = ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) + ).find((event) => event.id === 'evt-default-reconcile') + expect(defaultReconcile?.needsEffectReconcile).toBe(false) + + const duplicate = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-1-dup', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T11:00:00.000Z', + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T11:00:00.000Z', + }, + }) + expect(duplicate.inserted).toBe(false) + + const stale = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-2', + messageId: message.id, + eventType: 'deferred', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-2', + createdAt: '2026-07-02T09:00:00.000Z', + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + }, + }) + expect(stale).toEqual({ inserted: true, updatedLatestStatus: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }) + + // Assert validation failures inside the DO isolate so remote RPC error + // plumbing does not surface uncaught worker exceptions. + const stub = stubFor(userId) + await runInDurableObject(stub, async (instance: Mailbox) => { + await assertMailboxThrows(/export cursor is invalid/, () => + instance.exportMailbox({ pageSize: 1, startAfter: 'not-a-cursor' }), + ) + await assertMailboxThrows(/list cursor is invalid/, () => + instance.listMessages({ limit: 1, cursor: '%%%' }), + ) + await assertMailboxThrows(/blob-reference cursor is invalid/, () => + instance.listBlobReferences({ pageSize: 1, startAfter: 'bad' }), + ) + await assertMailboxThrows(/canonical ISO-8601/, () => + instance.mirrorMessage({ + message: baseMessage({ + id: 'bad-iso', + createdAt: '2026-07-01T12:00:00Z', + }), + }), + ) + }) + + const kinds: Array = [] + let startAfter: string | null = null + for (let page = 0; page < 30; page += 1) { + const exported = await mailbox.exportMailbox({ + pageSize: 1, + startAfter, + }) + for (const row of exported.rows) { + kinds.push(row.kind) + switch (row.kind) { + case 'thread': + expect(row.row.id).toBe(thread.id) + break + case 'message': + expect(row.row.id).toBe(message.id) + break + case 'attachment': + expect(row.row.id).toBe(attachment.id) + break + case 'delivery_event': + expect(typeof row.row.id).toBe('string') + break + default: { + const exhaustive: never = row + throw new Error(`Unhandled export kind: ${String(exhaustive)}`) + } + } + } + if (!exported.truncated) break + expect(exported.nextStartAfter).not.toBe(startAfter) + startAfter = exported.nextStartAfter + } + expect(kinds.filter((kind) => kind === 'thread')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'message')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(1) + expect( + kinds.filter((kind) => kind === 'delivery_event').length, + ).toBeGreaterThanOrEqual(3) + + const blobKinds: Array = [] + let blobCursor: string | null = null + for (let page = 0; page < 10; page += 1) { + const pageResult = await mailbox.listBlobReferences({ + pageSize: 1, + startAfter: blobCursor, + }) + for (const reference of pageResult.references) { + blobKinds.push(reference.kind) + if (reference.kind === 'raw_mime') { + expect(reference.key).toBe(message.rawMimeKey) + expect(reference.messageId).toBe(message.id) + } else { + expect(reference.key).toBe(attachment.storageKey) + expect(reference.attachmentId).toBe(attachment.id) + } + } + if (!pageResult.truncated) break + expect(pageResult.nextStartAfter).not.toBe(blobCursor) + blobCursor = pageResult.nextStartAfter + } + expect(blobKinds).toEqual(['raw_mime', 'attachment']) + + await mailbox.purge() + expect(await mailbox.countMailbox()).toEqual({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) +}) + +test('Mailbox retention deletes R2 blobs before metadata and backs off on failure', async () => { + silenceIncidentalRuntimeWarnings() + consoleWarn.mockImplementation((...args: Array) => { + const message = String(args[0] ?? '') + if (message.includes('mailbox-retention-blob-delete-failed')) return + }) + + const userId = uniqueUserId('retention') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const oldMessageAt = new Date( + Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const oldEventAt = new Date( + Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const freshAt = new Date().toISOString() + + const keepMessage = baseMessage({ + id: 'keep-msg', + subject: 'fresh', + createdAt: freshAt, + updatedAt: freshAt, + rawMimeKey: 'email-raw:v1:owner/keep-msg', + }) + const dropMessage = baseMessage({ + id: 'drop-msg', + threadId: 'drop-thread', + subject: 'old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + rawMimeKey: 'email-raw:v1:owner/drop-msg', + }) + const failMessage = baseMessage({ + id: 'fail-msg', + subject: 'fail-old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + rawMimeKey: 'email-raw:v1:owner/fail-msg', + }) + const dropAttachment = baseAttachment(dropMessage.id, { + id: 'drop-att', + storageKey: 'email-attachment:v1:owner/drop-msg/drop-att', + createdAt: oldMessageAt, + }) + + await mailbox.mirrorMessage({ + thread: baseThread({ + id: 'drop-thread', + lastMessageAt: oldMessageAt, + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }), + message: dropMessage, + attachments: [dropAttachment], + }) + await mailbox.mirrorMessage({ message: keepMessage }) + await mailbox.mirrorMessage({ message: failMessage }) + await mailbox.upsertDeliveryEvent({ + event: { + id: 'old-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: oldEventAt, + needsEffectReconcile: false, + }, + }) + await mailbox.upsertDeliveryEvent({ + event: { + id: 'fresh-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: freshAt, + needsEffectReconcile: false, + }, + }) + + await env.EMAIL_BLOBS.put(dropMessage.rawMimeKey!, 'drop-raw') + await env.EMAIL_BLOBS.put(dropAttachment.storageKey!, 'drop-att') + await env.EMAIL_BLOBS.put(failMessage.rawMimeKey!, 'fail-raw') + await env.EMAIL_BLOBS.put(keepMessage.rawMimeKey!, 'keep-raw') + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { + const list = Array.isArray(keys) ? keys : [keys] + if (list.includes(failMessage.rawMimeKey!)) { + throw new Error('simulated R2 delete failure') + } + return await originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const alarmBefore = Date.now() + const alarmAfterFailure = await runInDurableObject( + stub, + async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + await state.storage.deleteAlarm() + await instance.alarm() + return await state.storage.getAlarm() + }, + ) + // Overdue retained work (failed R2 delete) must retry with hourly backoff, + // not every second. + expect(alarmAfterFailure).toBeTypeOf('number') + expect(alarmAfterFailure).toBeGreaterThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs - 5_000, + ) + expect(alarmAfterFailure).toBeLessThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs + 5_000, + ) + + expect(await env.EMAIL_BLOBS.get(dropMessage.rawMimeKey!)).toBeNull() + expect(await env.EMAIL_BLOBS.get(dropAttachment.storageKey!)).toBeNull() + expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(keepMessage.rawMimeKey!)).not.toBeNull() + + expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), + ).toHaveLength(0) + expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() + expect( + await mailbox.getMessage({ messageId: failMessage.id }), + ).toMatchObject({ + id: failMessage.id, + rawMimeKey: failMessage.rawMimeKey, + }) + expect( + await mailbox.getMessage({ messageId: keepMessage.id }), + ).toMatchObject({ + id: keepMessage.id, + }) + const events = await mailbox.listDeliveryEvents({ limit: 10 }) + expect(events.map((event) => event.id)).toEqual(['fresh-event']) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + } + + await runInDurableObject(stub, async (instance: Mailbox) => { + await instance.alarm() + }) + expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).toBeNull() + expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-export.ts b/packages/worker/src/email/mailbox-export.ts new file mode 100644 index 0000000000..e2202c6999 --- /dev/null +++ b/packages/worker/src/email/mailbox-export.ts @@ -0,0 +1,293 @@ +import { + mailboxBlobRefAttachmentCursorPrefix, + mailboxBlobRefRawMimeCursorPrefix, + mailboxExportCursorPrefixForPhase, + normalizeMailboxPageSize, + parseMailboxBlobRefCursor, + parseMailboxExportCursor, + type MailboxBlobReference, + type MailboxBlobReferencePage, + type MailboxExportPhase, + type MailboxExportResult, + type MailboxExportRow, +} from './mailbox-types.ts' +import { + mapMailboxAttachmentRow, + mapMailboxDeliveryEventRow, + mapMailboxMessageRow, + mapMailboxThreadRow, +} from './mailbox-mappers.ts' + +export function exportMailboxFromStore( + sql: SqlStorage, + input: { + pageSize?: number + startAfter?: string | null + }, +): MailboxExportResult { + const pageSize = normalizeMailboxPageSize(input.pageSize) + const startAfterRaw = + typeof input.startAfter === 'string' && input.startAfter.length > 0 + ? input.startAfter + : null + const phaseOrder: Array = [ + 'thread', + 'message', + 'attachment', + 'delivery_event', + ] + const result: MailboxExportResult = { + rows: [], + nextStartAfter: null, + truncated: false, + } + let { phase, startAfterId } = parseMailboxExportCursor(startAfterRaw) + let remaining = pageSize + let phaseIndex = phaseOrder.indexOf(phase) + const maxIterations = phaseOrder.length + 1 + let iterations = 0 + + while (phaseIndex < phaseOrder.length) { + iterations += 1 + if (iterations > maxIterations) { + throw new Error('exportMailbox exceeded phase iteration budget') + } + if (remaining <= 0) { + const nextPhase = phaseOrder[phaseIndex]! + const handoff = mailboxExportCursorPrefixForPhase(nextPhase) + if (handoff === startAfterRaw) { + phaseIndex += 1 + startAfterId = '' + continue + } + result.nextStartAfter = handoff + result.truncated = true + return result + } + + const currentPhase = phaseOrder[phaseIndex]! + const page = exportPhasePage(sql, { + phase: currentPhase, + startAfterId, + limit: remaining, + }) + result.rows.push(...page.rows) + remaining -= page.taken + + if (page.truncated) { + if (page.taken <= 0) { + phaseIndex += 1 + startAfterId = '' + continue + } + const nextStartAfter = `${mailboxExportCursorPrefixForPhase(currentPhase)}${page.nextId}` + if (nextStartAfter === startAfterRaw) { + phaseIndex += 1 + startAfterId = '' + continue + } + result.nextStartAfter = nextStartAfter + result.truncated = true + return result + } + + phaseIndex += 1 + startAfterId = '' + if (remaining <= 0 && phaseIndex < phaseOrder.length) { + const nextPhase = phaseOrder[phaseIndex]! + const handoff = mailboxExportCursorPrefixForPhase(nextPhase) + if (handoff === startAfterRaw) continue + result.nextStartAfter = handoff + result.truncated = true + return result + } + } + return result +} + +function exportPhasePage( + sql: SqlStorage, + input: { + phase: MailboxExportPhase + startAfterId: string + limit: number + }, +): { + rows: Array + taken: number + truncated: boolean + nextId: string +} { + if (input.limit <= 0) { + return { + rows: [], + taken: 0, + truncated: false, + nextId: input.startAfterId, + } + } + const limit = Math.trunc(input.limit) + const tableForPhase = (phase: MailboxExportPhase) => { + switch (phase) { + case 'thread': + return 'email_threads' + case 'message': + return 'email_messages' + case 'attachment': + return 'email_attachments' + case 'delivery_event': + return 'email_delivery_events' + default: { + const exhaustive: never = phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } + } + const rows = sql + .exec>( + `SELECT * FROM ${tableForPhase(input.phase)} + WHERE id > ? + ORDER BY id ASC + LIMIT ?`, + input.startAfterId, + limit + 1, + ) + .toArray() + const truncated = rows.length > limit + const pageRows = truncated ? rows.slice(0, limit) : rows + const last = pageRows[pageRows.length - 1] + const mapped: Array = pageRows.map((row) => { + switch (input.phase) { + case 'thread': + return { kind: 'thread', row: mapMailboxThreadRow(row) } + case 'message': + return { kind: 'message', row: mapMailboxMessageRow(row) } + case 'attachment': + return { kind: 'attachment', row: mapMailboxAttachmentRow(row) } + case 'delivery_event': + return { + kind: 'delivery_event', + row: mapMailboxDeliveryEventRow(row), + } + default: { + const exhaustive: never = input.phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } + }) + return { + rows: mapped, + taken: pageRows.length, + truncated, + nextId: last ? String(last['id']) : input.startAfterId, + } +} + +export function listMailboxBlobReferences( + sql: SqlStorage, + input: { + pageSize?: number + startAfter?: string | null + }, +): MailboxBlobReferencePage { + const pageSize = normalizeMailboxPageSize(input.pageSize) + const startAfterRaw = + typeof input.startAfter === 'string' && input.startAfter.length > 0 + ? input.startAfter + : null + const { phase, startAfterId } = parseMailboxBlobRefCursor(startAfterRaw) + const references: Array = [] + let remaining = pageSize + + if (phase === 'raw_mime') { + const rows = sql + .exec<{ id: string; raw_mime_key: string }>( + `SELECT id, raw_mime_key FROM email_messages + WHERE raw_mime_key IS NOT NULL AND id > ? + ORDER BY id ASC + LIMIT ?`, + startAfterId, + remaining + 1, + ) + .toArray() + const truncated = rows.length > remaining + const pageRows = truncated ? rows.slice(0, remaining) : rows + for (const row of pageRows) { + references.push({ + kind: 'raw_mime', + key: row.raw_mime_key, + messageId: row.id, + attachmentId: null, + }) + } + if (truncated) { + const last = pageRows[pageRows.length - 1]! + return { + references, + nextStartAfter: `${mailboxBlobRefRawMimeCursorPrefix}${last.id}`, + truncated: true, + } + } + remaining -= pageRows.length + if (remaining <= 0) { + return { + references, + nextStartAfter: mailboxBlobRefAttachmentCursorPrefix, + truncated: true, + } + } + const attachmentPage = listAttachmentBlobReferences(sql, { + startAfterId: '', + limit: remaining, + }) + references.push(...attachmentPage.references) + return { + references, + nextStartAfter: attachmentPage.nextStartAfter, + truncated: attachmentPage.truncated, + } + } + + return listAttachmentBlobReferences(sql, { + startAfterId, + limit: remaining, + }) +} + +function listAttachmentBlobReferences( + sql: SqlStorage, + input: { + startAfterId: string + limit: number + }, +): MailboxBlobReferencePage { + if (input.limit <= 0) { + return { references: [], nextStartAfter: null, truncated: false } + } + const rows = sql + .exec<{ id: string; message_id: string; storage_key: string }>( + `SELECT id, message_id, storage_key FROM email_attachments + WHERE storage_key IS NOT NULL AND id > ? + ORDER BY id ASC + LIMIT ?`, + input.startAfterId, + input.limit + 1, + ) + .toArray() + const truncated = rows.length > input.limit + const pageRows = truncated ? rows.slice(0, input.limit) : rows + const last = pageRows[pageRows.length - 1] + return { + references: pageRows.map((row) => ({ + kind: 'attachment' as const, + key: row.storage_key, + messageId: row.message_id, + attachmentId: row.id, + })), + nextStartAfter: + truncated && last + ? `${mailboxBlobRefAttachmentCursorPrefix}${last.id}` + : null, + truncated, + } +} diff --git a/packages/worker/src/email/mailbox-mappers.ts b/packages/worker/src/email/mailbox-mappers.ts new file mode 100644 index 0000000000..9934469b5c --- /dev/null +++ b/packages/worker/src/email/mailbox-mappers.ts @@ -0,0 +1,319 @@ +import { + maxRestorableTextColumnBytes, + truncateToUtf8Bytes, + utf8ByteLength, +} from '@kody-internal/shared/backup-restore-safety.ts' +import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' +import { + emailClassificationValues, + emailDeliveryEventTypeValues, + emailDeliveryStatusValues, + emailDirectionValues, + emailProcessingStatusValues, + type EmailClassification, + type EmailDeliveryEventType, + type EmailDeliveryStatus, + type EmailDirection, + type EmailProcessingStatus, +} from './types.ts' +import { + mailboxInboundDeliveryStateValues, + mailboxSubscriptionEffectStateValues, + type MailboxAttachmentRecord, + type MailboxDeliveryEventRecord, + type MailboxInboundDeliveryState, + type MailboxMessageRecord, + type MailboxSubscriptionEffectState, + type MailboxThreadRecord, +} from './mailbox-types.ts' + +const emailBodyTruncationNotice = + '\n[truncated for backup-safe storage; the full message is retained in the raw MIME object]' + +export function boundedEmailBody( + body: string | null | undefined, +): string | null { + if (body == null) return null + if (utf8ByteLength(body) <= maxRestorableTextColumnBytes) return body + return ( + truncateToUtf8Bytes( + body, + maxRestorableTextColumnBytes - utf8ByteLength(emailBodyTruncationNotice), + ) + emailBodyTruncationNotice + ) +} + +function parseOptionalJsonRecord(value: string | null) { + if (!value) return null + try { + const parsed = JSON.parse(value) as unknown + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return null + } + return parsed as Record + } catch { + return null + } +} + +export function escapeLikePattern(value: string) { + return value.replaceAll(/[\\%_]/g, (character) => `\\${character}`) +} + +function mapInboundState(value: unknown): MailboxInboundDeliveryState | null { + if (value == null) return null + const state = String(value) + return (mailboxInboundDeliveryStateValues as ReadonlyArray).includes( + state, + ) + ? (state as MailboxInboundDeliveryState) + : null +} + +function mapSubscriptionEffectState( + value: unknown, +): MailboxSubscriptionEffectState | null { + if (value == null) return null + const state = String(value) + return ( + mailboxSubscriptionEffectStateValues as ReadonlyArray + ).includes(state) + ? (state as MailboxSubscriptionEffectState) + : null +} + +export function mapMailboxThreadRow( + row: Record, +): MailboxThreadRecord { + return { + id: String(row['id']), + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + subjectNormalized: + row['subject_normalized'] == null + ? null + : String(row['subject_normalized']), + rootMessageIdHeader: + row['root_message_id_header'] == null + ? null + : String(row['root_message_id_header']), + lastMessageAt: String(row['last_message_at']), + createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), + } +} + +export function mapMailboxMessageRow( + row: Record, +): MailboxMessageRecord { + const direction = String(row['direction'] ?? '') + const processingStatus = String(row['processing_status'] ?? '') + const classificationRaw = String(row['classification'] ?? 'accepted') + const deliveryStatusRaw = + row['delivery_status'] == null ? null : String(row['delivery_status']) + return { + id: String(row['id']), + direction: (emailDirectionValues as ReadonlyArray).includes( + direction, + ) + ? (direction as EmailDirection) + : 'inbound', + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + threadId: row['thread_id'] == null ? null : String(row['thread_id']), + senderIdentityId: + row['sender_identity_id'] == null + ? null + : String(row['sender_identity_id']), + fromAddress: + row['from_address'] == null ? null : String(row['from_address']), + envelopeFrom: + row['envelope_from'] == null ? null : String(row['envelope_from']), + toAddresses: parseJsonArray(String(row['to_addresses_json'] ?? '[]')), + ccAddresses: parseJsonArray(String(row['cc_addresses_json'] ?? '[]')), + bccAddresses: parseJsonArray(String(row['bcc_addresses_json'] ?? '[]')), + replyToAddresses: parseJsonArray( + String(row['reply_to_addresses_json'] ?? '[]'), + ), + subject: row['subject'] == null ? null : String(row['subject']), + messageIdHeader: + row['message_id_header'] == null + ? null + : String(row['message_id_header']), + inReplyToHeader: + row['in_reply_to_header'] == null + ? null + : String(row['in_reply_to_header']), + references: parseJsonArray(String(row['references_json'] ?? '[]')), + headers: parseOptionalJsonRecord( + row['headers_json'] == null ? null : String(row['headers_json']), + ), + authResults: + row['auth_results'] == null ? null : String(row['auth_results']), + textBody: row['text_body'] == null ? null : String(row['text_body']), + htmlBody: row['html_body'] == null ? null : String(row['html_body']), + rawMimeKey: + row['raw_mime_key'] == null ? null : String(row['raw_mime_key']), + rawSize: row['raw_size'] == null ? null : Number(row['raw_size']) || 0, + processingStatus: ( + emailProcessingStatusValues as ReadonlyArray + ).includes(processingStatus) + ? (processingStatus as EmailProcessingStatus) + : 'stored', + classification: ( + emailClassificationValues as ReadonlyArray + ).includes(classificationRaw) + ? (classificationRaw as EmailClassification) + : 'accepted', + classificationReason: + row['classification_reason'] == null + ? null + : String(row['classification_reason']), + providerMessageId: + row['provider_message_id'] == null + ? null + : String(row['provider_message_id']), + deliveryStatus: + deliveryStatusRaw != null && + (emailDeliveryStatusValues as ReadonlyArray).includes( + deliveryStatusRaw, + ) + ? (deliveryStatusRaw as EmailDeliveryStatus) + : null, + deliveryStatusAt: + row['delivery_status_at'] == null + ? null + : String(row['delivery_status_at']), + error: row['error'] == null ? null : String(row['error']), + receivedAt: row['received_at'] == null ? null : String(row['received_at']), + sentAt: row['sent_at'] == null ? null : String(row['sent_at']), + createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), + } +} + +export function mapMailboxAttachmentRow( + row: Record, +): MailboxAttachmentRecord { + return { + id: String(row['id']), + messageId: String(row['message_id']), + filename: row['filename'] == null ? null : String(row['filename']), + contentType: + row['content_type'] == null ? null : String(row['content_type']), + contentId: row['content_id'] == null ? null : String(row['content_id']), + disposition: row['disposition'] == null ? null : String(row['disposition']), + size: Number(row['size'] ?? 0) || 0, + storageKind: String(row['storage_kind']), + storageKey: row['storage_key'] == null ? null : String(row['storage_key']), + createdAt: String(row['created_at']), + } +} + +export function mapMailboxDeliveryEventRow( + row: Record, +): MailboxDeliveryEventRecord { + const eventType = String(row['event_type'] ?? '') + return { + id: String(row['id']), + messageId: row['message_id'] == null ? null : String(row['message_id']), + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + eventType: (emailDeliveryEventTypeValues as ReadonlyArray).includes( + eventType, + ) + ? (eventType as EmailDeliveryEventType) + : 'received', + provider: row['provider'] == null ? null : String(row['provider']), + providerMessageId: + row['provider_message_id'] == null + ? null + : String(row['provider_message_id']), + providerEventId: + row['provider_event_id'] == null + ? null + : String(row['provider_event_id']), + detailJson: String(row['detail_json'] ?? '{}'), + needsEffectReconcile: Number(row['needs_effect_reconcile'] ?? 0) === 1, + state: mapInboundState(row['state']), + fingerprint: row['fingerprint'] == null ? null : String(row['fingerprint']), + storageLease: + row['storage_lease'] == null ? null : String(row['storage_lease']), + storageLeaseAt: + row['storage_lease_at'] == null ? null : String(row['storage_lease_at']), + cleanupLease: + row['cleanup_lease'] == null ? null : String(row['cleanup_lease']), + cleanupLeaseAt: + row['cleanup_lease_at'] == null ? null : String(row['cleanup_lease_at']), + cleanupRetryAt: + row['cleanup_retry_at'] == null ? null : String(row['cleanup_retry_at']), + expectedAttachmentCount: + row['expected_attachment_count'] == null + ? null + : Number(row['expected_attachment_count']) || 0, + finalizationToken: + row['finalization_token'] == null + ? null + : String(row['finalization_token']), + reconcileAfter: + row['reconcile_after'] == null ? null : String(row['reconcile_after']), + dedupeExpiresAt: + row['dedupe_expires_at'] == null + ? null + : String(row['dedupe_expires_at']), + usageEffectRecordedAt: + row['usage_effect_recorded_at'] == null + ? null + : String(row['usage_effect_recorded_at']), + usageEffectSuppressedAt: + row['usage_effect_suppressed_at'] == null + ? null + : String(row['usage_effect_suppressed_at']), + usageStartedAt: + row['usage_started_at'] == null ? null : String(row['usage_started_at']), + usageMonth: row['usage_month'] == null ? null : String(row['usage_month']), + usageBytes: + row['usage_bytes'] == null ? null : Number(row['usage_bytes']) || 0, + usageDurationMs: + row['usage_duration_ms'] == null + ? null + : Number(row['usage_duration_ms']) || 0, + usageEffectRetryAt: + row['usage_effect_retry_at'] == null + ? null + : String(row['usage_effect_retry_at']), + usageEffectLease: + row['usage_effect_lease'] == null + ? null + : String(row['usage_effect_lease']), + usageEffectLeaseAt: + row['usage_effect_lease_at'] == null + ? null + : String(row['usage_effect_lease_at']), + subscriptionEffectState: mapSubscriptionEffectState( + row['subscription_effect_state'], + ), + subscriptionEffectLease: + row['subscription_effect_lease'] == null + ? null + : String(row['subscription_effect_lease']), + subscriptionEffectLeaseAt: + row['subscription_effect_lease_at'] == null + ? null + : String(row['subscription_effect_lease_at']), + subscriptionEffectRetryAt: + row['subscription_effect_retry_at'] == null + ? null + : String(row['subscription_effect_retry_at']), + subscriptionEffectAttemptCount: + row['subscription_effect_attempt_count'] == null + ? null + : Number(row['subscription_effect_attempt_count']) || 0, + subscriptionEffectDeadLetterAt: + row['subscription_effect_dead_letter_at'] == null + ? null + : String(row['subscription_effect_dead_letter_at']), + subscriptionEffectLastError: + row['subscription_effect_last_error'] == null + ? null + : String(row['subscription_effect_last_error']), + createdAt: String(row['created_at']), + } +} diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts new file mode 100644 index 0000000000..2a86713797 --- /dev/null +++ b/packages/worker/src/email/mailbox-retention.ts @@ -0,0 +1,137 @@ +import { + mailboxBlobDeleteMaxKeys, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionBatchSize, + mailboxRetentionRetryDelayMs, +} from './mailbox-types.ts' +import { type MailboxStore } from './mailbox-store.ts' + +const messageRetentionMs = mailboxMessageRetentionDays * 24 * 60 * 60 * 1000 +const deliveryEventRetentionMs = + mailboxDeliveryEventRetentionDays * 24 * 60 * 60 * 1000 + +export type MailboxRetentionPassResult = { + /** True when at least one message blob delete failed (row retained). */ + hadBlobDeleteFailures: boolean +} + +/** + * Soonest retention due-time from stored rows, or null when empty. + * Overdue rows yield timestamps in the past — callers must apply retry backoff + * instead of scheduling at now+1s. + */ +export function nextMailboxRetentionDueAtMs( + store: MailboxStore, +): number | null { + let next: number | null = null + const consider = (at: number) => { + if (!Number.isFinite(at)) return + if (next == null || at < next) next = at + } + + const oldestMessage = store.oldestMessageCreatedAt() + if (oldestMessage) { + consider(Date.parse(oldestMessage) + messageRetentionMs) + } + const oldestEvent = store.oldestDeliveryEventCreatedAt() + if (oldestEvent) { + consider(Date.parse(oldestEvent) + deliveryEventRetentionMs) + } + return next +} + +/** + * Schedule time for the next retention alarm. + * - Future due-time → schedule at due-time + * - Overdue work → bounded hourly backoff (avoids 1s storms on persistent R2 failure) + */ +export function mailboxRetentionAlarmAtMs(input: { + dueAtMs: number | null + nowMs?: number +}): number | null { + if (input.dueAtMs == null) return null + const nowMs = input.nowMs ?? Date.now() + if (input.dueAtMs <= nowMs) { + return nowMs + mailboxRetentionRetryDelayMs + } + return input.dueAtMs +} + +async function deleteBlobKeys( + blobs: Pick, + keys: Array, +): Promise { + if (keys.length === 0) return true + try { + for ( + let index = 0; + index < keys.length; + index += mailboxBlobDeleteMaxKeys + ) { + const chunk = keys.slice(index, index + mailboxBlobDeleteMaxKeys) + await blobs.delete(chunk) + } + return true + } catch (error) { + console.warn('mailbox-retention-blob-delete-failed', { error }) + return false + } +} + +async function pruneExpiredMessages(input: { + store: MailboxStore + blobs: Pick +}): Promise { + const cutoff = new Date(Date.now() - messageRetentionMs).toISOString() + const rows = input.store.listExpiredMessagesForRetention({ + cutoff, + limit: mailboxRetentionBatchSize, + }) + if (rows.length === 0) return false + + const attachmentRows = input.store.listAttachmentStorageKeysForMessages( + rows.map((row) => row.id), + ) + const attachmentKeysByMessageId = new Map>() + for (const attachment of attachmentRows) { + const keys = attachmentKeysByMessageId.get(attachment.message_id) + if (keys) keys.push(attachment.storage_key) + else { + attachmentKeysByMessageId.set(attachment.message_id, [ + attachment.storage_key, + ]) + } + } + + let hadBlobDeleteFailures = false + for (const row of rows) { + const keys = [ + ...(row.raw_mime_key ? [row.raw_mime_key] : []), + ...(attachmentKeysByMessageId.get(row.id) ?? []), + ] + const blobsDeleted = await deleteBlobKeys(input.blobs, keys) + if (!blobsDeleted) { + hadBlobDeleteFailures = true + continue + } + input.store.deleteMessageCascade(row.id) + } + return hadBlobDeleteFailures +} + +export async function enforceMailboxRetention(input: { + store: MailboxStore + blobs: Pick +}): Promise { + const hadBlobDeleteFailures = await pruneExpiredMessages(input) + const eventCutoff = new Date( + Date.now() - deliveryEventRetentionMs, + ).toISOString() + input.store.pruneExpiredDeliveryEvents({ + cutoff: eventCutoff, + limit: mailboxRetentionBatchSize, + }) + input.store.pruneOrphanThreads(mailboxRetentionBatchSize) + return { hadBlobDeleteFailures } +} diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts new file mode 100644 index 0000000000..ef8bcdda5d --- /dev/null +++ b/packages/worker/src/email/mailbox-schema.ts @@ -0,0 +1,328 @@ +import { + mailboxMetaSchemaVersionKey, + mailboxSchemaVersion, +} from './mailbox-types.ts' + +export function getMailboxMeta( + storage: DurableObjectStorage, + key: string, +): number | null { + const row = storage.sql + .exec<{ value: number }>( + `SELECT value FROM mailbox_meta WHERE key = ? LIMIT 1`, + key, + ) + .toArray()[0] + return row == null ? null : Number(row.value) || 0 +} + +export function setMailboxMeta( + storage: DurableObjectStorage, + key: string, + value: number, +) { + storage.sql.exec( + `INSERT INTO mailbox_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + key, + value, + ) +} + +/** Warm v1 objects lack promoted inbound columns. */ +function ensureDeliveryEventPromotedColumns( + sql: SqlStorage, + installedVersion: number | null, +) { + if (installedVersion == null || installedVersion >= mailboxSchemaVersion) { + return + } + const columns: Array = [ + `state TEXT`, + `fingerprint TEXT`, + `storage_lease TEXT`, + `storage_lease_at TEXT`, + `cleanup_lease TEXT`, + `cleanup_lease_at TEXT`, + `cleanup_retry_at TEXT`, + `expected_attachment_count INTEGER`, + `finalization_token TEXT`, + `reconcile_after TEXT`, + `dedupe_expires_at TEXT`, + `usage_effect_suppressed_at TEXT`, + `usage_started_at TEXT`, + `usage_effect_retry_at TEXT`, + `usage_effect_lease TEXT`, + `usage_effect_lease_at TEXT`, + `subscription_effect_state TEXT`, + `subscription_effect_lease TEXT`, + `subscription_effect_lease_at TEXT`, + `subscription_effect_retry_at TEXT`, + `subscription_effect_attempt_count INTEGER`, + `subscription_effect_dead_letter_at TEXT`, + `subscription_effect_last_error TEXT`, + ] + for (const column of columns) { + try { + sql.exec(`ALTER TABLE email_delivery_events ADD COLUMN ${column}`) + } catch { + // Column already present on a partially migrated object. + } + } +} + +export function initializeMailboxSchema(storage: DurableObjectStorage) { + const sql = storage.sql + sql.exec(` + CREATE TABLE IF NOT EXISTS mailbox_meta ( + key TEXT PRIMARY KEY NOT NULL, + value INTEGER NOT NULL + ) + `) + const installedVersion = getMailboxMeta(storage, mailboxMetaSchemaVersionKey) + if (installedVersion === mailboxSchemaVersion) return + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_threads ( + id TEXT PRIMARY KEY NOT NULL, + inbox_id TEXT, + subject_normalized TEXT NOT NULL DEFAULT '', + root_message_id_header TEXT, + last_message_at TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_last_message_at + ON email_threads(last_message_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_root_message_id + ON email_threads(root_message_id_header) + WHERE root_message_id_header IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_created_at + ON email_threads(created_at ASC, id ASC)`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_messages ( + id TEXT PRIMARY KEY NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ('inbound', 'outbound')), + inbox_id TEXT, + thread_id TEXT, + sender_identity_id TEXT, + from_address TEXT NOT NULL DEFAULT '', + envelope_from TEXT, + to_addresses_json TEXT NOT NULL DEFAULT '[]', + cc_addresses_json TEXT NOT NULL DEFAULT '[]', + bcc_addresses_json TEXT NOT NULL DEFAULT '[]', + reply_to_addresses_json TEXT NOT NULL DEFAULT '[]', + subject TEXT NOT NULL DEFAULT '', + message_id_header TEXT, + in_reply_to_header TEXT, + references_json TEXT NOT NULL DEFAULT '[]', + headers_json TEXT NOT NULL DEFAULT '{}', + auth_results TEXT, + text_body TEXT, + html_body TEXT, + raw_mime_key TEXT, + raw_size INTEGER NOT NULL DEFAULT 0, + processing_status TEXT NOT NULL CHECK ( + processing_status IN ('stored', 'sent', 'failed') + ), + classification TEXT NOT NULL DEFAULT 'accepted' CHECK ( + classification IN ('accepted', 'quarantined') + ), + classification_reason TEXT, + provider_message_id TEXT, + delivery_status TEXT CHECK ( + delivery_status IS NULL OR delivery_status IN ( + 'delivered', 'deferred', 'bounced', 'failed', + 'rejected', 'complained' + ) + ), + delivery_status_at TEXT, + error TEXT, + received_at TEXT, + sent_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_created_at + ON email_messages(created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_created_at_asc + ON email_messages(created_at ASC, id ASC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_inbox_created_at + ON email_messages(inbox_id, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_thread_created_at + ON email_messages(thread_id, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_message_id_header + ON email_messages(message_id_header) + WHERE message_id_header IS NOT NULL`, + ) + sql.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_messages_provider_message_id + ON email_messages(provider_message_id) + WHERE direction = 'outbound' AND provider_message_id IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_direction_created_at + ON email_messages(direction, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_delivery_status_created_at + ON email_messages(delivery_status, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_classification_created_at + ON email_messages(classification, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_raw_mime_key + ON email_messages(id ASC) + WHERE raw_mime_key IS NOT NULL`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_attachments ( + id TEXT PRIMARY KEY NOT NULL, + message_id TEXT NOT NULL, + filename TEXT, + content_type TEXT NOT NULL DEFAULT 'application/octet-stream', + content_id TEXT, + disposition TEXT, + size INTEGER NOT NULL DEFAULT 0, + storage_kind TEXT NOT NULL CHECK ( + storage_kind IN ('raw-mime', 'external', 'unavailable') + ), + storage_key TEXT, + created_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_attachments_message_id + ON email_attachments(message_id)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_attachments_storage_key + ON email_attachments(id ASC) + WHERE storage_key IS NOT NULL`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_delivery_events ( + id TEXT PRIMARY KEY NOT NULL, + message_id TEXT, + inbox_id TEXT, + event_type TEXT NOT NULL CHECK ( + event_type IN ( + 'receive_started', 'received', 'rejected', 'send_requested', + 'sent', 'failed', 'delivered', 'deferred', 'bounced', 'complained' + ) + ), + provider TEXT NOT NULL DEFAULT 'kody', + provider_message_id TEXT, + provider_event_id TEXT, + detail_json TEXT NOT NULL DEFAULT '{}', + needs_effect_reconcile INTEGER NOT NULL DEFAULT 0 CHECK ( + needs_effect_reconcile IN (0, 1) + ), + state TEXT CHECK ( + state IS NULL OR state IN ( + 'pending', 'storing', 'cleaning', 'received', + 'rejected', 'orphan-cleaned' + ) + ), + fingerprint TEXT, + storage_lease TEXT, + storage_lease_at TEXT, + cleanup_lease TEXT, + cleanup_lease_at TEXT, + cleanup_retry_at TEXT, + expected_attachment_count INTEGER, + finalization_token TEXT, + reconcile_after TEXT, + dedupe_expires_at TEXT, + usage_effect_recorded_at TEXT, + usage_effect_suppressed_at TEXT, + usage_started_at TEXT, + usage_month TEXT, + usage_bytes INTEGER, + usage_duration_ms INTEGER, + usage_effect_retry_at TEXT, + usage_effect_lease TEXT, + usage_effect_lease_at TEXT, + subscription_effect_state TEXT CHECK ( + subscription_effect_state IS NULL OR subscription_effect_state IN ( + 'pending', 'processing', 'complete', 'dead-letter' + ) + ), + subscription_effect_lease TEXT, + subscription_effect_lease_at TEXT, + subscription_effect_retry_at TEXT, + subscription_effect_attempt_count INTEGER, + subscription_effect_dead_letter_at TEXT, + subscription_effect_last_error TEXT, + created_at TEXT NOT NULL + ) + `) + ensureDeliveryEventPromotedColumns(sql, installedVersion) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_message_id + ON email_delivery_events(message_id)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_created_at + ON email_delivery_events(created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_created_at_asc + ON email_delivery_events(created_at ASC, id ASC)`, + ) + sql.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_delivery_events_provider_event_id + ON email_delivery_events(provider_event_id) + WHERE provider_event_id IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_pending_effects + ON email_delivery_events(created_at ASC, id ASC) + WHERE needs_effect_reconcile = 1 + AND fingerprint IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_state_created + ON email_delivery_events(state, created_at ASC) + WHERE state IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_fingerprint + ON email_delivery_events(fingerprint) + WHERE fingerprint IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_dedupe_expires + ON email_delivery_events(dedupe_expires_at ASC) + WHERE dedupe_expires_at IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_recorded_usage_month + ON email_delivery_events(usage_month, created_at ASC) + WHERE usage_effect_recorded_at IS NOT NULL`, + ) + + setMailboxMeta(storage, mailboxMetaSchemaVersionKey, mailboxSchemaVersion) +} diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts new file mode 100644 index 0000000000..ce5c591766 --- /dev/null +++ b/packages/worker/src/email/mailbox-store.ts @@ -0,0 +1,840 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxClassification, + assertMailboxDeliveryEventType, + assertMailboxDeliveryStatus, + assertMailboxDirection, + assertMailboxInboundDeliveryState, + assertMailboxNonEmptyString, + assertMailboxProcessingStatus, + assertMailboxStorageKind, + assertMailboxSubscriptionEffectState, + assertOptionalMailboxCanonicalIsoTimestamp, + decodeMailboxListCursor, + encodeMailboxListCursor, + mailboxNowIso, + normalizeMailboxPageSize, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxListMessagesInput, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxSearchMessagesInput, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' +import { + type EmailDeliveryEventType, + type EmailDeliveryStatus, +} from './types.ts' +import { + boundedEmailBody, + escapeLikePattern, + mapMailboxAttachmentRow, + mapMailboxDeliveryEventRow, + mapMailboxMessageRow, + mapMailboxThreadRow, +} from './mailbox-mappers.ts' +import { + exportMailboxFromStore, + listMailboxBlobReferences, +} from './mailbox-export.ts' +import { initializeMailboxSchema } from './mailbox-schema.ts' + +/** + * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. + */ +export class MailboxStore { + private readonly storage: DurableObjectStorage + + constructor(storage: DurableObjectStorage) { + this.storage = storage + } + + private get sql() { + return this.storage.sql + } + + initializeSchema() { + initializeMailboxSchema(this.storage) + } + + upsertThreadRow(thread: MailboxThreadInput) { + const id = assertMailboxNonEmptyString(thread.id, 'thread.id') + const lastMessageAt = assertMailboxCanonicalIsoTimestamp( + thread.lastMessageAt, + 'thread.lastMessageAt', + ) + const createdAt = assertMailboxCanonicalIsoTimestamp( + thread.createdAt?.trim() || mailboxNowIso(), + 'thread.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + thread.updatedAt?.trim() || mailboxNowIso(), + 'thread.updatedAt', + ) + this.sql.exec( + `INSERT INTO email_threads ( + id, inbox_id, subject_normalized, root_message_id_header, + last_message_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + inbox_id = excluded.inbox_id, + subject_normalized = excluded.subject_normalized, + root_message_id_header = excluded.root_message_id_header, + last_message_at = excluded.last_message_at, + updated_at = excluded.updated_at`, + id, + thread.inboxId ?? null, + thread.subjectNormalized ?? '', + thread.rootMessageIdHeader ?? null, + lastMessageAt, + createdAt, + updatedAt, + ) + } + + /** + * Upsert message metadata. Delivery status is monotonic by + * `delivery_status_at` (`>=` / equal timestamps may update; older must not + * regress a newer status from dual-write replay). + */ + upsertMessageRow(message: MailboxMessageInput) { + const id = assertMailboxNonEmptyString(message.id, 'message.id') + const direction = assertMailboxDirection(message.direction) + const processingStatus = assertMailboxProcessingStatus( + message.processingStatus, + ) + const classification = assertMailboxClassification( + message.classification ?? 'accepted', + ) + const deliveryStatus = + message.deliveryStatus == null + ? null + : assertMailboxDeliveryStatus(message.deliveryStatus) + const deliveryStatusAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.deliveryStatusAt, + 'message.deliveryStatusAt', + ) + if (deliveryStatus != null && deliveryStatusAt == null) { + throw new Error( + 'Mailbox message.deliveryStatusAt is required when deliveryStatus is set.', + ) + } + const createdAt = assertMailboxCanonicalIsoTimestamp( + message.createdAt?.trim() || mailboxNowIso(), + 'message.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + message.updatedAt?.trim() || mailboxNowIso(), + 'message.updatedAt', + ) + const receivedAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.receivedAt, + 'message.receivedAt', + ) + const sentAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.sentAt, + 'message.sentAt', + ) + this.sql.exec( + `INSERT INTO email_messages ( + id, direction, inbox_id, thread_id, sender_identity_id, + from_address, envelope_from, to_addresses_json, cc_addresses_json, + bcc_addresses_json, reply_to_addresses_json, subject, + message_id_header, in_reply_to_header, references_json, headers_json, + auth_results, text_body, html_body, raw_mime_key, raw_size, + processing_status, classification, classification_reason, + provider_message_id, delivery_status, delivery_status_at, error, + received_at, sent_at, created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + direction = excluded.direction, + inbox_id = excluded.inbox_id, + thread_id = excluded.thread_id, + sender_identity_id = excluded.sender_identity_id, + from_address = excluded.from_address, + envelope_from = excluded.envelope_from, + to_addresses_json = excluded.to_addresses_json, + cc_addresses_json = excluded.cc_addresses_json, + bcc_addresses_json = excluded.bcc_addresses_json, + reply_to_addresses_json = excluded.reply_to_addresses_json, + subject = excluded.subject, + message_id_header = excluded.message_id_header, + in_reply_to_header = excluded.in_reply_to_header, + references_json = excluded.references_json, + headers_json = excluded.headers_json, + auth_results = excluded.auth_results, + text_body = excluded.text_body, + html_body = excluded.html_body, + raw_mime_key = excluded.raw_mime_key, + raw_size = excluded.raw_size, + processing_status = excluded.processing_status, + classification = excluded.classification, + classification_reason = excluded.classification_reason, + provider_message_id = excluded.provider_message_id, + delivery_status = CASE + WHEN excluded.delivery_status_at IS NULL + THEN email_messages.delivery_status + WHEN email_messages.delivery_status_at IS NULL + THEN excluded.delivery_status + WHEN excluded.delivery_status_at >= email_messages.delivery_status_at + THEN excluded.delivery_status + ELSE email_messages.delivery_status + END, + delivery_status_at = CASE + WHEN excluded.delivery_status_at IS NULL + THEN email_messages.delivery_status_at + WHEN email_messages.delivery_status_at IS NULL + THEN excluded.delivery_status_at + WHEN excluded.delivery_status_at >= email_messages.delivery_status_at + THEN excluded.delivery_status_at + ELSE email_messages.delivery_status_at + END, + error = excluded.error, + received_at = excluded.received_at, + sent_at = excluded.sent_at, + updated_at = excluded.updated_at`, + id, + direction, + message.inboxId ?? null, + message.threadId ?? null, + message.senderIdentityId ?? null, + message.fromAddress ?? '', + message.envelopeFrom ?? null, + JSON.stringify(message.toAddresses ?? []), + JSON.stringify(message.ccAddresses ?? []), + JSON.stringify(message.bccAddresses ?? []), + JSON.stringify(message.replyToAddresses ?? []), + message.subject ?? '', + message.messageIdHeader ?? null, + message.inReplyToHeader ?? null, + JSON.stringify(message.references ?? []), + message.headers ? JSON.stringify(message.headers) : '{}', + message.authResults ?? null, + boundedEmailBody(message.textBody), + boundedEmailBody(message.htmlBody), + message.rawMimeKey ?? null, + message.rawSize ?? 0, + processingStatus, + classification, + message.classificationReason ?? null, + message.providerMessageId ?? null, + deliveryStatus, + deliveryStatusAt, + message.error ?? null, + receivedAt, + sentAt, + createdAt, + updatedAt, + ) + } + + replaceAttachmentsForMessage( + messageId: string, + attachments: Array, + ) { + this.sql.exec( + `DELETE FROM email_attachments WHERE message_id = ?`, + messageId, + ) + for (const attachment of attachments) { + const id = assertMailboxNonEmptyString(attachment.id, 'attachment.id') + const attachmentMessageId = assertMailboxNonEmptyString( + attachment.messageId, + 'attachment.messageId', + ) + if (attachmentMessageId !== messageId) { + throw new Error( + 'Mailbox attachment.messageId must match the mirrored message id.', + ) + } + const storageKind = assertMailboxStorageKind( + String(attachment.storageKind), + ) + const createdAt = assertMailboxCanonicalIsoTimestamp( + attachment.createdAt?.trim() || mailboxNowIso(), + 'attachment.createdAt', + ) + this.sql.exec( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, content_id, disposition, + size, storage_kind, storage_key, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + id, + attachmentMessageId, + attachment.filename ?? null, + attachment.contentType ?? 'application/octet-stream', + attachment.contentId ?? null, + attachment.disposition ?? null, + Math.max(0, Math.trunc(Number(attachment.size) || 0)), + storageKind, + attachment.storageKey ?? null, + createdAt, + ) + } + } + + findDeliveryEventByProviderEventId( + providerEventId: string, + ): MailboxDeliveryEventRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider_event_id = ? + LIMIT 1`, + providerEventId, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null + } + + writeDeliveryEventRow(event: MailboxDeliveryEventInput): boolean { + const id = assertMailboxNonEmptyString(event.id, 'event.id') + const eventType = assertMailboxDeliveryEventType(event.eventType) + const providerEventId = event.providerEventId ?? null + if (providerEventId) { + const existingByProvider = + this.findDeliveryEventByProviderEventId(providerEventId) + if (existingByProvider && existingByProvider.id !== id) { + return false + } + } + const existed = this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + const createdAt = assertMailboxCanonicalIsoTimestamp( + event.createdAt?.trim() || mailboxNowIso(), + 'event.createdAt', + ) + const detailJson = + typeof event.detailJson === 'string' && event.detailJson.length > 0 + ? event.detailJson + : '{}' + const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + const state = + event.state == null + ? null + : assertMailboxInboundDeliveryState(event.state) + const subscriptionEffectState = + event.subscriptionEffectState == null + ? null + : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) + + this.sql.exec( + `INSERT INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, + usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, + usage_month, usage_bytes, usage_duration_ms, + usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, + subscription_effect_state, subscription_effect_lease, + subscription_effect_lease_at, subscription_effect_retry_at, + subscription_effect_attempt_count, subscription_effect_dead_letter_at, + subscription_effect_last_error, created_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider = excluded.provider, + provider_message_id = excluded.provider_message_id, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error`, + id, + event.messageId ?? null, + event.inboxId ?? null, + eventType, + event.provider ?? 'kody', + event.providerMessageId ?? null, + providerEventId, + detailJson, + needsEffectReconcile, + state, + event.fingerprint ?? null, + event.storageLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.storageLeaseAt, + 'event.storageLeaseAt', + ), + event.cleanupLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupLeaseAt, + 'event.cleanupLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupRetryAt, + 'event.cleanupRetryAt', + ), + event.expectedAttachmentCount ?? null, + event.finalizationToken ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.reconcileAfter, + 'event.reconcileAfter', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.dedupeExpiresAt, + 'event.dedupeExpiresAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRecordedAt, + 'event.usageEffectRecordedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectSuppressedAt, + 'event.usageEffectSuppressedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageStartedAt, + 'event.usageStartedAt', + ), + event.usageMonth ?? null, + event.usageBytes ?? null, + event.usageDurationMs ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRetryAt, + 'event.usageEffectRetryAt', + ), + event.usageEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectLeaseAt, + 'event.usageEffectLeaseAt', + ), + subscriptionEffectState, + event.subscriptionEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectLeaseAt, + 'event.subscriptionEffectLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectRetryAt, + 'event.subscriptionEffectRetryAt', + ), + event.subscriptionEffectAttemptCount ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectDeadLetterAt, + 'event.subscriptionEffectDeadLetterAt', + ), + event.subscriptionEffectLastError ?? null, + createdAt, + ) + return existed == null + } + + updateLatestDeliveryStatus(input: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + }): boolean { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const deliveryStatus = assertMailboxDeliveryStatus(input.deliveryStatus) + const deliveryStatusAt = assertMailboxCanonicalIsoTimestamp( + input.deliveryStatusAt, + 'deliveryStatusAt', + ) + const cursor = this.sql.exec( + `UPDATE email_messages + SET delivery_status = ?, + delivery_status_at = ?, + updated_at = ? + WHERE id = ? + AND (delivery_status_at IS NULL OR delivery_status_at <= ?)`, + deliveryStatus, + deliveryStatusAt, + mailboxNowIso(), + messageId, + deliveryStatusAt, + ) + return cursor.rowsWritten > 0 + } + + deliveryEventOwnsMessage(eventId: string, messageId: string) { + return ( + this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE id = ? AND message_id = ? + LIMIT 1`, + eventId, + messageId, + ) + .toArray()[0] != null + ) + } + + getThread(threadId: string): MailboxThreadRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_threads WHERE id = ? LIMIT 1`, + assertMailboxNonEmptyString(threadId, 'threadId'), + ) + .toArray()[0] + return row ? mapMailboxThreadRow(row) : null + } + + getMessage(messageId: string): MailboxMessageRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_messages WHERE id = ? LIMIT 1`, + assertMailboxNonEmptyString(messageId, 'messageId'), + ) + .toArray()[0] + return row ? mapMailboxMessageRow(row) : null + } + + getMessageByMessageIdHeader( + messageIdHeader: string, + ): MailboxMessageRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE message_id_header = ? + LIMIT 1`, + assertMailboxNonEmptyString(messageIdHeader, 'messageIdHeader'), + ) + .toArray()[0] + return row ? mapMailboxMessageRow(row) : null + } + + getOutboundMessageByProviderMessageId( + providerMessageId: string, + ): MailboxMessageRecord | null { + const id = assertMailboxNonEmptyString( + providerMessageId, + 'providerMessageId', + ) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE direction = 'outbound' AND provider_message_id = ? + LIMIT 2`, + id, + ) + .toArray() + if (rows.length > 1) { + throw new Error( + `Multiple outbound email messages share provider id: ${id}`, + ) + } + const row = rows[0] + return row ? mapMailboxMessageRow(row) : null + } + + listMessages(input: MailboxListMessagesInput): { + messages: Array + nextCursor: string | null + } { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.inboxId) { + clauses.push('inbox_id = ?') + params.push(input.inboxId) + } + if (input.direction) { + clauses.push('direction = ?') + params.push(assertMailboxDirection(input.direction)) + } + if (input.processingStatus) { + clauses.push('processing_status = ?') + params.push(assertMailboxProcessingStatus(input.processingStatus)) + } + if (input.deliveryStatus) { + clauses.push('delivery_status = ?') + params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) + } + if (input.classification) { + clauses.push('classification = ?') + params.push(assertMailboxClassification(input.classification)) + } + if (input.cursor) { + const cursor = decodeMailboxListCursor(input.cursor) + clauses.push('(created_at, id) < (?, ?)') + params.push(cursor.createdAt, cursor.id) + } + params.push(limit + 1) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const last = pageRows[pageRows.length - 1] + return { + messages: pageRows.map(mapMailboxMessageRow), + nextCursor: + hasMore && last + ? encodeMailboxListCursor({ + createdAt: String(last['created_at']), + id: String(last['id']), + }) + : null, + } + } + + searchMessages(input: MailboxSearchMessagesInput): { + messages: Array + } { + if (typeof input.query !== 'string') { + throw new Error('Mailbox search query must be a string.') + } + const limit = normalizeMailboxPageSize(input.limit) + const pattern = `%${escapeLikePattern(input.query.trim().toLowerCase())}%` + const clauses: Array = [ + `( + LOWER(subject) LIKE ? ESCAPE '\\' + OR LOWER(from_address) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' + )`, + ] + const params: Array = [pattern, pattern, pattern] + if (input.inboxId) { + clauses.push('inbox_id = ?') + params.push(input.inboxId) + } + if (input.direction) { + clauses.push('direction = ?') + params.push(assertMailboxDirection(input.direction)) + } + if (input.processingStatus) { + clauses.push('processing_status = ?') + params.push(assertMailboxProcessingStatus(input.processingStatus)) + } + if (input.deliveryStatus) { + clauses.push('delivery_status = ?') + params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) + } + params.push(limit) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + return { messages: rows.map(mapMailboxMessageRow) } + } + + listAttachmentsForMessage(messageId: string): Array { + return this.sql + .exec>( + `SELECT * FROM email_attachments + WHERE message_id = ? + ORDER BY id ASC`, + assertMailboxNonEmptyString(messageId, 'messageId'), + ) + .toArray() + .map(mapMailboxAttachmentRow) + } + + listDeliveryEvents(input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }): Array { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.messageId) { + clauses.push('message_id = ?') + params.push(input.messageId) + } + if (input.eventType) { + clauses.push('event_type = ?') + params.push(assertMailboxDeliveryEventType(input.eventType)) + } + params.push(limit) + return this.sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + .map(mapMailboxDeliveryEventRow) + } + + countMailbox(): MailboxCountResult { + const threads = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_threads`) + .one() + const messages = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_messages`) + .one() + const attachments = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_attachments`) + .one() + const deliveryEvents = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_delivery_events`) + .one() + return { + threads: Number(threads.n ?? 0) || 0, + messages: Number(messages.n ?? 0) || 0, + attachments: Number(attachments.n ?? 0) || 0, + deliveryEvents: Number(deliveryEvents.n ?? 0) || 0, + } + } + + exportMailbox(input: { + pageSize?: number + startAfter?: string | null + }): MailboxExportResult { + return exportMailboxFromStore(this.sql, input) + } + + listBlobReferences(input: { + pageSize?: number + startAfter?: string | null + }): MailboxBlobReferencePage { + return listMailboxBlobReferences(this.sql, input) + } + + oldestMessageCreatedAt(): string | null { + const row = this.sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_messages + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null + } + + oldestDeliveryEventCreatedAt(): string | null { + const row = this.sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_delivery_events + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null + } + + listExpiredMessagesForRetention(input: { + cutoff: string + limit: number + }): Array<{ id: string; raw_mime_key: string | null; created_at: string }> { + return this.sql + .exec<{ + id: string + raw_mime_key: string | null + created_at: string + }>( + `SELECT id, raw_mime_key, created_at FROM email_messages + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + input.cutoff, + input.limit, + ) + .toArray() + } + + listAttachmentStorageKeysForMessages( + messageIds: Array, + ): Array<{ message_id: string; storage_key: string }> { + if (messageIds.length === 0) return [] + return this.sql + .exec<{ message_id: string; storage_key: string }>( + `SELECT message_id, storage_key FROM email_attachments + WHERE storage_key IS NOT NULL + AND message_id IN (${messageIds.map(() => '?').join(', ')})`, + ...messageIds, + ) + .toArray() + } + + deleteMessageCascade(messageId: string) { + this.sql.exec( + `DELETE FROM email_attachments WHERE message_id = ?`, + messageId, + ) + this.sql.exec(`DELETE FROM email_messages WHERE id = ?`, messageId) + } + + pruneExpiredDeliveryEvents(input: { cutoff: string; limit: number }) { + this.sql.exec( + `DELETE FROM email_delivery_events + WHERE id IN ( + SELECT id FROM email_delivery_events + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ? + )`, + input.cutoff, + input.limit, + ) + } + + pruneOrphanThreads(limit: number) { + this.sql.exec( + `DELETE FROM email_threads + WHERE id IN ( + SELECT thread.id FROM email_threads thread + WHERE NOT EXISTS ( + SELECT 1 FROM email_messages message + WHERE message.thread_id = thread.id + ) + ORDER BY thread.created_at ASC, thread.id ASC + LIMIT ? + )`, + limit, + ) + } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts new file mode 100644 index 0000000000..a93016b2ab --- /dev/null +++ b/packages/worker/src/email/mailbox-types.ts @@ -0,0 +1,631 @@ +import { + emailClassificationValues, + emailDeliveryEventTypeValues, + emailDeliveryStatusValues, + emailDirectionValues, + emailProcessingStatusValues, + type EmailClassification, + type EmailDeliveryEventType, + type EmailDeliveryStatus, + type EmailDirection, + type EmailProcessingStatus, +} from './types.ts' + +/** + * Mailbox Durable Object wire types, constants, and boundary validators. + * Object identity is ownership — records have no `user_id`. + */ + +export const mailboxMessageRetentionDays = 365 +export const mailboxDeliveryEventRetentionDays = 90 +export const mailboxDefaultPageSize = 100 +export const mailboxMaxPageSize = 500 +export const mailboxRetentionBatchSize = 100 +export const mailboxRetentionAlarmSkewMs = 60_000 +/** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ +export const mailboxRetentionRetryDelayMs = 60 * 60 * 1000 +export const mailboxBlobDeleteMaxKeys = 1000 + +/** Bump when initializeSchema's DDL set changes; warm objects skip DDL. */ +export const mailboxSchemaVersion = 2 +export const mailboxMetaSchemaVersionKey = 'schema_version' + +export const mailboxExportThreadCursorPrefix = 'thread:' +export const mailboxExportMessageCursorPrefix = 'message:' +export const mailboxExportAttachmentCursorPrefix = 'attachment:' +export const mailboxExportDeliveryEventCursorPrefix = 'delivery_event:' + +export const mailboxBlobRefRawMimeCursorPrefix = 'raw_mime:' +export const mailboxBlobRefAttachmentCursorPrefix = 'attachment:' + +export const mailboxStorageKindValues = [ + 'raw-mime', + 'external', + 'unavailable', +] as const +export type MailboxStorageKind = (typeof mailboxStorageKindValues)[number] + +export const mailboxInboundDeliveryStateValues = [ + 'pending', + 'storing', + 'cleaning', + 'received', + 'rejected', + 'orphan-cleaned', +] as const +export type MailboxInboundDeliveryState = + (typeof mailboxInboundDeliveryStateValues)[number] + +export const mailboxSubscriptionEffectStateValues = [ + 'pending', + 'processing', + 'complete', + 'dead-letter', +] as const +export type MailboxSubscriptionEffectState = + (typeof mailboxSubscriptionEffectStateValues)[number] + +/** Canonical UTC ISO-8601 with milliseconds — lexical order matches time order. */ +export const mailboxCanonicalIsoTimestampPattern = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ + +export type MailboxThreadRecord = { + id: string + inboxId: string | null + subjectNormalized: string | null + rootMessageIdHeader: string | null + lastMessageAt: string + createdAt: string + updatedAt: string +} + +export type MailboxMessageRecord = { + id: string + direction: EmailDirection + inboxId: string | null + threadId: string | null + senderIdentityId: string | null + fromAddress: string | null + envelopeFrom: string | null + toAddresses: Array + ccAddresses: Array + bccAddresses: Array + replyToAddresses: Array + subject: string | null + messageIdHeader: string | null + inReplyToHeader: string | null + references: Array + headers: Record | null + authResults: string | null + textBody: string | null + htmlBody: string | null + rawMimeKey: string | null + rawSize: number | null + processingStatus: EmailProcessingStatus + classification: EmailClassification + classificationReason: string | null + providerMessageId: string | null + deliveryStatus: EmailDeliveryStatus | null + deliveryStatusAt: string | null + error: string | null + receivedAt: string | null + sentAt: string | null + createdAt: string + updatedAt: string +} + +export type MailboxAttachmentRecord = { + id: string + messageId: string + filename: string | null + contentType: string | null + contentId: string | null + disposition: string | null + size: number + storageKind: string + storageKey: string | null + createdAt: string +} + +/** + * Delivery-event row including promoted inbound idempotency / effect columns. + * `detailJson` remains for compatibility with fields not yet promoted. + */ +export type MailboxDeliveryEventRecord = { + id: string + messageId: string | null + inboxId: string | null + eventType: EmailDeliveryEventType + provider: string | null + providerMessageId: string | null + providerEventId: string | null + detailJson: string + needsEffectReconcile: boolean + state: MailboxInboundDeliveryState | null + fingerprint: string | null + storageLease: string | null + storageLeaseAt: string | null + cleanupLease: string | null + cleanupLeaseAt: string | null + cleanupRetryAt: string | null + expectedAttachmentCount: number | null + finalizationToken: string | null + reconcileAfter: string | null + dedupeExpiresAt: string | null + usageEffectRecordedAt: string | null + usageEffectSuppressedAt: string | null + usageStartedAt: string | null + usageMonth: string | null + usageBytes: number | null + usageDurationMs: number | null + usageEffectRetryAt: string | null + usageEffectLease: string | null + usageEffectLeaseAt: string | null + subscriptionEffectState: MailboxSubscriptionEffectState | null + subscriptionEffectLease: string | null + subscriptionEffectLeaseAt: string | null + subscriptionEffectRetryAt: string | null + subscriptionEffectAttemptCount: number | null + subscriptionEffectDeadLetterAt: string | null + subscriptionEffectLastError: string | null + createdAt: string +} + +export type MailboxThreadInput = { + id: string + inboxId?: string | null + subjectNormalized?: string | null + rootMessageIdHeader?: string | null + lastMessageAt: string + createdAt?: string + updatedAt?: string +} + +export type MailboxMessageInput = { + id: string + direction: EmailDirection + inboxId?: string | null + threadId?: string | null + senderIdentityId?: string | null + fromAddress?: string | null + envelopeFrom?: string | null + toAddresses?: Array + ccAddresses?: Array + bccAddresses?: Array + replyToAddresses?: Array + subject?: string | null + messageIdHeader?: string | null + inReplyToHeader?: string | null + references?: Array + headers?: Record | null + authResults?: string | null + textBody?: string | null + htmlBody?: string | null + rawMimeKey?: string | null + rawSize?: number | null + processingStatus: EmailProcessingStatus + classification?: EmailClassification + classificationReason?: string | null + providerMessageId?: string | null + deliveryStatus?: EmailDeliveryStatus | null + deliveryStatusAt?: string | null + error?: string | null + receivedAt?: string | null + sentAt?: string | null + createdAt?: string + updatedAt?: string +} + +export type MailboxAttachmentInput = { + id: string + messageId: string + filename?: string | null + contentType?: string | null + contentId?: string | null + disposition?: string | null + size?: number + storageKind: MailboxStorageKind | string + storageKey?: string | null + createdAt?: string +} + +export type MailboxDeliveryEventInput = { + id: string + messageId?: string | null + inboxId?: string | null + eventType: EmailDeliveryEventType + provider?: string | null + providerMessageId?: string | null + providerEventId?: string | null + detailJson?: string + /** Defaults false — set true only when inbound effects still need reconcile. */ + needsEffectReconcile?: boolean + state?: MailboxInboundDeliveryState | null + fingerprint?: string | null + storageLease?: string | null + storageLeaseAt?: string | null + cleanupLease?: string | null + cleanupLeaseAt?: string | null + cleanupRetryAt?: string | null + expectedAttachmentCount?: number | null + finalizationToken?: string | null + reconcileAfter?: string | null + dedupeExpiresAt?: string | null + usageEffectRecordedAt?: string | null + usageEffectSuppressedAt?: string | null + usageStartedAt?: string | null + usageMonth?: string | null + usageBytes?: number | null + usageDurationMs?: number | null + usageEffectRetryAt?: string | null + usageEffectLease?: string | null + usageEffectLeaseAt?: string | null + subscriptionEffectState?: MailboxSubscriptionEffectState | null + subscriptionEffectLease?: string | null + subscriptionEffectLeaseAt?: string | null + subscriptionEffectRetryAt?: string | null + subscriptionEffectAttemptCount?: number | null + subscriptionEffectDeadLetterAt?: string | null + subscriptionEffectLastError?: string | null + createdAt?: string +} + +export type MailboxExportRow = + | { kind: 'thread'; row: MailboxThreadRecord } + | { kind: 'message'; row: MailboxMessageRecord } + | { kind: 'attachment'; row: MailboxAttachmentRecord } + | { kind: 'delivery_event'; row: MailboxDeliveryEventRecord } + +export type MailboxExportResult = { + rows: Array + nextStartAfter: string | null + truncated: boolean +} + +export type MailboxBlobReference = { + kind: 'raw_mime' | 'attachment' + key: string + messageId: string + attachmentId: string | null +} + +export type MailboxBlobReferencePage = { + references: Array + nextStartAfter: string | null + truncated: boolean +} + +export type MailboxCountResult = { + threads: number + messages: number + attachments: number + deliveryEvents: number +} + +export type MailboxListMessagesInput = { + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + limit?: number + cursor?: string | null +} + +export type MailboxSearchMessagesInput = { + query: string + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + limit?: number +} + +export type MailboxListCursorPayload = { + createdAt: string + id: string +} + +export type MailboxExportPhase = + | 'thread' + | 'message' + | 'attachment' + | 'delivery_event' + +export type MailboxRpc = { + mirrorMessage: (input: { + thread?: MailboxThreadInput | null + message: MailboxMessageInput + attachments?: Array + }) => Promise<{ ok: true }> + upsertDeliveryEvent: (input: { + event: MailboxDeliveryEventInput + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null + }) => Promise<{ inserted: boolean; updatedLatestStatus: boolean }> + getThread: (input: { + threadId: string + }) => Promise + getMessage: (input: { + messageId: string + }) => Promise + getMessageByMessageIdHeader: (input: { + messageIdHeader: string + }) => Promise + getOutboundMessageByProviderMessageId: (input: { + providerMessageId: string + }) => Promise + listMessages: (input: MailboxListMessagesInput) => Promise<{ + messages: Array + nextCursor: string | null + }> + searchMessages: (input: MailboxSearchMessagesInput) => Promise<{ + messages: Array + }> + listAttachmentsForMessage: (input: { + messageId: string + }) => Promise> + listDeliveryEvents: (input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }) => Promise> + countMailbox: () => Promise + exportMailbox: (input: { + pageSize?: number + startAfter?: string | null + }) => Promise + listBlobReferences: (input: { + pageSize?: number + startAfter?: string | null + }) => Promise + purge: () => Promise<{ ok: true }> +} + +export function mailboxNowIso() { + return new Date().toISOString() +} + +export function normalizeMailboxPageSize(pageSize: number | undefined) { + const requested = + typeof pageSize === 'number' && Number.isFinite(pageSize) + ? Math.trunc(pageSize) + : mailboxDefaultPageSize + return Math.min(Math.max(requested, 1), mailboxMaxPageSize) +} + +export function assertMailboxNonEmptyString( + value: unknown, + label: string, +): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`Mailbox ${label} must be a non-empty string.`) + } + return value +} + +export function assertMailboxCanonicalIsoTimestamp( + value: string, + label: string, +): string { + if (!mailboxCanonicalIsoTimestampPattern.test(value)) { + throw new Error( + `Mailbox ${label} must be a canonical ISO-8601 UTC timestamp (YYYY-MM-DDTHH:mm:ss.sssZ).`, + ) + } + if (!Number.isFinite(Date.parse(value))) { + throw new Error(`Mailbox ${label} is not a valid timestamp.`) + } + return value +} + +export function assertOptionalMailboxCanonicalIsoTimestamp( + value: string | null | undefined, + label: string, +): string | null { + if (value == null) return null + return assertMailboxCanonicalIsoTimestamp(value, label) +} + +export function assertMailboxDirection(value: string): EmailDirection { + if (!(emailDirectionValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox direction must be inbound or outbound; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDirection +} + +export function assertMailboxProcessingStatus( + value: string, +): EmailProcessingStatus { + if (!(emailProcessingStatusValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox processingStatus is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailProcessingStatus +} + +export function assertMailboxClassification( + value: string, +): EmailClassification { + if (!(emailClassificationValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox classification is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailClassification +} + +export function assertMailboxDeliveryStatus( + value: string, +): EmailDeliveryStatus { + if (!(emailDeliveryStatusValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox deliveryStatus is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDeliveryStatus +} + +export function assertMailboxDeliveryEventType( + value: string, +): EmailDeliveryEventType { + if ( + !(emailDeliveryEventTypeValues as ReadonlyArray).includes(value) + ) { + throw new Error( + `Mailbox eventType is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDeliveryEventType +} + +export function assertMailboxStorageKind(value: string): MailboxStorageKind { + if (!(mailboxStorageKindValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox storageKind is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxStorageKind +} + +export function assertMailboxInboundDeliveryState( + value: string, +): MailboxInboundDeliveryState { + if ( + !(mailboxInboundDeliveryStateValues as ReadonlyArray).includes( + value, + ) + ) { + throw new Error( + `Mailbox delivery event state is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxInboundDeliveryState +} + +export function assertMailboxSubscriptionEffectState( + value: string, +): MailboxSubscriptionEffectState { + if ( + !(mailboxSubscriptionEffectStateValues as ReadonlyArray).includes( + value, + ) + ) { + throw new Error( + `Mailbox subscriptionEffectState is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxSubscriptionEffectState +} + +export function encodeMailboxListCursor(payload: MailboxListCursorPayload) { + return btoa(JSON.stringify(payload)) +} + +export function decodeMailboxListCursor( + cursor: string, +): MailboxListCursorPayload { + let parsed: unknown + try { + parsed = JSON.parse(atob(cursor)) as unknown + } catch { + throw new Error('Mailbox list cursor is invalid.') + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Mailbox list cursor is invalid.') + } + const record = parsed as Record + const createdAt = record['createdAt'] + const id = record['id'] + if (typeof createdAt !== 'string' || typeof id !== 'string') { + throw new Error('Mailbox list cursor is invalid.') + } + assertMailboxCanonicalIsoTimestamp(createdAt, 'list cursor createdAt') + assertMailboxNonEmptyString(id, 'list cursor id') + return { createdAt, id } +} + +export function parseMailboxExportCursor(cursor: string | null): { + phase: MailboxExportPhase + startAfterId: string +} { + if (cursor == null || cursor.length === 0) { + return { phase: 'thread', startAfterId: '' } + } + if (cursor.startsWith(mailboxExportThreadCursorPrefix)) { + return { + phase: 'thread', + startAfterId: cursor.slice(mailboxExportThreadCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportMessageCursorPrefix)) { + return { + phase: 'message', + startAfterId: cursor.slice(mailboxExportMessageCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportAttachmentCursorPrefix)) { + return { + phase: 'attachment', + startAfterId: cursor.slice(mailboxExportAttachmentCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportDeliveryEventCursorPrefix)) { + return { + phase: 'delivery_event', + startAfterId: cursor.slice(mailboxExportDeliveryEventCursorPrefix.length), + } + } + throw new Error( + `Mailbox export cursor is invalid; got ${JSON.stringify(cursor)}.`, + ) +} + +export function mailboxExportCursorPrefixForPhase( + phase: MailboxExportPhase, +): string { + switch (phase) { + case 'thread': + return mailboxExportThreadCursorPrefix + case 'message': + return mailboxExportMessageCursorPrefix + case 'attachment': + return mailboxExportAttachmentCursorPrefix + case 'delivery_event': + return mailboxExportDeliveryEventCursorPrefix + default: { + const exhaustive: never = phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } +} + +export function parseMailboxBlobRefCursor(cursor: string | null): { + phase: 'raw_mime' | 'attachment' + startAfterId: string +} { + if (cursor == null || cursor.length === 0) { + return { phase: 'raw_mime', startAfterId: '' } + } + if (cursor.startsWith(mailboxBlobRefAttachmentCursorPrefix)) { + return { + phase: 'attachment', + startAfterId: cursor.slice(mailboxBlobRefAttachmentCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxBlobRefRawMimeCursorPrefix)) { + return { + phase: 'raw_mime', + startAfterId: cursor.slice(mailboxBlobRefRawMimeCursorPrefix.length), + } + } + throw new Error( + `Mailbox blob-reference cursor is invalid; got ${JSON.stringify(cursor)}.`, + ) +} diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 0d50120601..402234b801 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -7,6 +7,7 @@ import { JobManager } from './jobs/manager-do.ts' import { StorageRunner } from './storage-runner.ts' import { RunLog } from './run-records/run-log-do.ts' import { UserMeter } from './entitlements/user-meter-do.ts' +import { Mailbox } from './email/mailbox-do.ts' import { RepoSession } from './repo/repo-session-do.ts' import { PackageRealtimeSession } from '#worker/package-runtime/realtime-session.ts' import { PackageServiceInstance } from '#worker/package-runtime/package-service.ts' @@ -110,6 +111,7 @@ export { StorageRunner, RunLog, UserMeter, + Mailbox, OAuthPurgeCoordinator, } diff --git a/packages/worker/src/user-scoped-durable-object-name.node.test.ts b/packages/worker/src/user-scoped-durable-object-name.node.test.ts index c21190f1f2..04d484a79c 100644 --- a/packages/worker/src/user-scoped-durable-object-name.node.test.ts +++ b/packages/worker/src/user-scoped-durable-object-name.node.test.ts @@ -5,6 +5,7 @@ import { expect, test } from 'vitest' import { durableObjectNameFromParts, jobManagerDurableObjectName, + mailboxDurableObjectName, mcpClientHubDurableObjectName, packageRealtimeSessionDurableObjectName, packageServiceInstanceDurableObjectName, @@ -31,6 +32,10 @@ test('user-scoped Durable Object name helpers preserve frozen idFromName contrac // UserMeter matches RunLog: untrimmed userId is the frozen idFromName. expect(userMeterDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(mailboxDurableObjectName('user-aaa')).toBe('user-aaa') + // Mailbox matches RunLog: untrimmed userId is the frozen idFromName. + expect(mailboxDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(mcpClientHubDurableObjectName(' user-aaa ')).toBe('user-aaa') expect(storageRunnerDurableObjectName('user-aaa', 'job:1')).toBe( diff --git a/packages/worker/src/user-scoped-durable-object-name.ts b/packages/worker/src/user-scoped-durable-object-name.ts index 6fa7e52a20..3b08310189 100644 --- a/packages/worker/src/user-scoped-durable-object-name.ts +++ b/packages/worker/src/user-scoped-durable-object-name.ts @@ -35,6 +35,11 @@ export function userMeterDurableObjectName(userId: string) { return userId } +/** Mailbox — one email-metadata DO per user (untrimmed, like RunLog). */ +export function mailboxDurableObjectName(userId: string) { + return userId +} + /** McpClientHub — one client-hub DO per user (trimmed). */ export function mcpClientHubDurableObjectName(userId: string) { return userId.trim() diff --git a/packages/worker/worker-configuration.d.ts b/packages/worker/worker-configuration.d.ts index 038d2c60c1..e2f197c8e2 100644 --- a/packages/worker/worker-configuration.d.ts +++ b/packages/worker/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: 7b65a7fb6d2acc30d81d3977328dfcba) +// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: 326802848e273a6c986ec73d8417c98a) // Runtime types generated with workerd@1.20260701.1 2026-04-16 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { OAUTH_KV: KVNamespace; @@ -30,15 +30,6 @@ interface __BaseEnv_Env { DR_EXPORT_ENABLED: "true"; DR_BACKUP_ACCOUNT_ID: "a41d50ecaf0ae0f86dd1824ef6729cb2"; DR_BACKUP_BUCKET_NAME: "kody-production-backups"; - COOKIE_SECRET: string; - SECRET_STORE_KEY: string; - APP_BASE_URL: string; - GITHUB_CLIENT_ID: string; - GITHUB_CLIENT_SECRET: string; - GOOGLE_CLIENT_ID: string; - GOOGLE_CLIENT_SECRET: string; - X_CLIENT_ID: string; - X_CLIENT_SECRET: string; MCP_OBJECT: DurableObjectNamespace; REMOTE_CONNECTOR_SESSION: DurableObjectNamespace; MCP_CLIENT_HUB: DurableObjectNamespace; @@ -47,6 +38,7 @@ interface __BaseEnv_Env { STORAGE_RUNNER: DurableObjectNamespace; RUN_LOG: DurableObjectNamespace; USER_METER: DurableObjectNamespace; + MAILBOX: DurableObjectNamespace; REPO_SESSION: DurableObjectNamespace; PACKAGE_REALTIME_SESSION: DurableObjectNamespace; PACKAGE_SERVICE_INSTANCE: DurableObjectNamespace; @@ -55,7 +47,7 @@ interface __BaseEnv_Env { declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/index"); - durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog" | "UserMeter"; + durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog" | "UserMeter" | "Mailbox"; } interface Env extends __BaseEnv_Env {} } @@ -64,7 +56,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.md" { const value: string; diff --git a/packages/worker/wrangler.jsonc b/packages/worker/wrangler.jsonc index 5f286f0d50..c98af70488 100644 --- a/packages/worker/wrangler.jsonc +++ b/packages/worker/wrangler.jsonc @@ -95,6 +95,10 @@ "new_sqlite_classes": ["UserMeter"], "tag": "v21", }, + { + "new_sqlite_classes": ["Mailbox"], + "tag": "v22", + }, ], "observability": { @@ -201,6 +205,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -428,6 +436,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -572,6 +584,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", From c21a0fe18dc1b52995ed15f42f68157450bee141 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:59:07 +0000 Subject: [PATCH 02/20] fix(email): harden Mailbox storage contracts Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 51 +- .../contributing/architecture/primitives.yaml | 1 + packages/worker/src/email/blob-keys.ts | 24 + .../src/email/mailbox-delivery-events.ts | 308 +++++++++ packages/worker/src/email/mailbox-do.ts | 72 ++- .../src/email/mailbox-do.workers.test.ts | 611 ++++++++++-------- packages/worker/src/email/mailbox-export.ts | 72 ++- packages/worker/src/email/mailbox-mappers.ts | 112 ++-- .../worker/src/email/mailbox-retention.ts | 123 +++- .../email/mailbox-retention.workers.test.ts | 243 +++++++ packages/worker/src/email/mailbox-schema.ts | 64 +- packages/worker/src/email/mailbox-store.ts | 438 ++++++------- .../worker/src/email/mailbox-test-helpers.ts | 122 ++++ packages/worker/src/email/mailbox-types.ts | 34 +- packages/worker/src/email/repo.ts | 25 +- 15 files changed, 1565 insertions(+), 735 deletions(-) create mode 100644 packages/worker/src/email/blob-keys.ts create mode 100644 packages/worker/src/email/mailbox-delivery-events.ts create mode 100644 packages/worker/src/email/mailbox-retention.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-test-helpers.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index c56f08bf91..99179b9d8a 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -456,6 +456,9 @@ transaction, the rebuild snapshots and restores not lost. Runtime types and `ensurePlatformSenderIdentity` provision verified rows only. +- Canonical key builders are `emailRawMimeKey` / `emailAttachmentBlobKey` in + `packages/worker/src/email/blob-keys.ts` (re-exported from + `packages/worker/src/email/repo.ts`). - All reads go through `loadRawMime` in `packages/worker/src/email/repo.ts`, which fetches the blob by `raw_mime_key` only. Attachment content extraction re-parses the resolved MIME from that blob. @@ -582,27 +585,45 @@ SQLite (`packages/worker/src/email/mailbox-do.ts` and siblings under Worker binding is `MAILBOX` (class `Mailbox`; Wrangler SQLite migration tag `v22` via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Raw MIME and outbound attachment bytes stay in `EMAIL_BLOBS` R2; the DO stores object -keys, not payload bytes. +keys, not payload bytes. Canonical key builders live in +`packages/worker/src/email/blob-keys.ts`. Naming matches `RunLog`, `JobManager`, and `UserMeter`: one object per untrimmed stable MCP `userId` via `mailboxDurableObjectName(userId)` → `idFromName(userId)` in -`packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` -column inside the DO because the object identity is the user. - -**SQLite ownership** (schema version in `mailbox_meta`, currently -`mailboxSchemaVersion = 2`): - +`packages/worker/src/user-scoped-durable-object-name.ts`. Data rows have no +`user_id` column (object identity is the user). Because a Durable Object cannot +introspect its `idFromName` string, a singleton `mailbox_owner_identity` row +persists `ownerId` on first write and rejects cross-owner RPCs. That persisted +owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` +/ `emailAttachmentBlobKey`). + +**SQLite ownership** (schema version in `mailbox_meta`; ships as +`mailboxSchemaVersion = 1`): + +- `mailbox_owner_identity` — singleton `owner_id` for blob-key validation and + cross-user write rejection - `email_threads`, `email_messages`, `email_attachments`, and `email_delivery_events` for the owning user (same logical names as the D1 tables they will eventually replace) - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` +**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` require +`ownerId` and apply equal-or-newer `updatedAt` snapshots (stale snapshots are +ignored, not applied). On `mirrorMessage`, omitting `attachments` preserves +existing attachment metadata; an explicit `attachments: []` clears them. +Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment +`storageKey` values against the canonical builders for that `ownerId`. + **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). -Alarm-driven deletes are strict blob-before-row for `EMAIL_BLOBS` keys (failed -blob deletes skip the row for retry), matching today's D1 retention ordering. +Alarm-driven deletes derive canonical blob keys from `ownerId` + row ids (rather +than trusting stored key strings), then apply strict blob-before-row ordering +for `EMAIL_BLOBS` (failed blob deletes skip the row for retry). After a +retention pass: successful work with expired rows remaining schedules a +near-immediate continuation (`mailboxRetentionContinuationDelayMs`); R2 delete +failures use hourly backoff (`mailboxRetentionRetryDelayMs`). Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state @@ -1103,9 +1124,15 @@ App-owned R2 keys are: prefix, including historical replacements left by earlier cleanup failures. - `email-raw:v1:{userId}/{messageId}` — raw email MIME for the message row that - stores this key in `email_messages.raw_mime_key`. The `userId` prefix is part - of the per-user isolation contract; account deletion deletes a user's blobs by - the keys stored on their rows. + stores this key in `email_messages.raw_mime_key`. Built by `emailRawMimeKey` + in `packages/worker/src/email/blob-keys.ts`. The `userId` prefix is part of + the per-user isolation contract; account deletion removes a user's blobs under + the matching prefix (and any remaining inventoried keys). + +- `email-attachment:v1:{userId}/{messageId}/{attachmentId}` — standalone + attachment bytes (`storage_kind = 'external'`). Built by + `emailAttachmentBlobKey` in `packages/worker/src/email/blob-keys.ts`. Same + per-user prefix isolation and account-deletion coverage as raw MIME. New R2 key prefixes must add corresponding account-deletion coverage or a deliberate retention note, same as KV. All currently registered R2 surfaces use diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index 6a0731c858..fcc9c477e2 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -536,6 +536,7 @@ primitives: name: Email blob storage (R2) summary: EMAIL_BLOBS R2 for raw MIME and outbound attachments. code: + - packages/worker/src/email/blob-keys.ts - packages/worker/src/email/repo.ts docs: - docs/contributing/architecture/data-storage.md diff --git a/packages/worker/src/email/blob-keys.ts b/packages/worker/src/email/blob-keys.ts new file mode 100644 index 0000000000..34dd318edf --- /dev/null +++ b/packages/worker/src/email/blob-keys.ts @@ -0,0 +1,24 @@ +/** + * R2 object key for a message's raw MIME payload in the EMAIL_BLOBS + * bucket. The userId prefix is part of the per-user isolation contract: + * account deletion enumerates and deletes a user's blobs by these stored + * keys, and the key can never be forged to point at another user's mail. + * Writers always store this canonical key in `raw_mime_key`. + */ +export function emailRawMimeKey(userId: string, messageId: string) { + return `email-raw:v1:${userId}/${messageId}` +} + +/** + * R2 object key for an attachment stored on its own (storage_kind + * 'external'), used by outbound mail whose bytes never exist as raw MIME. + * The userId prefix follows the same per-user isolation contract as + * emailRawMimeKey. + */ +export function emailAttachmentBlobKey( + userId: string, + messageId: string, + attachmentId: string, +) { + return `email-attachment:v1:${userId}/${messageId}/${attachmentId}` +} diff --git a/packages/worker/src/email/mailbox-delivery-events.ts b/packages/worker/src/email/mailbox-delivery-events.ts new file mode 100644 index 0000000000..8be19c23cc --- /dev/null +++ b/packages/worker/src/email/mailbox-delivery-events.ts @@ -0,0 +1,308 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxDeliveryEventType, + assertMailboxInboundDeliveryState, + assertMailboxNonEmptyString, + assertMailboxSubscriptionEffectState, + assertOptionalMailboxCanonicalIsoTimestamp, + mailboxNowIso, + normalizeMailboxPageSize, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, +} from './mailbox-types.ts' +import { type EmailDeliveryEventType } from './types.ts' +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' + +export function findDeliveryEventByProviderEventId( + sql: SqlStorage, + providerEventId: string, +): MailboxDeliveryEventRecord | null { + const row = sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider_event_id = ? + LIMIT 1`, + providerEventId, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null +} + +export function writeMailboxDeliveryEventRow( + sql: SqlStorage, + event: MailboxDeliveryEventInput, +): { inserted: boolean; accepted: boolean } { + const id = assertMailboxNonEmptyString(event.id, 'event.id') + const eventType = assertMailboxDeliveryEventType(event.eventType) + const providerEventId = event.providerEventId ?? null + if (providerEventId) { + const existingByProvider = findDeliveryEventByProviderEventId( + sql, + providerEventId, + ) + if (existingByProvider && existingByProvider.id !== id) { + return { inserted: false, accepted: false } + } + } + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_delivery_events WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + const createdAt = assertMailboxCanonicalIsoTimestamp( + event.createdAt?.trim() || mailboxNowIso(), + 'event.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + event.updatedAt, + 'event.updatedAt', + ) + if (existing && updatedAt < existing.updated_at) { + return { inserted: false, accepted: false } + } + const detailJson = + typeof event.detailJson === 'string' && event.detailJson.length > 0 + ? event.detailJson + : '{}' + const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + const state = + event.state == null ? null : assertMailboxInboundDeliveryState(event.state) + const subscriptionEffectState = + event.subscriptionEffectState == null + ? null + : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) + + sql.exec( + `INSERT INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, + usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, + usage_month, usage_bytes, usage_duration_ms, + usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, + subscription_effect_state, subscription_effect_lease, + subscription_effect_lease_at, subscription_effect_retry_at, + subscription_effect_attempt_count, subscription_effect_dead_letter_at, + subscription_effect_last_error, created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider = excluded.provider, + provider_message_id = excluded.provider_message_id, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_delivery_events.updated_at`, + id, + event.messageId ?? null, + event.inboxId ?? null, + eventType, + event.provider ?? 'kody', + event.providerMessageId ?? null, + providerEventId, + detailJson, + needsEffectReconcile, + state, + event.fingerprint ?? null, + event.storageLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.storageLeaseAt, + 'event.storageLeaseAt', + ), + event.cleanupLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupLeaseAt, + 'event.cleanupLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupRetryAt, + 'event.cleanupRetryAt', + ), + event.expectedAttachmentCount ?? null, + event.finalizationToken ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.reconcileAfter, + 'event.reconcileAfter', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.dedupeExpiresAt, + 'event.dedupeExpiresAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRecordedAt, + 'event.usageEffectRecordedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectSuppressedAt, + 'event.usageEffectSuppressedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageStartedAt, + 'event.usageStartedAt', + ), + event.usageMonth ?? null, + event.usageBytes ?? null, + event.usageDurationMs ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRetryAt, + 'event.usageEffectRetryAt', + ), + event.usageEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectLeaseAt, + 'event.usageEffectLeaseAt', + ), + subscriptionEffectState, + event.subscriptionEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectLeaseAt, + 'event.subscriptionEffectLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectRetryAt, + 'event.subscriptionEffectRetryAt', + ), + event.subscriptionEffectAttemptCount ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectDeadLetterAt, + 'event.subscriptionEffectDeadLetterAt', + ), + event.subscriptionEffectLastError ?? null, + createdAt, + updatedAt, + ) + return { inserted: existing == null, accepted: true } +} + +export function deliveryEventOwnsMessage( + sql: SqlStorage, + eventId: string, + messageId: string, +) { + return ( + sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE id = ? AND message_id = ? + LIMIT 1`, + eventId, + messageId, + ) + .toArray()[0] != null + ) +} + +export function listMailboxDeliveryEvents( + sql: SqlStorage, + input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }, +): Array { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.messageId) { + clauses.push('message_id = ?') + params.push(input.messageId) + } + if (input.eventType) { + clauses.push('event_type = ?') + params.push(assertMailboxDeliveryEventType(input.eventType)) + } + params.push(limit) + return sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + .map(mapMailboxDeliveryEventRow) +} + +export function oldestMailboxDeliveryEventCreatedAt( + sql: SqlStorage, +): string | null { + const row = sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_delivery_events + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null +} + +export function pruneExpiredMailboxDeliveryEvents( + sql: SqlStorage, + input: { cutoff: string; limit: number }, +) { + sql.exec( + `DELETE FROM email_delivery_events + WHERE id IN ( + SELECT id FROM email_delivery_events + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ? + )`, + input.cutoff, + input.limit, + ) +} + +export function hasExpiredMailboxDeliveryEvents( + sql: SqlStorage, + cutoff: string, +): boolean { + return ( + sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE created_at < ? + LIMIT 1`, + cutoff, + ) + .toArray()[0] != null + ) +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index cce457aca3..146a9ac7bf 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -6,6 +6,7 @@ import { type EmailDeliveryStatus, } from './types.ts' import { + computeMailboxRetentionReschedule, enforceMailboxRetention, mailboxRetentionAlarmAtMs, nextMailboxRetentionDueAtMs, @@ -24,6 +25,7 @@ import { type MailboxListMessagesInput, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxRpc, type MailboxSearchMessagesInput, type MailboxThreadInput, type MailboxThreadRecord, @@ -32,13 +34,15 @@ import { /** * Per-owner Mailbox Durable Object: SQLite metadata for threads, messages, * attachments, and delivery events. Object identity is ownership — no - * `user_id` columns. Raw MIME / external attachment bytes stay in - * `EMAIL_BLOBS`; rows retain keys. `system:email` stays in D1 by design. + * `user_id` columns on data rows. Owner binding for blob-key validation is + * a singleton identity row (DO name is not introspectable). Raw MIME / + * external attachment bytes stay in `EMAIL_BLOBS`; rows retain keys. + * `system:email` stays in D1 by design. * * Scaffold only: dual-write / cutover callers are not wired here. */ -class MailboxBase extends DurableObject { +class MailboxBase extends DurableObject implements MailboxRpc { private readonly store: MailboxStore /** * In-isolate cache: once an alarm is scheduled, hot writes skip @@ -66,8 +70,9 @@ class MailboxBase extends DurableObject { * Arming rule (keep this comment accurate — reviewers rely on it): * * - Schedule at most one alarm for the soonest retention due-time. - * - Overdue work uses an hourly retry delay (not now+1s) so persistent R2 - * failures cannot storm alarm wakes; future rows keep due-time scheduling. + * - Overdue work on the write path uses an hourly retry delay (not now+1s). + * - After an alarm: blob failures → hourly backoff; successful pass with + * expired rows remaining → near-immediate continuation; else future due. * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. * - `retentionIdleConfirmed` skips re-evaluation only while no new row has * been written; every write clears it so the next ensure re-arms. @@ -104,43 +109,57 @@ class MailboxBase extends DurableObject { } async alarm(): Promise { - await enforceMailboxRetention({ + const nowMs = Date.now() + const result = await enforceMailboxRetention({ store: this.store, blobs: this.env.EMAIL_BLOBS, }) - const dueAtMs = nextMailboxRetentionDueAtMs(this.store) - const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) - if (alarmAt == null) { + const nextDueAtMs = nextMailboxRetentionDueAtMs(this.store) + const reschedule = computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: result.hadBlobDeleteFailures, + expiredWorkRemaining: result.expiredWorkRemaining, + nextDueAtMs, + }) + if (reschedule.atMs == null) { this.retentionAlarmArmed = false this.retentionIdleConfirmed = true return } - await this.ctx.storage.setAlarm(alarmAt) + await this.ctx.storage.setAlarm(reschedule.atMs) this.retentionAlarmArmed = true this.retentionIdleConfirmed = false } /** Atomic mirror of thread + message + attachments for dual-write. */ async mirrorMessage(input: { + ownerId: string thread?: MailboxThreadInput | null message: MailboxMessageInput attachments?: Array - }): Promise<{ ok: true }> { + }): Promise<{ ok: true; accepted: boolean }> { const message = input.message assertMailboxNonEmptyString(message.id, 'message.id') - const attachments = Array.isArray(input.attachments) - ? input.attachments - : [] + let accepted = false this.ctx.storage.transactionSync(() => { + const ownerId = this.store.assertOwner(input.ownerId) + this.store.validateMessageBlobKeys({ + ownerId, + message, + attachments: input.attachments, + }) if (input.thread) { this.store.upsertThreadRow(input.thread) } - this.store.upsertMessageRow(message) - this.store.replaceAttachmentsForMessage(message.id, attachments) + const messageResult = this.store.upsertMessageRow(message) + accepted = messageResult.accepted + if (accepted && input.attachments !== undefined) { + this.store.replaceAttachmentsForMessage(message.id, input.attachments) + } }) this.markRetentionDirty() await this.ensureRetentionAlarm() - return { ok: true } + return { ok: true, accepted } } /** @@ -148,18 +167,27 @@ class MailboxBase extends DurableObject { * apply a monotonic latest `delivery_status` update on the message. */ async upsertDeliveryEvent(input: { + ownerId: string event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null - }): Promise<{ inserted: boolean; updatedLatestStatus: boolean }> { + }): Promise<{ + inserted: boolean + accepted: boolean + updatedLatestStatus: boolean + }> { let inserted = false + let accepted = false let updatedLatestStatus = false this.ctx.storage.transactionSync(() => { - inserted = this.store.writeDeliveryEventRow(input.event) - if (input.latestDeliveryStatus) { + this.store.assertOwner(input.ownerId) + const write = this.store.writeDeliveryEventRow(input.event) + inserted = write.inserted + accepted = write.accepted + if (accepted && input.latestDeliveryStatus) { const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') const messageId = assertMailboxNonEmptyString( input.latestDeliveryStatus.messageId, @@ -174,7 +202,7 @@ class MailboxBase extends DurableObject { }) this.markRetentionDirty() await this.ensureRetentionAlarm() - return { inserted, updatedLatestStatus } + return { inserted, accepted, updatedLatestStatus } } async getThread(input: { @@ -275,6 +303,7 @@ export type { MailboxRpc } from './mailbox-types.ts' export { mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, + mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, type MailboxAttachmentInput, type MailboxAttachmentRecord, @@ -290,3 +319,4 @@ export { type MailboxThreadInput, type MailboxThreadRecord, } from './mailbox-types.ts' +export { computeMailboxRetentionReschedule } from './mailbox-retention.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index fe6b423dd7..dfb70c369f 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -1,115 +1,18 @@ -import { env } from 'cloudflare:workers' import { runInDurableObject } from 'cloudflare:test' import { expect, test } from 'vitest' -import { consoleWarn } from '#worker/test-support/console-spies.ts' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' -import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' -import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { mapMailboxMessageRow } from './mailbox-mappers.ts' +import { Mailbox } from './mailbox-do.ts' import { - Mailbox, - mailboxDeliveryEventRetentionDays, - mailboxMessageRetentionDays, - mailboxRetentionRetryDelayMs, - type MailboxAttachmentInput, - type MailboxMessageInput, - type MailboxThreadInput, -} from './mailbox-do.ts' - -function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { - const mailbox = (env as MailboxEnv).MAILBOX - if (!mailbox) { - throw new Error( - 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', - ) - } - return { MAILBOX: mailbox } -} - -function stubFor(userId: string) { - const { MAILBOX } = mailboxEnv() - return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) -} - -function rpcFor(userId: string) { - return mailboxRpc({ env: mailboxEnv(), userId }) -} - -function uniqueUserId(label: string) { - return `mailbox-${label}-${crypto.randomUUID()}` -} - -async function assertMailboxThrows( - pattern: RegExp, - run: () => Promise, -) { - try { - await run() - throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) - } catch (error) { - expect(String(error)).toMatch(pattern) - } -} - -function baseThread( - overrides?: Partial, -): MailboxThreadInput { - const id = overrides?.id ?? crypto.randomUUID() - const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' - return { - id, - inboxId: 'inbox-1', - subjectNormalized: 'hello', - rootMessageIdHeader: ``, - lastMessageAt: at, - createdAt: at, - updatedAt: at, - ...overrides, - } -} - -function baseMessage( - overrides?: Partial, -): MailboxMessageInput { - const id = overrides?.id ?? crypto.randomUUID() - const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' - return { - id, - direction: 'inbound', - inboxId: 'inbox-1', - threadId: null, - fromAddress: 'sender@example.com', - envelopeFrom: 'envelope@example.com', - toAddresses: ['owner@example.com'], - subject: 'Hello mailbox', - messageIdHeader: ``, - rawMimeKey: `email-raw:v1:owner/${id}`, - rawSize: 128, - processingStatus: 'stored', - classification: 'accepted', - receivedAt: at, - createdAt: at, - updatedAt: at, - ...overrides, - } -} - -function baseAttachment( - messageId: string, - overrides?: Partial, -): MailboxAttachmentInput { - const id = overrides?.id ?? crypto.randomUUID() - return { - id, - messageId, - filename: 'note.txt', - contentType: 'text/plain', - size: 12, - storageKind: 'external', - storageKey: `email-attachment:v1:owner/${messageId}/${id}`, - createdAt: '2026-07-01T12:00:00.000Z', - ...overrides, - } -} + assertMailboxThrows, + baseAttachment, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { silenceIncidentalRuntimeWarnings() @@ -127,7 +30,8 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', WHERE type = 'table' AND name IN ( 'email_threads', 'email_messages', - 'email_attachments', 'email_delivery_events' + 'email_attachments', 'email_delivery_events', + 'mailbox_owner_identity' ) ORDER BY name ASC`, ) @@ -138,27 +42,34 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', 'email_delivery_events', 'email_messages', 'email_threads', + 'mailbox_owner_identity', ]) expect(await state.storage.getAlarm()).toBeNull() }) const thread = baseThread({ id: 'thread-1' }) - const message = baseMessage({ + const message = baseMessage(ownerA, { id: 'msg-1', threadId: thread.id, subject: 'Project update', fromAddress: 'alice@example.com', }) - const attachment = baseAttachment(message.id, { id: 'att-1' }) + const attachment = baseAttachment(ownerA, message.id, { id: 'att-1' }) await mailboxA.mirrorMessage({ + ownerId: ownerA, thread, message, attachments: [attachment], }) await mailboxA.mirrorMessage({ + ownerId: ownerA, thread, - message: { ...message, subject: 'Project update (edited)' }, + message: { + ...message, + subject: 'Project update (edited)', + updatedAt: '2026-07-01T12:00:01.000Z', + }, attachments: [attachment], }) @@ -168,7 +79,7 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', threadId: thread.id, subject: 'Project update (edited)', fromAddress: 'alice@example.com', - rawMimeKey: message.rawMimeKey, + rawMimeKey: emailRawMimeKey(ownerA, message.id), }) expect(await mailboxA.getThread({ threadId: thread.id })).toMatchObject({ id: thread.id, @@ -198,8 +109,9 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', ).toHaveLength(0) await mailboxB.mirrorMessage({ + ownerId: ownerB, thread: baseThread({ id: 'thread-b' }), - message: baseMessage({ + message: baseMessage(ownerB, { id: 'msg-b', threadId: 'thread-b', subject: 'Other owner', @@ -222,7 +134,8 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', const outboundId = 'msg-out-1' await mailboxA.mirrorMessage({ - message: baseMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { id: outboundId, direction: 'outbound', subject: 'Sent mail', @@ -236,6 +149,236 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', providerMessageId: 'provider-1', }), ).toMatchObject({ id: outboundId }) + + // Cross-user ownerId and forged blob keys must be rejected. + await runInDurableObject(stubA, async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.mirrorMessage({ + ownerId: ownerB, + message: baseMessage(ownerA, { id: 'cross-user' }), + }), + ) + await assertMailboxThrows(/rawMimeKey must equal/, () => + instance.mirrorMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { + id: 'bad-key', + rawMimeKey: emailRawMimeKey(ownerB, 'bad-key'), + }), + }), + ) + await assertMailboxThrows(/storageKey must equal/, () => + instance.mirrorMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { id: 'bad-att-msg' }), + attachments: [ + baseAttachment(ownerA, 'bad-att-msg', { + id: 'bad-att', + storageKey: emailAttachmentBlobKey( + ownerB, + 'bad-att-msg', + 'bad-att', + ), + }), + ], + }), + ) + }) +}) + +test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('stale') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'stale-thread' }) + const message = baseMessage(userId, { + id: 'stale-msg', + threadId: thread.id, + classification: 'accepted', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const attachment = baseAttachment(userId, message.id, { id: 'stale-att' }) + + await mailbox.mirrorMessage({ + ownerId: userId, + thread, + message, + attachments: [attachment], + }) + + // Omitted attachments leave existing metadata unchanged. + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...message, + subject: 'edited without attachments field', + updatedAt: '2026-07-01T12:00:01.000Z', + }, + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(1) + + // Stale snapshot must not regress classification, thread time, or attachments. + const staleMirror = await mailbox.mirrorMessage({ + ownerId: userId, + thread: { + ...thread, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-06-01T00:00:00.000Z', + }, + message: { + ...message, + classification: 'quarantined', + subject: 'stale subject', + updatedAt: '2026-07-01T11:00:00.000Z', + }, + attachments: [], + }) + expect(staleMirror.accepted).toBe(false) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'accepted', + subject: 'edited without attachments field', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: thread.lastMessageAt, + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(1) + + // Explicit [] clears only when the message snapshot is accepted. + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...message, + subject: 'cleared attachments', + updatedAt: '2026-07-01T12:00:02.000Z', + }, + attachments: [], + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(0) + + const outbound = baseMessage(userId, { + id: 'stale-out', + direction: 'outbound', + providerMessageId: 'prov-stale', + processingStatus: 'sent', + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ ownerId: userId, message: outbound }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...outbound, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }, + }) + expect(await mailbox.getMessage({ messageId: outbound.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + const first = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1', + messageId: outbound.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-stale', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + state: 'received', + fingerprint: 'fp-1', + }, + latestDeliveryStatus: { + messageId: outbound.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }, + }) + expect(first).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: true, + }) + + const staleEvent = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1', + messageId: outbound.id, + eventType: 'deferred', + provider: 'cloudflare-email', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T09:00:00.000Z', + needsEffectReconcile: true, + state: null, + fingerprint: null, + }, + }) + expect(staleEvent).toEqual({ + inserted: false, + accepted: false, + updatedLatestStatus: false, + }) + const eventRow = ( + await mailbox.listDeliveryEvents({ messageId: outbound.id, limit: 5 }) + ).find((row) => row.id === 'evt-1') + expect(eventRow).toMatchObject({ + eventType: 'delivered', + state: 'received', + fingerprint: 'fp-1', + needsEffectReconcile: false, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + + const duplicate = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1-dup', + messageId: outbound.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', + }, + }) + expect(duplicate).toEqual({ + inserted: false, + accepted: false, + updatedLatestStatus: false, + }) + + expect(() => + mapMailboxMessageRow({ + id: 'corrupt', + direction: 'bogus', + processing_status: 'stored', + classification: 'accepted', + to_addresses_json: '[]', + cc_addresses_json: '[]', + bcc_addresses_json: '[]', + reply_to_addresses_json: '[]', + references_json: '[]', + headers_json: '{}', + created_at: '2026-07-01T12:00:00.000Z', + updated_at: '2026-07-01T12:00:00.000Z', + }), + ).toThrow(/persisted direction is invalid/) }) test('Mailbox delivery status, promoted inbound fields, export paging, and cursor rejection', async () => { @@ -244,34 +387,47 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const mailbox = rpcFor(userId) const thread = baseThread({ id: 'export-thread' }) - const message = baseMessage({ + const message = baseMessage(userId, { id: 'export-msg', threadId: thread.id, direction: 'outbound', providerMessageId: 'prov-export', processingStatus: 'sent', - rawMimeKey: 'email-raw:v1:owner/export-msg', + rawMimeKey: null, deliveryStatus: 'delivered', deliveryStatusAt: '2026-07-02T10:00:00.000Z', }) - const attachment = baseAttachment(message.id, { + const attachment = baseAttachment(userId, message.id, { id: 'export-att', - storageKey: 'email-attachment:v1:owner/export-msg/export-att', + }) + const inboundBlobMessage = baseMessage(userId, { + id: 'export-inbound', + subject: 'inbound for blobs', + }) + const inboundAttachment = baseAttachment(userId, inboundBlobMessage.id, { + id: 'export-inbound-att', }) await mailbox.mirrorMessage({ + ownerId: userId, thread, message, attachments: [attachment], }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: inboundBlobMessage, + attachments: [inboundAttachment], + }) // Stale dual-write replay must not regress a newer delivery status. await mailbox.mirrorMessage({ + ownerId: userId, message: { ...message, deliveryStatus: 'deferred', deliveryStatusAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', }, - attachments: [attachment], }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'delivered', @@ -280,12 +436,13 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso // Equal timestamps may update (matches D1 <= semantics). await mailbox.mirrorMessage({ + ownerId: userId, message: { ...message, deliveryStatus: 'complained', deliveryStatusAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:02.000Z', }, - attachments: [attachment], }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'complained', @@ -293,6 +450,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }) const first = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-1', messageId: message.id, @@ -301,6 +459,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-1', createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, }, latestDeliveryStatus: { @@ -309,9 +468,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryStatusAt: '2026-07-02T10:30:00.000Z', }, }) - expect(first).toEqual({ inserted: true, updatedLatestStatus: true }) + expect(first).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: true, + }) const inbound = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'inbound-delivery-1', messageId: 'inbound-msg-1', @@ -319,6 +483,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso eventType: 'received', provider: 'cloudflare-email-routing', createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', needsEffectReconcile: true, state: 'received', fingerprint: 'fp-abc', @@ -356,6 +521,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso usageBytes: 2048, subscriptionEffectState: 'pending', subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', }) expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ recipient: 'owner@example.com', @@ -363,12 +529,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso // Default needsEffectReconcile is false unless explicitly true. await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-default-reconcile', messageId: message.id, eventType: 'sent', provider: 'kody', createdAt: '2026-07-02T11:30:00.000Z', + updatedAt: '2026-07-02T11:30:00.000Z', }, }) const defaultReconcile = ( @@ -377,6 +545,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso expect(defaultReconcile?.needsEffectReconcile).toBe(false) const duplicate = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-1-dup', messageId: message.id, @@ -385,6 +554,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', }, latestDeliveryStatus: { messageId: message.id, @@ -393,8 +563,10 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }, }) expect(duplicate.inserted).toBe(false) + expect(duplicate.accepted).toBe(false) const stale = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-2', messageId: message.id, @@ -403,6 +575,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-2', createdAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T09:00:00.000Z', }, latestDeliveryStatus: { messageId: message.id, @@ -410,14 +583,16 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryStatusAt: '2026-07-02T09:00:00.000Z', }, }) - expect(stale).toEqual({ inserted: true, updatedLatestStatus: false }) + expect(stale).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'delivered', deliveryStatusAt: '2026-07-02T10:30:00.000Z', }) - // Assert validation failures inside the DO isolate so remote RPC error - // plumbing does not surface uncaught worker exceptions. const stub = stubFor(userId) await runInDurableObject(stub, async (instance: Mailbox) => { await assertMailboxThrows(/export cursor is invalid/, () => @@ -431,9 +606,11 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso ) await assertMailboxThrows(/canonical ISO-8601/, () => instance.mirrorMessage({ - message: baseMessage({ + ownerId: userId, + message: baseMessage(userId, { id: 'bad-iso', createdAt: '2026-07-01T12:00:00Z', + updatedAt: '2026-07-01T12:00:00Z', }), }), ) @@ -441,7 +618,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const kinds: Array = [] let startAfter: string | null = null - for (let page = 0; page < 30; page += 1) { + for (let page = 0; page < 40; page += 1) { const exported = await mailbox.exportMailbox({ pageSize: 1, startAfter, @@ -453,10 +630,10 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso expect(row.row.id).toBe(thread.id) break case 'message': - expect(row.row.id).toBe(message.id) + expect(['export-msg', 'export-inbound']).toContain(row.row.id) break case 'attachment': - expect(row.row.id).toBe(attachment.id) + expect(['export-att', 'export-inbound-att']).toContain(row.row.id) break case 'delivery_event': expect(typeof row.row.id).toBe('string') @@ -472,13 +649,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso startAfter = exported.nextStartAfter } expect(kinds.filter((kind) => kind === 'thread')).toHaveLength(1) - expect(kinds.filter((kind) => kind === 'message')).toHaveLength(1) - expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'message')).toHaveLength(2) + expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(2) expect( kinds.filter((kind) => kind === 'delivery_event').length, ).toBeGreaterThanOrEqual(3) const blobKinds: Array = [] + const blobKeys: Array = [] let blobCursor: string | null = null for (let page = 0; page < 10; page += 1) { const pageResult = await mailbox.listBlobReferences({ @@ -487,19 +665,31 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }) for (const reference of pageResult.references) { blobKinds.push(reference.kind) + blobKeys.push(reference.key) if (reference.kind === 'raw_mime') { - expect(reference.key).toBe(message.rawMimeKey) - expect(reference.messageId).toBe(message.id) + expect(reference.key).toBe( + emailRawMimeKey(userId, inboundBlobMessage.id), + ) + expect(reference.messageId).toBe(inboundBlobMessage.id) } else { - expect(reference.key).toBe(attachment.storageKey) - expect(reference.attachmentId).toBe(attachment.id) + expect(reference.key).toBe( + emailAttachmentBlobKey( + userId, + reference.messageId, + reference.attachmentId!, + ), + ) } } if (!pageResult.truncated) break expect(pageResult.nextStartAfter).not.toBe(blobCursor) blobCursor = pageResult.nextStartAfter } - expect(blobKinds).toEqual(['raw_mime', 'attachment']) + expect(blobKinds[0]).toBe('raw_mime') + expect(blobKinds.filter((kind) => kind === 'attachment').length).toBe(2) + expect(blobKeys).toContain( + emailAttachmentBlobKey(userId, message.id, attachment.id), + ) await mailbox.purge() expect(await mailbox.countMailbox()).toEqual({ @@ -509,152 +699,3 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryEvents: 0, }) }) - -test('Mailbox retention deletes R2 blobs before metadata and backs off on failure', async () => { - silenceIncidentalRuntimeWarnings() - consoleWarn.mockImplementation((...args: Array) => { - const message = String(args[0] ?? '') - if (message.includes('mailbox-retention-blob-delete-failed')) return - }) - - const userId = uniqueUserId('retention') - const mailbox = rpcFor(userId) - const stub = stubFor(userId) - - const oldMessageAt = new Date( - Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, - ).toISOString() - const oldEventAt = new Date( - Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, - ).toISOString() - const freshAt = new Date().toISOString() - - const keepMessage = baseMessage({ - id: 'keep-msg', - subject: 'fresh', - createdAt: freshAt, - updatedAt: freshAt, - rawMimeKey: 'email-raw:v1:owner/keep-msg', - }) - const dropMessage = baseMessage({ - id: 'drop-msg', - threadId: 'drop-thread', - subject: 'old', - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - rawMimeKey: 'email-raw:v1:owner/drop-msg', - }) - const failMessage = baseMessage({ - id: 'fail-msg', - subject: 'fail-old', - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - rawMimeKey: 'email-raw:v1:owner/fail-msg', - }) - const dropAttachment = baseAttachment(dropMessage.id, { - id: 'drop-att', - storageKey: 'email-attachment:v1:owner/drop-msg/drop-att', - createdAt: oldMessageAt, - }) - - await mailbox.mirrorMessage({ - thread: baseThread({ - id: 'drop-thread', - lastMessageAt: oldMessageAt, - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - }), - message: dropMessage, - attachments: [dropAttachment], - }) - await mailbox.mirrorMessage({ message: keepMessage }) - await mailbox.mirrorMessage({ message: failMessage }) - await mailbox.upsertDeliveryEvent({ - event: { - id: 'old-event', - messageId: keepMessage.id, - eventType: 'received', - provider: 'kody', - createdAt: oldEventAt, - needsEffectReconcile: false, - }, - }) - await mailbox.upsertDeliveryEvent({ - event: { - id: 'fresh-event', - messageId: keepMessage.id, - eventType: 'received', - provider: 'kody', - createdAt: freshAt, - needsEffectReconcile: false, - }, - }) - - await env.EMAIL_BLOBS.put(dropMessage.rawMimeKey!, 'drop-raw') - await env.EMAIL_BLOBS.put(dropAttachment.storageKey!, 'drop-att') - await env.EMAIL_BLOBS.put(failMessage.rawMimeKey!, 'fail-raw') - await env.EMAIL_BLOBS.put(keepMessage.rawMimeKey!, 'keep-raw') - - const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) - env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { - const list = Array.isArray(keys) ? keys : [keys] - if (list.includes(failMessage.rawMimeKey!)) { - throw new Error('simulated R2 delete failure') - } - return await originalDelete(keys) - }) as typeof env.EMAIL_BLOBS.delete - - try { - const alarmBefore = Date.now() - const alarmAfterFailure = await runInDurableObject( - stub, - async (instance: Mailbox, state) => { - expect(instance).toBeInstanceOf(Mailbox) - await state.storage.deleteAlarm() - await instance.alarm() - return await state.storage.getAlarm() - }, - ) - // Overdue retained work (failed R2 delete) must retry with hourly backoff, - // not every second. - expect(alarmAfterFailure).toBeTypeOf('number') - expect(alarmAfterFailure).toBeGreaterThanOrEqual( - alarmBefore + mailboxRetentionRetryDelayMs - 5_000, - ) - expect(alarmAfterFailure).toBeLessThanOrEqual( - alarmBefore + mailboxRetentionRetryDelayMs + 5_000, - ) - - expect(await env.EMAIL_BLOBS.get(dropMessage.rawMimeKey!)).toBeNull() - expect(await env.EMAIL_BLOBS.get(dropAttachment.storageKey!)).toBeNull() - expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).not.toBeNull() - expect(await env.EMAIL_BLOBS.get(keepMessage.rawMimeKey!)).not.toBeNull() - - expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() - expect( - await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), - ).toHaveLength(0) - expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() - expect( - await mailbox.getMessage({ messageId: failMessage.id }), - ).toMatchObject({ - id: failMessage.id, - rawMimeKey: failMessage.rawMimeKey, - }) - expect( - await mailbox.getMessage({ messageId: keepMessage.id }), - ).toMatchObject({ - id: keepMessage.id, - }) - const events = await mailbox.listDeliveryEvents({ limit: 10 }) - expect(events.map((event) => event.id)).toEqual(['fresh-event']) - } finally { - env.EMAIL_BLOBS.delete = originalDelete - } - - await runInDurableObject(stub, async (instance: Mailbox) => { - await instance.alarm() - }) - expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).toBeNull() - expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() -}) diff --git a/packages/worker/src/email/mailbox-export.ts b/packages/worker/src/email/mailbox-export.ts index e2202c6999..a0dc656c72 100644 --- a/packages/worker/src/email/mailbox-export.ts +++ b/packages/worker/src/email/mailbox-export.ts @@ -1,3 +1,4 @@ +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxBlobRefAttachmentCursorPrefix, mailboxBlobRefRawMimeCursorPrefix, @@ -183,13 +184,23 @@ function exportPhasePage( } } +/** + * List owner-safe blob references. Inbound raw MIME keys are always derived + * from the canonical helper (even when the stored column is null). External + * attachment keys are included only when they match the canonical helper. + */ export function listMailboxBlobReferences( sql: SqlStorage, input: { + ownerId: string | null pageSize?: number startAfter?: string | null }, ): MailboxBlobReferencePage { + if (input.ownerId == null) { + return { references: [], nextStartAfter: null, truncated: false } + } + const ownerId = input.ownerId const pageSize = normalizeMailboxPageSize(input.pageSize) const startAfterRaw = typeof input.startAfter === 'string' && input.startAfter.length > 0 @@ -201,9 +212,9 @@ export function listMailboxBlobReferences( if (phase === 'raw_mime') { const rows = sql - .exec<{ id: string; raw_mime_key: string }>( - `SELECT id, raw_mime_key FROM email_messages - WHERE raw_mime_key IS NOT NULL AND id > ? + .exec<{ id: string }>( + `SELECT id FROM email_messages + WHERE direction = 'inbound' AND id > ? ORDER BY id ASC LIMIT ?`, startAfterId, @@ -215,7 +226,7 @@ export function listMailboxBlobReferences( for (const row of pageRows) { references.push({ kind: 'raw_mime', - key: row.raw_mime_key, + key: emailRawMimeKey(ownerId, row.id), messageId: row.id, attachmentId: null, }) @@ -237,6 +248,7 @@ export function listMailboxBlobReferences( } } const attachmentPage = listAttachmentBlobReferences(sql, { + ownerId, startAfterId: '', limit: remaining, }) @@ -249,6 +261,7 @@ export function listMailboxBlobReferences( } return listAttachmentBlobReferences(sql, { + ownerId, startAfterId, limit: remaining, }) @@ -257,6 +270,7 @@ export function listMailboxBlobReferences( function listAttachmentBlobReferences( sql: SqlStorage, input: { + ownerId: string startAfterId: string limit: number }, @@ -264,30 +278,48 @@ function listAttachmentBlobReferences( if (input.limit <= 0) { return { references: [], nextStartAfter: null, truncated: false } } + // Scan past non-canonical keys so callers never receive forgeable refs. + const scanLimit = Math.max(input.limit * 4, 32) const rows = sql - .exec<{ id: string; message_id: string; storage_key: string }>( + .exec<{ id: string; message_id: string; storage_key: string | null }>( `SELECT id, message_id, storage_key FROM email_attachments - WHERE storage_key IS NOT NULL AND id > ? + WHERE storage_kind = 'external' AND id > ? ORDER BY id ASC LIMIT ?`, input.startAfterId, - input.limit + 1, + scanLimit, ) .toArray() - const truncated = rows.length > input.limit - const pageRows = truncated ? rows.slice(0, input.limit) : rows - const last = pageRows[pageRows.length - 1] - return { - references: pageRows.map((row) => ({ - kind: 'attachment' as const, - key: row.storage_key, + const references: Array = [] + let lastScannedId = input.startAfterId + for (const row of rows) { + lastScannedId = row.id + const expected = emailAttachmentBlobKey( + input.ownerId, + row.message_id, + row.id, + ) + if (row.storage_key !== expected) continue + references.push({ + kind: 'attachment', + key: expected, messageId: row.message_id, attachmentId: row.id, - })), - nextStartAfter: - truncated && last - ? `${mailboxBlobRefAttachmentCursorPrefix}${last.id}` - : null, - truncated, + }) + if (references.length >= input.limit) { + return { + references, + nextStartAfter: `${mailboxBlobRefAttachmentCursorPrefix}${row.id}`, + truncated: true, + } + } + } + const maybeMore = rows.length >= scanLimit + return { + references, + nextStartAfter: maybeMore + ? `${mailboxBlobRefAttachmentCursorPrefix}${lastScannedId}` + : null, + truncated: maybeMore, } } diff --git a/packages/worker/src/email/mailbox-mappers.ts b/packages/worker/src/email/mailbox-mappers.ts index 9934469b5c..f9cb3c9f28 100644 --- a/packages/worker/src/email/mailbox-mappers.ts +++ b/packages/worker/src/email/mailbox-mappers.ts @@ -18,11 +18,13 @@ import { } from './types.ts' import { mailboxInboundDeliveryStateValues, + mailboxStorageKindValues, mailboxSubscriptionEffectStateValues, type MailboxAttachmentRecord, type MailboxDeliveryEventRecord, type MailboxInboundDeliveryState, type MailboxMessageRecord, + type MailboxStorageKind, type MailboxSubscriptionEffectState, type MailboxThreadRecord, } from './mailbox-types.ts' @@ -60,26 +62,27 @@ export function escapeLikePattern(value: string) { return value.replaceAll(/[\\%_]/g, (character) => `\\${character}`) } -function mapInboundState(value: unknown): MailboxInboundDeliveryState | null { - if (value == null) return null - const state = String(value) - return (mailboxInboundDeliveryStateValues as ReadonlyArray).includes( - state, - ) - ? (state as MailboxInboundDeliveryState) - : null +function requirePersistedEnum( + values: ReadonlyArray, + value: unknown, + label: string, +): T { + const raw = String(value ?? '') + if (!(values as ReadonlyArray).includes(raw)) { + throw new Error( + `Mailbox persisted ${label} is invalid; got ${JSON.stringify(value)}.`, + ) + } + return raw as T } -function mapSubscriptionEffectState( +function requireOptionalPersistedEnum( + values: ReadonlyArray, value: unknown, -): MailboxSubscriptionEffectState | null { + label: string, +): T | null { if (value == null) return null - const state = String(value) - return ( - mailboxSubscriptionEffectStateValues as ReadonlyArray - ).includes(state) - ? (state as MailboxSubscriptionEffectState) - : null + return requirePersistedEnum(values, value, label) } export function mapMailboxThreadRow( @@ -105,18 +108,13 @@ export function mapMailboxThreadRow( export function mapMailboxMessageRow( row: Record, ): MailboxMessageRecord { - const direction = String(row['direction'] ?? '') - const processingStatus = String(row['processing_status'] ?? '') - const classificationRaw = String(row['classification'] ?? 'accepted') - const deliveryStatusRaw = - row['delivery_status'] == null ? null : String(row['delivery_status']) return { id: String(row['id']), - direction: (emailDirectionValues as ReadonlyArray).includes( - direction, - ) - ? (direction as EmailDirection) - : 'inbound', + direction: requirePersistedEnum( + emailDirectionValues, + row['direction'], + 'direction', + ) as EmailDirection, inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), threadId: row['thread_id'] == null ? null : String(row['thread_id']), senderIdentityId: @@ -153,16 +151,16 @@ export function mapMailboxMessageRow( rawMimeKey: row['raw_mime_key'] == null ? null : String(row['raw_mime_key']), rawSize: row['raw_size'] == null ? null : Number(row['raw_size']) || 0, - processingStatus: ( - emailProcessingStatusValues as ReadonlyArray - ).includes(processingStatus) - ? (processingStatus as EmailProcessingStatus) - : 'stored', - classification: ( - emailClassificationValues as ReadonlyArray - ).includes(classificationRaw) - ? (classificationRaw as EmailClassification) - : 'accepted', + processingStatus: requirePersistedEnum( + emailProcessingStatusValues, + row['processing_status'], + 'processingStatus', + ) as EmailProcessingStatus, + classification: requirePersistedEnum( + emailClassificationValues, + row['classification'], + 'classification', + ) as EmailClassification, classificationReason: row['classification_reason'] == null ? null @@ -171,13 +169,11 @@ export function mapMailboxMessageRow( row['provider_message_id'] == null ? null : String(row['provider_message_id']), - deliveryStatus: - deliveryStatusRaw != null && - (emailDeliveryStatusValues as ReadonlyArray).includes( - deliveryStatusRaw, - ) - ? (deliveryStatusRaw as EmailDeliveryStatus) - : null, + deliveryStatus: requireOptionalPersistedEnum( + emailDeliveryStatusValues, + row['delivery_status'], + 'deliveryStatus', + ) as EmailDeliveryStatus | null, deliveryStatusAt: row['delivery_status_at'] == null ? null @@ -202,7 +198,11 @@ export function mapMailboxAttachmentRow( contentId: row['content_id'] == null ? null : String(row['content_id']), disposition: row['disposition'] == null ? null : String(row['disposition']), size: Number(row['size'] ?? 0) || 0, - storageKind: String(row['storage_kind']), + storageKind: requirePersistedEnum( + mailboxStorageKindValues, + row['storage_kind'], + 'storageKind', + ) as MailboxStorageKind, storageKey: row['storage_key'] == null ? null : String(row['storage_key']), createdAt: String(row['created_at']), } @@ -211,16 +211,15 @@ export function mapMailboxAttachmentRow( export function mapMailboxDeliveryEventRow( row: Record, ): MailboxDeliveryEventRecord { - const eventType = String(row['event_type'] ?? '') return { id: String(row['id']), messageId: row['message_id'] == null ? null : String(row['message_id']), inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), - eventType: (emailDeliveryEventTypeValues as ReadonlyArray).includes( - eventType, - ) - ? (eventType as EmailDeliveryEventType) - : 'received', + eventType: requirePersistedEnum( + emailDeliveryEventTypeValues, + row['event_type'], + 'eventType', + ) as EmailDeliveryEventType, provider: row['provider'] == null ? null : String(row['provider']), providerMessageId: row['provider_message_id'] == null @@ -232,7 +231,11 @@ export function mapMailboxDeliveryEventRow( : String(row['provider_event_id']), detailJson: String(row['detail_json'] ?? '{}'), needsEffectReconcile: Number(row['needs_effect_reconcile'] ?? 0) === 1, - state: mapInboundState(row['state']), + state: requireOptionalPersistedEnum( + mailboxInboundDeliveryStateValues, + row['state'], + 'state', + ) as MailboxInboundDeliveryState | null, fingerprint: row['fingerprint'] == null ? null : String(row['fingerprint']), storageLease: row['storage_lease'] == null ? null : String(row['storage_lease']), @@ -287,9 +290,11 @@ export function mapMailboxDeliveryEventRow( row['usage_effect_lease_at'] == null ? null : String(row['usage_effect_lease_at']), - subscriptionEffectState: mapSubscriptionEffectState( + subscriptionEffectState: requireOptionalPersistedEnum( + mailboxSubscriptionEffectStateValues, row['subscription_effect_state'], - ), + 'subscriptionEffectState', + ) as MailboxSubscriptionEffectState | null, subscriptionEffectLease: row['subscription_effect_lease'] == null ? null @@ -315,5 +320,6 @@ export function mapMailboxDeliveryEventRow( ? null : String(row['subscription_effect_last_error']), createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), } } diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts index 2a86713797..cd1cc3b6a8 100644 --- a/packages/worker/src/email/mailbox-retention.ts +++ b/packages/worker/src/email/mailbox-retention.ts @@ -1,8 +1,10 @@ +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxBlobDeleteMaxKeys, mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, mailboxRetentionBatchSize, + mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, } from './mailbox-types.ts' import { type MailboxStore } from './mailbox-store.ts' @@ -14,12 +16,54 @@ const deliveryEventRetentionMs = export type MailboxRetentionPassResult = { /** True when at least one message blob delete failed (row retained). */ hadBlobDeleteFailures: boolean + /** True when expired rows remain after a successful bounded pass. */ + expiredWorkRemaining: boolean +} + +export type MailboxRetentionRescheduleKind = + | 'idle' + | 'backoff' + | 'continue' + | 'next-due' + +export type MailboxRetentionReschedule = { + kind: MailboxRetentionRescheduleKind + atMs: number | null +} + +/** + * Deterministic next-alarm choice after a retention pass. + * - Blob delete failures → hourly backoff only + * - Successful pass with expired rows remaining → near-immediate continuation + * - Otherwise → future due-time (or idle) + */ +export function computeMailboxRetentionReschedule(input: { + nowMs: number + hadBlobDeleteFailures: boolean + expiredWorkRemaining: boolean + nextDueAtMs: number | null + continuationDelayMs?: number + backoffMs?: number +}): MailboxRetentionReschedule { + const backoffMs = input.backoffMs ?? mailboxRetentionRetryDelayMs + const continuationDelayMs = + input.continuationDelayMs ?? mailboxRetentionContinuationDelayMs + if (input.hadBlobDeleteFailures) { + return { kind: 'backoff', atMs: input.nowMs + backoffMs } + } + if (input.expiredWorkRemaining) { + return { kind: 'continue', atMs: input.nowMs + continuationDelayMs } + } + if (input.nextDueAtMs == null) { + return { kind: 'idle', atMs: null } + } + return { kind: 'next-due', atMs: input.nextDueAtMs } } /** * Soonest retention due-time from stored rows, or null when empty. * Overdue rows yield timestamps in the past — callers must apply retry backoff - * instead of scheduling at now+1s. + * or continuation instead of scheduling at now+1s blindly. */ export function nextMailboxRetentionDueAtMs( store: MailboxStore, @@ -42,7 +86,7 @@ export function nextMailboxRetentionDueAtMs( } /** - * Schedule time for the next retention alarm. + * Schedule time for the next retention alarm on a write path. * - Future due-time → schedule at due-time * - Overdue work → bounded hourly backoff (avoids 1s storms on persistent R2 failure) */ @@ -79,9 +123,39 @@ async function deleteBlobKeys( } } +/** + * Build owner-safe R2 keys for an expired message. Never deletes arbitrary + * stored key strings — only canonical inbound raw MIME keys and matching + * external attachment keys. + */ +export function canonicalRetentionBlobKeys(input: { + ownerId: string + messageId: string + direction: 'inbound' | 'outbound' + attachments: Array<{ id: string; storage_key: string | null }> +}): Array { + const keys: Array = [] + if (input.direction === 'inbound') { + keys.push(emailRawMimeKey(input.ownerId, input.messageId)) + } + for (const attachment of input.attachments) { + if (attachment.storage_key == null) continue + const expected = emailAttachmentBlobKey( + input.ownerId, + input.messageId, + attachment.id, + ) + if (attachment.storage_key === expected) { + keys.push(expected) + } + } + return keys +} + async function pruneExpiredMessages(input: { store: MailboxStore blobs: Pick + ownerId: string }): Promise { const cutoff = new Date(Date.now() - messageRetentionMs).toISOString() const rows = input.store.listExpiredMessagesForRetention({ @@ -90,26 +164,29 @@ async function pruneExpiredMessages(input: { }) if (rows.length === 0) return false - const attachmentRows = input.store.listAttachmentStorageKeysForMessages( + const attachmentRows = input.store.listAttachmentsForRetention( rows.map((row) => row.id), ) - const attachmentKeysByMessageId = new Map>() + const attachmentsByMessageId = new Map< + string, + Array<{ id: string; storage_key: string | null }> + >() for (const attachment of attachmentRows) { - const keys = attachmentKeysByMessageId.get(attachment.message_id) - if (keys) keys.push(attachment.storage_key) + const list = attachmentsByMessageId.get(attachment.message_id) + if (list) list.push(attachment) else { - attachmentKeysByMessageId.set(attachment.message_id, [ - attachment.storage_key, - ]) + attachmentsByMessageId.set(attachment.message_id, [attachment]) } } let hadBlobDeleteFailures = false for (const row of rows) { - const keys = [ - ...(row.raw_mime_key ? [row.raw_mime_key] : []), - ...(attachmentKeysByMessageId.get(row.id) ?? []), - ] + const keys = canonicalRetentionBlobKeys({ + ownerId: input.ownerId, + messageId: row.id, + direction: row.direction, + attachments: attachmentsByMessageId.get(row.id) ?? [], + }) const blobsDeleted = await deleteBlobKeys(input.blobs, keys) if (!blobsDeleted) { hadBlobDeleteFailures = true @@ -124,14 +201,30 @@ export async function enforceMailboxRetention(input: { store: MailboxStore blobs: Pick }): Promise { - const hadBlobDeleteFailures = await pruneExpiredMessages(input) + const messageCutoff = new Date(Date.now() - messageRetentionMs).toISOString() const eventCutoff = new Date( Date.now() - deliveryEventRetentionMs, ).toISOString() + + const ownerId = input.store.getOwnerId() + let hadBlobDeleteFailures = false + if (ownerId) { + hadBlobDeleteFailures = await pruneExpiredMessages({ + store: input.store, + blobs: input.blobs, + ownerId, + }) + } + input.store.pruneExpiredDeliveryEvents({ cutoff: eventCutoff, limit: mailboxRetentionBatchSize, }) input.store.pruneOrphanThreads(mailboxRetentionBatchSize) - return { hadBlobDeleteFailures } + + const expiredWorkRemaining = + input.store.hasExpiredMessages(messageCutoff) || + input.store.hasExpiredDeliveryEvents(eventCutoff) + + return { hadBlobDeleteFailures, expiredWorkRemaining } } diff --git a/packages/worker/src/email/mailbox-retention.workers.test.ts b/packages/worker/src/email/mailbox-retention.workers.test.ts new file mode 100644 index 0000000000..f0871a0a38 --- /dev/null +++ b/packages/worker/src/email/mailbox-retention.workers.test.ts @@ -0,0 +1,243 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { + computeMailboxRetentionReschedule, + Mailbox, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionContinuationDelayMs, + mailboxRetentionRetryDelayMs, +} from './mailbox-do.ts' +import { + baseAttachment, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +test('computeMailboxRetentionReschedule prefers backoff, then continue, then due-time', () => { + const nowMs = 1_700_000_000_000 + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: true, + expiredWorkRemaining: true, + nextDueAtMs: nowMs + 10_000, + }), + ).toEqual({ + kind: 'backoff', + atMs: nowMs + mailboxRetentionRetryDelayMs, + }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: true, + nextDueAtMs: nowMs + 10_000, + }), + ).toEqual({ + kind: 'continue', + atMs: nowMs + mailboxRetentionContinuationDelayMs, + }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: false, + nextDueAtMs: nowMs + 60_000, + }), + ).toEqual({ kind: 'next-due', atMs: nowMs + 60_000 }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: false, + nextDueAtMs: null, + }), + ).toEqual({ kind: 'idle', atMs: null }) +}) + +test('Mailbox retention deletes canonical R2 keys before metadata and backs off on failure', async () => { + silenceIncidentalRuntimeWarnings() + consoleWarn.mockImplementation((...args: Array) => { + const message = String(args[0] ?? '') + if (message.includes('mailbox-retention-blob-delete-failed')) return + }) + + const userId = uniqueUserId('retention') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const oldMessageAt = new Date( + Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const oldEventAt = new Date( + Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const freshAt = new Date().toISOString() + + const keepMessage = baseMessage(userId, { + id: 'keep-msg', + subject: 'fresh', + createdAt: freshAt, + updatedAt: freshAt, + }) + const dropMessage = baseMessage(userId, { + id: 'drop-msg', + threadId: 'drop-thread', + subject: 'old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const failMessage = baseMessage(userId, { + id: 'fail-msg', + subject: 'fail-old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const missingKeyMessage = baseMessage(userId, { + id: 'missing-key-msg', + subject: 'missing-stored-key', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const dropAttachment = baseAttachment(userId, dropMessage.id, { + id: 'drop-att', + createdAt: oldMessageAt, + }) + + await mailbox.mirrorMessage({ + ownerId: userId, + thread: baseThread({ + id: 'drop-thread', + lastMessageAt: oldMessageAt, + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }), + message: dropMessage, + attachments: [dropAttachment], + }) + await mailbox.mirrorMessage({ ownerId: userId, message: keepMessage }) + await mailbox.mirrorMessage({ ownerId: userId, message: failMessage }) + await mailbox.mirrorMessage({ ownerId: userId, message: missingKeyMessage }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'old-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: oldEventAt, + updatedAt: oldEventAt, + needsEffectReconcile: false, + }, + }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'fresh-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: freshAt, + updatedAt: freshAt, + needsEffectReconcile: false, + }, + }) + + const dropRawKey = emailRawMimeKey(userId, dropMessage.id) + const dropAttKey = emailAttachmentBlobKey( + userId, + dropMessage.id, + dropAttachment.id, + ) + const failRawKey = emailRawMimeKey(userId, failMessage.id) + const keepRawKey = emailRawMimeKey(userId, keepMessage.id) + const missingRawKey = emailRawMimeKey(userId, missingKeyMessage.id) + + await env.EMAIL_BLOBS.put(dropRawKey, 'drop-raw') + await env.EMAIL_BLOBS.put(dropAttKey, 'drop-att') + await env.EMAIL_BLOBS.put(failRawKey, 'fail-raw') + await env.EMAIL_BLOBS.put(keepRawKey, 'keep-raw') + await env.EMAIL_BLOBS.put(missingRawKey, 'missing-raw') + + // Stored key missing must not prevent deleting the canonical inbound key. + await runInDurableObject(stub, async (_instance: Mailbox, state) => { + state.storage.sql.exec( + `UPDATE email_messages SET raw_mime_key = NULL WHERE id = ?`, + missingKeyMessage.id, + ) + }) + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { + const list = Array.isArray(keys) ? keys : [keys] + if (list.includes(failRawKey)) { + throw new Error('simulated R2 delete failure') + } + return await originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const alarmBefore = Date.now() + const alarmAfterFailure = await runInDurableObject( + stub, + async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + await state.storage.deleteAlarm() + await instance.alarm() + return await state.storage.getAlarm() + }, + ) + // Overdue retained work (failed R2 delete) must retry with hourly backoff, + // not every second. + expect(alarmAfterFailure).toBeTypeOf('number') + expect(alarmAfterFailure).toBeGreaterThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs - 5_000, + ) + expect(alarmAfterFailure).toBeLessThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs + 5_000, + ) + + expect(await env.EMAIL_BLOBS.get(dropRawKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(dropAttKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(missingRawKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(failRawKey)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(keepRawKey)).not.toBeNull() + + expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), + ).toHaveLength(0) + expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() + expect( + await mailbox.getMessage({ messageId: missingKeyMessage.id }), + ).toBeNull() + expect( + await mailbox.getMessage({ messageId: failMessage.id }), + ).toMatchObject({ + id: failMessage.id, + }) + expect( + await mailbox.getMessage({ messageId: keepMessage.id }), + ).toMatchObject({ + id: keepMessage.id, + }) + const events = await mailbox.listDeliveryEvents({ limit: 10 }) + expect(events.map((event) => event.id)).toEqual(['fresh-event']) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + } + + await runInDurableObject(stub, async (instance: Mailbox) => { + await instance.alarm() + }) + expect(await env.EMAIL_BLOBS.get(failRawKey)).toBeNull() + expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts index ef8bcdda5d..77a8c8948e 100644 --- a/packages/worker/src/email/mailbox-schema.ts +++ b/packages/worker/src/email/mailbox-schema.ts @@ -29,48 +29,10 @@ export function setMailboxMeta( ) } -/** Warm v1 objects lack promoted inbound columns. */ -function ensureDeliveryEventPromotedColumns( - sql: SqlStorage, - installedVersion: number | null, -) { - if (installedVersion == null || installedVersion >= mailboxSchemaVersion) { - return - } - const columns: Array = [ - `state TEXT`, - `fingerprint TEXT`, - `storage_lease TEXT`, - `storage_lease_at TEXT`, - `cleanup_lease TEXT`, - `cleanup_lease_at TEXT`, - `cleanup_retry_at TEXT`, - `expected_attachment_count INTEGER`, - `finalization_token TEXT`, - `reconcile_after TEXT`, - `dedupe_expires_at TEXT`, - `usage_effect_suppressed_at TEXT`, - `usage_started_at TEXT`, - `usage_effect_retry_at TEXT`, - `usage_effect_lease TEXT`, - `usage_effect_lease_at TEXT`, - `subscription_effect_state TEXT`, - `subscription_effect_lease TEXT`, - `subscription_effect_lease_at TEXT`, - `subscription_effect_retry_at TEXT`, - `subscription_effect_attempt_count INTEGER`, - `subscription_effect_dead_letter_at TEXT`, - `subscription_effect_last_error TEXT`, - ] - for (const column of columns) { - try { - sql.exec(`ALTER TABLE email_delivery_events ADD COLUMN ${column}`) - } catch { - // Column already present on a partially migrated object. - } - } -} - +/** + * Schema v1 — coherent ship shape (this DO class has never been deployed). + * No warm ALTER migrations; bump `mailboxSchemaVersion` only with a real plan. + */ export function initializeMailboxSchema(storage: DurableObjectStorage) { const sql = storage.sql sql.exec(` @@ -82,6 +44,20 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { const installedVersion = getMailboxMeta(storage, mailboxMetaSchemaVersionKey) if (installedVersion === mailboxSchemaVersion) return + /** + * Cloudflare DO runtime does not expose a reverse lookup from the object + * id to the `idFromName` string, so the human owner id is not + * introspectable inside the DO. Ownership for blob-key validation and + * cross-user rejection is an explicit write-contract field, persisted + * once in this singleton row. Data rows still have no `user_id` column. + */ + sql.exec(` + CREATE TABLE IF NOT EXISTS mailbox_owner_identity ( + singleton INTEGER PRIMARY KEY NOT NULL CHECK (singleton = 1), + owner_id TEXT NOT NULL + ) + `) + sql.exec(` CREATE TABLE IF NOT EXISTS email_threads ( id TEXT PRIMARY KEY NOT NULL, @@ -276,10 +252,10 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { subscription_effect_attempt_count INTEGER, subscription_effect_dead_letter_at TEXT, subscription_effect_last_error TEXT, - created_at TEXT NOT NULL + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL ) `) - ensureDeliveryEventPromotedColumns(sql, installedVersion) sql.exec( `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_message_id ON email_delivery_events(message_id)`, diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index ce5c591766..8bb0307f6a 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -1,14 +1,11 @@ import { assertMailboxCanonicalIsoTimestamp, assertMailboxClassification, - assertMailboxDeliveryEventType, assertMailboxDeliveryStatus, assertMailboxDirection, - assertMailboxInboundDeliveryState, assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertMailboxStorageKind, - assertMailboxSubscriptionEffectState, assertOptionalMailboxCanonicalIsoTimestamp, decodeMailboxListCursor, encodeMailboxListCursor, @@ -31,21 +28,35 @@ import { import { type EmailDeliveryEventType, type EmailDeliveryStatus, + type EmailDirection, } from './types.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { boundedEmailBody, escapeLikePattern, mapMailboxAttachmentRow, - mapMailboxDeliveryEventRow, mapMailboxMessageRow, mapMailboxThreadRow, } from './mailbox-mappers.ts' +import { + deliveryEventOwnsMessage, + hasExpiredMailboxDeliveryEvents, + listMailboxDeliveryEvents, + oldestMailboxDeliveryEventCreatedAt, + pruneExpiredMailboxDeliveryEvents, + writeMailboxDeliveryEventRow, +} from './mailbox-delivery-events.ts' import { exportMailboxFromStore, listMailboxBlobReferences, } from './mailbox-export.ts' import { initializeMailboxSchema } from './mailbox-schema.ts' +export type MailboxUpsertResult = { + created: boolean + accepted: boolean +} + /** * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. */ @@ -64,7 +75,97 @@ export class MailboxStore { initializeMailboxSchema(this.storage) } - upsertThreadRow(thread: MailboxThreadInput) { + getOwnerId(): string | null { + const row = this.sql + .exec<{ owner_id: string }>( + `SELECT owner_id FROM mailbox_owner_identity + WHERE singleton = 1 + LIMIT 1`, + ) + .toArray()[0] + return row?.owner_id ?? null + } + + /** + * Persist owner once; reject cross-user writes. DO name is not + * introspectable — see mailbox_owner_identity DDL comment. + */ + assertOwner(ownerId: string): string { + const id = assertMailboxNonEmptyString(ownerId, 'ownerId') + const existing = this.getOwnerId() + if (existing == null) { + this.sql.exec( + `INSERT INTO mailbox_owner_identity (singleton, owner_id) + VALUES (1, ?)`, + id, + ) + return id + } + if (existing !== id) { + throw new Error( + `Mailbox ownerId mismatch; this object is bound to a different owner.`, + ) + } + return id + } + + validateMessageBlobKeys(input: { + ownerId: string + message: MailboxMessageInput + attachments?: Array + }) { + const messageId = assertMailboxNonEmptyString( + input.message.id, + 'message.id', + ) + const direction = assertMailboxDirection(input.message.direction) + const rawMimeKey = input.message.rawMimeKey ?? null + if (direction === 'inbound') { + const expected = emailRawMimeKey(input.ownerId, messageId) + if (rawMimeKey !== expected) { + throw new Error( + `Mailbox inbound rawMimeKey must equal emailRawMimeKey(ownerId, messageId).`, + ) + } + } else if (rawMimeKey != null) { + const expected = emailRawMimeKey(input.ownerId, messageId) + if (rawMimeKey !== expected) { + throw new Error( + `Mailbox outbound rawMimeKey must be null or equal emailRawMimeKey(ownerId, messageId).`, + ) + } + } + + if (input.attachments === undefined) return + for (const attachment of input.attachments) { + const attachmentId = assertMailboxNonEmptyString( + attachment.id, + 'attachment.id', + ) + const storageKind = assertMailboxStorageKind( + String(attachment.storageKind), + ) + const storageKey = attachment.storageKey ?? null + if (storageKind === 'external') { + const expected = emailAttachmentBlobKey( + input.ownerId, + messageId, + attachmentId, + ) + if (storageKey !== expected) { + throw new Error( + `Mailbox external attachment storageKey must equal emailAttachmentBlobKey(ownerId, messageId, attachmentId).`, + ) + } + } else if (storageKey != null) { + throw new Error( + `Mailbox ${storageKind} attachment storageKey must be null.`, + ) + } + } + } + + upsertThreadRow(thread: MailboxThreadInput): MailboxUpsertResult { const id = assertMailboxNonEmptyString(thread.id, 'thread.id') const lastMessageAt = assertMailboxCanonicalIsoTimestamp( thread.lastMessageAt, @@ -75,9 +176,18 @@ export class MailboxStore { 'thread.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( - thread.updatedAt?.trim() || mailboxNowIso(), + thread.updatedAt, 'thread.updatedAt', ) + const existing = this.sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + if (existing && updatedAt < existing.updated_at) { + return { created: false, accepted: false } + } this.sql.exec( `INSERT INTO email_threads ( id, inbox_id, subject_normalized, root_message_id_header, @@ -88,7 +198,8 @@ export class MailboxStore { subject_normalized = excluded.subject_normalized, root_message_id_header = excluded.root_message_id_header, last_message_at = excluded.last_message_at, - updated_at = excluded.updated_at`, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_threads.updated_at`, id, thread.inboxId ?? null, thread.subjectNormalized ?? '', @@ -97,14 +208,15 @@ export class MailboxStore { createdAt, updatedAt, ) + return { created: existing == null, accepted: true } } /** - * Upsert message metadata. Delivery status is monotonic by - * `delivery_status_at` (`>=` / equal timestamps may update; older must not - * regress a newer status from dual-write replay). + * Upsert message metadata. Whole-row updates require equal/newer + * `updatedAt`. Delivery status remains monotonic by `delivery_status_at` + * within an accepted snapshot. */ - upsertMessageRow(message: MailboxMessageInput) { + upsertMessageRow(message: MailboxMessageInput): MailboxUpsertResult { const id = assertMailboxNonEmptyString(message.id, 'message.id') const direction = assertMailboxDirection(message.direction) const processingStatus = assertMailboxProcessingStatus( @@ -131,7 +243,7 @@ export class MailboxStore { 'message.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( - message.updatedAt?.trim() || mailboxNowIso(), + message.updatedAt, 'message.updatedAt', ) const receivedAt = assertOptionalMailboxCanonicalIsoTimestamp( @@ -142,6 +254,15 @@ export class MailboxStore { message.sentAt, 'message.sentAt', ) + const existing = this.sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + if (existing && updatedAt < existing.updated_at) { + return { created: false, accepted: false } + } this.sql.exec( `INSERT INTO email_messages ( id, direction, inbox_id, thread_id, sender_identity_id, @@ -202,7 +323,8 @@ export class MailboxStore { error = excluded.error, received_at = excluded.received_at, sent_at = excluded.sent_at, - updated_at = excluded.updated_at`, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_messages.updated_at`, id, direction, message.inboxId ?? null, @@ -236,6 +358,7 @@ export class MailboxStore { createdAt, updatedAt, ) + return { created: existing == null, accepted: true } } replaceAttachmentsForMessage( @@ -283,188 +406,11 @@ export class MailboxStore { } } - findDeliveryEventByProviderEventId( - providerEventId: string, - ): MailboxDeliveryEventRecord | null { - const row = this.sql - .exec>( - `SELECT * FROM email_delivery_events - WHERE provider_event_id = ? - LIMIT 1`, - providerEventId, - ) - .toArray()[0] - return row ? mapMailboxDeliveryEventRow(row) : null - } - - writeDeliveryEventRow(event: MailboxDeliveryEventInput): boolean { - const id = assertMailboxNonEmptyString(event.id, 'event.id') - const eventType = assertMailboxDeliveryEventType(event.eventType) - const providerEventId = event.providerEventId ?? null - if (providerEventId) { - const existingByProvider = - this.findDeliveryEventByProviderEventId(providerEventId) - if (existingByProvider && existingByProvider.id !== id) { - return false - } - } - const existed = this.sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, - id, - ) - .toArray()[0] - const createdAt = assertMailboxCanonicalIsoTimestamp( - event.createdAt?.trim() || mailboxNowIso(), - 'event.createdAt', - ) - const detailJson = - typeof event.detailJson === 'string' && event.detailJson.length > 0 - ? event.detailJson - : '{}' - const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 - const state = - event.state == null - ? null - : assertMailboxInboundDeliveryState(event.state) - const subscriptionEffectState = - event.subscriptionEffectState == null - ? null - : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) - - this.sql.exec( - `INSERT INTO email_delivery_events ( - id, message_id, inbox_id, event_type, provider, - provider_message_id, provider_event_id, detail_json, - needs_effect_reconcile, state, fingerprint, - storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, - cleanup_retry_at, expected_attachment_count, finalization_token, - reconcile_after, dedupe_expires_at, - usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, - usage_month, usage_bytes, usage_duration_ms, - usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, - subscription_effect_state, subscription_effect_lease, - subscription_effect_lease_at, subscription_effect_retry_at, - subscription_effect_attempt_count, subscription_effect_dead_letter_at, - subscription_effect_last_error, created_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(id) DO UPDATE SET - message_id = excluded.message_id, - inbox_id = excluded.inbox_id, - event_type = excluded.event_type, - provider = excluded.provider, - provider_message_id = excluded.provider_message_id, - provider_event_id = excluded.provider_event_id, - detail_json = excluded.detail_json, - needs_effect_reconcile = excluded.needs_effect_reconcile, - state = excluded.state, - fingerprint = excluded.fingerprint, - storage_lease = excluded.storage_lease, - storage_lease_at = excluded.storage_lease_at, - cleanup_lease = excluded.cleanup_lease, - cleanup_lease_at = excluded.cleanup_lease_at, - cleanup_retry_at = excluded.cleanup_retry_at, - expected_attachment_count = excluded.expected_attachment_count, - finalization_token = excluded.finalization_token, - reconcile_after = excluded.reconcile_after, - dedupe_expires_at = excluded.dedupe_expires_at, - usage_effect_recorded_at = excluded.usage_effect_recorded_at, - usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, - usage_started_at = excluded.usage_started_at, - usage_month = excluded.usage_month, - usage_bytes = excluded.usage_bytes, - usage_duration_ms = excluded.usage_duration_ms, - usage_effect_retry_at = excluded.usage_effect_retry_at, - usage_effect_lease = excluded.usage_effect_lease, - usage_effect_lease_at = excluded.usage_effect_lease_at, - subscription_effect_state = excluded.subscription_effect_state, - subscription_effect_lease = excluded.subscription_effect_lease, - subscription_effect_lease_at = excluded.subscription_effect_lease_at, - subscription_effect_retry_at = excluded.subscription_effect_retry_at, - subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, - subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, - subscription_effect_last_error = excluded.subscription_effect_last_error`, - id, - event.messageId ?? null, - event.inboxId ?? null, - eventType, - event.provider ?? 'kody', - event.providerMessageId ?? null, - providerEventId, - detailJson, - needsEffectReconcile, - state, - event.fingerprint ?? null, - event.storageLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.storageLeaseAt, - 'event.storageLeaseAt', - ), - event.cleanupLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.cleanupLeaseAt, - 'event.cleanupLeaseAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.cleanupRetryAt, - 'event.cleanupRetryAt', - ), - event.expectedAttachmentCount ?? null, - event.finalizationToken ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.reconcileAfter, - 'event.reconcileAfter', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.dedupeExpiresAt, - 'event.dedupeExpiresAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectRecordedAt, - 'event.usageEffectRecordedAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectSuppressedAt, - 'event.usageEffectSuppressedAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageStartedAt, - 'event.usageStartedAt', - ), - event.usageMonth ?? null, - event.usageBytes ?? null, - event.usageDurationMs ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectRetryAt, - 'event.usageEffectRetryAt', - ), - event.usageEffectLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectLeaseAt, - 'event.usageEffectLeaseAt', - ), - subscriptionEffectState, - event.subscriptionEffectLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectLeaseAt, - 'event.subscriptionEffectLeaseAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectRetryAt, - 'event.subscriptionEffectRetryAt', - ), - event.subscriptionEffectAttemptCount ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectDeadLetterAt, - 'event.subscriptionEffectDeadLetterAt', - ), - event.subscriptionEffectLastError ?? null, - createdAt, - ) - return existed == null + writeDeliveryEventRow(event: MailboxDeliveryEventInput): { + inserted: boolean + accepted: boolean + } { + return writeMailboxDeliveryEventRow(this.sql, event) } updateLatestDeliveryStatus(input: { @@ -495,17 +441,7 @@ export class MailboxStore { } deliveryEventOwnsMessage(eventId: string, messageId: string) { - return ( - this.sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events - WHERE id = ? AND message_id = ? - LIMIT 1`, - eventId, - messageId, - ) - .toArray()[0] != null - ) + return deliveryEventOwnsMessage(this.sql, eventId, messageId) } getThread(threadId: string): MailboxThreadRecord | null { @@ -685,28 +621,7 @@ export class MailboxStore { eventType?: EmailDeliveryEventType | null limit?: number }): Array { - const limit = normalizeMailboxPageSize(input.limit) - const clauses: Array = ['1 = 1'] - const params: Array = [] - if (input.messageId) { - clauses.push('message_id = ?') - params.push(input.messageId) - } - if (input.eventType) { - clauses.push('event_type = ?') - params.push(assertMailboxDeliveryEventType(input.eventType)) - } - params.push(limit) - return this.sql - .exec>( - `SELECT * FROM email_delivery_events - WHERE ${clauses.join(' AND ')} - ORDER BY created_at DESC, id DESC - LIMIT ?`, - ...params, - ) - .toArray() - .map(mapMailboxDeliveryEventRow) + return listMailboxDeliveryEvents(this.sql, input) } countMailbox(): MailboxCountResult { @@ -741,7 +656,11 @@ export class MailboxStore { pageSize?: number startAfter?: string | null }): MailboxBlobReferencePage { - return listMailboxBlobReferences(this.sql, input) + return listMailboxBlobReferences(this.sql, { + ownerId: this.getOwnerId(), + pageSize: input.pageSize, + startAfter: input.startAfter, + }) } oldestMessageCreatedAt(): string | null { @@ -756,27 +675,24 @@ export class MailboxStore { } oldestDeliveryEventCreatedAt(): string | null { - const row = this.sql - .exec<{ created_at: string }>( - `SELECT created_at FROM email_delivery_events - ORDER BY created_at ASC, id ASC - LIMIT 1`, - ) - .toArray()[0] - return row?.created_at ?? null + return oldestMailboxDeliveryEventCreatedAt(this.sql) } listExpiredMessagesForRetention(input: { cutoff: string limit: number - }): Array<{ id: string; raw_mime_key: string | null; created_at: string }> { + }): Array<{ + id: string + direction: EmailDirection + created_at: string + }> { return this.sql .exec<{ id: string - raw_mime_key: string | null + direction: EmailDirection created_at: string }>( - `SELECT id, raw_mime_key, created_at FROM email_messages + `SELECT id, direction, created_at FROM email_messages WHERE created_at < ? ORDER BY created_at ASC, id ASC LIMIT ?`, @@ -786,15 +702,18 @@ export class MailboxStore { .toArray() } - listAttachmentStorageKeysForMessages( + listAttachmentsForRetention( messageIds: Array, - ): Array<{ message_id: string; storage_key: string }> { + ): Array<{ id: string; message_id: string; storage_key: string | null }> { if (messageIds.length === 0) return [] return this.sql - .exec<{ message_id: string; storage_key: string }>( - `SELECT message_id, storage_key FROM email_attachments - WHERE storage_key IS NOT NULL - AND message_id IN (${messageIds.map(() => '?').join(', ')})`, + .exec<{ + id: string + message_id: string + storage_key: string | null + }>( + `SELECT id, message_id, storage_key FROM email_attachments + WHERE message_id IN (${messageIds.map(() => '?').join(', ')})`, ...messageIds, ) .toArray() @@ -809,19 +728,26 @@ export class MailboxStore { } pruneExpiredDeliveryEvents(input: { cutoff: string; limit: number }) { - this.sql.exec( - `DELETE FROM email_delivery_events - WHERE id IN ( - SELECT id FROM email_delivery_events - WHERE created_at < ? - ORDER BY created_at ASC, id ASC - LIMIT ? - )`, - input.cutoff, - input.limit, + pruneExpiredMailboxDeliveryEvents(this.sql, input) + } + + hasExpiredMessages(cutoff: string): boolean { + return ( + this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_messages + WHERE created_at < ? + LIMIT 1`, + cutoff, + ) + .toArray()[0] != null ) } + hasExpiredDeliveryEvents(cutoff: string): boolean { + return hasExpiredMailboxDeliveryEvents(this.sql, cutoff) + } + pruneOrphanThreads(limit: number) { this.sql.exec( `DELETE FROM email_threads diff --git a/packages/worker/src/email/mailbox-test-helpers.ts b/packages/worker/src/email/mailbox-test-helpers.ts new file mode 100644 index 0000000000..3578f312b4 --- /dev/null +++ b/packages/worker/src/email/mailbox-test-helpers.ts @@ -0,0 +1,122 @@ +import { env } from 'cloudflare:workers' +import { expect } from 'vitest' +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + type MailboxAttachmentInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-do.ts' + +export function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { + const mailbox = (env as MailboxEnv).MAILBOX + if (!mailbox) { + throw new Error( + 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', + ) + } + return { MAILBOX: mailbox } +} + +export function stubFor(userId: string) { + const { MAILBOX } = mailboxEnv() + return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) +} + +export function rpcFor(userId: string) { + return mailboxRpc({ env: mailboxEnv(), userId }) +} + +export function uniqueUserId(label: string) { + return `mailbox-${label}-${crypto.randomUUID()}` +} + +export async function assertMailboxThrows( + pattern: RegExp, + run: () => Promise, +) { + try { + await run() + throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) + } catch (error) { + expect(String(error)).toMatch(pattern) + } +} + +export function baseThread( + overrides?: Partial, +): MailboxThreadInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: ``, + lastMessageAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +export function baseMessage( + ownerId: string, + overrides?: Partial, +): MailboxMessageInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' + const direction = overrides?.direction ?? 'inbound' + const rawMimeKey = + overrides?.rawMimeKey !== undefined + ? overrides.rawMimeKey + : direction === 'outbound' + ? null + : emailRawMimeKey(ownerId, id) + return { + inboxId: 'inbox-1', + threadId: null, + fromAddress: 'sender@example.com', + envelopeFrom: 'envelope@example.com', + toAddresses: ['owner@example.com'], + subject: 'Hello mailbox', + messageIdHeader: ``, + rawSize: 128, + processingStatus: 'stored', + classification: 'accepted', + receivedAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + id, + direction, + rawMimeKey, + } +} + +export function baseAttachment( + ownerId: string, + messageId: string, + overrides?: Partial, +): MailboxAttachmentInput { + const id = overrides?.id ?? crypto.randomUUID() + const storageKind = overrides?.storageKind ?? 'external' + const storageKey = + overrides?.storageKey !== undefined + ? overrides.storageKey + : storageKind === 'external' + ? emailAttachmentBlobKey(ownerId, messageId, id) + : null + return { + filename: 'note.txt', + contentType: 'text/plain', + size: 12, + createdAt: '2026-07-01T12:00:00.000Z', + ...overrides, + id, + messageId, + storageKind, + storageKey, + } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index a93016b2ab..1702e2b1b1 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -24,10 +24,12 @@ export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ export const mailboxRetentionRetryDelayMs = 60 * 60 * 1000 +/** Near-immediate continuation when a retention pass succeeds but expired rows remain. */ +export const mailboxRetentionContinuationDelayMs = 1_000 export const mailboxBlobDeleteMaxKeys = 1000 /** Bump when initializeSchema's DDL set changes; warm objects skip DDL. */ -export const mailboxSchemaVersion = 2 +export const mailboxSchemaVersion = 1 export const mailboxMetaSchemaVersionKey = 'schema_version' export const mailboxExportThreadCursorPrefix = 'thread:' @@ -122,7 +124,7 @@ export type MailboxAttachmentRecord = { contentId: string | null disposition: string | null size: number - storageKind: string + storageKind: MailboxStorageKind storageKey: string | null createdAt: string } @@ -169,6 +171,7 @@ export type MailboxDeliveryEventRecord = { subscriptionEffectDeadLetterAt: string | null subscriptionEffectLastError: string | null createdAt: string + updatedAt: string } export type MailboxThreadInput = { @@ -178,7 +181,8 @@ export type MailboxThreadInput = { rootMessageIdHeader?: string | null lastMessageAt: string createdAt?: string - updatedAt?: string + /** Required for stale-snapshot rejection (equal/newer only). */ + updatedAt: string } export type MailboxMessageInput = { @@ -213,7 +217,8 @@ export type MailboxMessageInput = { receivedAt?: string | null sentAt?: string | null createdAt?: string - updatedAt?: string + /** Required for stale-snapshot rejection (equal/newer only). */ + updatedAt: string } export type MailboxAttachmentInput = { @@ -268,6 +273,8 @@ export type MailboxDeliveryEventInput = { subscriptionEffectDeadLetterAt?: string | null subscriptionEffectLastError?: string | null createdAt?: string + /** Required monotonic mirror timestamp (equal/newer accepted). */ + updatedAt: string } export type MailboxExportRow = @@ -334,18 +341,33 @@ export type MailboxExportPhase = export type MailboxRpc = { mirrorMessage: (input: { + /** + * Explicit owner binding. DO name is not introspectable at runtime, so + * writers must pass the owning user id for blob-key validation and + * singleton identity checks. + */ + ownerId: string thread?: MailboxThreadInput | null message: MailboxMessageInput + /** + * Omitted → leave existing attachment metadata unchanged. + * Explicit `[]` → clear attachments (only when the message snapshot is accepted). + */ attachments?: Array - }) => Promise<{ ok: true }> + }) => Promise<{ ok: true; accepted: boolean }> upsertDeliveryEvent: (input: { + ownerId: string event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null - }) => Promise<{ inserted: boolean; updatedLatestStatus: boolean }> + }) => Promise<{ + inserted: boolean + accepted: boolean + updatedLatestStatus: boolean + }> getThread: (input: { threadId: string }) => Promise diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 6cedacf759..3503580936 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -50,30 +50,9 @@ export type EmailInboundDeliveryFence = { storageLease: string } -/** - * R2 object key for a message's raw MIME payload in the EMAIL_BLOBS - * bucket. The userId prefix is part of the per-user isolation contract: - * account deletion enumerates and deletes a user's blobs by these stored - * keys, and the key can never be forged to point at another user's mail. - * Writers always store this canonical key in `raw_mime_key`. - */ -export function emailRawMimeKey(userId: string, messageId: string) { - return `email-raw:v1:${userId}/${messageId}` -} +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' -/** - * R2 object key for an attachment stored on its own (storage_kind - * 'external'), used by outbound mail whose bytes never exist as raw MIME. - * The userId prefix follows the same per-user isolation contract as - * emailRawMimeKey. - */ -export function emailAttachmentBlobKey( - userId: string, - messageId: string, - attachmentId: string, -) { - return `email-attachment:v1:${userId}/${messageId}/${attachmentId}` -} +export { emailAttachmentBlobKey, emailRawMimeKey } // One corrupt stored row must not fail an entire list/search response, so // the record parser degrades to an empty shape on malformed JSON. From 309fc692849d8821d168285071315339455b13c3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 07:15:25 +0000 Subject: [PATCH 03/20] fix(email): make Mailbox mirrors full snapshots Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 19 +- .../src/email/mailbox-delivery-events.ts | 52 +++--- packages/worker/src/email/mailbox-do.ts | 52 ++++-- .../src/email/mailbox-do.workers.test.ts | 40 ++-- .../worker/src/email/mailbox-retention.ts | 34 ++++ .../email/mailbox-retention.workers.test.ts | 46 ++++- packages/worker/src/email/mailbox-store.ts | 83 ++++----- .../worker/src/email/mailbox-test-helpers.ts | 92 +++++++++- packages/worker/src/email/mailbox-types.ts | 172 ++++++++++-------- 9 files changed, 388 insertions(+), 202 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 99179b9d8a..f270fb639b 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -609,12 +609,15 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` -**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` require -`ownerId` and apply equal-or-newer `updatedAt` snapshots (stale snapshots are -ignored, not applied). On `mirrorMessage`, omitting `attachments` preserves -existing attachment metadata; an explicit `attachments: []` clears them. -Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment -`storageKey` values against the canonical builders for that `ownerId`. +**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` take +complete snapshots — every persisted field is explicit (nullable fields use +explicit `null`). They are not patch APIs. Both require `ownerId` and apply +equal-or-newer `updatedAt` snapshots (stale snapshots are ignored, not applied). +The only omission exception is the `mirrorMessage` `attachments` bundle: +omitting it preserves existing attachment rows; an explicit `attachments: []` +clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and +external attachment `storageKey` values against the canonical builders for that +`ownerId`. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -623,7 +626,9 @@ than trusting stored key strings), then apply strict blob-before-row ordering for `EMAIL_BLOBS` (failed blob deletes skip the row for retry). After a retention pass: successful work with expired rows remaining schedules a near-immediate continuation (`mailboxRetentionContinuationDelayMs`); R2 delete -failures use hourly backoff (`mailboxRetentionRetryDelayMs`). +failures use hourly backoff (`mailboxRetentionRetryDelayMs`). Write-path alarm +selection never postpones an earlier existing alarm under sustained writes +(near-equal times within skew keep the existing alarm). Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state diff --git a/packages/worker/src/email/mailbox-delivery-events.ts b/packages/worker/src/email/mailbox-delivery-events.ts index 8be19c23cc..88e71b364e 100644 --- a/packages/worker/src/email/mailbox-delivery-events.ts +++ b/packages/worker/src/email/mailbox-delivery-events.ts @@ -5,7 +5,6 @@ import { assertMailboxNonEmptyString, assertMailboxSubscriptionEffectState, assertOptionalMailboxCanonicalIsoTimestamp, - mailboxNowIso, normalizeMailboxPageSize, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, @@ -34,7 +33,8 @@ export function writeMailboxDeliveryEventRow( ): { inserted: boolean; accepted: boolean } { const id = assertMailboxNonEmptyString(event.id, 'event.id') const eventType = assertMailboxDeliveryEventType(event.eventType) - const providerEventId = event.providerEventId ?? null + const provider = assertMailboxNonEmptyString(event.provider, 'event.provider') + const providerEventId = event.providerEventId if (providerEventId) { const existingByProvider = findDeliveryEventByProviderEventId( sql, @@ -51,7 +51,7 @@ export function writeMailboxDeliveryEventRow( ) .toArray()[0] const createdAt = assertMailboxCanonicalIsoTimestamp( - event.createdAt?.trim() || mailboxNowIso(), + event.createdAt, 'event.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -61,11 +61,13 @@ export function writeMailboxDeliveryEventRow( if (existing && updatedAt < existing.updated_at) { return { inserted: false, accepted: false } } - const detailJson = - typeof event.detailJson === 'string' && event.detailJson.length > 0 - ? event.detailJson - : '{}' - const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + if (typeof event.detailJson !== 'string') { + throw new Error('Mailbox event.detailJson must be a string.') + } + if (typeof event.needsEffectReconcile !== 'boolean') { + throw new Error('Mailbox event.needsEffectReconcile must be a boolean.') + } + const needsEffectReconcile = event.needsEffectReconcile ? 1 : 0 const state = event.state == null ? null : assertMailboxInboundDeliveryState(event.state) const subscriptionEffectState = @@ -131,22 +133,22 @@ export function writeMailboxDeliveryEventRow( updated_at = excluded.updated_at WHERE excluded.updated_at >= email_delivery_events.updated_at`, id, - event.messageId ?? null, - event.inboxId ?? null, + event.messageId, + event.inboxId, eventType, - event.provider ?? 'kody', - event.providerMessageId ?? null, + provider, + event.providerMessageId, providerEventId, - detailJson, + event.detailJson, needsEffectReconcile, state, - event.fingerprint ?? null, - event.storageLease ?? null, + event.fingerprint, + event.storageLease, assertOptionalMailboxCanonicalIsoTimestamp( event.storageLeaseAt, 'event.storageLeaseAt', ), - event.cleanupLease ?? null, + event.cleanupLease, assertOptionalMailboxCanonicalIsoTimestamp( event.cleanupLeaseAt, 'event.cleanupLeaseAt', @@ -155,8 +157,8 @@ export function writeMailboxDeliveryEventRow( event.cleanupRetryAt, 'event.cleanupRetryAt', ), - event.expectedAttachmentCount ?? null, - event.finalizationToken ?? null, + event.expectedAttachmentCount, + event.finalizationToken, assertOptionalMailboxCanonicalIsoTimestamp( event.reconcileAfter, 'event.reconcileAfter', @@ -177,20 +179,20 @@ export function writeMailboxDeliveryEventRow( event.usageStartedAt, 'event.usageStartedAt', ), - event.usageMonth ?? null, - event.usageBytes ?? null, - event.usageDurationMs ?? null, + event.usageMonth, + event.usageBytes, + event.usageDurationMs, assertOptionalMailboxCanonicalIsoTimestamp( event.usageEffectRetryAt, 'event.usageEffectRetryAt', ), - event.usageEffectLease ?? null, + event.usageEffectLease, assertOptionalMailboxCanonicalIsoTimestamp( event.usageEffectLeaseAt, 'event.usageEffectLeaseAt', ), subscriptionEffectState, - event.subscriptionEffectLease ?? null, + event.subscriptionEffectLease, assertOptionalMailboxCanonicalIsoTimestamp( event.subscriptionEffectLeaseAt, 'event.subscriptionEffectLeaseAt', @@ -199,12 +201,12 @@ export function writeMailboxDeliveryEventRow( event.subscriptionEffectRetryAt, 'event.subscriptionEffectRetryAt', ), - event.subscriptionEffectAttemptCount ?? null, + event.subscriptionEffectAttemptCount, assertOptionalMailboxCanonicalIsoTimestamp( event.subscriptionEffectDeadLetterAt, 'event.subscriptionEffectDeadLetterAt', ), - event.subscriptionEffectLastError ?? null, + event.subscriptionEffectLastError, createdAt, updatedAt, ) diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 146a9ac7bf..11cabd8d6f 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -10,11 +10,11 @@ import { enforceMailboxRetention, mailboxRetentionAlarmAtMs, nextMailboxRetentionDueAtMs, + selectMailboxRetentionWriteAlarm, } from './mailbox-retention.ts' import { MailboxStore } from './mailbox-store.ts' import { assertMailboxNonEmptyString, - mailboxRetentionAlarmSkewMs, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, @@ -71,6 +71,8 @@ class MailboxBase extends DurableObject implements MailboxRpc { * * - Schedule at most one alarm for the soonest retention due-time. * - Overdue work on the write path uses an hourly retry delay (not now+1s). + * - Write-path ensure never postpones an earlier existing alarm (skew-equal + * times keep the existing alarm to avoid churn). * - After an alarm: blob failures → hourly backoff; successful pass with * expired rows remaining → near-immediate continuation; else future due. * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. @@ -83,24 +85,33 @@ class MailboxBase extends DurableObject implements MailboxRpc { if (this.retentionIdleConfirmed) return const dueAtMs = nextMailboxRetentionDueAtMs(this.store) - const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) - if (alarmAt == null) { - this.retentionIdleConfirmed = true - this.retentionAlarmArmed = false - return - } - - this.retentionIdleConfirmed = false - const existing = await this.ctx.storage.getAlarm() - if ( - existing != null && - Math.abs(existing - alarmAt) < mailboxRetentionAlarmSkewMs - ) { - this.retentionAlarmArmed = true - return + const proposedAtMs = mailboxRetentionAlarmAtMs({ dueAtMs }) + const existingAtMs = await this.ctx.storage.getAlarm() + const selection = selectMailboxRetentionWriteAlarm({ + proposedAtMs, + existingAtMs, + }) + switch (selection.action) { + case 'idle': + this.retentionIdleConfirmed = true + this.retentionAlarmArmed = false + return + case 'keep-existing': + this.retentionIdleConfirmed = false + this.retentionAlarmArmed = true + return + case 'set': + this.retentionIdleConfirmed = false + await this.ctx.storage.setAlarm(selection.atMs) + this.retentionAlarmArmed = true + return + default: { + const exhaustive: never = selection + throw new Error( + `Unhandled retention write-alarm selection: ${String(exhaustive)}`, + ) + } } - await this.ctx.storage.setAlarm(alarmAt) - this.retentionAlarmArmed = true } private markRetentionDirty() { @@ -319,4 +330,7 @@ export { type MailboxThreadInput, type MailboxThreadRecord, } from './mailbox-types.ts' -export { computeMailboxRetentionReschedule } from './mailbox-retention.ts' +export { + computeMailboxRetentionReschedule, + selectMailboxRetentionWriteAlarm, +} from './mailbox-retention.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index dfb70c369f..c619df9701 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -7,6 +7,7 @@ import { Mailbox } from './mailbox-do.ts' import { assertMailboxThrows, baseAttachment, + baseDeliveryEvent, baseMessage, baseThread, rpcFor, @@ -289,7 +290,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const first = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: outbound.id, eventType: 'delivered', @@ -301,7 +302,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', needsEffectReconcile: false, state: 'received', fingerprint: 'fp-1', - }, + }), latestDeliveryStatus: { messageId: outbound.id, deliveryStatus: 'delivered', @@ -316,7 +317,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const staleEvent = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: outbound.id, eventType: 'deferred', @@ -327,7 +328,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', needsEffectReconcile: true, state: null, fingerprint: null, - }, + }), }) expect(staleEvent).toEqual({ inserted: false, @@ -347,7 +348,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const duplicate = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1-dup', messageId: outbound.id, eventType: 'delivered', @@ -355,7 +356,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', updatedAt: '2026-07-02T11:00:00.000Z', - }, + }), }) expect(duplicate).toEqual({ inserted: false, @@ -451,7 +452,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const first = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: message.id, eventType: 'delivered', @@ -461,7 +462,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso createdAt: '2026-07-02T10:00:00.000Z', updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'delivered', @@ -476,7 +477,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const inbound = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'inbound-delivery-1', messageId: 'inbound-msg-1', inboxId: 'inbox-1', @@ -487,10 +488,6 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso needsEffectReconcile: true, state: 'received', fingerprint: 'fp-abc', - storageLease: null, - storageLeaseAt: null, - cleanupLease: null, - cleanupLeaseAt: null, expectedAttachmentCount: 2, finalizationToken: 'lease-token-1', usageStartedAt: '2026-07-02T10:59:00.000Z', @@ -500,7 +497,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso subscriptionEffectState: 'pending', subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', detailJson: JSON.stringify({ recipient: 'owner@example.com' }), - }, + }), }) expect(inbound.inserted).toBe(true) const inboundRow = ( @@ -527,17 +524,18 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso recipient: 'owner@example.com', }) - // Default needsEffectReconcile is false unless explicitly true. + // Explicit needsEffectReconcile: false is stored as false. await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-default-reconcile', messageId: message.id, eventType: 'sent', provider: 'kody', createdAt: '2026-07-02T11:30:00.000Z', updatedAt: '2026-07-02T11:30:00.000Z', - }, + needsEffectReconcile: false, + }), }) const defaultReconcile = ( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) @@ -546,7 +544,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const duplicate = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1-dup', messageId: message.id, eventType: 'delivered', @@ -555,7 +553,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', updatedAt: '2026-07-02T11:00:00.000Z', - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'delivered', @@ -567,7 +565,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const stale = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-2', messageId: message.id, eventType: 'deferred', @@ -576,7 +574,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerEventId: 'provider-event-2', createdAt: '2026-07-02T09:00:00.000Z', updatedAt: '2026-07-02T09:00:00.000Z', - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'deferred', diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts index cd1cc3b6a8..3ce4c5bafc 100644 --- a/packages/worker/src/email/mailbox-retention.ts +++ b/packages/worker/src/email/mailbox-retention.ts @@ -3,6 +3,7 @@ import { mailboxBlobDeleteMaxKeys, mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, + mailboxRetentionAlarmSkewMs, mailboxRetentionBatchSize, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, @@ -102,6 +103,39 @@ export function mailboxRetentionAlarmAtMs(input: { return input.dueAtMs } +export type MailboxRetentionWriteAlarmSelection = + | { action: 'idle' } + | { action: 'keep-existing' } + | { action: 'set'; atMs: number } + +/** + * Choose whether a write-path ensure should set, keep, or clear interest in + * an alarm. Sustained writes must never postpone an earlier existing alarm; + * near-equal times within skew keep the existing alarm to avoid churn. + */ +export function selectMailboxRetentionWriteAlarm(input: { + proposedAtMs: number | null + existingAtMs: number | null + skewMs?: number +}): MailboxRetentionWriteAlarmSelection { + if (input.proposedAtMs == null) { + return { action: 'idle' } + } + const existingAtMs = input.existingAtMs + if (existingAtMs == null) { + return { action: 'set', atMs: input.proposedAtMs } + } + const skewMs = input.skewMs ?? mailboxRetentionAlarmSkewMs + if (Math.abs(existingAtMs - input.proposedAtMs) < skewMs) { + return { action: 'keep-existing' } + } + // Existing fires sooner — never postpone retention for a later proposal. + if (existingAtMs < input.proposedAtMs) { + return { action: 'keep-existing' } + } + return { action: 'set', atMs: input.proposedAtMs } +} + async function deleteBlobKeys( blobs: Pick, keys: Array, diff --git a/packages/worker/src/email/mailbox-retention.workers.test.ts b/packages/worker/src/email/mailbox-retention.workers.test.ts index f0871a0a38..d30cf646be 100644 --- a/packages/worker/src/email/mailbox-retention.workers.test.ts +++ b/packages/worker/src/email/mailbox-retention.workers.test.ts @@ -11,9 +11,12 @@ import { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, + selectMailboxRetentionWriteAlarm, } from './mailbox-do.ts' +import { mailboxRetentionAlarmSkewMs } from './mailbox-types.ts' import { baseAttachment, + baseDeliveryEvent, baseMessage, baseThread, rpcFor, @@ -63,6 +66,41 @@ test('computeMailboxRetentionReschedule prefers backoff, then continue, then due ).toEqual({ kind: 'idle', atMs: null }) }) +test('selectMailboxRetentionWriteAlarm never postpones an earlier existing alarm', () => { + const earlier = 1_700_000_100_000 + const later = earlier + 60_000 + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: later, + existingAtMs: earlier, + }), + ).toEqual({ action: 'keep-existing' }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier, + existingAtMs: later, + }), + ).toEqual({ action: 'set', atMs: earlier }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier + Math.floor(mailboxRetentionAlarmSkewMs / 2), + existingAtMs: earlier, + }), + ).toEqual({ action: 'keep-existing' }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier, + existingAtMs: null, + }), + ).toEqual({ action: 'set', atMs: earlier }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: null, + existingAtMs: earlier, + }), + ).toEqual({ action: 'idle' }) +}) + test('Mailbox retention deletes canonical R2 keys before metadata and backs off on failure', async () => { silenceIncidentalRuntimeWarnings() consoleWarn.mockImplementation((...args: Array) => { @@ -128,7 +166,7 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off await mailbox.mirrorMessage({ ownerId: userId, message: missingKeyMessage }) await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'old-event', messageId: keepMessage.id, eventType: 'received', @@ -136,11 +174,11 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off createdAt: oldEventAt, updatedAt: oldEventAt, needsEffectReconcile: false, - }, + }), }) await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'fresh-event', messageId: keepMessage.id, eventType: 'received', @@ -148,7 +186,7 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off createdAt: freshAt, updatedAt: freshAt, needsEffectReconcile: false, - }, + }), }) const dropRawKey = emailRawMimeKey(userId, dropMessage.id) diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 8bb0307f6a..75b38f9927 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -119,7 +119,7 @@ export class MailboxStore { 'message.id', ) const direction = assertMailboxDirection(input.message.direction) - const rawMimeKey = input.message.rawMimeKey ?? null + const rawMimeKey = input.message.rawMimeKey if (direction === 'inbound') { const expected = emailRawMimeKey(input.ownerId, messageId) if (rawMimeKey !== expected) { @@ -142,10 +142,8 @@ export class MailboxStore { attachment.id, 'attachment.id', ) - const storageKind = assertMailboxStorageKind( - String(attachment.storageKind), - ) - const storageKey = attachment.storageKey ?? null + const storageKind = assertMailboxStorageKind(attachment.storageKind) + const storageKey = attachment.storageKey if (storageKind === 'external') { const expected = emailAttachmentBlobKey( input.ownerId, @@ -172,7 +170,7 @@ export class MailboxStore { 'thread.lastMessageAt', ) const createdAt = assertMailboxCanonicalIsoTimestamp( - thread.createdAt?.trim() || mailboxNowIso(), + thread.createdAt, 'thread.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -201,9 +199,9 @@ export class MailboxStore { updated_at = excluded.updated_at WHERE excluded.updated_at >= email_threads.updated_at`, id, - thread.inboxId ?? null, - thread.subjectNormalized ?? '', - thread.rootMessageIdHeader ?? null, + thread.inboxId, + thread.subjectNormalized, + thread.rootMessageIdHeader, lastMessageAt, createdAt, updatedAt, @@ -222,9 +220,7 @@ export class MailboxStore { const processingStatus = assertMailboxProcessingStatus( message.processingStatus, ) - const classification = assertMailboxClassification( - message.classification ?? 'accepted', - ) + const classification = assertMailboxClassification(message.classification) const deliveryStatus = message.deliveryStatus == null ? null @@ -239,7 +235,7 @@ export class MailboxStore { ) } const createdAt = assertMailboxCanonicalIsoTimestamp( - message.createdAt?.trim() || mailboxNowIso(), + message.createdAt, 'message.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -327,32 +323,32 @@ export class MailboxStore { WHERE excluded.updated_at >= email_messages.updated_at`, id, direction, - message.inboxId ?? null, - message.threadId ?? null, - message.senderIdentityId ?? null, - message.fromAddress ?? '', - message.envelopeFrom ?? null, - JSON.stringify(message.toAddresses ?? []), - JSON.stringify(message.ccAddresses ?? []), - JSON.stringify(message.bccAddresses ?? []), - JSON.stringify(message.replyToAddresses ?? []), - message.subject ?? '', - message.messageIdHeader ?? null, - message.inReplyToHeader ?? null, - JSON.stringify(message.references ?? []), - message.headers ? JSON.stringify(message.headers) : '{}', - message.authResults ?? null, + message.inboxId, + message.threadId, + message.senderIdentityId, + message.fromAddress, + message.envelopeFrom, + JSON.stringify(message.toAddresses), + JSON.stringify(message.ccAddresses), + JSON.stringify(message.bccAddresses), + JSON.stringify(message.replyToAddresses), + message.subject, + message.messageIdHeader, + message.inReplyToHeader, + JSON.stringify(message.references), + JSON.stringify(message.headers), + message.authResults, boundedEmailBody(message.textBody), boundedEmailBody(message.htmlBody), - message.rawMimeKey ?? null, - message.rawSize ?? 0, + message.rawMimeKey, + message.rawSize, processingStatus, classification, - message.classificationReason ?? null, - message.providerMessageId ?? null, + message.classificationReason, + message.providerMessageId, deliveryStatus, deliveryStatusAt, - message.error ?? null, + message.error, receivedAt, sentAt, createdAt, @@ -380,13 +376,14 @@ export class MailboxStore { 'Mailbox attachment.messageId must match the mirrored message id.', ) } - const storageKind = assertMailboxStorageKind( - String(attachment.storageKind), - ) + const storageKind = assertMailboxStorageKind(attachment.storageKind) const createdAt = assertMailboxCanonicalIsoTimestamp( - attachment.createdAt?.trim() || mailboxNowIso(), + attachment.createdAt, 'attachment.createdAt', ) + if (!Number.isFinite(attachment.size)) { + throw new Error('Mailbox attachment.size must be a finite number.') + } this.sql.exec( `INSERT INTO email_attachments ( id, message_id, filename, content_type, content_id, disposition, @@ -394,13 +391,13 @@ export class MailboxStore { ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, id, attachmentMessageId, - attachment.filename ?? null, - attachment.contentType ?? 'application/octet-stream', - attachment.contentId ?? null, - attachment.disposition ?? null, - Math.max(0, Math.trunc(Number(attachment.size) || 0)), + attachment.filename, + attachment.contentType, + attachment.contentId, + attachment.disposition, + Math.trunc(attachment.size), storageKind, - attachment.storageKey ?? null, + attachment.storageKey, createdAt, ) } diff --git a/packages/worker/src/email/mailbox-test-helpers.ts b/packages/worker/src/email/mailbox-test-helpers.ts index 3578f312b4..b647ea9d6a 100644 --- a/packages/worker/src/email/mailbox-test-helpers.ts +++ b/packages/worker/src/email/mailbox-test-helpers.ts @@ -5,6 +5,7 @@ import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' import { type MailboxAttachmentInput, + type MailboxDeliveryEventInput, type MailboxMessageInput, type MailboxThreadInput, } from './mailbox-do.ts' @@ -44,12 +45,13 @@ export async function assertMailboxThrows( } } +/** Complete thread snapshot; overrides may be Partial. */ export function baseThread( overrides?: Partial, ): MailboxThreadInput { const id = overrides?.id ?? crypto.randomUUID() const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' - return { + const defaults: MailboxThreadInput = { id, inboxId: 'inbox-1', subjectNormalized: 'hello', @@ -57,10 +59,11 @@ export function baseThread( lastMessageAt: at, createdAt: at, updatedAt: at, - ...overrides, } + return { ...defaults, ...overrides, id: overrides?.id ?? id } } +/** Complete message snapshot; overrides may be Partial. */ export function baseMessage( ownerId: string, overrides?: Partial, @@ -74,20 +77,42 @@ export function baseMessage( : direction === 'outbound' ? null : emailRawMimeKey(ownerId, id) - return { + const defaults: MailboxMessageInput = { + id, + direction, inboxId: 'inbox-1', threadId: null, + senderIdentityId: null, fromAddress: 'sender@example.com', envelopeFrom: 'envelope@example.com', toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], subject: 'Hello mailbox', messageIdHeader: ``, + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: null, + htmlBody: null, + rawMimeKey, rawSize: 128, processingStatus: 'stored', classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, receivedAt: at, + sentAt: null, createdAt: at, updatedAt: at, + } + return { + ...defaults, ...overrides, id, direction, @@ -95,6 +120,7 @@ export function baseMessage( } } +/** Complete attachment snapshot; overrides may be Partial. */ export function baseAttachment( ownerId: string, messageId: string, @@ -108,11 +134,20 @@ export function baseAttachment( : storageKind === 'external' ? emailAttachmentBlobKey(ownerId, messageId, id) : null - return { + const defaults: MailboxAttachmentInput = { + id, + messageId, filename: 'note.txt', contentType: 'text/plain', + contentId: null, + disposition: null, size: 12, + storageKind, + storageKey, createdAt: '2026-07-01T12:00:00.000Z', + } + return { + ...defaults, ...overrides, id, messageId, @@ -120,3 +155,52 @@ export function baseAttachment( storageKey, } } + +/** Complete delivery-event snapshot; overrides may be Partial. */ +export function baseDeliveryEvent( + overrides?: Partial, +): MailboxDeliveryEventInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-02T10:00:00.000Z' + const defaults: MailboxDeliveryEventInput = { + id, + messageId: null, + inboxId: null, + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: at, + updatedAt: at, + } + return { ...defaults, ...overrides, id: overrides?.id ?? id } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 1702e2b1b1..6e281c850c 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -174,106 +174,120 @@ export type MailboxDeliveryEventRecord = { updatedAt: string } +/** + * Full thread snapshot for mirror writes — every persisted field is required. + * Nullable columns use `T | null` (no omitted/defaulted fields). + */ export type MailboxThreadInput = { id: string - inboxId?: string | null - subjectNormalized?: string | null - rootMessageIdHeader?: string | null + inboxId: string | null + subjectNormalized: string + rootMessageIdHeader: string | null lastMessageAt: string - createdAt?: string - /** Required for stale-snapshot rejection (equal/newer only). */ + createdAt: string + /** Stale-snapshot rejection: equal/newer only. */ updatedAt: string } +/** + * Full message snapshot for mirror writes — every persisted field is required. + * Nullable columns use `T | null` (no omitted/defaulted fields). + */ export type MailboxMessageInput = { id: string direction: EmailDirection - inboxId?: string | null - threadId?: string | null - senderIdentityId?: string | null - fromAddress?: string | null - envelopeFrom?: string | null - toAddresses?: Array - ccAddresses?: Array - bccAddresses?: Array - replyToAddresses?: Array - subject?: string | null - messageIdHeader?: string | null - inReplyToHeader?: string | null - references?: Array - headers?: Record | null - authResults?: string | null - textBody?: string | null - htmlBody?: string | null - rawMimeKey?: string | null - rawSize?: number | null + inboxId: string | null + threadId: string | null + senderIdentityId: string | null + fromAddress: string + envelopeFrom: string | null + toAddresses: Array + ccAddresses: Array + bccAddresses: Array + replyToAddresses: Array + subject: string + messageIdHeader: string | null + inReplyToHeader: string | null + references: Array + headers: Record + authResults: string | null + textBody: string | null + htmlBody: string | null + rawMimeKey: string | null + rawSize: number processingStatus: EmailProcessingStatus - classification?: EmailClassification - classificationReason?: string | null - providerMessageId?: string | null - deliveryStatus?: EmailDeliveryStatus | null - deliveryStatusAt?: string | null - error?: string | null - receivedAt?: string | null - sentAt?: string | null - createdAt?: string - /** Required for stale-snapshot rejection (equal/newer only). */ + classification: EmailClassification + classificationReason: string | null + providerMessageId: string | null + deliveryStatus: EmailDeliveryStatus | null + deliveryStatusAt: string | null + error: string | null + receivedAt: string | null + sentAt: string | null + createdAt: string + /** Stale-snapshot rejection: equal/newer only. */ updatedAt: string } +/** + * Full attachment snapshot — every persisted field is required. + * The RPC `attachments` array itself may still be omitted (preserve) or `[]` (clear). + */ export type MailboxAttachmentInput = { id: string messageId: string - filename?: string | null - contentType?: string | null - contentId?: string | null - disposition?: string | null - size?: number - storageKind: MailboxStorageKind | string - storageKey?: string | null - createdAt?: string + filename: string | null + contentType: string + contentId: string | null + disposition: string | null + size: number + storageKind: MailboxStorageKind + storageKey: string | null + createdAt: string } +/** + * Full delivery-event snapshot — every persisted field is required. + * `updatedAt` is the monotonic mirror timestamp (equal/newer accepted). + */ export type MailboxDeliveryEventInput = { id: string - messageId?: string | null - inboxId?: string | null + messageId: string | null + inboxId: string | null eventType: EmailDeliveryEventType - provider?: string | null - providerMessageId?: string | null - providerEventId?: string | null - detailJson?: string - /** Defaults false — set true only when inbound effects still need reconcile. */ - needsEffectReconcile?: boolean - state?: MailboxInboundDeliveryState | null - fingerprint?: string | null - storageLease?: string | null - storageLeaseAt?: string | null - cleanupLease?: string | null - cleanupLeaseAt?: string | null - cleanupRetryAt?: string | null - expectedAttachmentCount?: number | null - finalizationToken?: string | null - reconcileAfter?: string | null - dedupeExpiresAt?: string | null - usageEffectRecordedAt?: string | null - usageEffectSuppressedAt?: string | null - usageStartedAt?: string | null - usageMonth?: string | null - usageBytes?: number | null - usageDurationMs?: number | null - usageEffectRetryAt?: string | null - usageEffectLease?: string | null - usageEffectLeaseAt?: string | null - subscriptionEffectState?: MailboxSubscriptionEffectState | null - subscriptionEffectLease?: string | null - subscriptionEffectLeaseAt?: string | null - subscriptionEffectRetryAt?: string | null - subscriptionEffectAttemptCount?: number | null - subscriptionEffectDeadLetterAt?: string | null - subscriptionEffectLastError?: string | null - createdAt?: string - /** Required monotonic mirror timestamp (equal/newer accepted). */ + provider: string + providerMessageId: string | null + providerEventId: string | null + detailJson: string + needsEffectReconcile: boolean + state: MailboxInboundDeliveryState | null + fingerprint: string | null + storageLease: string | null + storageLeaseAt: string | null + cleanupLease: string | null + cleanupLeaseAt: string | null + cleanupRetryAt: string | null + expectedAttachmentCount: number | null + finalizationToken: string | null + reconcileAfter: string | null + dedupeExpiresAt: string | null + usageEffectRecordedAt: string | null + usageEffectSuppressedAt: string | null + usageStartedAt: string | null + usageMonth: string | null + usageBytes: number | null + usageDurationMs: number | null + usageEffectRetryAt: string | null + usageEffectLease: string | null + usageEffectLeaseAt: string | null + subscriptionEffectState: MailboxSubscriptionEffectState | null + subscriptionEffectLease: string | null + subscriptionEffectLeaseAt: string | null + subscriptionEffectRetryAt: string | null + subscriptionEffectAttemptCount: number | null + subscriptionEffectDeadLetterAt: string | null + subscriptionEffectLastError: string | null + createdAt: string updatedAt: string } From cebf5b37512606d1d5a0d0aad605b14de0726f67 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:12:54 +0000 Subject: [PATCH 04/20] feat(email): add Mailbox dual-write primitives Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 116 +++++- packages/worker/src/email/email-owner.ts | 14 + packages/worker/src/email/mailbox-do.ts | 129 +++++++ .../src/email/mailbox-mirror.node.test.ts | 333 ++++++++++++++++ packages/worker/src/email/mailbox-mirror.ts | 261 +++++++++++++ .../worker/src/email/mailbox-mutations.ts | 228 +++++++++++ .../email/mailbox-mutations.workers.test.ts | 357 +++++++++++++++++ .../email/mailbox-parity-events.node.test.ts | 152 ++++++++ .../worker/src/email/mailbox-parity-events.ts | 87 +++++ .../src/email/mailbox-snapshots.node.test.ts | 309 +++++++++++++++ .../worker/src/email/mailbox-snapshots.ts | 362 ++++++++++++++++++ packages/worker/src/email/mailbox-store.ts | 45 +++ packages/worker/src/email/mailbox-types.ts | 93 +++++ packages/worker/src/email/system-email.ts | 3 +- 14 files changed, 2472 insertions(+), 17 deletions(-) create mode 100644 packages/worker/src/email/email-owner.ts create mode 100644 packages/worker/src/email/mailbox-mirror.node.test.ts create mode 100644 packages/worker/src/email/mailbox-mirror.ts create mode 100644 packages/worker/src/email/mailbox-mutations.ts create mode 100644 packages/worker/src/email/mailbox-mutations.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-parity-events.node.test.ts create mode 100644 packages/worker/src/email/mailbox-parity-events.ts create mode 100644 packages/worker/src/email/mailbox-snapshots.node.test.ts create mode 100644 packages/worker/src/email/mailbox-snapshots.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 6bbeda7bb7..e7c38ddfcf 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -218,10 +218,10 @@ Durable Object export behavior: - `Mailbox` exports per-user email metadata (threads, messages, attachments, delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest - counts use `countMailbox`. Phase 1 registers this consumption without - dual-write or a change of D1 authority — D1 `email_*` rows remain the live - source of truth and are still exported in the `d1` section. See - [Mailbox](#durable-objects-mailbox). + counts use `countMailbox`. Phase 1 registers this consumption; phase 2a adds + dual-write primitives without live callers or a change of D1 authority — D1 + `email_*` rows remain the live source of truth and are still exported in the + `d1` section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -376,6 +376,28 @@ When Analytics Engine SQL is unreachable, these two charts zero-fill while the rest of the page renders. Local development uses the existing D1 counters and delivery-event table because Wrangler's emulated dataset has no SQL API. +**Mailbox expand-phase parity events** (additive; no live writers in the +phase-2a PR) reuse the same `EMAIL_EVENTS` dataset with a separate row shape +defined in `packages/worker/src/email/mailbox-parity-events.ts` +(`recordMailboxParityEvent`). These rows are for future dual-write observability +and reconciliation — they do not feed the admin insights charts today: + +- `index1` — stable user id (per-user isolation; `system:email` is excluded) +- `blob1` — mirror or parity operation name +- `blob2` — operation outcome +- `blob3` — source event timestamp (ISO 8601) +- `double1` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; + `0` when absent or not applicable) +- `double2` — event weight (always `1`) + +Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, +`update_message_delivery`, `set_message_classification`, +`delete_message_metadata`, `delete_delivery_event`. Mirror outcomes: `mirrored`, +`stale`, `skipped`, `error`. Parity operations: `compare_threads`, +`compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity +outcomes: `match`, `mismatch`. Writes are best-effort and never throw into D1 +authority paths. + Two D1 reporting projections deliberately remain: - `usage_rollups` keeps 24 months of per-user monthly aggregates. Analytics @@ -640,6 +662,56 @@ clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. +**Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no +R2 or alarm side effects beyond the usual retention dirty mark): + +- `touchThread` — advance `last_message_at` / `updated_at` without a full + snapshot; `last_message_at` never moves backward +- `updateMessageDelivery` — outbound processing fields (`processing_status`, + `provider_message_id`, `error`, `sent_at`) +- `setMessageClassification` — inbound classification fields +- `deleteMessageMetadata` — metadata-only delete (attachments + message + orphan + thread); never deletes R2; distinguishes missing (idempotent) from stale + (newer `updated_at` retained) +- `deleteDeliveryEvent` — metadata-only delivery-event delete with the same + missing vs stale semantics + +Implementation lives in `mailbox-mutations.ts` (SQLite helpers) and is exposed +through `mailbox-do.ts` RPCs. All require `ownerId` and reject cross-owner +calls. + +**Email owner constant** (`packages/worker/src/email/email-owner.ts`): a +lightweight module exporting `systemEmailOwnerId` (`'system:email'`) and +`isSystemEmailOwner`. It stays free of system-email service imports so +dual-write helpers can skip the reserved operator inbox without import cycles. +Operator mail is never mirrored into per-user Mailbox objects. + +**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`): pure converters +that turn D1 rows into complete Mailbox wire inputs — `toMailboxThreadInput`, +`toMailboxMessageInput`, `toMailboxAttachmentInput`, and +`toMailboxDeliveryEventInput`. They normalize nulls to Mailbox SQLite defaults +(empty strings, `[]`, `{}`, `0`, `application/octet-stream`, `kody`) and fail +clearly on invalid persisted JSON/enums rather than fabricating state. Delivery +events take an additive `EmailDeliveryEventMirrorSnapshot`: D1-promoted columns +(`needs_effect_reconcile`, usage-effect fields) are authoritative; remaining +inbound/effect lease fields come from `detail_json`. D1 delivery events lack +`updated_at`; callers pass it explicitly for stale rejection. + +**Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers +around the DO RPCs for future D1-authoritative dual-write. Each returns a +structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, +`{ status: 'stale' }`, `{ status: 'skipped', reason }` (`system-email` | +`mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. +Failures log with stable tags (for example `mailbox-mirror-message-failed`) and +never propagate into D1 commit paths. Helpers cover full snapshots +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`) and +partial mutations (`mirrorMailboxTouchThread`, +`mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, +`mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`). +**There are no live dual-write callers in the phase-2a PR** — only unit/worker +tests invoke these helpers. D1 remains sole authority for all live mail +read/write paths. + **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). Alarm-driven deletes derive canonical blob keys from `ownerId` + row ids (rather @@ -660,13 +732,17 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **The current PR is phase 1 only:** -additive scaffold (DO class, binding, naming, client, mirror/export/purge RPCs, -and alarm retention) **plus registered account-deletion `purge` and -account-export `mailbox` section consumption**. It does **not** dual-write, does -**not** change live email read/write authority, and leaves D1 `email_*` tables -authoritative for all live paths. D1 email rows and existing R2 inventory -deletion stay authoritative during expand. +This is an expand/contract migration. **The current PR is phase 2a — +prerequisite scaffolding within phase 2:** additive dual-write **primitives** +(email-owner constant, D1→Mailbox snapshot adapters, best-effort mirror helpers +with structured outcomes, owner-bound partial-mutation and stale-safe +metadata-delete RPCs, and the Analytics Engine parity event schema). It does +**not** wire live dual-write callers, does **not** change live email read/write +authority, and leaves D1 `email_*` tables authoritative for all live paths. D1 +email rows and existing R2 inventory deletion stay authoritative during expand. +**Phase 2 is complete only when every user-mail mutation that writes D1 also +writes the DO and parity/reconciliation are wired** — this PR ships the library +surface that later live-path wiring will call. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -675,9 +751,16 @@ deletion stay authoritative during expand. authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO - completeness against D1. D1 stays authoritative for reads. -3. **Reads cut over after production parity** — user-mail reads move to the DO - only after production parity is verified. D1 writes continue. + completeness against D1. D1 stays authoritative for reads. Phase 2a (current + PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, `mailbox-mirror.ts`, + `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, + `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, + `deleteDeliveryEvent`), and `mailbox-parity-events.ts`. Those helpers and + parity-event writers are library-only today — **no production path invokes + them yet**; live-path dual-write and parity wiring remain for the rest of + phase 2. +3. **Reads cut over after production soak** — user-mail reads move to the DO + only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. @@ -723,7 +806,7 @@ metadata store: message (logged, non-retry) rather than risking a duplicate on Email Routing retry. -Phase 1 scaffolding does not change that live D1 boundary. +Phase 1 and phase 2a scaffolding does not change that live D1 boundary. ### Package state model @@ -844,7 +927,8 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption without changing D1 authority) + consumption; phase 2a adds dual-write primitives without changing D1 + authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) diff --git a/packages/worker/src/email/email-owner.ts b/packages/worker/src/email/email-owner.ts new file mode 100644 index 0000000000..697abd5a3c --- /dev/null +++ b/packages/worker/src/email/email-owner.ts @@ -0,0 +1,14 @@ +/** + * Lightweight email ownership constants. + * + * Kept free of system-email / service imports so dual-write helpers + * (`mailbox-mirror`) can skip the reserved operator inbox without creating + * import cycles (service → mailbox-mirror → … → service). + */ + +/** Reserved owner id for the platform operator inbox (stays in D1 only). */ +export const systemEmailOwnerId = 'system:email' + +export function isSystemEmailOwner(ownerId: string): boolean { + return ownerId === systemEmailOwnerId +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 11cabd8d6f..5222063275 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -15,10 +15,15 @@ import { import { MailboxStore } from './mailbox-store.ts' import { assertMailboxNonEmptyString, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -27,8 +32,11 @@ import { type MailboxMessageRecord, type MailboxRpc, type MailboxSearchMessagesInput, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' /** @@ -216,6 +224,119 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { inserted, accepted, updatedLatestStatus } } + /** + * Advance thread activity without a full snapshot. Equal/newer `updatedAt` + * only; `last_message_at` never moves backward. + */ + async touchThread( + input: MailboxTouchThreadInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.touchThread({ + threadId: input.threadId, + lastMessageAt: input.lastMessageAt, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Partial delivery/processing update for dual-write parity. Equal/newer + * `updatedAt` only. + */ + async updateMessageDelivery( + input: MailboxUpdateMessageDeliveryInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.updateMessageDelivery({ + messageId: input.messageId, + processingStatus: input.processingStatus, + providerMessageId: input.providerMessageId, + error: input.error, + sentAt: input.sentAt, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Partial classification update for dual-write parity. Equal/newer + * `updatedAt` only. + */ + async setMessageClassification( + input: MailboxSetMessageClassificationInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.setMessageClassification({ + messageId: input.messageId, + classification: input.classification, + classificationReason: input.classificationReason, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Metadata-only delete (attachments + message + orphan thread). Never + * deletes R2. Stale when the message `updated_at` is newer than `deletedAt`. + */ + async deleteMessageMetadata( + input: MailboxDeleteMessageMetadataInput, + ): Promise { + let result: MailboxDeleteMessageMetadataResult = { + deleted: false, + stale: false, + } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = this.store.deleteMessageMetadata({ + messageId: input.messageId, + deletedAt: input.deletedAt, + }) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + + /** + * Metadata-only delivery-event delete. Distinguishes missing (idempotent) + * from stale (newer `updated_at` retained). + */ + async deleteDeliveryEvent( + input: MailboxDeleteDeliveryEventInput, + ): Promise { + let result: MailboxDeleteDeliveryEventResult = { + deleted: false, + stale: false, + } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = this.store.deleteDeliveryEvent({ + eventId: input.eventId, + deletedAt: input.deletedAt, + }) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + async getThread(input: { threadId: string }): Promise { @@ -316,19 +437,27 @@ export { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReference, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxExportRow, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' export { computeMailboxRetentionReschedule, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts new file mode 100644 index 0000000000..4668e5bb64 --- /dev/null +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -0,0 +1,333 @@ +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { + mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeleteDeliveryEvent, + mirrorMailboxDeleteMessageMetadata, + mirrorMailboxMessageSnapshot, + mirrorMailboxSetMessageClassification, + mirrorMailboxTouchThread, + mirrorMailboxUpdateMessageDelivery, +} from './mailbox-mirror.ts' +import { + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' + +function fakeMailboxEnv(stub: Record) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => stub) + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + }, + idFromName, + get, + } +} + +function baseMessage( + overrides?: Partial, +): EmailMessageRecord { + return { + id: 'msg-1', + direction: 'inbound', + userId: 'user-aaa', + inboxId: 'inbox-1', + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: 'sender@example.com', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: 'Hello', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: null, + htmlBody: null, + rawMimeKey: 'user-aaa/msg-1.eml', + rawSize: 32, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +test('mailbox mirror helpers scope by user idFromName, convert payloads, and report stale/skip/error', async () => { + consoleWarn.mockImplementation(() => {}) + + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const touchThread = vi.fn(async () => ({ accepted: true })) + const updateMessageDelivery = vi.fn(async () => ({ accepted: false })) + const setMessageClassification = vi.fn(async () => ({ accepted: true })) + const deleteMessageMetadata = vi.fn( + async (input: { messageId: string; deletedAt: string }) => { + if (input.messageId === 'missing-msg') { + return { deleted: false as const, stale: false as const } + } + if (input.deletedAt < '2026-07-01T12:00:00.000Z') { + return { deleted: false as const, stale: true as const } + } + return { + deleted: true as const, + stale: false as const, + orphanThreadDeleted: false, + } + }, + ) + const deleteDeliveryEvent = vi.fn(async () => { + throw new Error('do unavailable') + }) + + const { env, idFromName } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvent, + touchThread, + updateMessageDelivery, + setMessageClassification, + deleteMessageMetadata, + deleteDeliveryEvent, + }) + + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + subjectNormalized: null, + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + const message = baseMessage() + + expect( + await mirrorMailboxMessageSnapshot({ + env, + thread, + message, + attachments: [ + { + id: 'att-1', + messageId: 'msg-1', + filename: null, + contentType: null, + contentId: null, + disposition: null, + size: 0, + storageKind: 'unavailable', + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + }, + ], + }), + ).toEqual({ status: 'mirrored' }) + expect(idFromName).toHaveBeenCalledWith('user-aaa') + expect(mirrorMessage).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + thread: { + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: '', + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }, + message: expect.objectContaining({ + id: 'msg-1', + fromAddress: 'sender@example.com', + subject: 'Hello', + rawSize: 32, + headers: {}, + }), + attachments: [ + expect.objectContaining({ + id: 'att-1', + contentType: 'application/octet-stream', + storageKind: 'unavailable', + }), + ], + }) + + const detail = { + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + // Stale JSON copies of promoted columns — must not win. + usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', + usageMonth: 'json-month', + usageBytes: 1, + usageDurationMs: 1, + } + const event: EmailDeliveryEventRecord = { + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + eventType: 'receive_started', + provider: 'cloudflare-email-routing', + providerMessageId: null, + providerEventId: null, + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + } + expect( + await mirrorMailboxDeliveryEventSnapshot({ + env, + snapshot: { + event, + updatedAt: '2026-07-02T10:00:01.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + }, + }), + ).toEqual({ status: 'mirrored' }) + expect(upsertDeliveryEvent).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + event: expect.objectContaining({ + id: 'evt-1', + provider: 'cloudflare-email-routing', + needsEffectReconcile: true, + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + updatedAt: '2026-07-02T10:00:01.000Z', + }), + latestDeliveryStatus: undefined, + }) + + expect( + await mirrorMailboxTouchThread({ + env, + ownerId: 'user-aaa', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + expect(touchThread).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }) + + expect( + await mirrorMailboxUpdateMessageDelivery({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + + expect( + await mirrorMailboxSetMessageClassification({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + classification: 'quarantined', + classificationReason: 'manual', + updatedAt: '2026-07-01T14:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + deletedAt: '2026-07-01T15:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'missing-msg', + deletedAt: '2026-07-01T15:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteDeliveryEvent({ + env, + ownerId: 'user-aaa', + eventId: 'evt-1', + deletedAt: '2026-07-02T11:00:00.000Z', + }), + ).toEqual({ + status: 'error', + error: expect.objectContaining({ message: 'do unavailable' }), + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-mirror-delete-delivery-event-failed', + expect.objectContaining({ message: 'do unavailable' }), + ) + + expect( + await mirrorMailboxMessageSnapshot({ + env, + message: baseMessage({ userId: systemEmailOwnerId }), + }), + ).toEqual({ status: 'skipped', reason: 'system-email' }) + expect(mirrorMessage).toHaveBeenCalledTimes(1) + + expect( + await mirrorMailboxMessageSnapshot({ + env: {}, + message: baseMessage(), + }), + ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) +}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts new file mode 100644 index 0000000000..e27b3174d2 --- /dev/null +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -0,0 +1,261 @@ +import { isSystemEmailOwner } from './email-owner.ts' +import { + mailboxNamespace, + mailboxRpc, + type MailboxEnv, +} from './mailbox-client.ts' +import { + toMailboxAttachmentInput, + toMailboxDeliveryEventInput, + toMailboxMessageInput, + toMailboxThreadInput, + type EmailDeliveryEventMirrorSnapshot, +} from './mailbox-snapshots.ts' +import { + type EmailAttachmentRecord, + type EmailDeliveryStatus, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, + type MailboxSetMessageClassificationInput, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, +} from './mailbox-types.ts' + +/** + * Best-effort D1 → Mailbox dual-write helpers. + * + * Never throw into D1 authority paths: catch/log with stable tags and return + * structured `mirrored | stale | skipped | error` outcomes. `system:email` + * stays in D1 only. + */ + +export type MailboxMirrorEnv = MailboxEnv + +export type MailboxMirrorSkipReason = + | 'system-email' + | 'mailbox-unconfigured' + | 'missing-owner' + +export type MailboxMirrorResult = + | { status: 'mirrored' } + | { status: 'stale' } + | { status: 'skipped'; reason: MailboxMirrorSkipReason } + | { status: 'error'; error: unknown } + +function resolveMirrorOwner( + ownerId: string | null | undefined, +): { ok: true; ownerId: string } | { ok: false; result: MailboxMirrorResult } { + if (ownerId == null || ownerId.length === 0) { + return { ok: false, result: { status: 'skipped', reason: 'missing-owner' } } + } + if (isSystemEmailOwner(ownerId)) { + return { + ok: false, + result: { status: 'skipped', reason: 'system-email' }, + } + } + return { ok: true, ownerId } +} + +function skipIfMailboxMissing( + env: MailboxMirrorEnv, +): MailboxMirrorResult | null { + if (!mailboxNamespace(env)) { + return { status: 'skipped', reason: 'mailbox-unconfigured' } + } + return null +} + +function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { + return accepted ? { status: 'mirrored' } : { status: 'stale' } +} + +/** + * Map delete RPC outcome: + * - `deleted: true` → mirrored + * - `deleted: false, stale: true` → stale + * - `deleted: false, stale: false` (missing) → mirrored (idempotent) + */ +function outcomeFromDeleteResult( + result: MailboxDeleteMessageMetadataResult | MailboxDeleteDeliveryEventResult, +): MailboxMirrorResult { + if (result.deleted) return { status: 'mirrored' } + if (result.stale) return { status: 'stale' } + return { status: 'mirrored' } +} + +async function runMirror( + tag: string, + run: () => Promise, +): Promise { + try { + return await run() + } catch (error) { + console.warn(tag, error) + return { status: 'error', error } + } +} + +/** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ +export async function mirrorMailboxMessageSnapshot(input: { + env: MailboxMirrorEnv + thread?: EmailThreadRecord | null + message: EmailMessageRecord + attachments?: Array +}): Promise { + const owner = resolveMirrorOwner(input.message.userId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-message-failed', async () => { + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).mirrorMessage({ + ownerId: owner.ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort delivery-event snapshot mirror. */ +export async function mirrorMailboxDeliveryEventSnapshot(input: { + env: MailboxMirrorEnv + snapshot: EmailDeliveryEventMirrorSnapshot + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null +}): Promise { + const owner = resolveMirrorOwner(input.snapshot.event.userId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delivery-event-failed', async () => { + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvent({ + ownerId: owner.ownerId, + event: toMailboxDeliveryEventInput(input.snapshot), + latestDeliveryStatus: input.latestDeliveryStatus, + }) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort thread touch mirror. */ +export async function mirrorMailboxTouchThread( + input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-touch-thread-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).touchThread(rpcInput) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort outbound delivery/processing update mirror. */ +export async function mirrorMailboxUpdateMessageDelivery( + input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-update-message-delivery-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).updateMessageDelivery(rpcInput) + return outcomeFromAccepted(result.accepted) + }, + ) +} + +/** Best-effort inbound classification update mirror. */ +export async function mirrorMailboxSetMessageClassification( + input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-set-message-classification-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).setMessageClassification(rpcInput) + return outcomeFromAccepted(result.accepted) + }, + ) +} + +/** Best-effort metadata-only message delete mirror (no R2). */ +export async function mirrorMailboxDeleteMessageMetadata( + input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-delete-message-metadata-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteMessageMetadata(rpcInput) + return outcomeFromDeleteResult(result) + }, + ) +} + +/** Best-effort delivery-event delete mirror. */ +export async function mirrorMailboxDeleteDeliveryEvent( + input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delete-delivery-event-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteDeliveryEvent(rpcInput) + return outcomeFromDeleteResult(result) + }) +} diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts new file mode 100644 index 0000000000..af139f7cbc --- /dev/null +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -0,0 +1,228 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxClassification, + assertMailboxNonEmptyString, + assertMailboxProcessingStatus, + assertOptionalMailboxCanonicalIsoTimestamp, + type MailboxAcceptedResult, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataResult, + type MailboxSetMessageClassificationInput, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, +} from './mailbox-types.ts' + +/** + * Owner-bound partial mutation helpers for Mailbox SQLite metadata. + * No R2 / alarm side effects — callers own transactions and retention. + */ + +export function touchMailboxThread( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') + const lastMessageAt = assertMailboxCanonicalIsoTimestamp( + input.lastMessageAt, + 'lastMessageAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const cursor = sql.exec( + `UPDATE email_threads + SET last_message_at = CASE + WHEN last_message_at < ? THEN ? + ELSE last_message_at + END, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + lastMessageAt, + lastMessageAt, + updatedAt, + threadId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +export function updateMailboxMessageDelivery( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const processingStatus = assertMailboxProcessingStatus(input.processingStatus) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const sentAt = assertOptionalMailboxCanonicalIsoTimestamp( + input.sentAt, + 'sentAt', + ) + const cursor = sql.exec( + `UPDATE email_messages + SET processing_status = ?, + provider_message_id = ?, + error = ?, + sent_at = ?, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + processingStatus, + input.providerMessageId, + input.error, + sentAt, + updatedAt, + messageId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +export function setMailboxMessageClassification( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const classification = assertMailboxClassification(input.classification) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const cursor = sql.exec( + `UPDATE email_messages + SET classification = ?, + classification_reason = ?, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + classification, + input.classificationReason, + updatedAt, + messageId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +/** + * Delete message + attachment metadata and any orphaned thread. Never deletes + * R2 objects. Distinguishes missing (idempotent) from stale (newer retained). + */ +export function deleteMailboxMessageMetadata( + sql: SqlStorage, + input: { messageId: string; deletedAt: string }, +): MailboxDeleteMessageMetadataResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ thread_id: string | null; updated_at: string }>( + `SELECT thread_id, updated_at FROM email_messages + WHERE id = ? + LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) { + return { deleted: false, stale: false } + } + if (existing.updated_at > deletedAt) { + return { deleted: false, stale: true } + } + + sql.exec(`DELETE FROM email_attachments WHERE message_id = ?`, messageId) + const messageDelete = sql.exec( + `DELETE FROM email_messages + WHERE id = ? + AND updated_at <= ?`, + messageId, + deletedAt, + ) + if (messageDelete.rowsWritten === 0) { + const stillThere = sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + return stillThere == null + ? { deleted: false, stale: false } + : { deleted: false, stale: true } + } + + const threadId = existing.thread_id + if (threadId == null || threadId.length === 0) { + return { deleted: true, stale: false, orphanThreadDeleted: false } + } + + const orphanDelete = sql.exec( + `DELETE FROM email_threads + WHERE id = ? + AND NOT EXISTS ( + SELECT 1 FROM email_messages + WHERE thread_id = ? + )`, + threadId, + threadId, + ) + return { + deleted: true, + stale: false, + orphanThreadDeleted: orphanDelete.rowsWritten > 0, + } +} + +/** + * Delete a delivery-event row. SELECT `updated_at` first so missing and stale + * are distinguishable; missing is idempotent success for best-effort callers. + */ +export function deleteMailboxDeliveryEvent( + sql: SqlStorage, + input: { eventId: string; deletedAt: string }, +): MailboxDeleteDeliveryEventResult { + const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_delivery_events + WHERE id = ? + LIMIT 1`, + eventId, + ) + .toArray()[0] + if (existing == null) { + return { deleted: false, stale: false } + } + if (existing.updated_at > deletedAt) { + return { deleted: false, stale: true } + } + + const cursor = sql.exec( + `DELETE FROM email_delivery_events + WHERE id = ? + AND updated_at <= ?`, + eventId, + deletedAt, + ) + if (cursor.rowsWritten === 0) { + const stillThere = sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, + eventId, + ) + .toArray()[0] + return stillThere == null + ? { deleted: false, stale: false } + : { deleted: false, stale: true } + } + return { deleted: true, stale: false } +} diff --git a/packages/worker/src/email/mailbox-mutations.workers.test.ts b/packages/worker/src/email/mailbox-mutations.workers.test.ts new file mode 100644 index 0000000000..721617ff1b --- /dev/null +++ b/packages/worker/src/email/mailbox-mutations.workers.test.ts @@ -0,0 +1,357 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { type Mailbox } from './mailbox-do.ts' +import { + assertMailboxThrows, + baseAttachment, + baseDeliveryEvent, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection', async () => { + silenceIncidentalRuntimeWarnings() + const ownerA = uniqueUserId('mut-a') + const ownerB = uniqueUserId('mut-b') + const mailbox = rpcFor(ownerA) + const stub = stubFor(ownerA) + + const thread = baseThread({ + id: 'mut-thread', + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const message = baseMessage(ownerA, { + id: 'mut-msg', + threadId: thread.id, + processingStatus: 'stored', + classification: 'accepted', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + await mailbox.mirrorMessage({ + ownerId: ownerA, + thread, + message, + attachments: [baseAttachment(ownerA, message.id, { id: 'mut-att' })], + }) + + await runInDurableObject(stub, async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.touchThread({ + ownerId: ownerB, + threadId: thread.id, + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.updateMessageDelivery({ + ownerId: ownerB, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov', + error: null, + sentAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.setMessageClassification({ + ownerId: ownerB, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'spam', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteMessageMetadata({ + ownerId: ownerB, + messageId: message.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteDeliveryEvent({ + ownerId: ownerB, + eventId: 'missing', + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) + }) + + const staleTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T14:00:00.000Z', + updatedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleTouch).toEqual({ accepted: false }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const regressTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + expect(regressTouch).toEqual({ accepted: true }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + + const advanceTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }) + expect(advanceTouch).toEqual({ accepted: true }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }) + + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: 'missing-thread', + lastMessageAt: '2026-07-01T16:00:00.000Z', + updatedAt: '2026-07-01T16:00:00.000Z', + }), + ).toEqual({ accepted: false }) + + const staleDelivery = await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'failed', + providerMessageId: 'stale-prov', + error: 'stale', + sentAt: null, + updatedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleDelivery).toEqual({ accepted: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + processingStatus: 'stored', + providerMessageId: null, + error: null, + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const deliveryUpdate = await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + expect(deliveryUpdate).toEqual({ accepted: true }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + processingStatus: 'sent', + providerMessageId: 'prov-1', + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const staleClass = await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'stale', + updatedAt: '2026-06-01T00:00:00.000Z', + }) + expect(staleClass).toEqual({ accepted: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'accepted', + classificationReason: null, + }) + + const classUpdate = await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }) + expect(classUpdate).toEqual({ accepted: true }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }) +}) + +test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no R2', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('del-msg') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'del-thread' }) + const alone = baseMessage(userId, { + id: 'del-alone', + threadId: thread.id, + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const attachment = baseAttachment(userId, alone.id, { id: 'del-att' }) + await mailbox.mirrorMessage({ + ownerId: userId, + thread, + message: alone, + attachments: [attachment], + }) + + const rawKey = emailRawMimeKey(userId, alone.id) + const attKey = emailAttachmentBlobKey(userId, alone.id, attachment.id) + await env.EMAIL_BLOBS.put(rawKey, 'raw-bytes') + await env.EMAIL_BLOBS.put(attKey, 'att-bytes') + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + let blobDeleteCalls = 0 + env.EMAIL_BLOBS.delete = ((keys: string | Array) => { + blobDeleteCalls += 1 + return originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const staleDelete = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: alone.id }), + ).toHaveLength(1) + + const deleted = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }) + expect(deleted).toEqual({ + deleted: true, + stale: false, + orphanThreadDeleted: true, + }) + expect(await mailbox.getMessage({ messageId: alone.id })).toBeNull() + expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: alone.id }), + ).toHaveLength(0) + expect(await env.EMAIL_BLOBS.get(rawKey)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(attKey)).not.toBeNull() + expect(blobDeleteCalls).toBe(0) + + const again = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }) + // Missing (`!deleted && !stale`): idempotent success for best-effort callers. + expect(again).toEqual({ deleted: false, stale: false }) + expect(blobDeleteCalls).toBe(0) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + await env.EMAIL_BLOBS.delete(rawKey) + await env.EMAIL_BLOBS.delete(attKey) + } + + const sharedThread = baseThread({ id: 'shared-thread' }) + const keep = baseMessage(userId, { + id: 'keep-msg', + threadId: sharedThread.id, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + const drop = baseMessage(userId, { + id: 'drop-msg', + threadId: sharedThread.id, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ + ownerId: userId, + thread: sharedThread, + message: keep, + }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: drop, + }) + const dropOnly = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: drop.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }) + expect(dropOnly).toEqual({ + deleted: true, + stale: false, + orphanThreadDeleted: false, + }) + expect(await mailbox.getMessage({ messageId: keep.id })).not.toBeNull() + expect(await mailbox.getThread({ threadId: sharedThread.id })).not.toBeNull() +}) + +test('Mailbox deleteDeliveryEvent: stale, deleted, and missing outcomes', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('del-evt') + const mailbox = rpcFor(userId) + + const message = baseMessage(userId, { + id: 'evt-msg', + direction: 'outbound', + processingStatus: 'sent', + rawMimeKey: null, + }) + await mailbox.mirrorMessage({ ownerId: userId, message }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: baseDeliveryEvent({ + id: 'evt-del', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + }) + + const staleDelete = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T09:00:00.000Z', + }) + expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), + ).toHaveLength(1) + + const deleted = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T10:00:00.000Z', + }) + expect(deleted).toEqual({ deleted: true, stale: false }) + expect( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), + ).toHaveLength(0) + + const again = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T11:00:00.000Z', + }) + expect(again).toEqual({ deleted: false, stale: false }) +}) diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts new file mode 100644 index 0000000000..1cc170a450 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -0,0 +1,152 @@ +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { recordMailboxParityEvent } from './mailbox-parity-events.ts' + +test('mailbox parity writes compact indexed mirror and parity data points', () => { + const writeDataPoint = vi.fn() + const env = { + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + } + + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + timestamp: '2026-07-30T10:00:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'touch_thread', + outcome: 'stale', + timestamp: '2026-07-30T10:05:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'set_message_classification', + outcome: 'skipped', + timestamp: '2026-07-30T10:06:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'delete_message_metadata', + outcome: 'error', + timestamp: '2026-07-30T10:07:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-2', + category: 'parity', + operation: 'compare_messages', + outcome: 'mismatch', + d1Count: 12, + doCount: 10, + timestamp: '2026-07-30T11:00:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-3', + category: 'parity', + operation: 'compare_threads', + outcome: 'match', + timestamp: '2026-07-30T12:00:00.000Z', + }) + + expect(writeDataPoint).toHaveBeenCalledTimes(6) + expect(writeDataPoint).toHaveBeenNthCalledWith(1, { + indexes: ['user-1'], + blobs: ['mirror_message', 'mirrored', '2026-07-30T10:00:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(2, { + indexes: ['user-1'], + blobs: ['touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(3, { + indexes: ['user-1'], + blobs: [ + 'set_message_classification', + 'skipped', + '2026-07-30T10:06:00.000Z', + ], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(4, { + indexes: ['user-1'], + blobs: ['delete_message_metadata', 'error', '2026-07-30T10:07:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(5, { + indexes: ['user-2'], + blobs: ['compare_messages', 'mismatch', '2026-07-30T11:00:00.000Z'], + doubles: [2, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + indexes: ['user-3'], + blobs: ['compare_threads', 'match', '2026-07-30T12:00:00.000Z'], + doubles: [0, 1], + }) +}) + +test('mailbox parity skips system email and never throws when the binding is absent or fails', () => { + const writeDataPoint = vi.fn() + consoleWarn.mockImplementation(() => {}) + + recordMailboxParityEvent( + { + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + }, + { + userId: systemEmailOwnerId, + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + }, + ) + expect(writeDataPoint).not.toHaveBeenCalled() + + expect(() => + recordMailboxParityEvent( + {}, + { + userId: 'user-1', + category: 'mirror', + operation: 'update_message_delivery', + outcome: 'mirrored', + }, + ), + ).not.toThrow() + + expect(() => + recordMailboxParityEvent( + { + EMAIL_EVENTS: { + writeDataPoint() { + throw new Error('analytics unavailable') + }, + } as unknown as AnalyticsEngineDataset, + }, + { + userId: 'user-1', + category: 'parity', + operation: 'compare_delivery_events', + outcome: 'mismatch', + d1Count: 3, + doCount: 1, + }, + ), + ).not.toThrow() + + expect(consoleWarn).toHaveBeenCalledTimes(1) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-event-write-failed', + expect.objectContaining({ message: 'analytics unavailable' }), + ) +}) diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts new file mode 100644 index 0000000000..e3dcd74229 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -0,0 +1,87 @@ +import { isSystemEmailOwner } from './email-owner.ts' + +export type MailboxParityEnv = { + EMAIL_EVENTS?: AnalyticsEngineDataset +} + +export type MailboxMirrorOperation = + | 'mirror_message' + | 'upsert_delivery_event' + | 'touch_thread' + | 'update_message_delivery' + | 'set_message_classification' + | 'delete_message_metadata' + | 'delete_delivery_event' + +export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' + +export type MailboxParityOperation = + | 'compare_threads' + | 'compare_messages' + | 'compare_attachments' + | 'compare_delivery_events' + +export type MailboxParityOutcome = 'match' | 'mismatch' + +export type MailboxParityEvent = + | { + userId: string + category: 'mirror' + operation: MailboxMirrorOperation + outcome: MailboxMirrorOutcome + timestamp?: string + } + | { + userId: string + category: 'parity' + operation: MailboxParityOperation + outcome: MailboxParityOutcome + d1Count?: number + doCount?: number + timestamp?: string + } + +function resolveCountDelta(event: MailboxParityEvent): number { + if ( + event.category === 'parity' && + event.d1Count !== undefined && + event.doCount !== undefined + ) { + return event.d1Count - event.doCount + } + return 0 +} + +/** + * Additive Analytics Engine schema (same EMAIL_EVENTS dataset as reporting): + * + * - index1: stable user id (per-user isolation) + * - blob1: mirror or parity operation + * - blob2: operation outcome + * - blob3: source event timestamp (ISO 8601) + * - double1: D1-minus-DO count delta (0 when absent or not applicable) + * - double2: event weight (always 1) + * + * `system:email` is excluded — operator mail stays in D1 only. + */ +export function recordMailboxParityEvent( + env: MailboxParityEnv, + event: MailboxParityEvent, +): void { + try { + if (!env.EMAIL_EVENTS || !event.userId) return + if (isSystemEmailOwner(event.userId)) return + + env.EMAIL_EVENTS.writeDataPoint({ + indexes: [event.userId], + blobs: [ + event.operation, + event.outcome, + event.timestamp ?? new Date().toISOString(), + ], + doubles: [resolveCountDelta(event), 1], + }) + } catch (error) { + console.warn('mailbox-parity-event-write-failed', error) + } +} diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts new file mode 100644 index 0000000000..bd26855c74 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -0,0 +1,309 @@ +import { expect, test } from 'vitest' +import { + type EmailAttachmentRecord, + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + toMailboxAttachmentInput, + toMailboxDeliveryEventInput, + toMailboxMessageInput, + toMailboxThreadInput, + type EmailDeliveryEventMirrorSnapshot, +} from './mailbox-snapshots.ts' + +test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted columns', () => { + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-1', + inboxId: 'inbox-1', + subjectNormalized: null, + rootMessageIdHeader: null, + lastMessageAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:30:00.000Z', + } + expect(toMailboxThreadInput(thread)).toEqual({ + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: '', + rootMessageIdHeader: null, + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:30:00.000Z', + }) + + const message: EmailMessageRecord = { + id: 'msg-1', + direction: 'inbound', + userId: 'user-1', + inboxId: null, + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: null, + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: null, + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: null, + authResults: null, + textBody: 'hello', + htmlBody: null, + rawMimeKey: 'user-1/msg-1.eml', + rawSize: null, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + expect(toMailboxMessageInput(message)).toEqual({ + id: 'msg-1', + direction: 'inbound', + inboxId: null, + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: '', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: '', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: 'hello', + htmlBody: null, + rawMimeKey: 'user-1/msg-1.eml', + rawSize: 0, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const attachment: EmailAttachmentRecord = { + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: null, + contentId: null, + disposition: null, + size: 12, + storageKind: 'external', + storageKey: 'user-1/msg-1/att-1', + createdAt: '2026-07-01T12:00:00.000Z', + } + expect(toMailboxAttachmentInput(attachment)).toEqual({ + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: 'application/octet-stream', + contentId: null, + disposition: null, + size: 12, + storageKind: 'external', + storageKey: 'user-1/msg-1/att-1', + createdAt: '2026-07-01T12:00:00.000Z', + }) + + // detail_json still owns inbound/effect lease fields, but also may contain + // stale copies of promoted columns — converter must prefer D1 columns. + const detail = { + state: 'received', + fingerprint: 'fp-1', + storageLease: 'lease-1', + storageLeaseAt: '2026-07-02T10:00:00.000Z', + cleanupLease: 'cleanup-1', + cleanupLeaseAt: '2026-07-02T10:05:00.000Z', + cleanupRetryAt: '2026-07-02T11:00:00.000Z', + expectedAttachmentCount: 2, + finalizationToken: 'token-1', + reconcileAfter: '2026-07-02T12:00:00.000Z', + dedupeExpiresAt: '2026-07-04T10:00:00.000Z', + usageEffectRecordedAt: '2026-07-02T09:00:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: '2026-07-02T09:59:00.000Z', + usageMonth: 'stale-from-json', + usageBytes: 1, + usageDurationMs: 1, + usageEffectRetryAt: null, + usageEffectLease: 'usage-lease', + usageEffectLeaseAt: '2026-07-02T10:00:30.000Z', + subscriptionEffectState: 'pending', + subscriptionEffectLease: 'sub-lease', + subscriptionEffectLeaseAt: '2026-07-02T10:02:00.000Z', + subscriptionEffectRetryAt: '2026-07-02T13:00:00.000Z', + subscriptionEffectAttemptCount: 1, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + recipient: 'owner@example.com', + } + const event: EmailDeliveryEventRecord = { + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: null, + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + } + const snapshot: EmailDeliveryEventMirrorSnapshot = { + event, + updatedAt: '2026-07-02T10:00:01.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + } + expect(toMailboxDeliveryEventInput(snapshot)).toEqual({ + id: 'evt-1', + messageId: 'msg-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: event.detailJson, + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-1', + storageLease: 'lease-1', + storageLeaseAt: '2026-07-02T10:00:00.000Z', + cleanupLease: 'cleanup-1', + cleanupLeaseAt: '2026-07-02T10:05:00.000Z', + cleanupRetryAt: '2026-07-02T11:00:00.000Z', + expectedAttachmentCount: 2, + finalizationToken: 'token-1', + reconcileAfter: '2026-07-02T12:00:00.000Z', + dedupeExpiresAt: '2026-07-04T10:00:00.000Z', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: '2026-07-02T09:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + usageEffectRetryAt: null, + usageEffectLease: 'usage-lease', + usageEffectLeaseAt: '2026-07-02T10:00:30.000Z', + subscriptionEffectState: 'pending', + subscriptionEffectLease: 'sub-lease', + subscriptionEffectLeaseAt: '2026-07-02T10:02:00.000Z', + subscriptionEffectRetryAt: '2026-07-02T13:00:00.000Z', + subscriptionEffectAttemptCount: 1, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }) +}) + +test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { + const baseEvent: EmailDeliveryEventRecord = { + id: 'evt-bad', + messageId: null, + userId: 'user-1', + inboxId: null, + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + createdAt: '2026-07-02T10:00:00.000Z', + } + const baseSnapshot = { + updatedAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + usageEffectRecordedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + } satisfies Omit + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { ...baseEvent, detailJson: '{not-json' }, + }), + ).toThrow(/detailJson is not valid JSON/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { ...baseEvent, detailJson: '[]' }, + }), + ).toThrow(/detailJson must be a JSON object/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ state: 'not-a-state' }), + }, + }), + ).toThrow(/detail\.state is invalid/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ + subscriptionEffectState: 'bogus', + }), + }, + }), + ).toThrow(/detail\.subscriptionEffectState is invalid/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ expectedAttachmentCount: 'lots' }), + }, + }), + ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) + + expect(() => + toMailboxAttachmentInput({ + id: 'att-bad', + messageId: 'msg-1', + filename: null, + contentType: 'text/plain', + contentId: null, + disposition: null, + size: 1, + storageKind: 'not-a-kind', + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + }), + ).toThrow(/storageKind is invalid/) +}) diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts new file mode 100644 index 0000000000..f2bd487315 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -0,0 +1,362 @@ +import { + type EmailAttachmentRecord, + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + assertMailboxClassification, + assertMailboxDeliveryEventType, + assertMailboxDeliveryStatus, + assertMailboxDirection, + assertMailboxProcessingStatus, + assertMailboxStorageKind, + mailboxInboundDeliveryStateValues, + mailboxSubscriptionEffectStateValues, + type MailboxAttachmentInput, + type MailboxDeliveryEventInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-types.ts' + +/** + * Pure D1 → Mailbox complete-snapshot converters. + * + * Normalize nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, + * `application/octet-stream`, `kody`). Delivery-event conversion takes an + * additive {@link EmailDeliveryEventMirrorSnapshot}: D1-promoted columns are + * authoritative; `detail_json` supplies only fields that remain JSON-owned. + * Invalid persisted JSON/enums fail clearly rather than fabricating state. + */ + +/** + * Additive delivery-event mirror input. D1 `EmailDeliveryEventRecord` does not + * yet expose promoted columns or `updated_at`; callers pass those explicitly. + * + * Authoritative D1 columns (do not read these from `detail_json`): + * - `needs_effect_reconcile` + * - `usage_effect_recorded_at` + * - `usage_month` + * - `usage_bytes` + * - `usage_duration_ms` + * + * Still JSON-owned in `detail_json` (inbound/effect lease fields, etc.). + */ +export type EmailDeliveryEventMirrorSnapshot = { + event: EmailDeliveryEventRecord + /** D1 delivery events lack `updated_at`; required for stale rejection. */ + updatedAt: string + /** D1 `needs_effect_reconcile` column. */ + needsEffectReconcile: boolean + /** D1 `usage_effect_recorded_at` column. */ + usageEffectRecordedAt: string | null + /** D1 `usage_month` column. */ + usageMonth: string | null + /** D1 `usage_bytes` column. */ + usageBytes: number | null + /** D1 `usage_duration_ms` column. */ + usageDurationMs: number | null +} + +function requireOptionalString(value: unknown, label: string): string | null { + if (value == null) return null + if (typeof value !== 'string') { + throw new Error( + `Mailbox snapshot ${label} must be a string or null; got ${JSON.stringify(value)}.`, + ) + } + return value +} + +function requireOptionalNumber(value: unknown, label: string): number | null { + if (value == null) return null + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error( + `Mailbox snapshot ${label} must be a finite number or null; got ${JSON.stringify(value)}.`, + ) + } + return value +} + +function requireOptionalEnum( + values: ReadonlyArray, + value: unknown, + label: string, +): T | null { + if (value == null) return null + if ( + typeof value !== 'string' || + !(values as ReadonlyArray).includes(value) + ) { + throw new Error( + `Mailbox snapshot ${label} is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as T +} + +function parseDetailRecord(detailJson: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(detailJson) as unknown + } catch { + throw new Error( + `Mailbox snapshot detailJson is not valid JSON; got ${JSON.stringify(detailJson)}.`, + ) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error( + `Mailbox snapshot detailJson must be a JSON object; got ${JSON.stringify(parsed)}.`, + ) + } + return parsed as Record +} + +/** Convert a D1 thread row into a full Mailbox thread snapshot. */ +export function toMailboxThreadInput( + thread: EmailThreadRecord, +): MailboxThreadInput { + return { + id: thread.id, + inboxId: thread.inboxId, + subjectNormalized: thread.subjectNormalized ?? '', + rootMessageIdHeader: thread.rootMessageIdHeader, + lastMessageAt: thread.lastMessageAt ?? thread.createdAt, + createdAt: thread.createdAt, + updatedAt: thread.updatedAt, + } +} + +/** Convert a D1 message row into a full Mailbox message snapshot. */ +export function toMailboxMessageInput( + message: EmailMessageRecord, +): MailboxMessageInput { + const deliveryStatus = + message.deliveryStatus == null + ? null + : assertMailboxDeliveryStatus(message.deliveryStatus) + return { + id: message.id, + direction: assertMailboxDirection(message.direction), + inboxId: message.inboxId, + threadId: message.threadId, + senderIdentityId: message.senderIdentityId, + fromAddress: message.fromAddress ?? '', + envelopeFrom: message.envelopeFrom, + toAddresses: message.toAddresses ?? [], + ccAddresses: message.ccAddresses ?? [], + bccAddresses: message.bccAddresses ?? [], + replyToAddresses: message.replyToAddresses ?? [], + subject: message.subject ?? '', + messageIdHeader: message.messageIdHeader, + inReplyToHeader: message.inReplyToHeader, + references: message.references ?? [], + headers: message.headers ?? {}, + authResults: message.authResults, + textBody: message.textBody, + htmlBody: message.htmlBody, + rawMimeKey: message.rawMimeKey, + rawSize: message.rawSize ?? 0, + processingStatus: assertMailboxProcessingStatus(message.processingStatus), + classification: assertMailboxClassification(message.classification), + classificationReason: message.classificationReason, + providerMessageId: message.providerMessageId, + deliveryStatus, + deliveryStatusAt: message.deliveryStatusAt, + error: message.error, + receivedAt: message.receivedAt, + sentAt: message.sentAt, + createdAt: message.createdAt, + updatedAt: message.updatedAt, + } +} + +/** Convert a D1 attachment row into a full Mailbox attachment snapshot. */ +export function toMailboxAttachmentInput( + attachment: EmailAttachmentRecord, +): MailboxAttachmentInput { + return { + id: attachment.id, + messageId: attachment.messageId, + filename: attachment.filename, + contentType: attachment.contentType ?? 'application/octet-stream', + contentId: attachment.contentId, + disposition: attachment.disposition, + size: attachment.size, + storageKind: assertMailboxStorageKind(attachment.storageKind), + storageKey: attachment.storageKey, + createdAt: attachment.createdAt, + } +} + +/** + * Convert a D1 delivery-event mirror snapshot into a full Mailbox input. + * Promoted D1 columns win; remaining inbound/effect fields come from JSON. + */ +export function toMailboxDeliveryEventInput( + snapshot: EmailDeliveryEventMirrorSnapshot, +): MailboxDeliveryEventInput { + const { + event, + updatedAt, + needsEffectReconcile, + usageEffectRecordedAt, + usageMonth, + usageBytes, + usageDurationMs, + } = snapshot + if (typeof needsEffectReconcile !== 'boolean') { + throw new Error( + `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(needsEffectReconcile)}.`, + ) + } + if (typeof updatedAt !== 'string' || updatedAt.length === 0) { + throw new Error( + `Mailbox snapshot updatedAt must be a non-empty string; got ${JSON.stringify(updatedAt)}.`, + ) + } + if ( + usageEffectRecordedAt != null && + typeof usageEffectRecordedAt !== 'string' + ) { + throw new Error( + `Mailbox snapshot usageEffectRecordedAt must be a string or null; got ${JSON.stringify(usageEffectRecordedAt)}.`, + ) + } + if (usageMonth != null && typeof usageMonth !== 'string') { + throw new Error( + `Mailbox snapshot usageMonth must be a string or null; got ${JSON.stringify(usageMonth)}.`, + ) + } + if ( + usageBytes != null && + (typeof usageBytes !== 'number' || !Number.isFinite(usageBytes)) + ) { + throw new Error( + `Mailbox snapshot usageBytes must be a finite number or null; got ${JSON.stringify(usageBytes)}.`, + ) + } + if ( + usageDurationMs != null && + (typeof usageDurationMs !== 'number' || !Number.isFinite(usageDurationMs)) + ) { + throw new Error( + `Mailbox snapshot usageDurationMs must be a finite number or null; got ${JSON.stringify(usageDurationMs)}.`, + ) + } + + const detail = parseDetailRecord(event.detailJson) + + return { + id: event.id, + messageId: event.messageId, + inboxId: event.inboxId, + eventType: assertMailboxDeliveryEventType(event.eventType), + provider: event.provider ?? 'kody', + providerMessageId: event.providerMessageId, + providerEventId: event.providerEventId, + detailJson: event.detailJson, + needsEffectReconcile, + state: requireOptionalEnum( + mailboxInboundDeliveryStateValues, + detail['state'], + 'detail.state', + ), + fingerprint: requireOptionalString( + detail['fingerprint'], + 'detail.fingerprint', + ), + storageLease: requireOptionalString( + detail['storageLease'], + 'detail.storageLease', + ), + storageLeaseAt: requireOptionalString( + detail['storageLeaseAt'], + 'detail.storageLeaseAt', + ), + cleanupLease: requireOptionalString( + detail['cleanupLease'], + 'detail.cleanupLease', + ), + cleanupLeaseAt: requireOptionalString( + detail['cleanupLeaseAt'], + 'detail.cleanupLeaseAt', + ), + cleanupRetryAt: requireOptionalString( + detail['cleanupRetryAt'], + 'detail.cleanupRetryAt', + ), + expectedAttachmentCount: requireOptionalNumber( + detail['expectedAttachmentCount'], + 'detail.expectedAttachmentCount', + ), + finalizationToken: requireOptionalString( + detail['finalizationToken'], + 'detail.finalizationToken', + ), + reconcileAfter: requireOptionalString( + detail['reconcileAfter'], + 'detail.reconcileAfter', + ), + dedupeExpiresAt: requireOptionalString( + detail['dedupeExpiresAt'], + 'detail.dedupeExpiresAt', + ), + // Authoritative D1 columns — ignore any JSON copies of these keys. + usageEffectRecordedAt, + usageEffectSuppressedAt: requireOptionalString( + detail['usageEffectSuppressedAt'], + 'detail.usageEffectSuppressedAt', + ), + usageStartedAt: requireOptionalString( + detail['usageStartedAt'], + 'detail.usageStartedAt', + ), + usageMonth, + usageBytes, + usageDurationMs, + usageEffectRetryAt: requireOptionalString( + detail['usageEffectRetryAt'], + 'detail.usageEffectRetryAt', + ), + usageEffectLease: requireOptionalString( + detail['usageEffectLease'], + 'detail.usageEffectLease', + ), + usageEffectLeaseAt: requireOptionalString( + detail['usageEffectLeaseAt'], + 'detail.usageEffectLeaseAt', + ), + subscriptionEffectState: requireOptionalEnum( + mailboxSubscriptionEffectStateValues, + detail['subscriptionEffectState'], + 'detail.subscriptionEffectState', + ), + subscriptionEffectLease: requireOptionalString( + detail['subscriptionEffectLease'], + 'detail.subscriptionEffectLease', + ), + subscriptionEffectLeaseAt: requireOptionalString( + detail['subscriptionEffectLeaseAt'], + 'detail.subscriptionEffectLeaseAt', + ), + subscriptionEffectRetryAt: requireOptionalString( + detail['subscriptionEffectRetryAt'], + 'detail.subscriptionEffectRetryAt', + ), + subscriptionEffectAttemptCount: requireOptionalNumber( + detail['subscriptionEffectAttemptCount'], + 'detail.subscriptionEffectAttemptCount', + ), + subscriptionEffectDeadLetterAt: requireOptionalString( + detail['subscriptionEffectDeadLetterAt'], + 'detail.subscriptionEffectDeadLetterAt', + ), + subscriptionEffectLastError: requireOptionalString( + detail['subscriptionEffectLastError'], + 'detail.subscriptionEffectLastError', + ), + createdAt: event.createdAt, + updatedAt, + } +} diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 75b38f9927..3d9ea8b629 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -11,10 +11,13 @@ import { encodeMailboxListCursor, mailboxNowIso, normalizeMailboxPageSize, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -22,9 +25,19 @@ import { type MailboxMessageInput, type MailboxMessageRecord, type MailboxSearchMessagesInput, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' +import { + deleteMailboxDeliveryEvent, + deleteMailboxMessageMetadata, + setMailboxMessageClassification, + touchMailboxThread, + updateMailboxMessageDelivery, +} from './mailbox-mutations.ts' import { type EmailDeliveryEventType, type EmailDeliveryStatus, @@ -441,6 +454,38 @@ export class MailboxStore { return deliveryEventOwnsMessage(this.sql, eventId, messageId) } + touchThread( + input: Omit, + ): MailboxAcceptedResult { + return touchMailboxThread(this.sql, input) + } + + updateMessageDelivery( + input: Omit, + ): MailboxAcceptedResult { + return updateMailboxMessageDelivery(this.sql, input) + } + + setMessageClassification( + input: Omit, + ): MailboxAcceptedResult { + return setMailboxMessageClassification(this.sql, input) + } + + deleteMessageMetadata(input: { + messageId: string + deletedAt: string + }): MailboxDeleteMessageMetadataResult { + return deleteMailboxMessageMetadata(this.sql, input) + } + + deleteDeliveryEvent(input: { + eventId: string + deletedAt: string + }): MailboxDeleteDeliveryEventResult { + return deleteMailboxDeliveryEvent(this.sql, input) + } + getThread(threadId: string): MailboxThreadRecord | null { const row = this.sql .exec>( diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 6e281c850c..6178ce59b6 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -353,6 +353,84 @@ export type MailboxExportPhase = | 'attachment' | 'delivery_event' +/** Partial thread touch — equal/newer `updatedAt` only; never regresses `lastMessageAt`. */ +export type MailboxTouchThreadInput = { + ownerId: string + threadId: string + lastMessageAt: string + updatedAt: string +} + +/** Partial delivery/processing update — equal/newer `updatedAt` only. */ +export type MailboxUpdateMessageDeliveryInput = { + ownerId: string + messageId: string + processingStatus: EmailProcessingStatus + providerMessageId: string | null + error: string | null + sentAt: string | null + updatedAt: string +} + +/** Partial classification update — equal/newer `updatedAt` only. */ +export type MailboxSetMessageClassificationInput = { + ownerId: string + messageId: string + classification: EmailClassification + classificationReason: string | null + updatedAt: string +} + +/** + * Metadata-only message delete. Never touches R2. Stale when the row's + * `updated_at` is newer than `deletedAt`. Idempotent when already absent. + */ +export type MailboxDeleteMessageMetadataInput = { + ownerId: string + messageId: string + deletedAt: string +} + +/** + * Metadata-only delivery-event delete. Stale when the row's `updated_at` is + * newer than `deletedAt`. Idempotent when already absent. + */ +export type MailboxDeleteDeliveryEventInput = { + ownerId: string + eventId: string + deletedAt: string +} + +export type MailboxAcceptedResult = { + accepted: boolean +} + +/** + * Message metadata delete outcome. + * - `deleted: true` — row removed; includes `orphanThreadDeleted` + * - `deleted: false, stale: false` — missing (idempotent success) + * - `deleted: false, stale: true` — newer row retained + * + * Best-effort callers map missing (`!deleted && !stale`) to success. + */ +export type MailboxDeleteMessageMetadataResult = + | { deleted: true; stale: false; orphanThreadDeleted: boolean } + | { deleted: false; stale: true } + | { deleted: false; stale: false } + +/** + * Delivery-event delete outcome. + * - `deleted: true` — row removed + * - `deleted: false, stale: false` — missing (idempotent success) + * - `deleted: false, stale: true` — newer row retained + * + * Best-effort callers map missing (`!deleted && !stale`) to success. + */ +export type MailboxDeleteDeliveryEventResult = + | { deleted: true; stale: false } + | { deleted: false; stale: true } + | { deleted: false; stale: false } + export type MailboxRpc = { mirrorMessage: (input: { /** @@ -382,6 +460,21 @@ export type MailboxRpc = { accepted: boolean updatedLatestStatus: boolean }> + touchThread: ( + input: MailboxTouchThreadInput, + ) => Promise + updateMessageDelivery: ( + input: MailboxUpdateMessageDeliveryInput, + ) => Promise + setMessageClassification: ( + input: MailboxSetMessageClassificationInput, + ) => Promise + deleteMessageMetadata: ( + input: MailboxDeleteMessageMetadataInput, + ) => Promise + deleteDeliveryEvent: ( + input: MailboxDeleteDeliveryEventInput, + ) => Promise getThread: (input: { threadId: string }) => Promise diff --git a/packages/worker/src/email/system-email.ts b/packages/worker/src/email/system-email.ts index 73484a2052..f373e17aff 100644 --- a/packages/worker/src/email/system-email.ts +++ b/packages/worker/src/email/system-email.ts @@ -10,9 +10,10 @@ import { getEmailInboxByName, } from './repo.ts' import { buildPlatformEmailAddress } from './platform-address.ts' +import { systemEmailOwnerId } from './email-owner.ts' import { type EmailInboxAddressRecord, type EmailInboxRecord } from './types.ts' -export const systemEmailOwnerId = 'system:email' +export { systemEmailOwnerId } export const systemEmailLocals = [ 'kody', From 861524852bf9a9d074a49fb6e60e5c344a75af8f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:24:23 +0000 Subject: [PATCH 05/20] fix(email): align Mailbox mutation parity Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 84 +++-- packages/worker/src/email/mailbox-do.ts | 121 ++++--- .../src/email/mailbox-mirror.node.test.ts | 123 ++++--- packages/worker/src/email/mailbox-mirror.ts | 100 ++++-- .../worker/src/email/mailbox-mutations.ts | 178 +++++----- .../email/mailbox-mutations.workers.test.ts | 314 +++++++++++------- .../email/mailbox-parity-events.node.test.ts | 40 ++- .../worker/src/email/mailbox-parity-events.ts | 19 +- .../email/mailbox-snapshot-repo.node.test.ts | 120 +++++++ .../worker/src/email/mailbox-snapshot-repo.ts | 126 +++++++ .../src/email/mailbox-snapshots.node.test.ts | 53 ++- .../worker/src/email/mailbox-snapshots.ts | 136 +++----- packages/worker/src/email/mailbox-store.ts | 45 --- packages/worker/src/email/mailbox-types.ts | 64 ++-- 14 files changed, 946 insertions(+), 577 deletions(-) create mode 100644 packages/worker/src/email/mailbox-snapshot-repo.node.test.ts create mode 100644 packages/worker/src/email/mailbox-snapshot-repo.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e7c38ddfcf..402bf79a48 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -383,20 +383,22 @@ defined in `packages/worker/src/email/mailbox-parity-events.ts` and reconciliation — they do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) -- `blob1` — mirror or parity operation name +- `blob1` — namespaced event type (`mailbox_mirror:` or + `mailbox_parity:`) so unfiltered consumers can distinguish these + from `email_send` / `email_receive` / `email_delivery` - `blob2` — operation outcome - `blob3` — source event timestamp (ISO 8601) -- `double1` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; +- `double1` — event weight (always `1`, matching reporting) +- `double2` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; `0` when absent or not applicable) -- `double2` — event weight (always `1`) Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, `update_message_delivery`, `set_message_classification`, -`delete_message_metadata`, `delete_delivery_event`. Mirror outcomes: `mirrored`, -`stale`, `skipped`, `error`. Parity operations: `compare_threads`, -`compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity -outcomes: `match`, `mismatch`. Writes are best-effort and never throw into D1 -authority paths. +`delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. +Mirror outcomes: `mirrored`, `stale`, `skipped`, `error`. Parity operations: +`compare_threads`, `compare_messages`, `compare_attachments`, +`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Writes are +best-effort and never throw into D1 authority paths. Two D1 reporting projections deliberately remain: @@ -663,19 +665,23 @@ external attachment `storageKey` values against the canonical builders for that `ownerId`. **Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no -R2 or alarm side effects beyond the usual retention dirty mark): +R2 or retention-alarm side effects — unlike full snapshot mirrors, these do not +mark retention dirty or reschedule alarms): - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward - `updateMessageDelivery` — outbound processing fields (`processing_status`, `provider_message_id`, `error`, `sent_at`) - `setMessageClassification` — inbound classification fields -- `deleteMessageMetadata` — metadata-only delete (attachments + message + orphan - thread); never deletes R2; distinguishes missing (idempotent) from stale - (newer `updated_at` retained) -- `deleteDeliveryEvent` — metadata-only delivery-event delete with the same - missing vs stale semantics - +- `deleteMessageMetadata` — metadata-only delete (null delivery-event + `message_id`, then attachments + message); never deletes R2 or empty threads +- `deleteDeliveryEvent` — metadata-only delivery-event delete +- `deleteThreadIfEmpty` — deferred empty-thread cleanup (D1 + `deleteEmptyEmailThreads` parity); stale-safe by `thread.updated_at` + +Partial touch/update/classify RPCs return `accepted`, `missing` (target absent — +idempotent for best-effort callers), or `stale` (newer `updated_at` retained). +Delete RPCs return `deleted`, `missing`, or `stale` with the same semantics. Implementation lives in `mailbox-mutations.ts` (SQLite helpers) and is exposed through `mailbox-do.ts` RPCs. All require `ownerId` and reject cross-owner calls. @@ -686,16 +692,20 @@ lightweight module exporting `systemEmailOwnerId` (`'system:email'`) and dual-write helpers can skip the reserved operator inbox without import cycles. Operator mail is never mirrored into per-user Mailbox objects. -**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`): pure converters -that turn D1 rows into complete Mailbox wire inputs — `toMailboxThreadInput`, -`toMailboxMessageInput`, `toMailboxAttachmentInput`, and -`toMailboxDeliveryEventInput`. They normalize nulls to Mailbox SQLite defaults -(empty strings, `[]`, `{}`, `0`, `application/octet-stream`, `kody`) and fail -clearly on invalid persisted JSON/enums rather than fabricating state. Delivery -events take an additive `EmailDeliveryEventMirrorSnapshot`: D1-promoted columns -(`needs_effect_reconcile`, usage-effect fields) are authoritative; remaining -inbound/effect lease fields come from `detail_json`. D1 delivery events lack -`updated_at`; callers pass it explicitly for stale rejection. +**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`; delivery loads in +`mailbox-snapshot-repo.ts`): pure converters that turn D1 rows into complete +Mailbox wire inputs — `toMailboxThreadInput`, `toMailboxMessageInput`, +`toMailboxAttachmentInput`, and `toMailboxDeliveryEventInput`. They normalize +nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, +`application/octet-stream`, `kody`) and fail clearly on invalid persisted +JSON/enums rather than fabricating state. Delivery events consume a complete +`EmailDeliveryEventMirrorProjection` loaded as one cohesive D1 row (base columns +plus promoted `needs_effect_reconcile` and usage-effect fields); inbound/effect +lease fields still come from `detail_json`. Callers load via +`getMailboxDeliveryEventMirrorInput` / `getEmailDeliveryEventMirrorProjection` +and supply only `sourceMutationAt` — the canonical mirror `updatedAt` from the +D1 mutation (inserts use `created_at`). Callers must not stitch promoted columns +field-by-field. **Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers around the DO RPCs for future D1-authoritative dual-write. Each returns a @@ -704,13 +714,14 @@ structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. Failures log with stable tags (for example `mailbox-mirror-message-failed`) and never propagate into D1 commit paths. Helpers cover full snapshots -(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`) and -partial mutations (`mirrorMailboxTouchThread`, -`mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, -`mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`). -**There are no live dual-write callers in the phase-2a PR** — only unit/worker -tests invoke these helpers. D1 remains sole authority for all live mail -read/write paths. +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot` — prefer +loading delivery events with `getMailboxDeliveryEventMirrorInput`) and partial +mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, +`mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, +`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). **There +are no live dual-write callers in the phase-2a PR** — only unit/worker tests +invoke these helpers. D1 remains sole authority for all live mail read/write +paths. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -752,10 +763,11 @@ surface that later live-path wiring will call. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, `mailbox-mirror.ts`, - `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, - `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, - `deleteDeliveryEvent`), and `mailbox-parity-events.ts`. Those helpers and + PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, + `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + + matching DO RPCs (`touchThread`, `updateMessageDelivery`, + `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, + `deleteThreadIfEmpty`), and `mailbox-parity-events.ts`. Those helpers and parity-event writers are library-only today — **no production path invokes them yet**; live-path dual-write and parity wiring remain for the rest of phase 2. diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 5222063275..c8b06ce2f4 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -13,23 +13,31 @@ import { selectMailboxRetentionWriteAlarm, } from './mailbox-retention.ts' import { MailboxStore } from './mailbox-store.ts' +import { + deleteMailboxDeliveryEvent, + deleteMailboxMessageMetadata, + deleteMailboxThreadIfEmpty, + setMailboxMessageClassification, + touchMailboxThread, + updateMailboxMessageDelivery, +} from './mailbox-mutations.ts' import { assertMailboxNonEmptyString, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxListMessagesInput, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxPartialMutationResult, type MailboxRpc, type MailboxSearchMessagesInput, type MailboxSetMessageClassificationInput, @@ -230,19 +238,14 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async touchThread( input: MailboxTouchThreadInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.touchThread({ - threadId: input.threadId, - lastMessageAt: input.lastMessageAt, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = touchMailboxThread(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** @@ -251,22 +254,14 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async updateMessageDelivery( input: MailboxUpdateMessageDeliveryInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.updateMessageDelivery({ - messageId: input.messageId, - processingStatus: input.processingStatus, - providerMessageId: input.providerMessageId, - error: input.error, - sentAt: input.sentAt, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = updateMailboxMessageDelivery(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** @@ -275,42 +270,32 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async setMessageClassification( input: MailboxSetMessageClassificationInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.setMessageClassification({ - messageId: input.messageId, - classification: input.classification, - classificationReason: input.classificationReason, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = setMailboxMessageClassification( + this.ctx.storage.sql, + mutationInput, + ) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** - * Metadata-only delete (attachments + message + orphan thread). Never - * deletes R2. Stale when the message `updated_at` is newer than `deletedAt`. + * Metadata-only delete (null delivery-event message_id + attachments + + * message). Never deletes R2 or empty threads. */ async deleteMessageMetadata( input: MailboxDeleteMessageMetadataInput, - ): Promise { - let result: MailboxDeleteMessageMetadataResult = { - deleted: false, - stale: false, - } + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - result = this.store.deleteMessageMetadata({ - messageId: input.messageId, - deletedAt: input.deletedAt, - }) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxMessageMetadata(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() return result } @@ -320,20 +305,28 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async deleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput, - ): Promise { - let result: MailboxDeleteDeliveryEventResult = { - deleted: false, - stale: false, - } + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - result = this.store.deleteDeliveryEvent({ - eventId: input.eventId, - deletedAt: input.deletedAt, - }) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxDeliveryEvent(this.ctx.storage.sql, mutationInput) + }) + return result + } + + /** + * Deferred empty-thread cleanup. Stale-safe by `thread.updated_at`. + */ + async deleteThreadIfEmpty( + input: MailboxDeleteThreadIfEmptyInput, + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxThreadIfEmpty(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() return result } @@ -437,22 +430,22 @@ export { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReference, type MailboxBlobReferencePage, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxExportRow, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 4668e5bb64..b7bc8344e0 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -5,16 +5,14 @@ import { mirrorMailboxDeliveryEventSnapshot, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, + mirrorMailboxDeleteThreadIfEmpty, mirrorMailboxMessageSnapshot, mirrorMailboxSetMessageClassification, mirrorMailboxTouchThread, mirrorMailboxUpdateMessageDelivery, } from './mailbox-mirror.ts' -import { - type EmailDeliveryEventRecord, - type EmailMessageRecord, - type EmailThreadRecord, -} from './types.ts' +import { type EmailMessageRecord, type EmailThreadRecord } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' function fakeMailboxEnv(stub: Record) { const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) @@ -84,27 +82,30 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep accepted: true, updatedLatestStatus: false, })) - const touchThread = vi.fn(async () => ({ accepted: true })) - const updateMessageDelivery = vi.fn(async () => ({ accepted: false })) - const setMessageClassification = vi.fn(async () => ({ accepted: true })) + const touchThread = vi.fn(async () => ({ status: 'accepted' as const })) + const updateMessageDelivery = vi.fn(async () => ({ + status: 'stale' as const, + })) + const setMessageClassification = vi.fn(async () => ({ + status: 'missing' as const, + })) const deleteMessageMetadata = vi.fn( async (input: { messageId: string; deletedAt: string }) => { if (input.messageId === 'missing-msg') { - return { deleted: false as const, stale: false as const } + return { status: 'missing' as const } } if (input.deletedAt < '2026-07-01T12:00:00.000Z') { - return { deleted: false as const, stale: true as const } - } - return { - deleted: true as const, - stale: false as const, - orphanThreadDeleted: false, + return { status: 'stale' as const } } + return { status: 'deleted' as const } }, ) const deleteDeliveryEvent = vi.fn(async () => { throw new Error('do unavailable') }) + const deleteThreadIfEmpty = vi.fn(async () => ({ + status: 'missing' as const, + })) const { env, idFromName } = fakeMailboxEnv({ mirrorMessage, @@ -114,6 +115,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep setMessageClassification, deleteMessageMetadata, deleteDeliveryEvent, + deleteThreadIfEmpty, }) const thread: EmailThreadRecord = { @@ -177,59 +179,61 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep ], }) - const detail = { - state: 'storing', - fingerprint: 'fp-mirror', - storageLease: 'lease', - subscriptionEffectState: 'processing', - // Stale JSON copies of promoted columns — must not win. - usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', - usageMonth: 'json-month', - usageBytes: 1, - usageDurationMs: 1, - } - const event: EmailDeliveryEventRecord = { + const deliveryEvent: MailboxDeliveryEventInput = { id: 'evt-1', messageId: 'msg-1', - userId: 'user-aaa', inboxId: 'inbox-1', eventType: 'receive_started', provider: 'cloudflare-email-routing', providerMessageId: null, providerEventId: null, - detailJson: JSON.stringify(detail), + detailJson: JSON.stringify({ + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + }), + needsEffectReconcile: true, + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: 'processing', + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', } expect( await mirrorMailboxDeliveryEventSnapshot({ env, - snapshot: { - event, - updatedAt: '2026-07-02T10:00:01.000Z', - needsEffectReconcile: true, - usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', - usageMonth: '2026-07', - usageBytes: 99, - usageDurationMs: 50, - }, + ownerId: 'user-aaa', + event: deliveryEvent, }), ).toEqual({ status: 'mirrored' }) expect(upsertDeliveryEvent).toHaveBeenCalledWith({ ownerId: 'user-aaa', - event: expect.objectContaining({ - id: 'evt-1', - provider: 'cloudflare-email-routing', - needsEffectReconcile: true, - state: 'storing', - fingerprint: 'fp-mirror', - storageLease: 'lease', - subscriptionEffectState: 'processing', - usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', - usageMonth: '2026-07', - usageBytes: 99, - usageDurationMs: 50, - updatedAt: '2026-07-02T10:00:01.000Z', - }), + event: deliveryEvent, latestDeliveryStatus: undefined, }) @@ -262,6 +266,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) + // missing partial mutation is idempotent success for best-effort dual-write expect( await mirrorMailboxSetMessageClassification({ env, @@ -300,6 +305,20 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'mirrored' }) + expect( + await mirrorMailboxDeleteThreadIfEmpty({ + env, + ownerId: 'user-aaa', + threadId: 'thread-1', + deletedAt: '2026-07-01T16:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + expect(deleteThreadIfEmpty).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + threadId: 'thread-1', + deletedAt: '2026-07-01T16:00:00.000Z', + }) + expect( await mirrorMailboxDeleteDeliveryEvent({ env, diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index e27b3174d2..b72b4f4fe3 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -6,10 +6,8 @@ import { } from './mailbox-client.ts' import { toMailboxAttachmentInput, - toMailboxDeliveryEventInput, toMailboxMessageInput, toMailboxThreadInput, - type EmailDeliveryEventMirrorSnapshot, } from './mailbox-snapshots.ts' import { type EmailAttachmentRecord, @@ -19,9 +17,11 @@ import { } from './types.ts' import { type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, + type MailboxDeliveryEventInput, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, @@ -33,6 +33,10 @@ import { * Never throw into D1 authority paths: catch/log with stable tags and return * structured `mirrored | stale | skipped | error` outcomes. `system:email` * stays in D1 only. + * + * Delivery-event snapshots should be loaded via + * `getMailboxDeliveryEventMirrorInput` (mailbox-snapshot-repo) so callers do + * not supply promoted D1 columns individually. */ export type MailboxMirrorEnv = MailboxEnv @@ -72,22 +76,46 @@ function skipIfMailboxMissing( return null } -function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { - return accepted ? { status: 'mirrored' } : { status: 'stale' } +/** Map touch/update/classify: accepted/missing → mirrored; stale → stale. */ +function outcomeFromPartialMutation( + result: MailboxPartialMutationResult, +): MailboxMirrorResult { + switch (result.status) { + case 'accepted': + case 'missing': + return { status: 'mirrored' } + case 'stale': + return { status: 'stale' } + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox partial mutation status: ${JSON.stringify(exhaustive)}`, + ) + } + } } -/** - * Map delete RPC outcome: - * - `deleted: true` → mirrored - * - `deleted: false, stale: true` → stale - * - `deleted: false, stale: false` (missing) → mirrored (idempotent) - */ +/** Map delete RPC: deleted/missing → mirrored; stale → stale. */ function outcomeFromDeleteResult( - result: MailboxDeleteMessageMetadataResult | MailboxDeleteDeliveryEventResult, + result: MailboxDeleteResult, ): MailboxMirrorResult { - if (result.deleted) return { status: 'mirrored' } - if (result.stale) return { status: 'stale' } - return { status: 'mirrored' } + switch (result.status) { + case 'deleted': + case 'missing': + return { status: 'mirrored' } + case 'stale': + return { status: 'stale' } + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox delete status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { + return accepted ? { status: 'mirrored' } : { status: 'stale' } } async function runMirror( @@ -128,17 +156,24 @@ export async function mirrorMailboxMessageSnapshot(input: { }) } -/** Best-effort delivery-event snapshot mirror. */ +/** + * Best-effort delivery-event snapshot mirror. + * + * Prefer loading `event` with `getMailboxDeliveryEventMirrorInput` so promoted + * D1 columns and detail decoding stay cohesive. `event.updatedAt` must be the + * canonical `sourceMutationAt` from that load (inserts: `created_at`). + */ export async function mirrorMailboxDeliveryEventSnapshot(input: { env: MailboxMirrorEnv - snapshot: EmailDeliveryEventMirrorSnapshot + ownerId: string + event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null }): Promise { - const owner = resolveMirrorOwner(input.snapshot.event.userId) + const owner = resolveMirrorOwner(input.ownerId) if (!owner.ok) return owner.result const skippedMailbox = skipIfMailboxMissing(input.env) if (skippedMailbox) return skippedMailbox @@ -149,7 +184,7 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { userId: owner.ownerId, }).upsertDeliveryEvent({ ownerId: owner.ownerId, - event: toMailboxDeliveryEventInput(input.snapshot), + event: input.event, latestDeliveryStatus: input.latestDeliveryStatus, }) return outcomeFromAccepted(result.accepted) @@ -171,7 +206,7 @@ export async function mirrorMailboxTouchThread( env: input.env, userId: owner.ownerId, }).touchThread(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }) } @@ -192,7 +227,7 @@ export async function mirrorMailboxUpdateMessageDelivery( env: input.env, userId: owner.ownerId, }).updateMessageDelivery(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }, ) } @@ -214,7 +249,7 @@ export async function mirrorMailboxSetMessageClassification( env: input.env, userId: owner.ownerId, }).setMessageClassification(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }, ) } @@ -259,3 +294,22 @@ export async function mirrorMailboxDeleteDeliveryEvent( return outcomeFromDeleteResult(result) }) } + +/** Best-effort empty-thread cleanup mirror (D1 deleteEmptyEmailThreads parity). */ +export async function mirrorMailboxDeleteThreadIfEmpty( + input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delete-thread-if-empty-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteThreadIfEmpty(rpcInput) + return outcomeFromDeleteResult(result) + }) +} diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts index af139f7cbc..039985d632 100644 --- a/packages/worker/src/email/mailbox-mutations.ts +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -4,9 +4,9 @@ import { assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertOptionalMailboxCanonicalIsoTimestamp, - type MailboxAcceptedResult, - type MailboxDeleteDeliveryEventResult, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, @@ -14,13 +14,13 @@ import { /** * Owner-bound partial mutation helpers for Mailbox SQLite metadata. - * No R2 / alarm side effects — callers own transactions and retention. + * No R2 / alarm side effects — callers own transactions. */ export function touchMailboxThread( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') const lastMessageAt = assertMailboxCanonicalIsoTimestamp( input.lastMessageAt, @@ -30,7 +30,16 @@ export function touchMailboxThread( input.updatedAt, 'updatedAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads WHERE id = ? LIMIT 1`, + threadId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_threads SET last_message_at = CASE WHEN last_message_at < ? THEN ? @@ -45,13 +54,13 @@ export function touchMailboxThread( threadId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } export function updateMailboxMessageDelivery( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const processingStatus = assertMailboxProcessingStatus(input.processingStatus) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -62,7 +71,16 @@ export function updateMailboxMessageDelivery( input.sentAt, 'sentAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_messages SET processing_status = ?, provider_message_id = ?, @@ -79,20 +97,29 @@ export function updateMailboxMessageDelivery( messageId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } export function setMailboxMessageClassification( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const classification = assertMailboxClassification(input.classification) const updatedAt = assertMailboxCanonicalIsoTimestamp( input.updatedAt, 'updatedAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_messages SET classification = ?, classification_reason = ?, @@ -105,87 +132,58 @@ export function setMailboxMessageClassification( messageId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } /** - * Delete message + attachment metadata and any orphaned thread. Never deletes - * R2 objects. Distinguishes missing (idempotent) from stale (newer retained). + * Delete message + attachment metadata. Nulls delivery-event `message_id` + * first (D1 parity). Never deletes R2 or empty threads. */ export function deleteMailboxMessageMetadata( sql: SqlStorage, input: { messageId: string; deletedAt: string }, -): MailboxDeleteMessageMetadataResult { +): MailboxDeleteResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const deletedAt = assertMailboxCanonicalIsoTimestamp( input.deletedAt, 'deletedAt', ) const existing = sql - .exec<{ thread_id: string | null; updated_at: string }>( - `SELECT thread_id, updated_at FROM email_messages + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, messageId, ) .toArray()[0] - if (existing == null) { - return { deleted: false, stale: false } - } - if (existing.updated_at > deletedAt) { - return { deleted: false, stale: true } - } + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } + sql.exec( + `UPDATE email_delivery_events + SET message_id = NULL + WHERE message_id = ?`, + messageId, + ) sql.exec(`DELETE FROM email_attachments WHERE message_id = ?`, messageId) - const messageDelete = sql.exec( + sql.exec( `DELETE FROM email_messages WHERE id = ? AND updated_at <= ?`, messageId, deletedAt, ) - if (messageDelete.rowsWritten === 0) { - const stillThere = sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_messages WHERE id = ? LIMIT 1`, - messageId, - ) - .toArray()[0] - return stillThere == null - ? { deleted: false, stale: false } - : { deleted: false, stale: true } - } - - const threadId = existing.thread_id - if (threadId == null || threadId.length === 0) { - return { deleted: true, stale: false, orphanThreadDeleted: false } - } - - const orphanDelete = sql.exec( - `DELETE FROM email_threads - WHERE id = ? - AND NOT EXISTS ( - SELECT 1 FROM email_messages - WHERE thread_id = ? - )`, - threadId, - threadId, - ) - return { - deleted: true, - stale: false, - orphanThreadDeleted: orphanDelete.rowsWritten > 0, - } + return { status: 'deleted' } } /** * Delete a delivery-event row. SELECT `updated_at` first so missing and stale - * are distinguishable; missing is idempotent success for best-effort callers. + * are distinguishable. */ export function deleteMailboxDeliveryEvent( sql: SqlStorage, input: { eventId: string; deletedAt: string }, -): MailboxDeleteDeliveryEventResult { +): MailboxDeleteResult { const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') const deletedAt = assertMailboxCanonicalIsoTimestamp( input.deletedAt, @@ -199,30 +197,54 @@ export function deleteMailboxDeliveryEvent( eventId, ) .toArray()[0] - if (existing == null) { - return { deleted: false, stale: false } - } - if (existing.updated_at > deletedAt) { - return { deleted: false, stale: true } - } + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } - const cursor = sql.exec( + sql.exec( `DELETE FROM email_delivery_events WHERE id = ? AND updated_at <= ?`, eventId, deletedAt, ) - if (cursor.rowsWritten === 0) { - const stillThere = sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, - eventId, - ) - .toArray()[0] - return stillThere == null - ? { deleted: false, stale: false } - : { deleted: false, stale: true } - } - return { deleted: true, stale: false } + return { status: 'deleted' } +} + +/** + * Delete a thread only when it has no messages. Stale-safe by + * `thread.updated_at`. Not-empty / already-absent → `missing` (idempotent). + */ +export function deleteMailboxThreadIfEmpty( + sql: SqlStorage, + input: Omit, +): MailboxDeleteResult { + const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads + WHERE id = ? + LIMIT 1`, + threadId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } + + const cursor = sql.exec( + `DELETE FROM email_threads + WHERE id = ? + AND updated_at <= ? + AND NOT EXISTS ( + SELECT 1 FROM email_messages + WHERE thread_id = ? + )`, + threadId, + deletedAt, + threadId, + ) + return cursor.rowsWritten > 0 ? { status: 'deleted' } : { status: 'missing' } } diff --git a/packages/worker/src/email/mailbox-mutations.workers.test.ts b/packages/worker/src/email/mailbox-mutations.workers.test.ts index 721617ff1b..f36a0bc28a 100644 --- a/packages/worker/src/email/mailbox-mutations.workers.test.ts +++ b/packages/worker/src/email/mailbox-mutations.workers.test.ts @@ -15,7 +15,7 @@ import { uniqueUserId, } from './mailbox-test-helpers.ts' -test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection', async () => { +test('Mailbox mutation RPCs: owner bind, accepted/missing/stale updates', async () => { silenceIncidentalRuntimeWarnings() const ownerA = uniqueUserId('mut-a') const ownerB = uniqueUserId('mut-b') @@ -84,39 +84,49 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' deletedAt: '2026-07-01T13:00:00.000Z', }), ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteThreadIfEmpty({ + ownerId: ownerB, + threadId: thread.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) }) - const staleTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-07-01T14:00:00.000Z', - updatedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleTouch).toEqual({ accepted: false }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T14:00:00.000Z', + updatedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T12:00:00.000Z', updatedAt: '2026-07-01T12:00:00.000Z', }) - const regressTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-06-01T00:00:00.000Z', - updatedAt: '2026-07-01T12:00:01.000Z', - }) - expect(regressTouch).toEqual({ accepted: true }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T12:00:00.000Z', updatedAt: '2026-07-01T12:00:01.000Z', }) - const advanceTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-07-01T15:00:00.000Z', - updatedAt: '2026-07-01T12:00:02.000Z', - }) - expect(advanceTouch).toEqual({ accepted: true }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T15:00:00.000Z', updatedAt: '2026-07-01T12:00:02.000Z', @@ -129,18 +139,19 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' lastMessageAt: '2026-07-01T16:00:00.000Z', updatedAt: '2026-07-01T16:00:00.000Z', }), - ).toEqual({ accepted: false }) + ).toEqual({ status: 'missing' }) - const staleDelivery = await mailbox.updateMessageDelivery({ - ownerId: ownerA, - messageId: message.id, - processingStatus: 'failed', - providerMessageId: 'stale-prov', - error: 'stale', - sentAt: null, - updatedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleDelivery).toEqual({ accepted: false }) + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'failed', + providerMessageId: 'stale-prov', + error: 'stale', + sentAt: null, + updatedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ processingStatus: 'stored', providerMessageId: null, @@ -148,16 +159,29 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' updatedAt: '2026-07-01T12:00:00.000Z', }) - const deliveryUpdate = await mailbox.updateMessageDelivery({ - ownerId: ownerA, - messageId: message.id, - processingStatus: 'sent', - providerMessageId: 'prov-1', - error: null, - sentAt: '2026-07-01T12:05:00.000Z', - updatedAt: '2026-07-01T12:00:00.000Z', - }) - expect(deliveryUpdate).toEqual({ accepted: true }) + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: 'missing-msg', + processingStatus: 'sent', + providerMessageId: null, + error: null, + sentAt: null, + updatedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) + + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ processingStatus: 'sent', providerMessageId: 'prov-1', @@ -165,27 +189,25 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' updatedAt: '2026-07-01T12:00:00.000Z', }) - const staleClass = await mailbox.setMessageClassification({ - ownerId: ownerA, - messageId: message.id, - classification: 'quarantined', - classificationReason: 'stale', - updatedAt: '2026-06-01T00:00:00.000Z', - }) - expect(staleClass).toEqual({ accepted: false }) - expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ - classification: 'accepted', - classificationReason: null, - }) + expect( + await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'stale', + updatedAt: '2026-06-01T00:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) - const classUpdate = await mailbox.setMessageClassification({ - ownerId: ownerA, - messageId: message.id, - classification: 'quarantined', - classificationReason: 'Sender matched quarantine rule.', - updatedAt: '2026-07-01T12:00:03.000Z', - }) - expect(classUpdate).toEqual({ accepted: true }) + expect( + await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ classification: 'quarantined', classificationReason: 'Sender matched quarantine rule.', @@ -193,12 +215,15 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' }) }) -test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no R2', async () => { +test('Mailbox deleteMessageMetadata: nulls delivery message_id, no orphan/R2', async () => { silenceIncidentalRuntimeWarnings() const userId = uniqueUserId('del-msg') const mailbox = rpcFor(userId) - const thread = baseThread({ id: 'del-thread' }) + const thread = baseThread({ + id: 'del-thread', + updatedAt: '2026-07-01T12:00:00.000Z', + }) const alone = baseMessage(userId, { id: 'del-alone', threadId: thread.id, @@ -211,6 +236,16 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no message: alone, attachments: [attachment], }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: baseDeliveryEvent({ + id: 'del-evt', + messageId: alone.id, + eventType: 'received', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }), + }) const rawKey = emailRawMimeKey(userId, alone.id) const attKey = emailAttachmentBlobKey(userId, alone.id, attachment.id) @@ -225,52 +260,80 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no }) as typeof env.EMAIL_BLOBS.delete try { - const staleDelete = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleDelete).toEqual({ deleted: false, stale: true }) - expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() - expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() expect( - await mailbox.listAttachmentsForMessage({ messageId: alone.id }), - ).toHaveLength(1) + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() - const deleted = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T12:00:00.000Z', - }) - expect(deleted).toEqual({ - deleted: true, - stale: false, - orphanThreadDeleted: true, - }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect(await mailbox.getMessage({ messageId: alone.id })).toBeNull() - expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + // Thread remains for deferred empty-thread cleanup. + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() expect( await mailbox.listAttachmentsForMessage({ messageId: alone.id }), ).toHaveLength(0) + const events = await mailbox.listDeliveryEvents({ limit: 5 }) + expect(events).toEqual([ + expect.objectContaining({ id: 'del-evt', messageId: null }), + ]) expect(await env.EMAIL_BLOBS.get(rawKey)).not.toBeNull() expect(await env.EMAIL_BLOBS.get(attKey)).not.toBeNull() expect(blobDeleteCalls).toBe(0) - const again = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T13:00:00.000Z', - }) - // Missing (`!deleted && !stale`): idempotent success for best-effort callers. - expect(again).toEqual({ deleted: false, stale: false }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) expect(blobDeleteCalls).toBe(0) + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) + expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) } finally { env.EMAIL_BLOBS.delete = originalDelete await env.EMAIL_BLOBS.delete(rawKey) await env.EMAIL_BLOBS.delete(attKey) } - const sharedThread = baseThread({ id: 'shared-thread' }) + const sharedThread = baseThread({ + id: 'shared-thread', + updatedAt: '2026-07-02T10:00:00.000Z', + }) const keep = baseMessage(userId, { id: 'keep-msg', threadId: sharedThread.id, @@ -290,17 +353,21 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no ownerId: userId, message: drop, }) - const dropOnly = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: drop.id, - deletedAt: '2026-07-02T10:00:00.000Z', - }) - expect(dropOnly).toEqual({ - deleted: true, - stale: false, - orphanThreadDeleted: false, - }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: drop.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect(await mailbox.getMessage({ messageId: keep.id })).not.toBeNull() + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: sharedThread.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) expect(await mailbox.getThread({ threadId: sharedThread.id })).not.toBeNull() }) @@ -328,30 +395,33 @@ test('Mailbox deleteDeliveryEvent: stale, deleted, and missing outcomes', async }), }) - const staleDelete = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T09:00:00.000Z', - }) - expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T09:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), ).toHaveLength(1) - const deleted = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T10:00:00.000Z', - }) - expect(deleted).toEqual({ deleted: true, stale: false }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), ).toHaveLength(0) - const again = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T11:00:00.000Z', - }) - expect(again).toEqual({ deleted: false, stale: false }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T11:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) }) diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 1cc170a450..2b70d35ecc 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -59,37 +59,53 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = expect(writeDataPoint).toHaveBeenCalledTimes(6) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], - blobs: ['mirror_message', 'mirrored', '2026-07-30T10:00:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_mirror:mirror_message', + 'mirrored', + '2026-07-30T10:00:00.000Z', + ], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(2, { indexes: ['user-1'], - blobs: ['touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], - doubles: [0, 1], + blobs: ['mailbox_mirror:touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(3, { indexes: ['user-1'], blobs: [ - 'set_message_classification', + 'mailbox_mirror:set_message_classification', 'skipped', '2026-07-30T10:06:00.000Z', ], - doubles: [0, 1], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(4, { indexes: ['user-1'], - blobs: ['delete_message_metadata', 'error', '2026-07-30T10:07:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_mirror:delete_message_metadata', + 'error', + '2026-07-30T10:07:00.000Z', + ], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(5, { indexes: ['user-2'], - blobs: ['compare_messages', 'mismatch', '2026-07-30T11:00:00.000Z'], - doubles: [2, 1], + blobs: [ + 'mailbox_parity:compare_messages', + 'mismatch', + '2026-07-30T11:00:00.000Z', + ], + doubles: [1, 2], }) expect(writeDataPoint).toHaveBeenNthCalledWith(6, { indexes: ['user-3'], - blobs: ['compare_threads', 'match', '2026-07-30T12:00:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_parity:compare_threads', + 'match', + '2026-07-30T12:00:00.000Z', + ], + doubles: [1, 0], }) }) diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index e3dcd74229..66f394eee9 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -12,6 +12,7 @@ export type MailboxMirrorOperation = | 'set_message_classification' | 'delete_message_metadata' | 'delete_delivery_event' + | 'delete_thread_if_empty' export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' @@ -52,15 +53,23 @@ function resolveCountDelta(event: MailboxParityEvent): number { return 0 } +function mailboxParityEventType(event: MailboxParityEvent): string { + const prefix = + event.category === 'mirror' ? 'mailbox_mirror' : 'mailbox_parity' + return `${prefix}:${event.operation}` +} + /** * Additive Analytics Engine schema (same EMAIL_EVENTS dataset as reporting): * * - index1: stable user id (per-user isolation) - * - blob1: mirror or parity operation + * - blob1: namespaced event type (`mailbox_mirror:` or + * `mailbox_parity:`) so unfiltered consumers can distinguish these + * from `email_send` / `email_receive` / `email_delivery` * - blob2: operation outcome * - blob3: source event timestamp (ISO 8601) - * - double1: D1-minus-DO count delta (0 when absent or not applicable) - * - double2: event weight (always 1) + * - double1: event weight (always 1, matching reporting) + * - double2: D1-minus-DO count delta (0 when absent or not applicable) * * `system:email` is excluded — operator mail stays in D1 only. */ @@ -75,11 +84,11 @@ export function recordMailboxParityEvent( env.EMAIL_EVENTS.writeDataPoint({ indexes: [event.userId], blobs: [ - event.operation, + mailboxParityEventType(event), event.outcome, event.timestamp ?? new Date().toISOString(), ], - doubles: [resolveCountDelta(event), 1], + doubles: [1, resolveCountDelta(event)], }) } catch (error) { console.warn('mailbox-parity-event-write-failed', error) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts new file mode 100644 index 0000000000..fb55632566 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -0,0 +1,120 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { + getEmailDeliveryEventMirrorProjection, + getMailboxDeliveryEventMirrorInput, +} from './mailbox-snapshot-repo.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +async function createEmailDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +test('mailbox-snapshot-repo loads complete delivery-event projection and ready Mailbox input', async () => { + const { db } = await createEmailDb() + const detail = { + state: 'received', + fingerprint: 'fp-repo', + storageLease: 'lease-repo', + subscriptionEffectState: 'pending', + usageStartedAt: '2026-07-02T09:59:00.000Z', + // Stale JSON copies of promoted columns must not win after conversion. + usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', + usageMonth: 'json-month', + usageBytes: 1, + usageDurationMs: 1, + } + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, usage_effect_recorded_at, usage_month, + usage_bytes, usage_duration_ms, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'evt-1', + 'msg-1', + 'user-aaa', + 'inbox-1', + 'received', + 'cloudflare-email-routing', + null, + 'provider-evt-1', + JSON.stringify(detail), + 1, + '2026-07-02T10:01:00.000Z', + '2026-07', + 2048, + 120, + '2026-07-02T10:00:00.000Z', + ) + .run() + + const projection = await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'user-aaa', + eventId: 'evt-1', + }) + expect(projection).toEqual({ + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'cloudflare-email-routing', + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + }) + + expect( + await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'other-user', + eventId: 'evt-1', + }), + ).toBeNull() + expect( + await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'user-aaa', + eventId: 'missing', + }), + ).toBeNull() + + const mailboxInput = await getMailboxDeliveryEventMirrorInput({ + db, + ownerId: 'user-aaa', + eventId: 'evt-1', + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + expect(mailboxInput).toEqual( + expect.objectContaining({ + id: 'evt-1', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-repo', + storageLease: 'lease-repo', + subscriptionEffectState: 'pending', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + usageStartedAt: '2026-07-02T09:59:00.000Z', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + ) +}) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts new file mode 100644 index 0000000000..30df8d8ea3 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -0,0 +1,126 @@ +import { + toMailboxDeliveryEventInput, + type EmailDeliveryEventMirrorProjection, +} from './mailbox-snapshots.ts' +import { type EmailDeliveryEventType } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' + +/** + * D1 loaders for complete Mailbox mirror projections. + * + * Delivery-event dual-write callers load one cohesive projection here instead + * of stitching promoted columns into the converter by hand. Detail JSON + * decoding stays centralized in `mailbox-snapshots.ts`. + * + * Phase-2 high-risk lifecycle mutations must pass the same canonical + * `sourceMutationAt` used for the D1 mutation as the Mailbox `updatedAt`. + * Inserts use `created_at`. + */ + +type DeliveryEventMirrorRow = { + id: string + message_id: string | null + user_id: string | null + inbox_id: string | null + event_type: string + provider: string | null + provider_message_id: string | null + provider_event_id: string | null + detail_json: string + needs_effect_reconcile: number + usage_effect_recorded_at: string | null + usage_month: string | null + usage_bytes: number | null + usage_duration_ms: number | null + created_at: string +} + +function mapDeliveryEventMirrorRow( + row: DeliveryEventMirrorRow, +): EmailDeliveryEventMirrorProjection { + return { + id: String(row.id), + messageId: row.message_id == null ? null : String(row.message_id), + userId: row.user_id == null ? null : String(row.user_id), + inboxId: row.inbox_id == null ? null : String(row.inbox_id), + eventType: String(row.event_type) as EmailDeliveryEventType, + provider: row.provider == null ? null : String(row.provider), + providerMessageId: + row.provider_message_id == null ? null : String(row.provider_message_id), + providerEventId: + row.provider_event_id == null ? null : String(row.provider_event_id), + detailJson: String(row.detail_json ?? '{}'), + createdAt: String(row.created_at), + needsEffectReconcile: Number(row.needs_effect_reconcile ?? 0) === 1, + usageEffectRecordedAt: + row.usage_effect_recorded_at == null + ? null + : String(row.usage_effect_recorded_at), + usageMonth: row.usage_month == null ? null : String(row.usage_month), + usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes) || 0, + usageDurationMs: + row.usage_duration_ms == null ? null : Number(row.usage_duration_ms) || 0, + } +} + +/** + * Load one complete delivery-event mirror projection scoped by owner + id. + * Returns null when the row is absent or owned by a different user. + */ +export async function getEmailDeliveryEventMirrorProjection(input: { + db: D1Database + ownerId: string + eventId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT + id, + message_id, + user_id, + inbox_id, + event_type, + provider, + provider_message_id, + provider_event_id, + detail_json, + needs_effect_reconcile, + usage_effect_recorded_at, + usage_month, + usage_bytes, + usage_duration_ms, + created_at + FROM email_delivery_events + WHERE id = ? + AND user_id = ? + LIMIT 1`, + ) + .bind(input.eventId, input.ownerId) + .first() + return row ? mapDeliveryEventMirrorRow(row) : null +} + +/** + * Load and convert one ready Mailbox delivery-event snapshot for dual-write. + * + * @param sourceMutationAt - Canonical mirror `updatedAt`. D1 delivery events + * lack `updated_at`. Phase-2 high-risk lifecycle mutations must pass the + * same timestamp used for the D1 mutation; inserts use `created_at`. + */ +export async function getMailboxDeliveryEventMirrorInput(input: { + db: D1Database + ownerId: string + eventId: string + sourceMutationAt: string +}): Promise { + const projection = await getEmailDeliveryEventMirrorProjection({ + db: input.db, + ownerId: input.ownerId, + eventId: input.eventId, + }) + if (!projection) return null + return toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: input.sourceMutationAt, + }) +} diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts index bd26855c74..7b6ba6a95f 100644 --- a/packages/worker/src/email/mailbox-snapshots.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -1,7 +1,6 @@ import { expect, test } from 'vitest' import { type EmailAttachmentRecord, - type EmailDeliveryEventRecord, type EmailMessageRecord, type EmailThreadRecord, } from './types.ts' @@ -10,7 +9,7 @@ import { toMailboxDeliveryEventInput, toMailboxMessageInput, toMailboxThreadInput, - type EmailDeliveryEventMirrorSnapshot, + type EmailDeliveryEventMirrorProjection, } from './mailbox-snapshots.ts' test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted columns', () => { @@ -129,8 +128,6 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted createdAt: '2026-07-01T12:00:00.000Z', }) - // detail_json still owns inbound/effect lease fields, but also may contain - // stale copies of promoted columns — converter must prefer D1 columns. const detail = { state: 'received', fingerprint: 'fp-1', @@ -161,7 +158,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted subscriptionEffectLastError: null, recipient: 'owner@example.com', } - const event: EmailDeliveryEventRecord = { + const projection: EmailDeliveryEventMirrorProjection = { id: 'evt-1', messageId: 'msg-1', userId: 'user-1', @@ -172,17 +169,18 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted providerEventId: 'provider-evt-1', detailJson: JSON.stringify(detail), createdAt: '2026-07-02T10:00:00.000Z', - } - const snapshot: EmailDeliveryEventMirrorSnapshot = { - event, - updatedAt: '2026-07-02T10:00:01.000Z', needsEffectReconcile: true, usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', usageMonth: '2026-07', usageBytes: 2048, usageDurationMs: 120, } - expect(toMailboxDeliveryEventInput(snapshot)).toEqual({ + expect( + toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: '2026-07-02T10:00:01.000Z', + }), + ).toEqual({ id: 'evt-1', messageId: 'msg-1', inboxId: 'inbox-1', @@ -190,7 +188,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted provider: 'kody', providerMessageId: null, providerEventId: 'provider-evt-1', - detailJson: event.detailJson, + detailJson: projection.detailJson, needsEffectReconcile: true, state: 'received', fingerprint: 'fp-1', @@ -225,7 +223,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted }) test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { - const baseEvent: EmailDeliveryEventRecord = { + const baseProjection: EmailDeliveryEventMirrorProjection = { id: 'evt-bad', messageId: null, userId: 'user-1', @@ -236,59 +234,56 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or providerEventId: null, detailJson: '{}', createdAt: '2026-07-02T10:00:00.000Z', - } - const baseSnapshot = { - updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, usageEffectRecordedAt: null, usageMonth: null, usageBytes: null, usageDurationMs: null, - } satisfies Omit + } expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { ...baseEvent, detailJson: '{not-json' }, + projection: { ...baseProjection, detailJson: '{not-json' }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson is not valid JSON/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { ...baseEvent, detailJson: '[]' }, + projection: { ...baseProjection, detailJson: '[]' }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson must be a JSON object/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ state: 'not-a-state' }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.state is invalid/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ subscriptionEffectState: 'bogus', }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.subscriptionEffectState is invalid/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ expectedAttachmentCount: 'lots' }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts index f2bd487315..839001d35f 100644 --- a/packages/worker/src/email/mailbox-snapshots.ts +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -1,6 +1,6 @@ import { type EmailAttachmentRecord, - type EmailDeliveryEventRecord, + type EmailDeliveryEventType, type EmailMessageRecord, type EmailThreadRecord, } from './types.ts' @@ -23,38 +23,42 @@ import { * Pure D1 → Mailbox complete-snapshot converters. * * Normalize nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, - * `application/octet-stream`, `kody`). Delivery-event conversion takes an - * additive {@link EmailDeliveryEventMirrorSnapshot}: D1-promoted columns are - * authoritative; `detail_json` supplies only fields that remain JSON-owned. - * Invalid persisted JSON/enums fail clearly rather than fabricating state. + * `application/octet-stream`, `kody`). Delivery-event conversion consumes a + * complete {@link EmailDeliveryEventMirrorProjection} (base + promoted D1 + * columns) plus an explicit `sourceMutationAt`. Invalid persisted JSON/enums + * fail clearly rather than fabricating state. + * + * Load projections via `mailbox-snapshot-repo.ts` — callers must not assemble + * promoted columns field-by-field. */ /** - * Additive delivery-event mirror input. D1 `EmailDeliveryEventRecord` does not - * yet expose promoted columns or `updated_at`; callers pass those explicitly. + * Complete D1 delivery-event mirror projection: base row fields plus + * authoritative promoted columns. `detail_json` still owns inbound/effect + * lease fields that have not been promoted. * - * Authoritative D1 columns (do not read these from `detail_json`): + * Authoritative D1 columns (never read from `detail_json`): * - `needs_effect_reconcile` * - `usage_effect_recorded_at` * - `usage_month` * - `usage_bytes` * - `usage_duration_ms` - * - * Still JSON-owned in `detail_json` (inbound/effect lease fields, etc.). */ -export type EmailDeliveryEventMirrorSnapshot = { - event: EmailDeliveryEventRecord - /** D1 delivery events lack `updated_at`; required for stale rejection. */ - updatedAt: string - /** D1 `needs_effect_reconcile` column. */ +export type EmailDeliveryEventMirrorProjection = { + id: string + messageId: string | null + userId: string | null + inboxId: string | null + eventType: EmailDeliveryEventType + provider: string | null + providerMessageId: string | null + providerEventId: string | null + detailJson: string + createdAt: string needsEffectReconcile: boolean - /** D1 `usage_effect_recorded_at` column. */ usageEffectRecordedAt: string | null - /** D1 `usage_month` column. */ usageMonth: string | null - /** D1 `usage_bytes` column. */ usageBytes: number | null - /** D1 `usage_duration_ms` column. */ usageDurationMs: number | null } @@ -190,73 +194,41 @@ export function toMailboxAttachmentInput( } /** - * Convert a D1 delivery-event mirror snapshot into a full Mailbox input. - * Promoted D1 columns win; remaining inbound/effect fields come from JSON. + * Convert a complete D1 delivery-event projection into a Mailbox input. + * Promoted columns win; remaining inbound/effect fields come from JSON. + * + * `sourceMutationAt` becomes Mailbox `updatedAt`. D1 delivery events lack + * `updated_at`: phase-2 high-risk lifecycle mutations must pass the same + * canonical timestamp used for the D1 mutation; inserts use `created_at`. */ -export function toMailboxDeliveryEventInput( - snapshot: EmailDeliveryEventMirrorSnapshot, -): MailboxDeliveryEventInput { - const { - event, - updatedAt, - needsEffectReconcile, - usageEffectRecordedAt, - usageMonth, - usageBytes, - usageDurationMs, - } = snapshot - if (typeof needsEffectReconcile !== 'boolean') { - throw new Error( - `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(needsEffectReconcile)}.`, - ) - } - if (typeof updatedAt !== 'string' || updatedAt.length === 0) { - throw new Error( - `Mailbox snapshot updatedAt must be a non-empty string; got ${JSON.stringify(updatedAt)}.`, - ) - } - if ( - usageEffectRecordedAt != null && - typeof usageEffectRecordedAt !== 'string' - ) { +export function toMailboxDeliveryEventInput(input: { + projection: EmailDeliveryEventMirrorProjection + sourceMutationAt: string +}): MailboxDeliveryEventInput { + const { projection, sourceMutationAt } = input + if (typeof sourceMutationAt !== 'string' || sourceMutationAt.length === 0) { throw new Error( - `Mailbox snapshot usageEffectRecordedAt must be a string or null; got ${JSON.stringify(usageEffectRecordedAt)}.`, + `Mailbox snapshot sourceMutationAt must be a non-empty string; got ${JSON.stringify(sourceMutationAt)}.`, ) } - if (usageMonth != null && typeof usageMonth !== 'string') { - throw new Error( - `Mailbox snapshot usageMonth must be a string or null; got ${JSON.stringify(usageMonth)}.`, - ) - } - if ( - usageBytes != null && - (typeof usageBytes !== 'number' || !Number.isFinite(usageBytes)) - ) { - throw new Error( - `Mailbox snapshot usageBytes must be a finite number or null; got ${JSON.stringify(usageBytes)}.`, - ) - } - if ( - usageDurationMs != null && - (typeof usageDurationMs !== 'number' || !Number.isFinite(usageDurationMs)) - ) { + if (typeof projection.needsEffectReconcile !== 'boolean') { throw new Error( - `Mailbox snapshot usageDurationMs must be a finite number or null; got ${JSON.stringify(usageDurationMs)}.`, + `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(projection.needsEffectReconcile)}.`, ) } - const detail = parseDetailRecord(event.detailJson) + const detail = parseDetailRecord(projection.detailJson) return { - id: event.id, - messageId: event.messageId, - inboxId: event.inboxId, - eventType: assertMailboxDeliveryEventType(event.eventType), - provider: event.provider ?? 'kody', - providerMessageId: event.providerMessageId, - providerEventId: event.providerEventId, - detailJson: event.detailJson, - needsEffectReconcile, + id: projection.id, + messageId: projection.messageId, + inboxId: projection.inboxId, + eventType: assertMailboxDeliveryEventType(projection.eventType), + provider: projection.provider ?? 'kody', + providerMessageId: projection.providerMessageId, + providerEventId: projection.providerEventId, + detailJson: projection.detailJson, + needsEffectReconcile: projection.needsEffectReconcile, state: requireOptionalEnum( mailboxInboundDeliveryStateValues, detail['state'], @@ -303,7 +275,7 @@ export function toMailboxDeliveryEventInput( 'detail.dedupeExpiresAt', ), // Authoritative D1 columns — ignore any JSON copies of these keys. - usageEffectRecordedAt, + usageEffectRecordedAt: projection.usageEffectRecordedAt, usageEffectSuppressedAt: requireOptionalString( detail['usageEffectSuppressedAt'], 'detail.usageEffectSuppressedAt', @@ -312,9 +284,9 @@ export function toMailboxDeliveryEventInput( detail['usageStartedAt'], 'detail.usageStartedAt', ), - usageMonth, - usageBytes, - usageDurationMs, + usageMonth: projection.usageMonth, + usageBytes: projection.usageBytes, + usageDurationMs: projection.usageDurationMs, usageEffectRetryAt: requireOptionalString( detail['usageEffectRetryAt'], 'detail.usageEffectRetryAt', @@ -356,7 +328,7 @@ export function toMailboxDeliveryEventInput( detail['subscriptionEffectLastError'], 'detail.subscriptionEffectLastError', ), - createdAt: event.createdAt, - updatedAt, + createdAt: projection.createdAt, + updatedAt: sourceMutationAt, } } diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 3d9ea8b629..75b38f9927 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -11,13 +11,10 @@ import { encodeMailboxListCursor, mailboxNowIso, normalizeMailboxPageSize, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, - type MailboxDeleteDeliveryEventResult, - type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -25,19 +22,9 @@ import { type MailboxMessageInput, type MailboxMessageRecord, type MailboxSearchMessagesInput, - type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, - type MailboxTouchThreadInput, - type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' -import { - deleteMailboxDeliveryEvent, - deleteMailboxMessageMetadata, - setMailboxMessageClassification, - touchMailboxThread, - updateMailboxMessageDelivery, -} from './mailbox-mutations.ts' import { type EmailDeliveryEventType, type EmailDeliveryStatus, @@ -454,38 +441,6 @@ export class MailboxStore { return deliveryEventOwnsMessage(this.sql, eventId, messageId) } - touchThread( - input: Omit, - ): MailboxAcceptedResult { - return touchMailboxThread(this.sql, input) - } - - updateMessageDelivery( - input: Omit, - ): MailboxAcceptedResult { - return updateMailboxMessageDelivery(this.sql, input) - } - - setMessageClassification( - input: Omit, - ): MailboxAcceptedResult { - return setMailboxMessageClassification(this.sql, input) - } - - deleteMessageMetadata(input: { - messageId: string - deletedAt: string - }): MailboxDeleteMessageMetadataResult { - return deleteMailboxMessageMetadata(this.sql, input) - } - - deleteDeliveryEvent(input: { - eventId: string - deletedAt: string - }): MailboxDeleteDeliveryEventResult { - return deleteMailboxDeliveryEvent(this.sql, input) - } - getThread(threadId: string): MailboxThreadRecord | null { const row = this.sql .exec>( diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 6178ce59b6..5855ebce86 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -382,8 +382,9 @@ export type MailboxSetMessageClassificationInput = { } /** - * Metadata-only message delete. Never touches R2. Stale when the row's - * `updated_at` is newer than `deletedAt`. Idempotent when already absent. + * Metadata-only message delete. Never touches R2. Nulls matching delivery + * event `message_id` first; does not delete empty threads (use + * `deleteThreadIfEmpty`). Stale when `updated_at` is newer than `deletedAt`. */ export type MailboxDeleteMessageMetadataInput = { ownerId: string @@ -401,35 +402,37 @@ export type MailboxDeleteDeliveryEventInput = { deletedAt: string } -export type MailboxAcceptedResult = { - accepted: boolean +/** + * Deferred empty-thread cleanup (D1 `deleteEmptyEmailThreads` parity). + * Stale-safe by `thread.updated_at`; no-op when messages remain. + */ +export type MailboxDeleteThreadIfEmptyInput = { + ownerId: string + threadId: string + deletedAt: string } /** - * Message metadata delete outcome. - * - `deleted: true` — row removed; includes `orphanThreadDeleted` - * - `deleted: false, stale: false` — missing (idempotent success) - * - `deleted: false, stale: true` — newer row retained - * - * Best-effort callers map missing (`!deleted && !stale`) to success. + * Partial touch/update/classify outcome. + * - `accepted` — mutation applied + * - `missing` — target row absent (idempotent success for best-effort callers) + * - `stale` — newer `updated_at` retained */ -export type MailboxDeleteMessageMetadataResult = - | { deleted: true; stale: false; orphanThreadDeleted: boolean } - | { deleted: false; stale: true } - | { deleted: false; stale: false } +export type MailboxPartialMutationResult = + | { status: 'accepted' } + | { status: 'missing' } + | { status: 'stale' } /** - * Delivery-event delete outcome. - * - `deleted: true` — row removed - * - `deleted: false, stale: false` — missing (idempotent success) - * - `deleted: false, stale: true` — newer row retained - * - * Best-effort callers map missing (`!deleted && !stale`) to success. + * Metadata delete outcome (message, delivery event, or empty thread). + * - `deleted` — row removed + * - `missing` — already absent or nothing to delete (idempotent success) + * - `stale` — newer `updated_at` retained */ -export type MailboxDeleteDeliveryEventResult = - | { deleted: true; stale: false } - | { deleted: false; stale: true } - | { deleted: false; stale: false } +export type MailboxDeleteResult = + | { status: 'deleted' } + | { status: 'missing' } + | { status: 'stale' } export type MailboxRpc = { mirrorMessage: (input: { @@ -462,19 +465,22 @@ export type MailboxRpc = { }> touchThread: ( input: MailboxTouchThreadInput, - ) => Promise + ) => Promise updateMessageDelivery: ( input: MailboxUpdateMessageDeliveryInput, - ) => Promise + ) => Promise setMessageClassification: ( input: MailboxSetMessageClassificationInput, - ) => Promise + ) => Promise deleteMessageMetadata: ( input: MailboxDeleteMessageMetadataInput, - ) => Promise + ) => Promise deleteDeliveryEvent: ( input: MailboxDeleteDeliveryEventInput, - ) => Promise + ) => Promise + deleteThreadIfEmpty: ( + input: MailboxDeleteThreadIfEmptyInput, + ) => Promise getThread: (input: { threadId: string }) => Promise From ca0e6666e72521159b5ad6c4ebf671703af41b32 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:55:22 +0000 Subject: [PATCH 06/20] fix(email): harden Mailbox snapshot validation Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 2 +- .../src/email/mailbox-mirror.node.test.ts | 11 +++ .../worker/src/email/mailbox-mutations.ts | 6 +- .../email/mailbox-parity-events.node.test.ts | 13 ++++ .../worker/src/email/mailbox-snapshot-repo.ts | 4 +- .../src/email/mailbox-snapshots.node.test.ts | 70 ++++++++++++++++++- .../worker/src/email/mailbox-snapshots.ts | 34 +++++++-- 7 files changed, 127 insertions(+), 13 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 402bf79a48..0acb5fb677 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -763,7 +763,7 @@ surface that later live-path wiring will call. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, + PR) adds `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index b7bc8344e0..0bc60502e5 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -349,4 +349,15 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep message: baseMessage(), }), ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) + + expect( + await mirrorMailboxTouchThread({ + env, + ownerId: '', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'skipped', reason: 'missing-owner' }) + expect(touchThread).toHaveBeenCalledTimes(1) }) diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts index 039985d632..69bcd19e79 100644 --- a/packages/worker/src/email/mailbox-mutations.ts +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -4,6 +4,8 @@ import { assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertOptionalMailboxCanonicalIsoTimestamp, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteMessageMetadataInput, type MailboxDeleteResult, type MailboxDeleteThreadIfEmptyInput, type MailboxPartialMutationResult, @@ -141,7 +143,7 @@ export function setMailboxMessageClassification( */ export function deleteMailboxMessageMetadata( sql: SqlStorage, - input: { messageId: string; deletedAt: string }, + input: Omit, ): MailboxDeleteResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const deletedAt = assertMailboxCanonicalIsoTimestamp( @@ -182,7 +184,7 @@ export function deleteMailboxMessageMetadata( */ export function deleteMailboxDeliveryEvent( sql: SqlStorage, - input: { eventId: string; deletedAt: string }, + input: Omit, ): MailboxDeleteResult { const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') const deletedAt = assertMailboxCanonicalIsoTimestamp( diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 2b70d35ecc..93c755631c 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -128,6 +128,19 @@ test('mailbox parity skips system email and never throws when the binding is abs ) expect(writeDataPoint).not.toHaveBeenCalled() + recordMailboxParityEvent( + { + EMAIL_EVENTS: { writeDataPoint } as unknown as AnalyticsEngineDataset, + }, + { + userId: '', + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + }, + ) + expect(writeDataPoint).not.toHaveBeenCalled() + expect(() => recordMailboxParityEvent( {}, diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 30df8d8ea3..5fafe8ee09 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -57,9 +57,9 @@ function mapDeliveryEventMirrorRow( ? null : String(row.usage_effect_recorded_at), usageMonth: row.usage_month == null ? null : String(row.usage_month), - usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes) || 0, + usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes), usageDurationMs: - row.usage_duration_ms == null ? null : Number(row.usage_duration_ms) || 0, + row.usage_duration_ms == null ? null : Number(row.usage_duration_ms), } } diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts index 7b6ba6a95f..b944ff972b 100644 --- a/packages/worker/src/email/mailbox-snapshots.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -223,6 +223,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted }) test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { + const sentinelSecret = 'SENTINEL_SECRET_DO_NOT_LEAK_9f3a2b1c' const baseProjection: EmailDeliveryEventMirrorProjection = { id: 'evt-bad', messageId: null, @@ -243,10 +244,27 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or expect(() => toMailboxDeliveryEventInput({ - projection: { ...baseProjection, detailJson: '{not-json' }, + projection: baseProjection, + sourceMutationAt: '', + }), + ).toThrow(/sourceMutationAt must be a non-empty string/) + + const invalidJsonDetail = `{not-json,"apiKey":"${sentinelSecret}"` + expect(() => + toMailboxDeliveryEventInput({ + projection: { ...baseProjection, detailJson: invalidJsonDetail }, sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson is not valid JSON/) + try { + toMailboxDeliveryEventInput({ + projection: { ...baseProjection, detailJson: invalidJsonDetail }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/payload is \d+ bytes/) + } expect(() => toMailboxDeliveryEventInput({ @@ -254,6 +272,18 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson must be a JSON object/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify([sentinelSecret]), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/array \(1 items\)/) + } expect(() => toMailboxDeliveryEventInput({ @@ -264,6 +294,20 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.state is invalid/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify({ + state: 'not-a-state', + apiKey: sentinelSecret, + }), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + } expect(() => toMailboxDeliveryEventInput({ @@ -286,6 +330,30 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify({ + expectedAttachmentCount: sentinelSecret, + }), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/string \(\d+ chars\)/) + } + + expect(() => + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + usageBytes: Number.NaN, + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }), + ).toThrow(/usageBytes must be a finite number/) expect(() => toMailboxAttachmentInput({ diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts index 839001d35f..20d2c5a6c3 100644 --- a/packages/worker/src/email/mailbox-snapshots.ts +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -62,11 +62,28 @@ export type EmailDeliveryEventMirrorProjection = { usageDurationMs: number | null } +function describeSnapshotValue(value: unknown): string { + if (value === null) return 'null' + if (value === undefined) return 'undefined' + if (typeof value === 'string') { + return `string (${value.length} chars)` + } + if (typeof value === 'number') { + return Number.isFinite(value) ? 'number' : 'number (non-finite)' + } + if (typeof value === 'boolean') return 'boolean' + if (Array.isArray(value)) return `array (${value.length} items)` + if (typeof value === 'object') { + return `object (${Object.keys(value as Record).length} keys)` + } + return typeof value +} + function requireOptionalString(value: unknown, label: string): string | null { if (value == null) return null if (typeof value !== 'string') { throw new Error( - `Mailbox snapshot ${label} must be a string or null; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} must be a string or null; got ${describeSnapshotValue(value)}.`, ) } return value @@ -76,7 +93,7 @@ function requireOptionalNumber(value: unknown, label: string): number | null { if (value == null) return null if (typeof value !== 'number' || !Number.isFinite(value)) { throw new Error( - `Mailbox snapshot ${label} must be a finite number or null; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} must be a finite number or null; got ${describeSnapshotValue(value)}.`, ) } return value @@ -93,7 +110,7 @@ function requireOptionalEnum( !(values as ReadonlyArray).includes(value) ) { throw new Error( - `Mailbox snapshot ${label} is invalid; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} is invalid; got ${describeSnapshotValue(value)}.`, ) } return value as T @@ -105,12 +122,12 @@ function parseDetailRecord(detailJson: string): Record { parsed = JSON.parse(detailJson) as unknown } catch { throw new Error( - `Mailbox snapshot detailJson is not valid JSON; got ${JSON.stringify(detailJson)}.`, + `Mailbox snapshot detailJson is not valid JSON; payload is ${detailJson.length} bytes.`, ) } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { throw new Error( - `Mailbox snapshot detailJson must be a JSON object; got ${JSON.stringify(parsed)}.`, + `Mailbox snapshot detailJson must be a JSON object; got ${describeSnapshotValue(parsed)}.`, ) } return parsed as Record @@ -285,8 +302,11 @@ export function toMailboxDeliveryEventInput(input: { 'detail.usageStartedAt', ), usageMonth: projection.usageMonth, - usageBytes: projection.usageBytes, - usageDurationMs: projection.usageDurationMs, + usageBytes: requireOptionalNumber(projection.usageBytes, 'usageBytes'), + usageDurationMs: requireOptionalNumber( + projection.usageDurationMs, + 'usageDurationMs', + ), usageEffectRetryAt: requireOptionalString( detail['usageEffectRetryAt'], 'detail.usageEffectRetryAt', From ef268fe750d9e40c5bf7a5852d156a62a7e24a61 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 10:26:09 +0000 Subject: [PATCH 07/20] feat(email): dual-write non-inbound Mailbox state Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 146 ++++-- .../app/handlers/account-email.node.test.ts | 52 ++ .../worker/src/app/handlers/account-email.ts | 8 + packages/worker/src/email/delivery-events.ts | 43 +- .../src/email/delivery-events.workers.test.ts | 140 ++++++ .../src/email/delivery-queue.node.test.ts | 156 +++++- packages/worker/src/email/delivery-queue.ts | 19 +- .../email/mailbox-live-mirror.node.test.ts | 445 +++++++++++++++++ .../worker/src/email/mailbox-live-mirror.ts | 207 ++++++++ .../src/email/mailbox-mirror.node.test.ts | 95 +++- packages/worker/src/email/mailbox-mirror.ts | 464 ++++++++++++++---- .../email/mailbox-parity-events.node.test.ts | 44 +- .../worker/src/email/mailbox-parity-events.ts | 8 +- .../email/mailbox-snapshot-repo.node.test.ts | 55 +++ .../worker/src/email/mailbox-snapshot-repo.ts | 79 ++- packages/worker/src/email/outbound.ts | 50 +- .../worker/src/email/outbound.workers.test.ts | 242 +++++++++ .../email/repo-body-limits.workers.test.ts | 65 ++- packages/worker/src/email/repo.ts | 61 ++- .../email/email-message-classify.node.test.ts | 43 ++ .../email/email-message-classify.ts | 7 + 21 files changed, 2216 insertions(+), 213 deletions(-) create mode 100644 packages/worker/src/email/mailbox-live-mirror.node.test.ts create mode 100644 packages/worker/src/email/mailbox-live-mirror.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 1f2eb593bd..492f2bdb56 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -218,10 +218,11 @@ Durable Object export behavior: - `Mailbox` exports per-user email metadata (threads, messages, attachments, delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest - counts use `countMailbox`. Phase 1 registers this consumption; phase 2a adds - dual-write primitives without live callers or a change of D1 authority — D1 - `email_*` rows remain the live source of truth and are still exported in the - `d1` section. See [Mailbox](#durable-objects-mailbox). + counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires + non-inbound live dual-write for outbound terminals, provider-recorded delivery + events, and user classification without changing D1 authority — D1 `email_*` + rows remain the live source of truth and are still exported in the `d1` + section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -376,11 +377,12 @@ When Analytics Engine SQL is unreachable, these two charts zero-fill while the rest of the page renders. Local development uses the existing D1 counters and delivery-event table because Wrangler's emulated dataset has no SQL API. -**Mailbox expand-phase parity events** (additive; no live writers in the -phase-2a PR) reuse the same `EMAIL_EVENTS` dataset with a separate row shape -defined in `packages/worker/src/email/mailbox-parity-events.ts` -(`recordMailboxParityEvent`). These rows are for future dual-write observability -and reconciliation — they do not feed the admin insights charts today: +**Mailbox expand-phase parity events** reuse the same `EMAIL_EVENTS` dataset +with a separate row shape defined in +`packages/worker/src/email/mailbox-parity-events.ts` +(`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome +per attempted operation; scheduled parity compares are not wired yet. These rows +do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -395,10 +397,12 @@ and reconciliation — they do not feed the admin insights charts today: Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, `update_message_delivery`, `set_message_classification`, `delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. -Mirror outcomes: `mirrored`, `stale`, `skipped`, `error`. Parity operations: -`compare_threads`, `compare_messages`, `compare_attachments`, -`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Writes are -best-effort and never throw into D1 authority paths. +Mirror outcomes: `mirrored`, `stale`, `missing`, `timeout`, `skipped`, `error`. +Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, +`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome +writes are automatic from live dual-write helpers; parity compare writers remain +for future reconciliation. Writes are best-effort and never throw into D1 +authority paths. Two D1 reporting projections deliberately remain: @@ -681,9 +685,10 @@ clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. -**Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no -R2 or retention-alarm side effects — unlike full snapshot mirrors, these do not -mark retention dirty or reschedule alarms): +**Partial mutation RPCs** (library-only in the non-inbound phase-2 slice; +owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — +unlike full snapshot mirrors, these do not mark retention dirty or reschedule +alarms): - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward @@ -725,20 +730,51 @@ D1 mutation (inserts use `created_at`). Callers must not stitch promoted columns field-by-field. **Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers -around the DO RPCs for future D1-authoritative dual-write. Each returns a -structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, -`{ status: 'stale' }`, `{ status: 'skipped', reason }` (`system-email` | -`mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. -Failures log with stable tags (for example `mailbox-mirror-message-failed`) and -never propagate into D1 commit paths. Helpers cover full snapshots -(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot` — prefer -loading delivery events with `getMailboxDeliveryEventMirrorInput`) and partial -mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, +around the DO RPCs for D1-authoritative dual-write. Each awaited DO RPC is +bounded by `mailboxMirrorRpcTimeoutMs` (1 second). Each returns a structured +`MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, +`{ status: 'missing' }`, `{ status: 'timeout' }`, +`{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | +`missing-owner`), or `{ status: 'error', error }`. Every outcome is recorded +automatically via `recordMailboxParityEvent` (`mailbox_mirror:`) when +a user id is known; `system:email` is excluded. Failures log with stable tags +(for example `mailbox-mirror-message-failed`) and never propagate into D1 commit +paths. Helpers cover full snapshots (`mirrorMailboxMessageSnapshot`, +`mirrorMailboxDeliveryEventSnapshot` — prefer loading delivery events with +`getMailboxDeliveryEventMirrorInput`) and partial mutations +(`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, -`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). **There -are no live dual-write callers in the phase-2a PR** — only unit/worker tests -invoke these helpers. D1 remains sole authority for all live mail read/write -paths. +`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial +mutation helpers remain library-only; live paths prefer the graph orchestrator +below. D1 remains sole authority for all live mail read/write paths. + +**Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 +message graph (optional caller thread, message, attachments, then delivery +events) and mirrors it best-effort. `mirrorMailboxMessageGraphFromD1` settles +the message snapshot RPC first, then fans out delivery-event snapshot RPCs +concurrently so event-phase wall time stays near one RPC timeout (not limit × +timeout). Event load uses `max+1` rows so `eventsTruncated` is explicit; at most +`mailboxLiveMirrorMaxEvents` (`mailboxDefaultPageSize`, 100) events are mirrored +per call to stay under Analytics Engine's ~250 writes/request budget (1 message +outcome + up to 100 event outcomes). Never throws; returns a bounded summary. +**Live callers (non-inbound phase-2 slice):** + +- **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound + state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full + message/thread/attachment/event graph. Passes a just-created thread when + present; otherwise the orchestrator loads it from D1. Failures never affect + send/refund. +- **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue + ingestion persists a new provider event (`recorded`), schedules + `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not + blocked; full graph repair, not a partial delivery-event touch. +- **User classification** (`account-email.ts`, `email-message-classify.ts`) — + after D1 classification update, mirrors the full graph (not the partial + `setMessageClassification` RPC alone). + +Inbound commit/touch/classify paths, retention deletes, and partial mutation +mirrors are **not** wired in this slice. Parity compare/reconcile cron remains +future work. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -760,17 +796,21 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **The current PR is phase 2a — -prerequisite scaffolding within phase 2:** additive dual-write **primitives** -(email-owner constant, D1→Mailbox snapshot adapters, best-effort mirror helpers -with structured outcomes, owner-bound partial-mutation and stale-safe -metadata-delete RPCs, and the Analytics Engine parity event schema). It does -**not** wire live dual-write callers, does **not** change live email read/write -authority, and leaves D1 `email_*` tables authoritative for all live paths. D1 -email rows and existing R2 inventory deletion stay authoritative during expand. -**Phase 2 is complete only when every user-mail mutation that writes D1 also -writes the DO and parity/reconciliation are wired** — this PR ships the library -surface that later live-path wiring will call. +This is an expand/contract migration. **Phase 2 is partially wired (non-inbound +slice):** live dual-write covers outbound terminal +message/thread/attachment/event graphs, provider-recorded delivery events (queue +`waitUntil` full graph repair), and user classification (full graph repair after +D1 update). Mirror helpers emit automatic namespaced outcome telemetry +(`missing` and `timeout` included). Every awaited mirror RPC is bounded to 1s; +graph event fan-out is concurrent with explicit truncation when D1 has more than +`mailboxLiveMirrorMaxEvents` delivery events. **`system:email` stays in D1 +only.** Inbound lifecycle dual-write, retention/metadata deletes, and scheduled +parity reconcile are **not** wired in this slice. Live email read/write +authority is unchanged — D1 `email_*` tables remain authoritative for all live +paths. D1 email rows and existing R2 inventory deletion stay authoritative +during expand. **Phase 2 completes only when every user-mail mutation that +writes D1 also writes the DO and parity/reconciliation are wired** — inbound +paths and parity reconcile remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -779,15 +819,18 @@ surface that later live-path wiring will call. authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO - completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, - `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + - matching DO RPCs (`touchThread`, `updateMessageDelivery`, - `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, - `deleteThreadIfEmpty`), and `mailbox-parity-events.ts`. Those helpers and - parity-event writers are library-only today — **no production path invokes - them yet**; live-path dual-write and parity wiring remain for the rest of - phase 2. + completeness against D1. D1 stays authoritative for reads. Library surface: + `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, + `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-live-mirror.ts`, + `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, + `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, + `deleteDeliveryEvent`, `deleteThreadIfEmpty`), and + `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, + provider `recorded` queue events (`waitUntil`), and user classification call + `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` + outcomes automatically. **Still pending for phase-2 completion:** inbound + commit/touch/classify dual-write, retention and metadata-delete mirrors, and + scheduled parity compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata @@ -835,7 +878,8 @@ metadata store: message (logged, non-retry) rather than risking a duplicate on Email Routing retry. -Phase 1 and phase 2a scaffolding does not change that live D1 boundary. +Phase 1 scaffolding and the non-inbound phase-2 live slice do not change that +live D1 inbound boundary; inbound dual-write remains pending phase-2 completion. ### Package state model @@ -956,7 +1000,7 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption; phase 2a adds dual-write primitives without changing D1 + consumption; phase 2 wires non-inbound live dual-write without changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index fa111e18b0..e206514d67 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -155,6 +155,11 @@ const mockModule = vi.hoisted(() => ({ }, ]), setEmailMessageClassification: vi.fn(async () => true), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [], + })), prepare: vi.fn(), })) @@ -226,6 +231,11 @@ vi.mock('#worker/email/repo.ts', () => ({ mockModule.setEmailMessageClassification(...args), })) +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: (...args: Array) => + mockModule.mirrorMailboxMessageGraphFromD1(...args), +})) + const { createAccountEmailApiHandler } = await import('./account-email.ts') function createCountResult(total: number) { @@ -558,6 +568,7 @@ test('email API lists classification filters and classifies inbound messages', a }) mockModule.setEmailMessageClassification.mockClear() + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() // Reload after classify uses the default message list again. createEnv({ meter, messageRows: [messageRow] }) const quarantineResponse = await handler.handler({ @@ -579,12 +590,19 @@ test('email API lists classification filters and classifies inbound messages', a classification: 'quarantined', classificationReason: 'Reclassified by user.', }) + expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'stable-user-1', + messageId: 'msg-1', + }) await expect(quarantineResponse.json()).resolves.toMatchObject({ ok: true, selectedMessage: expect.objectContaining({ id: 'msg-1' }), }) mockModule.setEmailMessageClassification.mockClear() + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const acceptResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -604,8 +622,15 @@ test('email API lists classification filters and classifies inbound messages', a classification: 'accepted', classificationReason: null, }) + expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'stable-user-1', + messageId: 'msg-1', + }) mockModule.setEmailMessageClassification.mockResolvedValueOnce(false) + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const missingResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -618,6 +643,33 @@ test('email API lists classification filters and classifies inbound messages', a }), }) expect(missingResponse.status).toBe(404) + expect(mockModule.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + mockModule.setEmailMessageClassification.mockResolvedValueOnce(true) + mockModule.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + const mirrorFailureResponse = await handler.handler({ + request: new Request('https://example.com/account/email.json', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + action: 'classify', + message_id: 'msg-1', + classification: 'quarantined', + }), + }), + }) + expect(mirrorFailureResponse.status).toBe(200) + await expect(mirrorFailureResponse.json()).resolves.toMatchObject({ + ok: true, + selectedMessage: expect.objectContaining({ id: 'msg-1' }), + }) const invalidActionResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { diff --git a/packages/worker/src/app/handlers/account-email.ts b/packages/worker/src/app/handlers/account-email.ts index aa34b19234..62b8148201 100644 --- a/packages/worker/src/app/handlers/account-email.ts +++ b/packages/worker/src/app/handlers/account-email.ts @@ -7,6 +7,7 @@ import { requireAuthenticatedPageUser } from '#app/page-auth.ts' import { readTrimmedStringOrEmpty } from '#app/request-body.ts' import { type routes } from '#app/routes.ts' import { renderAppPage } from '#app/ssr-render.tsx' +import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' import { setEmailMessageClassification } from '#worker/email/repo.ts' import { emailClassificationValues, @@ -129,6 +130,13 @@ export function createAccountEmailApiHandler(env: Env) { ) } + await mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: user.mcpUser.userId, + messageId, + }) + const listUrl = new URL(request.url, 'http://localhost') listUrl.searchParams.set('selected', messageId) return jsonResponse( diff --git a/packages/worker/src/email/delivery-events.ts b/packages/worker/src/email/delivery-events.ts index 4f92ee6742..19aa3f8a10 100644 --- a/packages/worker/src/email/delivery-events.ts +++ b/packages/worker/src/email/delivery-events.ts @@ -94,30 +94,35 @@ export async function processCloudflareEmailDeliveryEvent(input: { reportingEnv?: EmailReportingEnv body: unknown }) { - const event = parseCloudflareEmailDeliveryEvent(input.body) - if (!event) { - return { outcome: 'invalid' as const, event: null, message: null } + const providerEvent = parseCloudflareEmailDeliveryEvent(input.body) + if (!providerEvent) { + return { + outcome: 'invalid' as const, + providerEvent: null, + event: null, + message: null, + } } const result = await recordProviderEmailDeliveryEvent({ db: input.db, reportingEnv: input.reportingEnv, - providerMessageId: event.payload.messageId, - providerEventId: event.payload.eventId, - deliveryStatus: event.payload.delivery.status, - eventTimestamp: event.metadata.eventTimestamp, + providerMessageId: providerEvent.payload.messageId, + providerEventId: providerEvent.payload.eventId, + deliveryStatus: providerEvent.payload.delivery.status, + eventTimestamp: providerEvent.metadata.eventTimestamp, detail: { - source: event.source, - sender: event.payload.sender, - recipient: event.payload.recipient, - subject: event.payload.subject ?? null, - terminal: event.payload.terminal, - delivery: event.payload.delivery, - bounce: event.payload.bounce ?? null, - failure: event.payload.failure ?? null, - rejection: event.payload.rejection ?? null, - complaint: event.payload.complaint ?? null, - metadata: event.metadata, + source: providerEvent.source, + sender: providerEvent.payload.sender, + recipient: providerEvent.payload.recipient, + subject: providerEvent.payload.subject ?? null, + terminal: providerEvent.payload.terminal, + delivery: providerEvent.payload.delivery, + bounce: providerEvent.payload.bounce ?? null, + failure: providerEvent.payload.failure ?? null, + rejection: providerEvent.payload.rejection ?? null, + complaint: providerEvent.payload.complaint ?? null, + metadata: providerEvent.metadata, }, }) - return { ...result, event } + return { ...result, providerEvent } } diff --git a/packages/worker/src/email/delivery-events.workers.test.ts b/packages/worker/src/email/delivery-events.workers.test.ts index 2e0957705f..e218031520 100644 --- a/packages/worker/src/email/delivery-events.workers.test.ts +++ b/packages/worker/src/email/delivery-events.workers.test.ts @@ -1,11 +1,13 @@ import { env } from 'cloudflare:workers' import { expect, test } from 'vitest' import { processCloudflareEmailDeliveryEvent } from './delivery-events.ts' +import { mirrorMailboxMessageGraphFromD1 } from './mailbox-live-mirror.ts' import { getEmailMessageById, insertEmailMessage, listEmailDeliveryEvents, } from './repo.ts' +import { mailboxEnv, rpcFor } from './mailbox-test-helpers.ts' import { ensureEmailTestSchema } from './test-schema.ts' function createDeliveryEvent(input: { @@ -90,6 +92,12 @@ test('provider delivery events are idempotent, ordered, and user scoped', async body: delivered, }) expect(recorded.outcome).toBe('recorded') + expect(recorded.event).toMatchObject({ + messageId: message.id, + userId, + eventType: 'delivered', + providerEventId: 'event-delivered', + }) expect(recorded.message).toMatchObject({ id: message.id, userId, @@ -209,3 +217,135 @@ test('provider delivery events are idempotent, ordered, and user scoped', async ).outcome, ).toBe('invalid') }) + +test('recorded provider delivery events mirror the D1 message graph to Mailbox when configured', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `delivery-mailbox-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Mailbox mirror', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-07-17T20:00:00.000Z', + }, + }) + const mailbox = rpcFor(userId) + // Warm the per-user DO before graph mirror (RPC timeout is ~1s). + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const recorded = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: createDeliveryEvent({ + eventId: 'event-mailbox-delivered', + messageId: providerMessageId, + status: 'delivered', + eventTimestamp: '2026-07-17T20:02:00.000Z', + }), + }) + expect(recorded.outcome).toBe('recorded') + if (!recorded.message) { + throw new Error('Expected recorded provider delivery message.') + } + + const summary = await mirrorMailboxMessageGraphFromD1({ + env: { ...env, ...mailboxEnv() }, + db: env.APP_DB, + userId, + messageId: message.id, + }) + expect(summary).toMatchObject({ + messageId: message.id, + message: { status: 'mirrored' }, + eventsTruncated: false, + }) + expect(summary.events).toEqual([ + expect.objectContaining({ + eventId: recorded.event!.id, + result: { status: 'mirrored' }, + }), + ]) + + const events = await mailbox.listDeliveryEvents({ + messageId: message.id, + limit: 10, + }) + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ + eventType: 'delivered', + provider: 'cloudflare-email', + providerEventId: 'event-mailbox-delivered', + providerMessageId, + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-17T20:02:00.000Z', + }) + expect( + await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: message.id, + }), + ).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-17T20:02:00.000Z', + }) +}, 30_000) + +test('recorded provider delivery graph mirror skips cleanly when MAILBOX is absent', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `delivery-no-mailbox-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'No mailbox binding', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-07-17T20:00:00.000Z', + }, + }) + + const recorded = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: createDeliveryEvent({ + eventId: 'event-no-mailbox', + messageId: providerMessageId, + status: 'delivered', + eventTimestamp: '2026-07-17T20:02:00.000Z', + }), + }) + expect(recorded.outcome).toBe('recorded') + if (!recorded.message) { + throw new Error('Expected recorded provider delivery message.') + } + + expect( + await mirrorMailboxMessageGraphFromD1({ + env: { APP_DB: env.APP_DB }, + db: env.APP_DB, + userId, + messageId: message.id, + }), + ).toEqual({ + messageId: message.id, + message: { status: 'skipped', reason: 'mailbox-unconfigured' }, + events: [ + { + eventId: recorded.event!.id, + result: { status: 'skipped', reason: 'mailbox-unconfigured' }, + }, + ], + eventsTruncated: false, + }) +}) diff --git a/packages/worker/src/email/delivery-queue.node.test.ts b/packages/worker/src/email/delivery-queue.node.test.ts index 6c4e64d044..8030c12213 100644 --- a/packages/worker/src/email/delivery-queue.node.test.ts +++ b/packages/worker/src/email/delivery-queue.node.test.ts @@ -7,6 +7,11 @@ import { const mocks = vi.hoisted(() => ({ processCloudflareEmailDeliveryEvent: vi.fn(), dispatchEmailDeliverySubscriptionEvents: vi.fn(async () => []), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + })), })) vi.mock('./delivery-events.ts', () => ({ @@ -14,6 +19,10 @@ vi.mock('./delivery-events.ts', () => ({ mocks.processCloudflareEmailDeliveryEvent, })) +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + vi.mock('./package-subscriptions.ts', () => ({ dispatchEmailDeliverySubscriptionEvents: mocks.dispatchEmailDeliverySubscriptionEvents, @@ -53,35 +62,50 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched }, } const storedMessage = { id: 'message-1', userId: 'user-1' } + const d1Event = { + id: 'd1-event-1', + messageId: 'message-1', + userId: 'user-1', + inboxId: 'inbox-1', + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'provider-1', + providerEventId: 'event-1', + detailJson: '{}', + createdAt: '2026-07-17T20:02:00.000Z', + } mocks.processCloudflareEmailDeliveryEvent .mockResolvedValueOnce({ outcome: 'recorded', - event: providerEvent, + providerEvent, + event: d1Event, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'duplicate', - event: providerEvent, + providerEvent, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'invalid', + providerEvent: null, event: null, message: null, }) .mockResolvedValueOnce({ outcome: 'stale', - event: providerEvent, + providerEvent, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'unmatched', - event: providerEvent, + providerEvent, message: null, }) .mockResolvedValueOnce({ outcome: 'recorded', - event: providerEvent, + providerEvent, + event: d1Event, message: storedMessage, }) mocks.dispatchEmailDeliverySubscriptionEvents @@ -135,6 +159,14 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched waitUntil: expect.any(Function), }, ) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(waitUntilPromises).toHaveLength(1) expect(consoleWarn).toHaveBeenCalledWith('email-delivery-event-unmatched', { queueMessageId: 'queue-unmatched', providerMessageId: 'provider-1', @@ -144,3 +176,117 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched expect.objectContaining({ message: expect.stringContaining('KV failure') }), ) }) + +test('recorded delivery events schedule mailbox graph mirror via waitUntil without blocking ack', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockResolvedValue([]) + + const providerEvent = { + payload: { + eventId: 'event-mirror', + messageId: 'provider-mirror', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-mirror', userId: 'user-mirror' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-mirror' }, + message: storedMessage, + }) + + let resolveMirror: (() => void) | undefined + const mirrorPromise = new Promise<{ messageId: string }>((resolve) => { + resolveMirror = () => + resolve({ + messageId: 'message-mirror', + message: { status: 'mirrored' }, + events: [], + }) + }) + mocks.mirrorMailboxMessageGraphFromD1.mockReturnValue(mirrorPromise) + + const queueMessage = createQueueMessage('queue-mirror', { kind: 'recorded' }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + expect(waitUntilPromises).toHaveLength(1) + resolveMirror?.() + await Promise.all(waitUntilPromises) +}) + +test('mailbox graph mirror failures scheduled in waitUntil do not reject or alter queue ack', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockResolvedValue([]) + + const providerEvent = { + payload: { + eventId: 'event-fail', + messageId: 'provider-fail', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-fail', userId: 'user-fail' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-fail' }, + message: storedMessage, + }) + mocks.mirrorMailboxMessageGraphFromD1.mockRejectedValue( + new Error('mailbox rpc unavailable'), + ) + + const queueMessage = createQueueMessage('queue-mirror-fail', { + kind: 'recorded', + }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + await expect(Promise.all(waitUntilPromises)).rejects.toThrow( + 'mailbox rpc unavailable', + ) +}) diff --git a/packages/worker/src/email/delivery-queue.ts b/packages/worker/src/email/delivery-queue.ts index f79ceea6b6..bb843ae987 100644 --- a/packages/worker/src/email/delivery-queue.ts +++ b/packages/worker/src/email/delivery-queue.ts @@ -1,4 +1,5 @@ import { processCloudflareEmailDeliveryEvent } from './delivery-events.ts' +import { mirrorMailboxMessageGraphFromD1 } from './mailbox-live-mirror.ts' import { applyOutboundEmailAbusePause } from './outbound-abuse.ts' import { dispatchEmailDeliverySubscriptionEvents } from './package-subscriptions.ts' import { type EmailReportingEnv } from './reporting-events.ts' @@ -26,7 +27,7 @@ export async function handleEmailDeliveryQueue( case 'unmatched': console.warn('email-delivery-event-unmatched', { queueMessageId: queueMessage.id, - providerMessageId: result.event?.payload.messageId ?? null, + providerMessageId: result.providerEvent?.payload.messageId ?? null, }) queueMessage.retry({ delaySeconds: unmatchedRetryDelaySeconds }) break @@ -39,7 +40,7 @@ export async function handleEmailDeliveryQueue( await applyOutboundEmailAbusePause({ env, userId: result.message.userId, - deliveryStatus: result.event.payload.delivery.status, + deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: false, }) queueMessage.ack() @@ -53,15 +54,25 @@ export async function handleEmailDeliveryQueue( await applyOutboundEmailAbusePause({ env, userId: result.message.userId, - deliveryStatus: result.event.payload.delivery.status, + deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: result.outcome === 'recorded', }) await dispatchEmailDeliverySubscriptionEvents({ env, message: result.message, - providerEvent: result.event, + providerEvent: result.providerEvent, waitUntil, }) + if (result.outcome === 'recorded' && result.message != null) { + waitUntil( + mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: result.message.userId, + messageId: result.message.id, + }), + ) + } queueMessage.ack() break } diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts new file mode 100644 index 0000000000..af8ae5ba64 --- /dev/null +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -0,0 +1,445 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { + mailboxLiveMirrorMaxAnalyticsWrites, + mailboxLiveMirrorMaxEvents, + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, +} from './mailbox-live-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' +import { type EmailThreadRecord } from './types.ts' + +async function createEmailDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +function fakeMailboxEnv(stub: Record) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => stub) + const writeDataPoint = vi.fn() + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + }, + writeDataPoint, + } +} + +async function seedMessage(db: D1Database) { + const at = '2026-07-01T12:00:00.000Z' + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, inbox_id, subject_normalized, root_message_id_header, + last_message_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'thread-1', + 'user-aaa', + 'inbox-1', + 'hello', + '', + at, + at, + at, + ) + .run() + await db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, inbox_id, thread_id, from_address, + to_addresses_json, subject, raw_mime_key, raw_size, + processing_status, classification, received_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'msg-1', + 'outbound', + 'user-aaa', + 'inbox-1', + 'thread-1', + 'sender@example.com', + '["owner@example.com"]', + 'Hello', + null, + 0, + 'sent', + 'accepted', + null, + at, + at, + ) + .run() + await db + .prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .bind('att-1', 'msg-1', 'note.txt', 'text/plain', 4, 'unavailable', at) + .run() +} + +async function seedDeliveryEvent( + db: D1Database, + input: { id: string; eventType: string; createdAt: string }, +) { + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + input.id, + 'msg-1', + 'user-aaa', + 'inbox-1', + input.eventType, + 'kody', + null, + null, + '{}', + 0, + input.createdAt, + ) + .run() +} + +async function seedMessageGraph(db: D1Database) { + await seedMessage(db) + await seedDeliveryEvent(db, { + id: 'evt-1', + eventType: 'send_requested', + createdAt: '2026-07-01T12:00:01.000Z', + }) + await seedDeliveryEvent(db, { + id: 'evt-2', + eventType: 'sent', + createdAt: '2026-07-01T12:00:02.000Z', + }) +} + +test('live-mirror AE write budget stays under Analytics Engine request cap', () => { + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe( + 1 + mailboxLiveMirrorMaxEvents, + ) +}) + +test('live-mirror graph mirrors message before events and uses createdAt for event updatedAt', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + let messageSettled = false + const mirrorMessage = vi.fn(async () => { + messageSettled = true + return { ok: true as const, accepted: true } + }) + const upsertDeliveryEvent = vi.fn( + async (input: { event: { id: string } }) => { + expect(messageSettled).toBe(true) + return { + inserted: true, + accepted: true, + updatedLatestStatus: false, + eventId: input.event.id, + } + }, + ) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + + const summary = await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + sourceMutationAt: '2026-07-01T12:00:03.000Z', + thread, + }) + + expect(summary).toEqual({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [ + { eventId: 'evt-1', result: { status: 'mirrored' } }, + { eventId: 'evt-2', result: { status: 'mirrored' } }, + ], + eventsTruncated: false, + }) + expect(mirrorMessage).toHaveBeenCalledWith( + expect.objectContaining({ + ownerId: 'user-aaa', + thread: expect.objectContaining({ id: 'thread-1' }), + message: expect.objectContaining({ id: 'msg-1' }), + attachments: [ + expect.objectContaining({ id: 'att-1', filename: 'note.txt' }), + ], + }), + ) + + // When thread is omitted, load it from D1 via getEmailThreadById. + mirrorMessage.mockClear() + upsertDeliveryEvent.mockClear() + await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + expect(mirrorMessage).toHaveBeenCalledWith( + expect.objectContaining({ + thread: expect.objectContaining({ + id: 'thread-1', + subjectNormalized: 'hello', + }), + }), + ) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-1', + createdAt: '2026-07-01T12:00:01.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + }), + ) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-2', + createdAt: '2026-07-01T12:00:02.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + }), + ) +}) + +test('live-mirror graph fans out event RPCs concurrently after message settles', async () => { + vi.useFakeTimers() + try { + const { db } = await createEmailDb() + await seedMessage(db) + const eventCount = 5 + for (let index = 0; index < eventCount; index += 1) { + await seedDeliveryEvent(db, { + id: `evt-${index + 1}`, + eventType: index === 0 ? 'send_requested' : 'sent', + createdAt: `2026-07-01T12:00:0${index + 1}.000Z`, + }) + } + + const eventDelayMs = 400 + let maxInFlight = 0 + let inFlight = 0 + let messageSettled = false + const mirrorMessage = vi.fn(async () => { + messageSettled = true + return { ok: true as const, accepted: true } + }) + const upsertDeliveryEvent = vi.fn(async () => { + expect(messageSettled).toBe(true) + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + await new Promise((resolve) => { + setTimeout(resolve, eventDelayMs) + }) + inFlight -= 1 + return { inserted: true, accepted: true, updatedLatestStatus: false } + }) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + + const pending = mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + + // One event-delay tick completes the whole fan-out (not eventCount × delay). + await vi.advanceTimersByTimeAsync(eventDelayMs) + const summary = await pending + + expect(summary.events).toHaveLength(eventCount) + expect(summary.eventsTruncated).toBe(false) + expect(maxInFlight).toBe(eventCount) + expect(upsertDeliveryEvent).toHaveBeenCalledTimes(eventCount) + // Wall budget stays near one RPC timeout for the event phase. + expect(eventDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) + } finally { + vi.useRealTimers() + } +}) + +test('live-mirror graph sets eventsTruncated and mirrors at most max events', async () => { + const { db } = await createEmailDb() + await seedMessage(db) + + const totalEvents = mailboxLiveMirrorMaxEvents + 1 + const baseMs = Date.parse('2026-07-01T12:00:00.000Z') + for (let index = 0; index < totalEvents; index += 1) { + const id = `evt-${String(index).padStart(3, '0')}` + await seedDeliveryEvent(db, { + id, + eventType: index === 0 ? 'send_requested' : 'sent', + createdAt: new Date(baseMs + index * 1000).toISOString(), + }) + } + + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env, writeDataPoint } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvent, + }) + + const summary = await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + + expect(summary.eventsTruncated).toBe(true) + expect(summary.events).toHaveLength(mailboxLiveMirrorMaxEvents) + expect(summary.events[0]?.eventId).toBe('evt-000') + expect(summary.events.at(-1)?.eventId).toBe( + `evt-${String(mailboxLiveMirrorMaxEvents - 1).padStart(3, '0')}`, + ) + expect(upsertDeliveryEvent).toHaveBeenCalledTimes(mailboxLiveMirrorMaxEvents) + // 1 message + max events AE writes; never the overflow row. + expect(writeDataPoint).toHaveBeenCalledTimes( + mailboxLiveMirrorMaxAnalyticsWrites, + ) + expect(writeDataPoint.mock.calls.length).toBeLessThan(250) +}) + +test('live-mirror graph returns missing when D1 message is absent', async () => { + const { db } = await createEmailDb() + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env } = fakeMailboxEnv({ mirrorMessage }) + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'missing-msg', + }), + ).resolves.toEqual({ + messageId: 'missing-msg', + message: { status: 'missing' }, + events: [], + eventsTruncated: false, + }) + expect(mirrorMessage).not.toHaveBeenCalled() +}) + +test('live-mirror graph catches unexpected errors without throwing', async () => { + consoleWarn.mockImplementation(() => {}) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env } = fakeMailboxEnv({ mirrorMessage }) + const db = { + prepare() { + throw new Error('d1 exploded') + }, + } as unknown as D1Database + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }), + ).resolves.toEqual({ + messageId: 'msg-1', + message: { + status: 'error', + error: expect.objectContaining({ message: 'd1 exploded' }), + }, + events: [], + eventsTruncated: false, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-live-mirror-message-graph-failed', + expect.objectContaining({ message: 'd1 exploded' }), + ) +}) + +test('live-mirror one-event helper loads cohesive projection and returns missing', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const { env } = fakeMailboxEnv({ upsertDeliveryEvent }) + + expect( + await mirrorMailboxDeliveryEventFromD1({ + env, + db, + userId: 'user-aaa', + eventId: 'evt-2', + }), + ).toEqual({ status: 'mirrored' }) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-2', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + }), + ) + + expect( + await mirrorMailboxDeliveryEventFromD1({ + env, + db, + userId: 'user-aaa', + eventId: 'missing-evt', + }), + ).toEqual({ status: 'missing' }) +}) diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts new file mode 100644 index 0000000000..062f915b8b --- /dev/null +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -0,0 +1,207 @@ +import { + mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxMessageSnapshot, + type MailboxMirrorEnv, + type MailboxMirrorResult, +} from './mailbox-mirror.ts' +import { + getEmailDeliveryEventMirrorProjection, + listMailboxDeliveryEventMirrorInputsForMessage, +} from './mailbox-snapshot-repo.ts' +import { toMailboxDeliveryEventInput } from './mailbox-snapshots.ts' +import { mailboxDefaultPageSize } from './mailbox-types.ts' +import { + getEmailMessageById, + getEmailThreadById, + listEmailAttachmentsForMessage, +} from './repo.ts' +import { type EmailThreadRecord } from './types.ts' + +/** + * High-level D1 → Mailbox live-mirror orchestration. + * + * D1 remains authoritative. These helpers load cohesive snapshots from D1 and + * call best-effort mirror RPCs; they never throw into caller paths and do not + * perform deletes or inbound-lifecycle special cases. + * + * Callers that just created a thread may pass it via `thread` to avoid a + * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. + * + * Graph event fan-out runs concurrently after the message RPC settles so event + * wall time stays near one RPC timeout (not limit × timeout). Analytics Engine + * parity writes stay under the ~250/request cap: 1 message + max events. + */ + +/** + * Max delivery events mirrored per graph call. + * `1 + mailboxLiveMirrorMaxEvents` AE writes must stay under ~250/request. + */ +export const mailboxLiveMirrorMaxEvents = mailboxDefaultPageSize + +/** AE write budget headroom check (message + event outcomes). */ +export const mailboxLiveMirrorMaxAnalyticsWrites = + 1 + mailboxLiveMirrorMaxEvents + +export type MailboxLiveMirrorEnv = MailboxMirrorEnv + +export type MailboxLiveMirrorEventResult = { + eventId: string + result: MailboxMirrorResult +} + +/** + * Bounded summary for one message graph mirror attempt. + * Always returned; never thrown. + */ +export type MailboxLiveMirrorGraphSummary = { + messageId: string + message: MailboxMirrorResult + events: Array + /** True when D1 had more than {@link mailboxLiveMirrorMaxEvents} events. */ + eventsTruncated: boolean +} + +function emptyGraphSummary( + messageId: string, + message: MailboxMirrorResult, +): MailboxLiveMirrorGraphSummary { + return { + messageId, + message, + events: [], + eventsTruncated: false, + } +} + +/** + * Load one ready delivery-event projection from D1 and mirror it. + * + * When `sourceMutationAt` is omitted, uses the event's `created_at` (immutable + * outbound/provider inserts). Returns `{ status: 'missing' }` when the D1 row + * is absent; never throws. + */ +export async function mirrorMailboxDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string + sourceMutationAt?: string +}): Promise { + try { + const projection = await getEmailDeliveryEventMirrorProjection({ + db: input.db, + ownerId: input.userId, + eventId: input.eventId, + }) + if (!projection) return { status: 'missing' } + const sourceMutationAt = + input.sourceMutationAt != null && input.sourceMutationAt.length > 0 + ? input.sourceMutationAt + : projection.createdAt + return await mirrorMailboxDeliveryEventSnapshot({ + env: input.env, + ownerId: input.userId, + event: toMailboxDeliveryEventInput({ + projection, + sourceMutationAt, + }), + }) + } catch (error) { + console.warn('mailbox-live-mirror-delivery-event-failed', error) + return { status: 'error', error } + } +} + +/** + * Load the authoritative D1 message graph and best-effort mirror it: + * message (+ optional caller-supplied thread, attachments) first, then all + * delivery events concurrently (cohesive projection; `created_at` as + * `sourceMutationAt`). + * + * Never throws. Returns a bounded structured summary. Queries `max+1` events + * so truncation is explicit; mirrors at most {@link mailboxLiveMirrorMaxEvents}. + * + * @param sourceMutationAt - Optional caller mutation timestamp for context; + * D1 row timestamps remain authoritative. Delivery events always use + * `created_at` for immutable outbound/provider inserts. + * @param thread - Optional thread from the caller. When omitted, loads via + * `getEmailThreadById` when `message.threadId` is set. Explicit `null` + * skips the thread snapshot. + */ +export async function mirrorMailboxMessageGraphFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + messageId: string + sourceMutationAt?: string + thread?: EmailThreadRecord | null +}): Promise { + // Caller mutation timestamp is accepted for API symmetry with dual-write + // sites; D1 row timestamps remain authoritative for the loaded graph. + void input.sourceMutationAt + try { + const message = await getEmailMessageById({ + db: input.db, + userId: input.userId, + messageId: input.messageId, + }) + if (!message) { + return emptyGraphSummary(input.messageId, { status: 'missing' }) + } + + const attachments = await listEmailAttachmentsForMessage({ + db: input.db, + messageId: input.messageId, + }) + + let thread = input.thread + if (thread === undefined && message.threadId) { + thread = await getEmailThreadById({ + db: input.db, + userId: input.userId, + threadId: message.threadId, + }) + } + + // Message settles before any event RPC starts (ordering guarantee). + const messageResult = await mirrorMailboxMessageSnapshot({ + env: input.env, + thread, + message, + attachments, + }) + + const loadedEvents = await listMailboxDeliveryEventMirrorInputsForMessage({ + db: input.db, + ownerId: input.userId, + messageId: input.messageId, + limit: mailboxLiveMirrorMaxEvents + 1, + }) + const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents + const eventInputs = eventsTruncated + ? loadedEvents.slice(0, mailboxLiveMirrorMaxEvents) + : loadedEvents + + // Concurrent fan-out: event-phase wall time ≈ one RPC timeout, not N×. + const events = await Promise.all( + eventInputs.map(async (event) => ({ + eventId: event.id, + result: await mirrorMailboxDeliveryEventSnapshot({ + env: input.env, + ownerId: input.userId, + event, + }), + })), + ) + + return { + messageId: input.messageId, + message: messageResult, + events, + eventsTruncated, + } + } catch (error) { + console.warn('mailbox-live-mirror-message-graph-failed', error) + return emptyGraphSummary(input.messageId, { status: 'error', error }) + } +} diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 0bc60502e5..0ede393f94 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -2,6 +2,7 @@ import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' import { + mailboxMirrorRpcTimeoutMs, mirrorMailboxDeliveryEventSnapshot, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, @@ -17,15 +18,20 @@ import { type MailboxDeliveryEventInput } from './mailbox-types.ts' function fakeMailboxEnv(stub: Record) { const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) const get = vi.fn(() => stub) + const writeDataPoint = vi.fn() return { env: { MAILBOX: { idFromName, get, } as unknown as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, }, idFromName, get, + writeDataPoint, } } @@ -107,7 +113,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep status: 'missing' as const, })) - const { env, idFromName } = fakeMailboxEnv({ + const { env, idFromName, writeDataPoint } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent, touchThread, @@ -266,7 +272,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) - // missing partial mutation is idempotent success for best-effort dual-write + // missing partial mutation is a distinct mirror outcome (not mirrored) expect( await mirrorMailboxSetMessageClassification({ env, @@ -276,7 +282,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep classificationReason: 'manual', updatedAt: '2026-07-01T14:00:00.000Z', }), - ).toEqual({ status: 'mirrored' }) + ).toEqual({ status: 'missing' }) expect( await mirrorMailboxDeleteMessageMetadata({ @@ -296,6 +302,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) + // delete missing remains mirrored — desired absence is achieved expect( await mirrorMailboxDeleteMessageMetadata({ env, @@ -345,7 +352,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep expect( await mirrorMailboxMessageSnapshot({ - env: {}, + env: { EMAIL_EVENTS: env.EMAIL_EVENTS }, message: baseMessage(), }), ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) @@ -360,4 +367,84 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'skipped', reason: 'missing-owner' }) expect(touchThread).toHaveBeenCalledTimes(1) + + const outcomes = writeDataPoint.mock.calls.map( + (call) => (call[0] as { blobs: Array }).blobs, + ) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:mirror_message', + 'mirrored', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:set_message_classification', + 'missing', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:update_message_delivery', + 'stale', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:delete_delivery_event', + 'error', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:mirror_message', + 'skipped', + expect.any(String), + ]) + // system email must not emit a parity data point + expect( + outcomes.filter( + (blobs) => + blobs[0] === 'mailbox_mirror:mirror_message' && blobs[1] === 'skipped', + ), + ).toHaveLength(1) +}) + +test('mailbox mirror RPC timeout returns timeout and late rejection is not unhandled', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + + let rejectRpc!: (error: unknown) => void + const mirrorMessage = vi.fn( + () => + new Promise<{ ok: true; accepted: boolean }>((_resolve, reject) => { + rejectRpc = reject + }), + ) + const { env, writeDataPoint } = fakeMailboxEnv({ mirrorMessage }) + + const pending = mirrorMailboxMessageSnapshot({ + env, + message: baseMessage(), + }) + const expectation = expect(pending).resolves.toEqual({ status: 'timeout' }) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await expectation + + expect(writeDataPoint).toHaveBeenCalledWith( + expect.objectContaining({ + indexes: ['user-aaa'], + blobs: ['mailbox_mirror:mirror_message', 'timeout', expect.any(String)], + }), + ) + + const unhandled: Array = [] + const onUnhandled = (reason: unknown) => { + unhandled.push(reason) + } + process.on('unhandledRejection', onUnhandled) + try { + rejectRpc(new Error('late rejection after timeout')) + await Promise.resolve() + await Promise.resolve() + expect(unhandled).toEqual([]) + } finally { + process.off('unhandledRejection', onUnhandled) + vi.useRealTimers() + } }) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index b72b4f4fe3..4a1c2b6bd3 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -4,6 +4,12 @@ import { mailboxRpc, type MailboxEnv, } from './mailbox-client.ts' +import { + recordMailboxParityEvent, + type MailboxMirrorOperation, + type MailboxMirrorOutcome, + type MailboxParityEnv, +} from './mailbox-parity-events.ts' import { toMailboxAttachmentInput, toMailboxMessageInput, @@ -31,15 +37,23 @@ import { * Best-effort D1 → Mailbox dual-write helpers. * * Never throw into D1 authority paths: catch/log with stable tags and return - * structured `mirrored | stale | skipped | error` outcomes. `system:email` - * stays in D1 only. + * structured outcomes (`mirrored | stale | missing | timeout | skipped | + * error`). `system:email` stays in D1 only. + * + * Every awaited DO RPC is bounded by {@link mailboxMirrorRpcTimeoutMs}. Partial + * mutation `missing` stays distinct from `mirrored`; delete `missing` maps to + * `mirrored` because desired absence is achieved. Outcomes are recorded via + * `recordMailboxParityEvent` when a user id is known. * * Delivery-event snapshots should be loaded via * `getMailboxDeliveryEventMirrorInput` (mailbox-snapshot-repo) so callers do * not supply promoted D1 columns individually. */ -export type MailboxMirrorEnv = MailboxEnv +/** Bound every awaited Mailbox DO mirror RPC (~1s). */ +export const mailboxMirrorRpcTimeoutMs = 1_000 + +export type MailboxMirrorEnv = MailboxEnv & MailboxParityEnv export type MailboxMirrorSkipReason = | 'system-email' @@ -49,6 +63,8 @@ export type MailboxMirrorSkipReason = export type MailboxMirrorResult = | { status: 'mirrored' } | { status: 'stale' } + | { status: 'missing' } + | { status: 'timeout' } | { status: 'skipped'; reason: MailboxMirrorSkipReason } | { status: 'error'; error: unknown } @@ -76,14 +92,49 @@ function skipIfMailboxMissing( return null } -/** Map touch/update/classify: accepted/missing → mirrored; stale → stale. */ +function mirrorOutcome(result: MailboxMirrorResult): MailboxMirrorOutcome { + switch (result.status) { + case 'mirrored': + case 'stale': + case 'missing': + case 'timeout': + case 'error': + return result.status + case 'skipped': + return 'skipped' + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox mirror result: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function recordMirrorOutcome( + env: MailboxMirrorEnv, + userId: string | null | undefined, + operation: MailboxMirrorOperation, + result: MailboxMirrorResult, +): void { + if (userId == null || userId.length === 0) return + recordMailboxParityEvent(env, { + userId, + category: 'mirror', + operation, + outcome: mirrorOutcome(result), + }) +} + +/** Map touch/update/classify: accepted → mirrored; missing/stale stay distinct. */ function outcomeFromPartialMutation( result: MailboxPartialMutationResult, ): MailboxMirrorResult { switch (result.status) { case 'accepted': - case 'missing': return { status: 'mirrored' } + case 'missing': + return { status: 'missing' } case 'stale': return { status: 'stale' } default: { @@ -118,16 +169,66 @@ function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { return accepted ? { status: 'mirrored' } : { status: 'stale' } } -async function runMirror( - tag: string, - run: () => Promise, -): Promise { +/** + * Race a DO RPC against {@link mailboxMirrorRpcTimeoutMs}. + * The settled wrapper ensures a late rejection cannot become unhandled. + */ +export async function awaitMailboxMirrorRpc( + rpc: Promise, + timeoutMs: number = mailboxMirrorRpcTimeoutMs, +): Promise<{ ok: true; value: T } | { ok: false; timedOut: true }> { + const settled = rpc.then( + (value) => ({ kind: 'fulfilled' as const, value }), + (error: unknown) => ({ kind: 'rejected' as const, error }), + ) + let timeoutId: ReturnType | undefined + try { + const raced = await Promise.race([ + settled, + new Promise<{ kind: 'timeout' }>((resolve) => { + timeoutId = setTimeout(() => { + resolve({ kind: 'timeout' }) + }, timeoutMs) + }), + ]) + if (raced.kind === 'timeout') { + return { ok: false, timedOut: true } + } + if (raced.kind === 'rejected') { + throw raced.error + } + return { ok: true, value: raced.value } + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId) + } +} + +async function runMirror(input: { + tag: string + env: MailboxMirrorEnv + userId: string | null | undefined + operation: MailboxMirrorOperation + run: () => Promise +}): Promise { + let result: MailboxMirrorResult try { - return await run() + result = await input.run() } catch (error) { - console.warn(tag, error) - return { status: 'error', error } + console.warn(input.tag, error) + result = { status: 'error', error } } + recordMirrorOutcome(input.env, input.userId, input.operation, result) + return result +} + +function finishEarly(input: { + env: MailboxMirrorEnv + userId: string | null | undefined + operation: MailboxMirrorOperation + result: MailboxMirrorResult +}): MailboxMirrorResult { + recordMirrorOutcome(input.env, input.userId, input.operation, input.result) + return input.result } /** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ @@ -137,22 +238,46 @@ export async function mirrorMailboxMessageSnapshot(input: { message: EmailMessageRecord attachments?: Array }): Promise { + const operation = 'mirror_message' as const const owner = resolveMirrorOwner(input.message.userId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.message.userId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-message-failed', async () => { - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).mirrorMessage({ - ownerId: owner.ownerId, - thread: input.thread ? toMailboxThreadInput(input.thread) : null, - message: toMailboxMessageInput(input.message), - attachments: input.attachments?.map(toMailboxAttachmentInput), + operation, + result: skippedMailbox, }) - return outcomeFromAccepted(result.accepted) + } + + return runMirror({ + tag: 'mailbox-mirror-message-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).mirrorMessage({ + ownerId: owner.ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromAccepted(raced.value.accepted) + }, }) } @@ -173,21 +298,45 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { deliveryStatusAt: string } | null }): Promise { + const operation = 'upsert_delivery_event' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delivery-event-failed', async () => { - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).upsertDeliveryEvent({ - ownerId: owner.ownerId, - event: input.event, - latestDeliveryStatus: input.latestDeliveryStatus, + operation, + result: skippedMailbox, }) - return outcomeFromAccepted(result.accepted) + } + + return runMirror({ + tag: 'mailbox-mirror-delivery-event-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvent({ + ownerId: owner.ownerId, + event: input.event, + latestDeliveryStatus: input.latestDeliveryStatus, + }), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromAccepted(raced.value.accepted) + }, }) } @@ -195,18 +344,42 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { export async function mirrorMailboxTouchThread( input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'touch_thread' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-touch-thread-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).touchThread(rpcInput) - return outcomeFromPartialMutation(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-touch-thread-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).touchThread(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) + }, }) } @@ -214,84 +387,171 @@ export async function mirrorMailboxTouchThread( export async function mirrorMailboxUpdateMessageDelivery( input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'update_message_delivery' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-update-message-delivery-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-update-message-delivery-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).updateMessageDelivery(rpcInput) - return outcomeFromPartialMutation(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).updateMessageDelivery(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) }, - ) + }) } /** Best-effort inbound classification update mirror. */ export async function mirrorMailboxSetMessageClassification( input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'set_message_classification' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-set-message-classification-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-set-message-classification-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).setMessageClassification(rpcInput) - return outcomeFromPartialMutation(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).setMessageClassification(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) }, - ) + }) } /** Best-effort metadata-only message delete mirror (no R2). */ export async function mirrorMailboxDeleteMessageMetadata( input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_message_metadata' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-delete-message-metadata-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-delete-message-metadata-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteMessageMetadata(rpcInput) - return outcomeFromDeleteResult(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteMessageMetadata(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) }, - ) + }) } /** Best-effort delivery-event delete mirror. */ export async function mirrorMailboxDeleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_delivery_event' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delete-delivery-event-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).deleteDeliveryEvent(rpcInput) - return outcomeFromDeleteResult(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-delete-delivery-event-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteDeliveryEvent(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) + }, }) } @@ -299,17 +559,41 @@ export async function mirrorMailboxDeleteDeliveryEvent( export async function mirrorMailboxDeleteThreadIfEmpty( input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_thread_if_empty' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delete-thread-if-empty-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).deleteThreadIfEmpty(rpcInput) - return outcomeFromDeleteResult(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-delete-thread-if-empty-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteThreadIfEmpty(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) + }, }) } diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 93c755631c..70cebc6313 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -29,16 +29,30 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = userId: 'user-1', category: 'mirror', operation: 'set_message_classification', - outcome: 'skipped', + outcome: 'missing', timestamp: '2026-07-30T10:06:00.000Z', }) recordMailboxParityEvent(env, { userId: 'user-1', category: 'mirror', operation: 'delete_message_metadata', - outcome: 'error', + outcome: 'timeout', timestamp: '2026-07-30T10:07:00.000Z', }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'touch_thread', + outcome: 'skipped', + timestamp: '2026-07-30T10:08:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'upsert_delivery_event', + outcome: 'error', + timestamp: '2026-07-30T10:09:00.000Z', + }) recordMailboxParityEvent(env, { userId: 'user-2', category: 'parity', @@ -56,7 +70,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = timestamp: '2026-07-30T12:00:00.000Z', }) - expect(writeDataPoint).toHaveBeenCalledTimes(6) + expect(writeDataPoint).toHaveBeenCalledTimes(8) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], blobs: [ @@ -75,7 +89,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = indexes: ['user-1'], blobs: [ 'mailbox_mirror:set_message_classification', - 'skipped', + 'missing', '2026-07-30T10:06:00.000Z', ], doubles: [1, 0], @@ -84,12 +98,30 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = indexes: ['user-1'], blobs: [ 'mailbox_mirror:delete_message_metadata', - 'error', + 'timeout', '2026-07-30T10:07:00.000Z', ], doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(5, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:touch_thread', + 'skipped', + '2026-07-30T10:08:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:upsert_delivery_event', + 'error', + '2026-07-30T10:09:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(7, { indexes: ['user-2'], blobs: [ 'mailbox_parity:compare_messages', @@ -98,7 +130,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = ], doubles: [1, 2], }) - expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + expect(writeDataPoint).toHaveBeenNthCalledWith(8, { indexes: ['user-3'], blobs: [ 'mailbox_parity:compare_threads', diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index 66f394eee9..bb2c564770 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -14,7 +14,13 @@ export type MailboxMirrorOperation = | 'delete_delivery_event' | 'delete_thread_if_empty' -export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' +export type MailboxMirrorOutcome = + | 'mirrored' + | 'stale' + | 'missing' + | 'timeout' + | 'skipped' + | 'error' export type MailboxParityOperation = | 'compare_threads' diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts index fb55632566..2357861335 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -4,6 +4,8 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { getEmailDeliveryEventMirrorProjection, getMailboxDeliveryEventMirrorInput, + listEmailDeliveryEventMirrorProjectionsForMessage, + listMailboxDeliveryEventMirrorInputsForMessage, } from './mailbox-snapshot-repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' @@ -117,4 +119,57 @@ test('mailbox-snapshot-repo loads complete delivery-event projection and ready M updatedAt: '2026-07-02T10:00:00.000Z', }), ) + + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, usage_effect_recorded_at, usage_month, + usage_bytes, usage_duration_ms, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'evt-2', + 'msg-1', + 'user-aaa', + 'inbox-1', + 'sent', + 'kody', + null, + null, + '{}', + 0, + null, + null, + null, + null, + '2026-07-02T10:05:00.000Z', + ) + .run() + + const projections = await listEmailDeliveryEventMirrorProjectionsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 10, + }) + expect(projections.map((row) => row.id)).toEqual(['evt-1', 'evt-2']) + + const mailboxInputs = await listMailboxDeliveryEventMirrorInputsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 10, + }) + expect(mailboxInputs).toEqual([ + expect.objectContaining({ + id: 'evt-1', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + expect.objectContaining({ + id: 'evt-2', + updatedAt: '2026-07-02T10:05:00.000Z', + }), + ]) }) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 5fafe8ee09..9d732c35a3 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -35,6 +35,24 @@ type DeliveryEventMirrorRow = { created_at: string } +const deliveryEventMirrorSelect = `SELECT + id, + message_id, + user_id, + inbox_id, + event_type, + provider, + provider_message_id, + provider_event_id, + detail_json, + needs_effect_reconcile, + usage_effect_recorded_at, + usage_month, + usage_bytes, + usage_duration_ms, + created_at + FROM email_delivery_events` + function mapDeliveryEventMirrorRow( row: DeliveryEventMirrorRow, ): EmailDeliveryEventMirrorProjection { @@ -74,23 +92,7 @@ export async function getEmailDeliveryEventMirrorProjection(input: { }): Promise { const row = await input.db .prepare( - `SELECT - id, - message_id, - user_id, - inbox_id, - event_type, - provider, - provider_message_id, - provider_event_id, - detail_json, - needs_effect_reconcile, - usage_effect_recorded_at, - usage_month, - usage_bytes, - usage_duration_ms, - created_at - FROM email_delivery_events + `${deliveryEventMirrorSelect} WHERE id = ? AND user_id = ? LIMIT 1`, @@ -124,3 +126,46 @@ export async function getMailboxDeliveryEventMirrorInput(input: { sourceMutationAt: input.sourceMutationAt, }) } + +/** + * Load complete delivery-event mirror projections for one message, oldest + * first, bounded by `limit`. Scoped by owner. + */ +export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { + db: D1Database + ownerId: string + messageId: string + limit: number +}): Promise> { + const result = await input.db + .prepare( + `${deliveryEventMirrorSelect} + WHERE message_id = ? + AND user_id = ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.messageId, input.ownerId, input.limit) + .all() + return (result.results ?? []).map(mapDeliveryEventMirrorRow) +} + +/** + * Load ready Mailbox delivery-event snapshots for one message. + * Uses each event's `created_at` as `sourceMutationAt` (immutable inserts). + */ +export async function listMailboxDeliveryEventMirrorInputsForMessage(input: { + db: D1Database + ownerId: string + messageId: string + limit: number +}): Promise> { + const projections = + await listEmailDeliveryEventMirrorProjectionsForMessage(input) + return projections.map((projection) => + toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: projection.createdAt, + }), + ) +} diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index b7a73592f8..ea135691fc 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -17,6 +17,10 @@ import { } from '#worker/entitlements/service.ts' import { recordUsage } from '#worker/usage/record-usage.ts' import { normalizeEmailAddress } from './address.ts' +import { + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' import { emailOutboundPausedMessage } from './outbound-abuse.ts' import { resolveUserPlatformSender } from './platform-address.ts' import { @@ -34,7 +38,11 @@ import { insertEmailMessageWithoutRawMime, updateEmailMessageDelivery, } from './service.ts' -import { type EmailMessageRecord, type EmailProcessingStatus } from './types.ts' +import { + type EmailMessageRecord, + type EmailProcessingStatus, + type EmailThreadRecord, +} from './types.ts' type SendEmailEnv = Pick< Env, @@ -48,8 +56,11 @@ type SendEmailEnv = Pick< | 'CLOUDFLARE_API_BASE_URL' | 'CLOUDFLARE_API_TOKEN' | 'USER_METER' + | 'MAILBOX' + | 'EMAIL_EVENTS' > & - EmailReportingEnv + EmailReportingEnv & + MailboxLiveMirrorEnv /** * Ceiling on the number of files per outbound message. Total size is @@ -324,6 +335,27 @@ async function requireStoredEmailMessage(input: { return stored } +/** + * One bounded never-throw D1 → Mailbox graph mirror at outbound terminals. + * Passes a just-created thread when present; otherwise omits `thread` so + * `mirrorMailboxMessageGraphFromD1` loads it from D1. Failures never affect + * send/refund. + */ +async function mirrorOutboundMailboxMessageGraph(input: { + env: SendEmailEnv + userId: string + messageId: string + createdThread: EmailThreadRecord | null +}) { + await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + messageId: input.messageId, + ...(input.createdThread ? { thread: input.createdThread } : {}), + }) +} + async function sendViaBinding(input: { env: SendEmailEnv from: string @@ -571,7 +603,7 @@ export async function sendOutboundEmail( }) const existingThreadId = original?.threadId ?? input.threadId ?? null - const thread = existingThreadId + const createdThread = existingThreadId ? null : await createEmailThread({ db: input.env.APP_DB, @@ -581,7 +613,7 @@ export async function sendOutboundEmail( rootMessageIdHeader: input.inReplyToHeader ?? null, lastMessageAt: new Date().toISOString(), }) - const threadId = existingThreadId ?? thread?.id ?? null + const threadId = existingThreadId ?? createdThread?.id ?? null const providerHeaders = buildProviderHeaders(storedHeaders) const message = await insertEmailMessageWithoutRawMime({ db: input.env.APP_DB, @@ -617,6 +649,13 @@ export async function sendOutboundEmail( sentAt: null, }, }) + const mirrorMailboxGraph = () => + mirrorOutboundMailboxMessageGraph({ + env: input.env, + userId: input.userId, + messageId: message.id, + createdThread, + }) try { await storeOutboundAttachments({ db: input.env.APP_DB, @@ -655,6 +694,7 @@ export async function sendOutboundEmail( }).catch((eventError) => { console.warn('email-attachment-failure-event-insert-failed', eventError) }) + await mirrorMailboxGraph() throw error } await insertEmailDeliveryEvent({ @@ -722,6 +762,7 @@ export async function sendOutboundEmail( providerMessageId, detail: { providerMessageId }, }) + await mirrorMailboxGraph() return { message: await requireStoredEmailMessage({ env: input.env, @@ -757,6 +798,7 @@ export async function sendOutboundEmail( }).catch((eventError) => { console.warn('email-delivery-failure-event-insert-failed', eventError) }) + await mirrorMailboxGraph() return { message: (await getEmailMessageById({ diff --git a/packages/worker/src/email/outbound.workers.test.ts b/packages/worker/src/email/outbound.workers.test.ts index 3f87a19883..a1be15b7f7 100644 --- a/packages/worker/src/email/outbound.workers.test.ts +++ b/packages/worker/src/email/outbound.workers.test.ts @@ -6,6 +6,8 @@ import { bytesToBase64 } from '@kody-internal/shared/base64.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' import { ensureEmailTestSchema } from './test-schema.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { rpcFor } from './mailbox-test-helpers.ts' import { deleteEmailMessageById, getEmailMessageById, @@ -48,6 +50,37 @@ function createBindingSendEnv() { } } +/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createMailboxStubEnv(input: { + mode: 'throw' | 'hang' + base?: typeof env +}) { + const base = input.base ?? env + const hang = () => new Promise(() => {}) + const fail = async () => { + throw new Error('simulated mailbox failure') + } + const method = input.mode === 'hang' ? hang : fail + const stub = { + mirrorMessage: method, + upsertDeliveryEvent: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + return { + ...base, + APP_BASE_URL: platformBaseUrl, + MAILBOX: { + idFromName: (name: string) => base.MAILBOX.idFromName(name), + get: () => stub, + } as unknown as DurableObjectNamespace, + } +} + async function seedVerifiedAccount(input: { email: string username?: string @@ -966,3 +999,212 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async maxPlanEmailLimits.email_sends_per_day, ) }, 30_000) + +test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) + const sendEnv = { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-sent' } + }, + }, + } + + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror sent', + text: 'Mirrored body', + attachments: [ + { + filename: 'note.pdf', + contentType: 'application/pdf', + contentBase64: bytesToBase64(pdfBytes), + }, + ], + }) + + expect(result.status).toBe('sent') + expect(result.message.threadId).toBeTruthy() + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'sent', + providerMessageId: 'provider-mailbox-sent', + subject: 'Mailbox mirror sent', + threadId: result.message.threadId, + }) + const mirroredThread = await mailbox.getThread({ + threadId: result.message.threadId!, + }) + expect(mirroredThread).toMatchObject({ + id: result.message.threadId, + subjectNormalized: 'mailbox mirror sent', + }) + const mirroredAttachments = await mailbox.listAttachmentsForMessage({ + messageId: result.message.id, + }) + expect(mirroredAttachments).toEqual([ + expect.objectContaining({ + filename: 'note.pdf', + contentType: 'application/pdf', + size: pdfBytes.byteLength, + storageKind: 'external', + }), + ]) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'send_requested', + 'sent', + ]) +}, 30_000) + +test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { + silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + + using _server = createMswNodeServer( + [ + http.post(cloudflareEmailApi, () => + HttpResponse.json( + { success: false, errors: [{ message: 'provider down' }] }, + { status: 500 }, + ), + ), + ], + { onUnhandledRequest: 'bypass' }, + ) + + const result = await sendOutboundEmail({ + env: { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: undefined as unknown as SendEmail, + CLOUDFLARE_ACCOUNT_ID: 'account-123', + CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', + CLOUDFLARE_API_TOKEN: 'token-123', + }, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror failed', + text: 'Body', + }) + + expect(result.status).toBe('failed') + expect(result.error).toBe('provider down') + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'failed', + error: 'provider down', + threadId: result.message.threadId, + }) + expect( + await mailbox.getThread({ threadId: result.message.threadId! }), + ).toMatchObject({ id: result.message.threadId }) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'failed', + 'send_requested', + ]) +}, 30_000) + +test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-live-mirror-message-graph-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + + { + const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'throw' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-err' } + }, + }, + } + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox throw ignored', + text: 'Body', + }) + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-err', + error: null, + }) + const stored = await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: result.message.id, + }) + expect(stored?.processingStatus).toBe('sent') + } + + { + const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'hang' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-hang' } + }, + }, + } + const startedAt = Date.now() + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox hang ignored', + text: 'Body', + }) + const elapsedMs = Date.now() - startedAt + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-hang', + error: null, + }) + // Message + send_requested + sent each race the mirror RPC bound. + expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) + } +}, 30_000) diff --git a/packages/worker/src/email/repo-body-limits.workers.test.ts b/packages/worker/src/email/repo-body-limits.workers.test.ts index 610d3c3c6a..cdb015dac8 100644 --- a/packages/worker/src/email/repo-body-limits.workers.test.ts +++ b/packages/worker/src/email/repo-body-limits.workers.test.ts @@ -4,7 +4,13 @@ import { maxRestorableTextColumnBytes, utf8ByteLength, } from '@kody-internal/shared/backup-restore-safety.ts' -import { emailBodyTruncationNotice, insertEmailMessage } from './repo.ts' +import { + createEmailThread, + emailBodyTruncationNotice, + getEmailThreadById, + insertEmailDeliveryEvent, + insertEmailMessage, +} from './repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' test('stored email bodies are bounded so D1 backups stay importable', async () => { @@ -32,3 +38,60 @@ test('stored email bodies are bounded so D1 backups stay importable', async () = maxRestorableTextColumnBytes, ) }) + +test('getEmailThreadById is owner-scoped and insertEmailDeliveryEvent returns the row', async () => { + await ensureEmailTestSchema(env.APP_DB) + const ownerId = `owner-${crypto.randomUUID()}` + const otherUserId = `other-${crypto.randomUUID()}` + const thread = await createEmailThread({ + db: env.APP_DB, + userId: ownerId, + subjectNormalized: 'hello thread', + rootMessageIdHeader: '', + }) + + expect( + await getEmailThreadById({ + db: env.APP_DB, + userId: ownerId, + threadId: thread.id, + }), + ).toMatchObject({ + id: thread.id, + userId: ownerId, + subjectNormalized: 'hello thread', + }) + expect( + await getEmailThreadById({ + db: env.APP_DB, + userId: otherUserId, + threadId: thread.id, + }), + ).toBeNull() + + const createdAt = '2026-07-15T12:00:00.000Z' + const inserted = await insertEmailDeliveryEvent({ + db: env.APP_DB, + messageId: null, + userId: ownerId, + inboxId: null, + eventType: 'send_requested', + provider: 'cloudflare-email', + detail: { subject: 'hello' }, + createdAt, + }) + expect(inserted).toMatchObject({ + id: expect.any(String), + userId: ownerId, + eventType: 'send_requested', + provider: 'cloudflare-email', + detailJson: JSON.stringify({ subject: 'hello' }), + createdAt, + }) + const row = await env.APP_DB.prepare( + `SELECT id, created_at FROM email_delivery_events WHERE id = ?`, + ) + .bind(inserted.id) + .first<{ id: string; created_at: string }>() + expect(row).toEqual({ id: inserted.id, created_at: createdAt }) +}) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 3503580936..c7fc822a29 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -524,6 +524,29 @@ async function findThreadForMessage(input: { export const findEmailThreadForInboundMessage = findThreadForMessage +/** + * Owner-scoped thread lookup for D1 → Mailbox graph mirrors and other + * strictly user-bound readers. Returns null when absent or owned by another + * user — never use for inbound delivery fencing. + */ +export async function getEmailThreadById(input: { + db: D1Database + userId: string + threadId: string +}) { + const row = await input.db + .prepare( + `SELECT * + FROM email_threads + WHERE id = ? + AND user_id = ? + LIMIT 1`, + ) + .bind(input.threadId, input.userId) + .first>() + return row ? mapThreadRow(row) : null +} + export async function createEmailThread(input: { db: D1Database id?: string @@ -1198,7 +1221,22 @@ export async function insertEmailDeliveryEvent(input: { providerEventId?: string | null detail?: Record | null createdAt?: string -}) { +}): Promise { + const id = crypto.randomUUID() + const createdAt = input.createdAt ?? nowIso() + const detailJson = JSON.stringify(input.detail ?? {}) + const record: EmailDeliveryEventRecord = { + id, + messageId: input.messageId ?? null, + userId: input.userId ?? null, + inboxId: input.inboxId ?? null, + eventType: input.eventType, + provider: input.provider ?? null, + providerMessageId: input.providerMessageId ?? null, + providerEventId: input.providerEventId ?? null, + detailJson, + createdAt, + } await input.db .prepare( `INSERT INTO email_delivery_events ( @@ -1207,16 +1245,17 @@ export async function insertEmailDeliveryEvent(input: { ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .bind( - crypto.randomUUID(), - input.messageId ?? null, - input.userId ?? null, - input.inboxId ?? null, - input.eventType, - input.provider ?? null, - input.providerMessageId ?? null, - input.providerEventId ?? null, - JSON.stringify(input.detail ?? {}), - input.createdAt ?? nowIso(), + record.id, + record.messageId, + record.userId, + record.inboxId, + record.eventType, + record.provider, + record.providerMessageId, + record.providerEventId, + record.detailJson, + record.createdAt, ) .run() + return record } diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts index 19979cd0c2..35f2299626 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts @@ -3,12 +3,21 @@ import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ setEmailMessageClassification: vi.fn(), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' }, + events: [], + })), })) vi.mock('#worker/email/repo.ts', () => ({ setEmailMessageClassification: mocks.setEmailMessageClassification, })) +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + const { emailMessageClassifyCapability } = await import('./email-message-classify.ts') @@ -56,8 +65,15 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'quarantined', classificationReason: 'Reclassified by user.', }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: context.env, + db: context.env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) mocks.setEmailMessageClassification.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'message-1', classification: 'accepted' }, @@ -74,12 +90,39 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'accepted', classificationReason: null, }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: context.env, + db: context.env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) mocks.setEmailMessageClassification.mockResolvedValueOnce(false) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'missing', classification: 'quarantined' }, context, ), ).rejects.toThrow('Email message not found: missing') + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + mocks.setEmailMessageClassification.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'message-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + await expect( + emailMessageClassifyCapability.handler( + { message_id: 'message-1', classification: 'quarantined' }, + context, + ), + ).resolves.toEqual({ + message_id: 'message-1', + classification: 'quarantined', + }) }) diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts index ea10869745..1a9562ddf9 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts @@ -1,6 +1,7 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' +import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' import { setEmailMessageClassification } from '#worker/email/repo.ts' import { emailClassificationValues } from '#worker/email/types.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' @@ -44,6 +45,12 @@ export const emailMessageClassifyCapability = defineDomainCapability( if (!updated) { throw new Error(`Email message not found: ${args.message_id}`) } + await mirrorMailboxMessageGraphFromD1({ + env: ctx.env, + db: ctx.env.APP_DB, + userId: user.userId, + messageId: args.message_id, + }) return { message_id: args.message_id, classification: args.classification, From e9e9f184968909212005b77708dd9e90faedc0c7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 10:44:13 +0000 Subject: [PATCH 08/20] fix(email): batch Mailbox graph event mirrors Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 123 +++-- .../app/handlers/account-email.node.test.ts | 57 +-- .../worker/src/app/handlers/account-email.ts | 11 +- packages/worker/src/email/mailbox-do.ts | 39 ++ .../src/email/mailbox-do.workers.test.ts | 158 ++++++ .../email/mailbox-live-mirror.node.test.ts | 241 ++++++--- .../worker/src/email/mailbox-live-mirror.ts | 61 +-- .../src/email/mailbox-mirror.node.test.ts | 157 ++++++ packages/worker/src/email/mailbox-mirror.ts | 470 +++++++----------- .../email/mailbox-parity-events.node.test.ts | 20 +- .../worker/src/email/mailbox-parity-events.ts | 1 + .../email/mailbox-snapshot-repo.node.test.ts | 9 + .../worker/src/email/mailbox-snapshot-repo.ts | 12 +- packages/worker/src/email/mailbox-types.ts | 22 + .../outbound-mailbox-mirror.workers.test.ts | 277 +++++++++++ .../worker/src/email/outbound.workers.test.ts | 242 --------- packages/worker/src/email/repo.ts | 6 +- .../email/service-classification.node.test.ts | 119 +++++ packages/worker/src/email/service.ts | 39 ++ .../email/email-message-classify.node.test.ts | 47 +- .../email/email-message-classify.ts | 10 +- 21 files changed, 1318 insertions(+), 803 deletions(-) create mode 100644 packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts create mode 100644 packages/worker/src/email/service-classification.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 492f2bdb56..3ec77dc79e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -220,9 +220,10 @@ Durable Object export behavior: RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires non-inbound live dual-write for outbound terminals, provider-recorded delivery - events, and user classification without changing D1 authority — D1 `email_*` - rows remain the live source of truth and are still exported in the `d1` - section. See [Mailbox](#durable-objects-mailbox). + events, and user classification (`service.ts#setEmailMessageClassification`) + without changing D1 authority — D1 `email_*` rows remain the live source of + truth and are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -394,11 +395,12 @@ do not feed the admin insights charts today: - `double2` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; `0` when absent or not applicable) -Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, -`update_message_delivery`, `set_message_classification`, -`delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. -Mirror outcomes: `mirrored`, `stale`, `missing`, `timeout`, `skipped`, `error`. -Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, +Mirror operations: `mirror_message`, `upsert_delivery_event`, +`upsert_delivery_event_batch`, `touch_thread`, `update_message_delivery`, +`set_message_classification`, `delete_message_metadata`, +`delete_delivery_event`, `delete_thread_if_empty`. Mirror outcomes: `mirrored`, +`stale`, `missing`, `timeout`, `skipped`, `error`. Parity operations: +`compare_threads`, `compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome writes are automatic from live dual-write helpers; parity compare writers remain for future reconciliation. Writes are best-effort and never throw into D1 @@ -675,15 +677,17 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` -**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` take -complete snapshots — every persisted field is explicit (nullable fields use -explicit `null`). They are not patch APIs. Both require `ownerId` and apply -equal-or-newer `updatedAt` snapshots (stale snapshots are ignored, not applied). -The only omission exception is the `mirrorMessage` `attachments` bundle: -omitting it preserves existing attachment rows; an explicit `attachments: []` -clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and -external attachment `storageKey` values against the canonical builders for that -`ownerId`. +**Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and +`upsertDeliveryEvents` take complete snapshots — every persisted field is +explicit (nullable fields use explicit `null`). They are not patch APIs. All +require `ownerId` and apply equal-or-newer `updatedAt` snapshots (stale +snapshots are ignored, not applied). `upsertDeliveryEvents` accepts at most +`mailboxUpsertDeliveryEventsMax` (100) events in one owner-bound batch RPC and +inserts them in caller order (chronological when loaded from D1). The only +omission exception is the `mirrorMessage` `attachments` bundle: omitting it +preserves existing attachment rows; an explicit `attachments: []` clears them. +Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment +`storageKey` values against the canonical builders for that `ownerId`. **Partial mutation RPCs** (library-only in the non-inbound phase-2 slice; owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — @@ -735,13 +739,17 @@ bounded by `mailboxMirrorRpcTimeoutMs` (1 second). Each returns a structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, `{ status: 'missing' }`, `{ status: 'timeout' }`, `{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | -`missing-owner`), or `{ status: 'error', error }`. Every outcome is recorded -automatically via `recordMailboxParityEvent` (`mailbox_mirror:`) when -a user id is known; `system:email` is excluded. Failures log with stable tags -(for example `mailbox-mirror-message-failed`) and never propagate into D1 commit -paths. Helpers cover full snapshots (`mirrorMailboxMessageSnapshot`, -`mirrorMailboxDeliveryEventSnapshot` — prefer loading delivery events with -`getMailboxDeliveryEventMirrorInput`) and partial mutations +`missing-owner`), or `{ status: 'error', error }`. Single-RPC helpers record one +`mailbox_mirror:` outcome automatically when a user id is known; +`mirrorMailboxDeliveryEventSnapshots` records one +`mailbox_mirror:upsert_delivery_event_batch` outcome per batch (timeout/error +apply uniformly to per-event summary entries). `system:email` is excluded. +Failures log with stable tags (for example `mailbox-mirror-message-failed`) and +never propagate into D1 commit paths. Helpers cover full snapshots +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`, +`mirrorMailboxDeliveryEventSnapshots` — prefer loading delivery events with +`getMailboxDeliveryEventMirrorInput` or +`listMailboxDeliveryEventMirrorInputsForMessage`) and partial mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial @@ -751,13 +759,16 @@ below. D1 remains sole authority for all live mail read/write paths. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery events) and mirrors it best-effort. `mirrorMailboxMessageGraphFromD1` settles -the message snapshot RPC first, then fans out delivery-event snapshot RPCs -concurrently so event-phase wall time stays near one RPC timeout (not limit × -timeout). Event load uses `max+1` rows so `eventsTruncated` is explicit; at most -`mailboxLiveMirrorMaxEvents` (`mailboxDefaultPageSize`, 100) events are mirrored -per call to stay under Analytics Engine's ~250 writes/request budget (1 message -outcome + up to 100 event outcomes). Never throws; returns a bounded summary. -**Live callers (non-inbound phase-2 slice):** +the message snapshot RPC first, then repairs delivery events with one +owner-bound `upsertDeliveryEvents` batch RPC (not concurrent per-event RPCs to +the same DO). Event load queries newest `max+1` rows from D1, restores +chronological order, and when truncated keeps the newest +`mailboxLiveMirrorMaxEvents` (`mailboxUpsertDeliveryEventsMax`, 100) — dropping +oldest overflow. The batch RPC shares one 1s timeout and one +`mailbox_mirror:upsert_delivery_event_batch` telemetry outcome (timeout/error +apply uniformly to per-event summary entries). Analytics Engine budget per graph +call is two writes (1 message outcome + 1 batch outcome). Never throws; returns +a bounded summary. **Live callers (non-inbound phase-2 slice):** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full @@ -767,10 +778,11 @@ outcome + up to 100 event outcomes). Never throws; returns a bounded summary. - **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue ingestion persists a new provider event (`recorded`), schedules `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not - blocked; full graph repair, not a partial delivery-event touch. -- **User classification** (`account-email.ts`, `email-message-classify.ts`) — - after D1 classification update, mirrors the full graph (not the partial - `setMessageClassification` RPC alone). + blocked; full graph repair via message + batch event upsert. +- **User classification** (`service.ts#setEmailMessageClassification`) — + transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate + here for the D1 mutation + full graph mirror invariant (mirror only after a + successful D1 update; failures never change the mutation response). Inbound commit/touch/classify paths, retention deletes, and partial mutation mirrors are **not** wired in this slice. Parity compare/reconcile cron remains @@ -800,23 +812,24 @@ This is an expand/contract migration. **Phase 2 is partially wired (non-inbound slice):** live dual-write covers outbound terminal message/thread/attachment/event graphs, provider-recorded delivery events (queue `waitUntil` full graph repair), and user classification (full graph repair after -D1 update). Mirror helpers emit automatic namespaced outcome telemetry -(`missing` and `timeout` included). Every awaited mirror RPC is bounded to 1s; -graph event fan-out is concurrent with explicit truncation when D1 has more than -`mailboxLiveMirrorMaxEvents` delivery events. **`system:email` stays in D1 -only.** Inbound lifecycle dual-write, retention/metadata deletes, and scheduled -parity reconcile are **not** wired in this slice. Live email read/write -authority is unchanged — D1 `email_*` tables remain authoritative for all live -paths. D1 email rows and existing R2 inventory deletion stay authoritative -during expand. **Phase 2 completes only when every user-mail mutation that -writes D1 also writes the DO and parity/reconciliation are wired** — inbound -paths and parity reconcile remain. +D1 update via `service.ts#setEmailMessageClassification`). Mirror helpers emit +automatic namespaced outcome telemetry (`missing` and `timeout` included). +Message and batch event repair each use one 1s-bounded RPC; graph event repair +batches up to 100 events with explicit truncation (newest retained, +chronological insertion restored) and one batch telemetry outcome. +**`system:email` stays in D1 only.** Inbound lifecycle dual-write, +retention/metadata deletes, and scheduled parity reconcile are **not** wired in +this slice. Live email read/write authority is unchanged — D1 `email_*` tables +remain authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. **Phase 2 completes only when every +user-mail mutation that writes D1 also writes the DO and parity/reconciliation +are wired** — inbound paths and parity reconcile remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / - `exportMailbox` / `countMailbox` / `purge` RPCs and alarm retention, and - register account deletion/export consumption. No dual-write; D1 remains sole - authority for live mail. + `upsertDeliveryEvents` / `exportMailbox` / `countMailbox` / `purge` RPCs and + alarm retention, and register account deletion/export consumption. No + dual-write; D1 remains sole authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Library surface: @@ -824,13 +837,15 @@ paths and parity reconcile remain. `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-live-mirror.ts`, `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, - `deleteDeliveryEvent`, `deleteThreadIfEmpty`), and + `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, - provider `recorded` queue events (`waitUntil`), and user classification call + provider `recorded` queue events (`waitUntil`), and user classification + (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` - outcomes automatically. **Still pending for phase-2 completion:** inbound - commit/touch/classify dual-write, retention and metadata-delete mirrors, and - scheduled parity compare/reconcile. + outcomes automatically (graph repair uses message RPC + one batch event RPC). + **Still pending for phase-2 completion:** inbound commit/touch/classify + dual-write, retention and metadata-delete mirrors, and scheduled parity + compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index e206514d67..69293c4201 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -155,11 +155,6 @@ const mockModule = vi.hoisted(() => ({ }, ]), setEmailMessageClassification: vi.fn(async () => true), - mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ - messageId: 'msg-1', - message: { status: 'mirrored' }, - events: [], - })), prepare: vi.fn(), })) @@ -227,13 +222,11 @@ vi.mock('#worker/email/repo.ts', () => ({ mockModule.listEmailAttachmentsForMessage(...args), listEmailDeliveryEvents: (...args: Array) => mockModule.listEmailDeliveryEvents(...args), - setEmailMessageClassification: (...args: Array) => - mockModule.setEmailMessageClassification(...args), })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: (...args: Array) => - mockModule.mirrorMailboxMessageGraphFromD1(...args), +vi.mock('#worker/email/service.ts', () => ({ + setEmailMessageClassification: (...args: Array) => + mockModule.setEmailMessageClassification(...args), })) const { createAccountEmailApiHandler } = await import('./account-email.ts') @@ -568,7 +561,6 @@ test('email API lists classification filters and classifies inbound messages', a }) mockModule.setEmailMessageClassification.mockClear() - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() // Reload after classify uses the default message list again. createEnv({ meter, messageRows: [messageRow] }) const quarantineResponse = await handler.handler({ @@ -584,25 +576,19 @@ test('email API lists classification filters and classifies inbound messages', a }) expect(quarantineResponse.status).toBe(200) expect(mockModule.setEmailMessageClassification).toHaveBeenCalledWith({ + env, db: env.APP_DB, userId: 'stable-user-1', messageId: 'msg-1', classification: 'quarantined', classificationReason: 'Reclassified by user.', }) - expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'stable-user-1', - messageId: 'msg-1', - }) await expect(quarantineResponse.json()).resolves.toMatchObject({ ok: true, selectedMessage: expect.objectContaining({ id: 'msg-1' }), }) mockModule.setEmailMessageClassification.mockClear() - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const acceptResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -616,21 +602,15 @@ test('email API lists classification filters and classifies inbound messages', a }) expect(acceptResponse.status).toBe(200) expect(mockModule.setEmailMessageClassification).toHaveBeenCalledWith({ + env, db: env.APP_DB, userId: 'stable-user-1', messageId: 'msg-1', classification: 'accepted', classificationReason: null, }) - expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'stable-user-1', - messageId: 'msg-1', - }) mockModule.setEmailMessageClassification.mockResolvedValueOnce(false) - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const missingResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -643,33 +623,6 @@ test('email API lists classification filters and classifies inbound messages', a }), }) expect(missingResponse.status).toBe(404) - expect(mockModule.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() - - mockModule.setEmailMessageClassification.mockResolvedValueOnce(true) - mockModule.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'msg-1', - message: { - status: 'error', - error: new Error('mailbox mirror timed out'), - }, - events: [], - }) - const mirrorFailureResponse = await handler.handler({ - request: new Request('https://example.com/account/email.json', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - action: 'classify', - message_id: 'msg-1', - classification: 'quarantined', - }), - }), - }) - expect(mirrorFailureResponse.status).toBe(200) - await expect(mirrorFailureResponse.json()).resolves.toMatchObject({ - ok: true, - selectedMessage: expect.objectContaining({ id: 'msg-1' }), - }) const invalidActionResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { diff --git a/packages/worker/src/app/handlers/account-email.ts b/packages/worker/src/app/handlers/account-email.ts index 62b8148201..e1e63b4551 100644 --- a/packages/worker/src/app/handlers/account-email.ts +++ b/packages/worker/src/app/handlers/account-email.ts @@ -7,8 +7,7 @@ import { requireAuthenticatedPageUser } from '#app/page-auth.ts' import { readTrimmedStringOrEmpty } from '#app/request-body.ts' import { type routes } from '#app/routes.ts' import { renderAppPage } from '#app/ssr-render.tsx' -import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' -import { setEmailMessageClassification } from '#worker/email/repo.ts' +import { setEmailMessageClassification } from '#worker/email/service.ts' import { emailClassificationValues, type EmailClassification, @@ -113,6 +112,7 @@ export function createAccountEmailApiHandler(env: Env) { } const updated = await setEmailMessageClassification({ + env, db: env.APP_DB, userId: user.mcpUser.userId, messageId, @@ -130,13 +130,6 @@ export function createAccountEmailApiHandler(env: Env) { ) } - await mirrorMailboxMessageGraphFromD1({ - env, - db: env.APP_DB, - userId: user.mcpUser.userId, - messageId, - }) - const listUrl = new URL(request.url, 'http://localhost') listUrl.searchParams.set('selected', messageId) return jsonResponse( diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index c8b06ce2f4..750ba38f96 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -23,6 +23,7 @@ import { } from './mailbox-mutations.ts' import { assertMailboxNonEmptyString, + mailboxUpsertDeliveryEventsMax, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, @@ -45,6 +46,8 @@ import { type MailboxThreadRecord, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, + type MailboxUpsertDeliveryEventBatchItemResult, + type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' /** @@ -232,6 +235,42 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { inserted, accepted, updatedLatestStatus } } + /** + * Upsert a bounded batch of complete immutable delivery-event snapshots in + * one transaction / one DO RPC. Validates non-empty and + * {@link mailboxUpsertDeliveryEventsMax}. Does not patch message latest + * delivery status. Marks retention dirty once for the batch. + */ + async upsertDeliveryEvents(input: { + ownerId: string + events: Array + }): Promise { + if (!Array.isArray(input.events) || input.events.length === 0) { + throw new Error('Mailbox upsertDeliveryEvents events must be non-empty.') + } + if (input.events.length > mailboxUpsertDeliveryEventsMax) { + throw new Error( + `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, + ) + } + const results: Array = [] + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + for (const event of input.events) { + const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + const write = this.store.writeDeliveryEventRow(event) + results.push({ + eventId, + inserted: write.inserted, + accepted: write.accepted, + }) + } + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { results } + } + /** * Advance thread activity without a full snapshot. Equal/newer `updatedAt` * only; `last_message_at` never moves backward. diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index c619df9701..b1f5aa4c10 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -14,6 +14,7 @@ import { stubFor, uniqueUserId, } from './mailbox-test-helpers.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { silenceIncidentalRuntimeWarnings() @@ -697,3 +698,160 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryEvents: 0, }) }) + +test('Mailbox upsertDeliveryEvents validates bounds, owns batch, and arms retention once', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('batch') + const otherUserId = uniqueUserId('batch-other') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const message = baseMessage(userId, { + id: 'batch-msg', + direction: 'outbound', + processingStatus: 'sent', + rawMimeKey: null, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ ownerId: userId, message }) + + await assertMailboxThrows(/events must be non-empty/, () => + mailbox.upsertDeliveryEvents({ ownerId: userId, events: [] }), + ) + const tooMany = Array.from( + { length: mailboxUpsertDeliveryEventsMax + 1 }, + (_, index) => + baseDeliveryEvent({ + id: `overflow-${index}`, + messageId: message.id, + eventType: 'sent', + provider: 'kody', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + ) + await assertMailboxThrows(/exceed max of/, () => + mailbox.upsertDeliveryEvents({ ownerId: userId, events: tooMany }), + ) + + await runInDurableObject(stub, async (_instance: Mailbox, state) => { + await state.storage.deleteAlarm() + }) + + const batch = [ + baseDeliveryEvent({ + id: 'batch-evt-1', + messageId: message.id, + eventType: 'send_requested', + provider: 'kody', + createdAt: '2026-07-02T10:00:01.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }), + baseDeliveryEvent({ + id: 'batch-evt-2', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + providerEventId: 'provider-batch-2', + createdAt: '2026-07-02T10:00:02.000Z', + updatedAt: '2026-07-02T10:00:02.000Z', + }), + ] + const first = await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: batch, + }) + expect(first).toEqual({ + results: [ + { eventId: 'batch-evt-1', inserted: true, accepted: true }, + { eventId: 'batch-evt-2', inserted: true, accepted: true }, + ], + }) + // Batch must not patch message latest delivery status. + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + expect( + new Set( + ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 10 }) + ).map((event) => event.id), + ), + ).toEqual(new Set(['batch-evt-1', 'batch-evt-2'])) + + const alarmAfterBatch = await runInDurableObject( + stub, + async (_instance: Mailbox, state) => state.storage.getAlarm(), + ) + expect(alarmAfterBatch).toBeTypeOf('number') + + const replay = await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + batch[0]!, + baseDeliveryEvent({ + id: 'batch-evt-2-dup', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + providerEventId: 'provider-batch-2', + createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', + }), + ], + }) + expect(replay).toEqual({ + results: [ + { eventId: 'batch-evt-1', inserted: false, accepted: true }, + { eventId: 'batch-evt-2-dup', inserted: false, accepted: false }, + ], + }) + + await assertMailboxThrows(/ownerId mismatch/, () => + mailbox.upsertDeliveryEvents({ + ownerId: otherUserId, + events: [ + baseDeliveryEvent({ + id: 'batch-evt-other', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + }), + ], + }), + ) + + // Transactionality: a mid-batch validation failure rolls back prior writes. + const beforeCount = (await mailbox.countMailbox()).deliveryEvents + await assertMailboxThrows(/canonical ISO-8601/, () => + mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + baseDeliveryEvent({ + id: 'batch-txn-ok', + messageId: message.id, + eventType: 'delivered', + provider: 'kody', + createdAt: '2026-07-02T12:00:00.000Z', + updatedAt: '2026-07-02T12:00:00.000Z', + }), + baseDeliveryEvent({ + id: 'batch-txn-bad', + messageId: message.id, + eventType: 'delivered', + provider: 'kody', + createdAt: '2026-07-02T12:00:00Z', + updatedAt: '2026-07-02T12:00:00Z', + }), + ], + }), + ) + expect((await mailbox.countMailbox()).deliveryEvents).toBe(beforeCount) + expect( + ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) + ).some((event) => event.id === 'batch-txn-ok'), + ).toBe(false) +}) diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts index af8ae5ba64..91be2c89f8 100644 --- a/packages/worker/src/email/mailbox-live-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -11,6 +11,7 @@ import { } from './mailbox-live-mirror.ts' import { ensureEmailTestSchema } from './test-schema.ts' import { type EmailThreadRecord } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' async function createEmailDb() { const sqlite = new DatabaseSync(':memory:') @@ -137,12 +138,10 @@ async function seedMessageGraph(db: D1Database) { test('live-mirror AE write budget stays under Analytics Engine request cap', () => { expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) - expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe( - 1 + mailboxLiveMirrorMaxEvents, - ) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe(2) }) -test('live-mirror graph mirrors message before events and uses createdAt for event updatedAt', async () => { +test('live-mirror graph mirrors message before one batch event RPC and uses createdAt for event updatedAt', async () => { const { db } = await createEmailDb() await seedMessageGraph(db) @@ -151,18 +150,28 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve messageSettled = true return { ok: true as const, accepted: true } }) - const upsertDeliveryEvent = vi.fn( - async (input: { event: { id: string } }) => { + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array<{ id: string }> }) => { expect(messageSettled).toBe(true) return { - inserted: true, - accepted: true, - updatedLatestStatus: false, - eventId: input.event.id, + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), } }, ) - const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const { env } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents, + upsertDeliveryEvent, + }) const thread: EmailThreadRecord = { id: 'thread-1', @@ -180,7 +189,6 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve db, userId: 'user-aaa', messageId: 'msg-1', - sourceMutationAt: '2026-07-01T12:00:03.000Z', thread, }) @@ -203,10 +211,12 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve ], }), ) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvent).not.toHaveBeenCalled() // When thread is omitted, load it from D1 via getEmailThreadById. mirrorMessage.mockClear() - upsertDeliveryEvent.mockClear() + upsertDeliveryEvents.mockClear() await mirrorMailboxMessageGraphFromD1({ env, db, @@ -221,27 +231,26 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve }), }), ) - expect(upsertDeliveryEvent).toHaveBeenCalledWith( - expect.objectContaining({ - event: expect.objectContaining({ - id: 'evt-1', - createdAt: '2026-07-01T12:00:01.000Z', - updatedAt: '2026-07-01T12:00:01.000Z', - }), - }), - ) - expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect(upsertDeliveryEvents).toHaveBeenCalledWith( expect.objectContaining({ - event: expect.objectContaining({ - id: 'evt-2', - createdAt: '2026-07-01T12:00:02.000Z', - updatedAt: '2026-07-01T12:00:02.000Z', - }), + ownerId: 'user-aaa', + events: [ + expect.objectContaining({ + id: 'evt-1', + createdAt: '2026-07-01T12:00:01.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + expect.objectContaining({ + id: 'evt-2', + createdAt: '2026-07-01T12:00:02.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + ], }), ) }) -test('live-mirror graph fans out event RPCs concurrently after message settles', async () => { +test('live-mirror graph uses one batch RPC after message settles, not per-event fan-out', async () => { vi.useFakeTimers() try { const { db } = await createEmailDb() @@ -255,25 +264,35 @@ test('live-mirror graph fans out event RPCs concurrently after message settles', }) } - const eventDelayMs = 400 - let maxInFlight = 0 - let inFlight = 0 + const batchDelayMs = 400 let messageSettled = false + let batchCalls = 0 const mirrorMessage = vi.fn(async () => { messageSettled = true return { ok: true as const, accepted: true } }) - const upsertDeliveryEvent = vi.fn(async () => { - expect(messageSettled).toBe(true) - inFlight += 1 - maxInFlight = Math.max(maxInFlight, inFlight) - await new Promise((resolve) => { - setTimeout(resolve, eventDelayMs) - }) - inFlight -= 1 - return { inserted: true, accepted: true, updatedLatestStatus: false } + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array<{ id: string }> }) => { + expect(messageSettled).toBe(true) + batchCalls += 1 + await new Promise((resolve) => { + setTimeout(resolve, batchDelayMs) + }) + return { + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), + } + }, + ) + const upsertDeliveryEvent = vi.fn() + const { env } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents, + upsertDeliveryEvent, }) - const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) const pending = mirrorMailboxMessageGraphFromD1({ env, @@ -282,22 +301,21 @@ test('live-mirror graph fans out event RPCs concurrently after message settles', messageId: 'msg-1', }) - // One event-delay tick completes the whole fan-out (not eventCount × delay). - await vi.advanceTimersByTimeAsync(eventDelayMs) + await vi.advanceTimersByTimeAsync(batchDelayMs) const summary = await pending expect(summary.events).toHaveLength(eventCount) expect(summary.eventsTruncated).toBe(false) - expect(maxInFlight).toBe(eventCount) - expect(upsertDeliveryEvent).toHaveBeenCalledTimes(eventCount) - // Wall budget stays near one RPC timeout for the event phase. - expect(eventDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) + expect(batchCalls).toBe(1) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvent).not.toHaveBeenCalled() + expect(batchDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) } finally { vi.useRealTimers() } }) -test('live-mirror graph sets eventsTruncated and mirrors at most max events', async () => { +test('live-mirror graph keeps newest events when truncated and sets eventsTruncated', async () => { const { db } = await createEmailDb() await seedMessage(db) @@ -312,18 +330,22 @@ test('live-mirror graph sets eventsTruncated and mirrors at most max events', as }) } - const upsertDeliveryEvent = vi.fn(async () => ({ - inserted: true, - accepted: true, - updatedLatestStatus: false, - })) + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array }) => ({ + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), + }), + ) const mirrorMessage = vi.fn(async () => ({ ok: true as const, accepted: true, })) const { env, writeDataPoint } = fakeMailboxEnv({ mirrorMessage, - upsertDeliveryEvent, + upsertDeliveryEvents, }) const summary = await mirrorMailboxMessageGraphFromD1({ @@ -335,16 +357,117 @@ test('live-mirror graph sets eventsTruncated and mirrors at most max events', as expect(summary.eventsTruncated).toBe(true) expect(summary.events).toHaveLength(mailboxLiveMirrorMaxEvents) - expect(summary.events[0]?.eventId).toBe('evt-000') + // Newest max kept: drop oldest overflow evt-000. + expect(summary.events[0]?.eventId).toBe('evt-001') expect(summary.events.at(-1)?.eventId).toBe( - `evt-${String(mailboxLiveMirrorMaxEvents - 1).padStart(3, '0')}`, + `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, + ) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + const batchEvents = upsertDeliveryEvents.mock.calls[0]?.[0]?.events ?? [] + expect(batchEvents).toHaveLength(mailboxLiveMirrorMaxEvents) + expect(batchEvents[0]?.id).toBe('evt-001') + expect(batchEvents.at(-1)?.id).toBe( + `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, ) - expect(upsertDeliveryEvent).toHaveBeenCalledTimes(mailboxLiveMirrorMaxEvents) - // 1 message + max events AE writes; never the overflow row. + // 1 message + 1 batch AE write; never per-event fan-out. expect(writeDataPoint).toHaveBeenCalledTimes( mailboxLiveMirrorMaxAnalyticsWrites, ) expect(writeDataPoint.mock.calls.length).toBeLessThan(250) + expect( + writeDataPoint.mock.calls.some( + (call) => + (call[0] as { blobs: Array }).blobs[0] === + 'mailbox_mirror:upsert_delivery_event_batch', + ), + ).toBe(true) +}) + +test('live-mirror graph maps batch timeout and error onto each event summary', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + try { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const hangBatch = vi.fn(() => new Promise(() => {})) + const { env: hangEnv, writeDataPoint: hangWrite } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents: hangBatch, + }) + + const pendingTimeout = mirrorMailboxMessageGraphFromD1({ + env: hangEnv, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + const timeoutExpectation = expect(pendingTimeout).resolves.toEqual({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [ + { eventId: 'evt-1', result: { status: 'timeout' } }, + { eventId: 'evt-2', result: { status: 'timeout' } }, + ], + eventsTruncated: false, + }) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await timeoutExpectation + expect(hangWrite).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'timeout', + expect.any(String), + ], + }), + ) + + const failingBatch = vi.fn(async () => { + throw new Error('batch exploded') + }) + const { env: failEnv, writeDataPoint: failWrite } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents: failingBatch, + }) + const failSummary = await mirrorMailboxMessageGraphFromD1({ + env: failEnv, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + expect(failSummary.events).toEqual([ + { + eventId: 'evt-1', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch exploded' }), + }, + }, + { + eventId: 'evt-2', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch exploded' }), + }, + }, + ]) + expect(failWrite).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'error', + expect.any(String), + ], + }), + ) + } finally { + vi.useRealTimers() + } }) test('live-mirror graph returns missing when D1 message is absent', async () => { diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index 062f915b8b..284f20d7fc 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -1,5 +1,6 @@ import { mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeliveryEventSnapshots, mirrorMailboxMessageSnapshot, type MailboxMirrorEnv, type MailboxMirrorResult, @@ -9,7 +10,7 @@ import { listMailboxDeliveryEventMirrorInputsForMessage, } from './mailbox-snapshot-repo.ts' import { toMailboxDeliveryEventInput } from './mailbox-snapshots.ts' -import { mailboxDefaultPageSize } from './mailbox-types.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' import { getEmailMessageById, getEmailThreadById, @@ -27,20 +28,19 @@ import { type EmailThreadRecord } from './types.ts' * Callers that just created a thread may pass it via `thread` to avoid a * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. * - * Graph event fan-out runs concurrently after the message RPC settles so event - * wall time stays near one RPC timeout (not limit × timeout). Analytics Engine - * parity writes stay under the ~250/request cap: 1 message + max events. + * Graph event mirroring uses one batch RPC after the message settles (not + * Promise.all of per-event RPCs). Analytics Engine parity writes stay small: + * 1 message + 1 batch outcome. */ /** * Max delivery events mirrored per graph call. - * `1 + mailboxLiveMirrorMaxEvents` AE writes must stay under ~250/request. + * Matches the DO batch RPC bound ({@link mailboxUpsertDeliveryEventsMax}). */ -export const mailboxLiveMirrorMaxEvents = mailboxDefaultPageSize +export const mailboxLiveMirrorMaxEvents = mailboxUpsertDeliveryEventsMax -/** AE write budget headroom check (message + event outcomes). */ -export const mailboxLiveMirrorMaxAnalyticsWrites = - 1 + mailboxLiveMirrorMaxEvents +/** AE write budget: message outcome + one batch outcome. */ +export const mailboxLiveMirrorMaxAnalyticsWrites = 2 export type MailboxLiveMirrorEnv = MailboxMirrorEnv @@ -114,16 +114,14 @@ export async function mirrorMailboxDeliveryEventFromD1(input: { /** * Load the authoritative D1 message graph and best-effort mirror it: - * message (+ optional caller-supplied thread, attachments) first, then all - * delivery events concurrently (cohesive projection; `created_at` as - * `sourceMutationAt`). + * message (+ optional caller-supplied thread, attachments) first, then a + * bounded batch of delivery events (cohesive projection; `created_at` as + * `sourceMutationAt`) via one RPC after the message settles. * - * Never throws. Returns a bounded structured summary. Queries `max+1` events - * so truncation is explicit; mirrors at most {@link mailboxLiveMirrorMaxEvents}. + * Never throws. Returns a bounded structured summary. Queries newest + * `max+1` events so truncation is explicit; mirrors at most + * {@link mailboxLiveMirrorMaxEvents} in chronological order. * - * @param sourceMutationAt - Optional caller mutation timestamp for context; - * D1 row timestamps remain authoritative. Delivery events always use - * `created_at` for immutable outbound/provider inserts. * @param thread - Optional thread from the caller. When omitted, loads via * `getEmailThreadById` when `message.threadId` is set. Explicit `null` * skips the thread snapshot. @@ -133,12 +131,8 @@ export async function mirrorMailboxMessageGraphFromD1(input: { db: D1Database userId: string messageId: string - sourceMutationAt?: string thread?: EmailThreadRecord | null }): Promise { - // Caller mutation timestamp is accepted for API symmetry with dual-write - // sites; D1 row timestamps remain authoritative for the loaded graph. - void input.sourceMutationAt try { const message = await getEmailMessageById({ db: input.db, @@ -163,7 +157,7 @@ export async function mirrorMailboxMessageGraphFromD1(input: { }) } - // Message settles before any event RPC starts (ordering guarantee). + // Message settles before the event-batch RPC starts (ordering guarantee). const messageResult = await mirrorMailboxMessageSnapshot({ env: input.env, thread, @@ -171,6 +165,8 @@ export async function mirrorMailboxMessageGraphFromD1(input: { attachments, }) + // Newest max+1 from D1 (chrono-restored). When truncated, keep the + // trailing newest max — drop the oldest overflow row at index 0. const loadedEvents = await listMailboxDeliveryEventMirrorInputsForMessage({ db: input.db, ownerId: input.userId, @@ -179,20 +175,17 @@ export async function mirrorMailboxMessageGraphFromD1(input: { }) const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents const eventInputs = eventsTruncated - ? loadedEvents.slice(0, mailboxLiveMirrorMaxEvents) + ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) : loadedEvents - // Concurrent fan-out: event-phase wall time ≈ one RPC timeout, not N×. - const events = await Promise.all( - eventInputs.map(async (event) => ({ - eventId: event.id, - result: await mirrorMailboxDeliveryEventSnapshot({ - env: input.env, - ownerId: input.userId, - event, - }), - })), - ) + const events = + eventInputs.length === 0 + ? [] + : await mirrorMailboxDeliveryEventSnapshots({ + env: input.env, + ownerId: input.userId, + events: eventInputs, + }) return { messageId: input.messageId, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 0ede393f94..29a6bae1e6 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -4,6 +4,7 @@ import { systemEmailOwnerId } from './email-owner.ts' import { mailboxMirrorRpcTimeoutMs, mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeliveryEventSnapshots, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, mirrorMailboxDeleteThreadIfEmpty, @@ -448,3 +449,159 @@ test('mailbox mirror RPC timeout returns timeout and late rejection is not unhan vi.useRealTimers() } }) + +function baseDeliveryEventInput( + overrides?: Partial, +): MailboxDeliveryEventInput { + return { + id: 'evt-1', + messageId: 'msg-1', + inboxId: 'inbox-1', + eventType: 'sent', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + ...overrides, + } +} + +test('mailbox mirror batch helper uses one RPC, one telemetry outcome, and maps per-event results', async () => { + consoleWarn.mockImplementation(() => {}) + const events = [ + baseDeliveryEventInput({ id: 'evt-a' }), + baseDeliveryEventInput({ + id: 'evt-b', + createdAt: '2026-07-02T10:00:01.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }), + ] + const upsertDeliveryEvents = vi.fn(async () => ({ + results: [ + { eventId: 'evt-a', inserted: true, accepted: true }, + { eventId: 'evt-b', inserted: false, accepted: false }, + ], + })) + const { env, writeDataPoint } = fakeMailboxEnv({ upsertDeliveryEvents }) + + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events, + }), + ).resolves.toEqual([ + { eventId: 'evt-a', result: { status: 'mirrored' } }, + { eventId: 'evt-b', result: { status: 'stale' } }, + ]) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvents).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + events, + }) + expect(writeDataPoint).toHaveBeenCalledTimes(1) + expect(writeDataPoint).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'stale', + expect.any(String), + ], + }), + ) + + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events: [], + }), + ).resolves.toEqual([]) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) +}) + +test('mailbox mirror batch helper maps timeout and error to every event', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + try { + const events = [ + baseDeliveryEventInput({ id: 'evt-1' }), + baseDeliveryEventInput({ id: 'evt-2' }), + ] + const hang = vi.fn(() => new Promise(() => {})) + const { env: hangEnv, writeDataPoint: hangWrite } = fakeMailboxEnv({ + upsertDeliveryEvents: hang, + }) + const pending = mirrorMailboxDeliveryEventSnapshots({ + env: hangEnv, + ownerId: 'user-aaa', + events, + }) + const expectation = expect(pending).resolves.toEqual([ + { eventId: 'evt-1', result: { status: 'timeout' } }, + { eventId: 'evt-2', result: { status: 'timeout' } }, + ]) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await expectation + expect(hangWrite).toHaveBeenCalledTimes(1) + + const fail = vi.fn(async () => { + throw new Error('batch unavailable') + }) + const { env: failEnv } = fakeMailboxEnv({ upsertDeliveryEvents: fail }) + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env: failEnv, + ownerId: 'user-aaa', + events, + }), + ).resolves.toEqual([ + { + eventId: 'evt-1', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch unavailable' }), + }, + }, + { + eventId: 'evt-2', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch unavailable' }), + }, + }, + ]) + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index 4a1c2b6bd3..b3869dd206 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -31,6 +31,7 @@ import { type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, + type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' /** @@ -68,6 +69,11 @@ export type MailboxMirrorResult = | { status: 'skipped'; reason: MailboxMirrorSkipReason } | { status: 'error'; error: unknown } +export type MailboxMirrorDeliveryEventBatchItemResult = { + eventId: string + result: MailboxMirrorResult +} + function resolveMirrorOwner( ownerId: string | null | undefined, ): { ok: true; ownerId: string } | { ok: false; result: MailboxMirrorResult } { @@ -169,6 +175,14 @@ function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { return accepted ? { status: 'mirrored' } : { status: 'stale' } } +function outcomeFromBatchAccepted( + results: MailboxUpsertDeliveryEventsResult['results'], +): MailboxMirrorResult { + return results.every((item) => item.accepted) + ? { status: 'mirrored' } + : { status: 'stale' } +} + /** * Race a DO RPC against {@link mailboxMirrorRpcTimeoutMs}. * The settled wrapper ensures a late rejection cannot become unhandled. @@ -203,32 +217,51 @@ export async function awaitMailboxMirrorRpc( } } -async function runMirror(input: { +/** + * Shared owner/config/timeout/telemetry pipeline for single-result mirror RPCs. + * Thin wrappers supply only the RPC call and result mapping. + */ +async function executeMailboxMirrorRpc(input: { tag: string env: MailboxMirrorEnv - userId: string | null | undefined + ownerId: string | null | undefined operation: MailboxMirrorOperation - run: () => Promise + call: (ownerId: string) => Promise + toResult: (value: T) => MailboxMirrorResult }): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) { + recordMirrorOutcome(input.env, input.ownerId, input.operation, owner.result) + return owner.result + } + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) { + recordMirrorOutcome( + input.env, + owner.ownerId, + input.operation, + skippedMailbox, + ) + return skippedMailbox + } + let result: MailboxMirrorResult try { - result = await input.run() + const raced = await awaitMailboxMirrorRpc(input.call(owner.ownerId)) + result = raced.ok ? input.toResult(raced.value) : { status: 'timeout' } } catch (error) { console.warn(input.tag, error) result = { status: 'error', error } } - recordMirrorOutcome(input.env, input.userId, input.operation, result) + recordMirrorOutcome(input.env, owner.ownerId, input.operation, result) return result } -function finishEarly(input: { - env: MailboxMirrorEnv - userId: string | null | undefined - operation: MailboxMirrorOperation - result: MailboxMirrorResult -}): MailboxMirrorResult { - recordMirrorOutcome(input.env, input.userId, input.operation, input.result) - return input.result +function mapUniformBatchResults( + events: Array<{ id: string }>, + result: MailboxMirrorResult, +): Array { + return events.map((event) => ({ eventId: event.id, result })) } /** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ @@ -238,46 +271,19 @@ export async function mirrorMailboxMessageSnapshot(input: { message: EmailMessageRecord attachments?: Array }): Promise { - const operation = 'mirror_message' as const - const owner = resolveMirrorOwner(input.message.userId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.message.userId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-message-failed', env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).mirrorMessage({ - ownerId: owner.ownerId, - thread: input.thread ? toMailboxThreadInput(input.thread) : null, - message: toMailboxMessageInput(input.message), - attachments: input.attachments?.map(toMailboxAttachmentInput), - }), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromAccepted(raced.value.accepted) - }, + ownerId: input.message.userId, + operation: 'mirror_message', + call: (ownerId) => + mailboxRpc({ env: input.env, userId: ownerId }).mirrorMessage({ + ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }), + toResult: (value) => outcomeFromAccepted(value.accepted), }) } @@ -298,88 +304,101 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { deliveryStatusAt: string } | null }): Promise { - const operation = 'upsert_delivery_event' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delivery-event-failed', env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).upsertDeliveryEvent({ - ownerId: owner.ownerId, - event: input.event, - latestDeliveryStatus: input.latestDeliveryStatus, - }), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromAccepted(raced.value.accepted) - }, + ownerId: input.ownerId, + operation: 'upsert_delivery_event', + call: (ownerId) => + mailboxRpc({ env: input.env, userId: ownerId }).upsertDeliveryEvent({ + ownerId, + event: input.event, + latestDeliveryStatus: input.latestDeliveryStatus, + }), + toResult: (value) => outcomeFromAccepted(value.accepted), }) } -/** Best-effort thread touch mirror. */ -export async function mirrorMailboxTouchThread( - input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, -): Promise { - const operation = 'touch_thread' as const +/** + * Best-effort batch delivery-event snapshot mirror. + * + * One 1s timeout and one `upsert_delivery_event_batch` telemetry outcome. + * Timeout/error/skip apply uniformly to each per-event summary entry. + * Empty `events` returns `[]` without RPC or telemetry. + */ +export async function mirrorMailboxDeliveryEventSnapshots(input: { + env: MailboxMirrorEnv + ownerId: string + events: Array +}): Promise> { + const operation = 'upsert_delivery_event_batch' as const + if (input.events.length === 0) return [] + const owner = resolveMirrorOwner(input.ownerId) if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) + recordMirrorOutcome(input.env, input.ownerId, operation, owner.result) + return mapUniformBatchResults(input.events, owner.result) } const skippedMailbox = skipIfMailboxMissing(input.env) if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, + recordMirrorOutcome(input.env, owner.ownerId, operation, skippedMailbox) + return mapUniformBatchResults(input.events, skippedMailbox) + } + + try { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvents({ + ownerId: owner.ownerId, + events: input.events, + }), + ) + if (!raced.ok) { + const result = { status: 'timeout' as const } + recordMirrorOutcome(input.env, owner.ownerId, operation, result) + return mapUniformBatchResults(input.events, result) + } + recordMirrorOutcome( + input.env, + owner.ownerId, operation, - result: skippedMailbox, + outcomeFromBatchAccepted(raced.value.results), + ) + const byEventId = new Map( + raced.value.results.map((item) => [item.eventId, item] as const), + ) + return input.events.map((event) => { + const item = byEventId.get(event.id) + return { + eventId: event.id, + result: item + ? outcomeFromAccepted(item.accepted) + : { status: 'missing' as const }, + } }) + } catch (error) { + console.warn('mailbox-mirror-delivery-event-batch-failed', error) + const result = { status: 'error' as const, error } + recordMirrorOutcome(input.env, owner.ownerId, operation, result) + return mapUniformBatchResults(input.events, result) } +} - return runMirror({ +/** Best-effort thread touch mirror. */ +export async function mirrorMailboxTouchThread( + input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, +): Promise { + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-touch-thread-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).touchThread(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'touch_thread', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).touchThread(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -387,42 +406,15 @@ export async function mirrorMailboxTouchThread( export async function mirrorMailboxUpdateMessageDelivery( input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'update_message_delivery' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-update-message-delivery-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).updateMessageDelivery(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'update_message_delivery', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).updateMessageDelivery(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -430,42 +422,15 @@ export async function mirrorMailboxUpdateMessageDelivery( export async function mirrorMailboxSetMessageClassification( input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'set_message_classification' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-set-message-classification-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).setMessageClassification(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'set_message_classification', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).setMessageClassification(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -473,42 +438,15 @@ export async function mirrorMailboxSetMessageClassification( export async function mirrorMailboxDeleteMessageMetadata( input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_message_metadata' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-message-metadata-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteMessageMetadata(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_message_metadata', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteMessageMetadata(rpcInput), + toResult: outcomeFromDeleteResult, }) } @@ -516,42 +454,15 @@ export async function mirrorMailboxDeleteMessageMetadata( export async function mirrorMailboxDeleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_delivery_event' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-delivery-event-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteDeliveryEvent(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_delivery_event', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteDeliveryEvent(rpcInput), + toResult: outcomeFromDeleteResult, }) } @@ -559,41 +470,14 @@ export async function mirrorMailboxDeleteDeliveryEvent( export async function mirrorMailboxDeleteThreadIfEmpty( input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_thread_if_empty' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-thread-if-empty-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteThreadIfEmpty(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_thread_if_empty', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteThreadIfEmpty(rpcInput), + toResult: outcomeFromDeleteResult, }) } diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 70cebc6313..0db8924499 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -53,6 +53,13 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = outcome: 'error', timestamp: '2026-07-30T10:09:00.000Z', }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'upsert_delivery_event_batch', + outcome: 'timeout', + timestamp: '2026-07-30T10:10:00.000Z', + }) recordMailboxParityEvent(env, { userId: 'user-2', category: 'parity', @@ -70,7 +77,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = timestamp: '2026-07-30T12:00:00.000Z', }) - expect(writeDataPoint).toHaveBeenCalledTimes(8) + expect(writeDataPoint).toHaveBeenCalledTimes(9) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], blobs: [ @@ -122,6 +129,15 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(7, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'timeout', + '2026-07-30T10:10:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(8, { indexes: ['user-2'], blobs: [ 'mailbox_parity:compare_messages', @@ -130,7 +146,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = ], doubles: [1, 2], }) - expect(writeDataPoint).toHaveBeenNthCalledWith(8, { + expect(writeDataPoint).toHaveBeenNthCalledWith(9, { indexes: ['user-3'], blobs: [ 'mailbox_parity:compare_threads', diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index bb2c564770..fde129a0ac 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -7,6 +7,7 @@ export type MailboxParityEnv = { export type MailboxMirrorOperation = | 'mirror_message' | 'upsert_delivery_event' + | 'upsert_delivery_event_batch' | 'touch_thread' | 'update_message_delivery' | 'set_message_classification' diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts index 2357861335..d79608b39f 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -172,4 +172,13 @@ test('mailbox-snapshot-repo loads complete delivery-event projection and ready M updatedAt: '2026-07-02T10:05:00.000Z', }), ]) + + // Newest-N selection: limit 1 returns the newest event, chrono-restored. + const newestOnly = await listMailboxDeliveryEventMirrorInputsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 1, + }) + expect(newestOnly.map((event) => event.id)).toEqual(['evt-2']) }) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 9d732c35a3..03d69a8fd8 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -128,8 +128,11 @@ export async function getMailboxDeliveryEventMirrorInput(input: { } /** - * Load complete delivery-event mirror projections for one message, oldest - * first, bounded by `limit`. Scoped by owner. + * Load complete delivery-event mirror projections for one message. + * + * Selects the newest `limit` rows (`ORDER BY created_at DESC, id DESC`) so + * truncation keeps recent events, then restores chronological order for + * insertion. Scoped by owner. */ export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { db: D1Database @@ -142,12 +145,13 @@ export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { `${deliveryEventMirrorSelect} WHERE message_id = ? AND user_id = ? - ORDER BY created_at ASC, id ASC + ORDER BY created_at DESC, id DESC LIMIT ?`, ) .bind(input.messageId, input.ownerId, input.limit) .all() - return (result.results ?? []).map(mapDeliveryEventMirrorRow) + const newestFirst = (result.results ?? []).map(mapDeliveryEventMirrorRow) + return newestFirst.reverse() } /** diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 5855ebce86..f506665e2d 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -20,6 +20,8 @@ export const mailboxMessageRetentionDays = 365 export const mailboxDeliveryEventRetentionDays = 90 export const mailboxDefaultPageSize = 100 export const mailboxMaxPageSize = 500 +/** Max complete delivery-event snapshots per `upsertDeliveryEvents` RPC. */ +export const mailboxUpsertDeliveryEventsMax = mailboxDefaultPageSize export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ @@ -434,6 +436,17 @@ export type MailboxDeleteResult = | { status: 'missing' } | { status: 'stale' } +/** Per-event outcome from {@link MailboxRpc.upsertDeliveryEvents}. */ +export type MailboxUpsertDeliveryEventBatchItemResult = { + eventId: string + inserted: boolean + accepted: boolean +} + +export type MailboxUpsertDeliveryEventsResult = { + results: Array +} + export type MailboxRpc = { mirrorMessage: (input: { /** @@ -463,6 +476,15 @@ export type MailboxRpc = { accepted: boolean updatedLatestStatus: boolean }> + /** + * Upsert a bounded batch of complete immutable delivery-event snapshots in + * one transaction. Does not patch message latest delivery status — that + * remains from the prior full message snapshot. + */ + upsertDeliveryEvents: (input: { + ownerId: string + events: Array + }) => Promise touchThread: ( input: MailboxTouchThreadInput, ) => Promise diff --git a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts new file mode 100644 index 0000000000..c93610214b --- /dev/null +++ b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts @@ -0,0 +1,277 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { http, HttpResponse } from 'msw' +import { bytesToBase64 } from '@kody-internal/shared/base64.ts' +import { ensureEmailTestSchema } from './test-schema.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { rpcFor } from './mailbox-test-helpers.ts' +import { getEmailMessageById } from './repo.ts' +import { sendOutboundEmail } from './outbound.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createMswNodeServer } from '#worker/test-support/msw-node-server.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' + +const cloudflareEmailApi = + 'https://api.cloudflare.test/client/v4/accounts/account-123/email/sending/send' + +const platformBaseUrl = 'https://kody.example.com' +// User mail lives on the inbox. subdomain derived from APP_BASE_URL. +const platformDomain = 'inbox.kody.example.com' + +/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createMailboxStubEnv(input: { + mode: 'throw' | 'hang' + base?: typeof env +}) { + const base = input.base ?? env + const hang = () => new Promise(() => {}) + const fail = async () => { + throw new Error('simulated mailbox failure') + } + const method = input.mode === 'hang' ? hang : fail + const stub = { + mirrorMessage: method, + upsertDeliveryEvent: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + return { + ...base, + APP_BASE_URL: platformBaseUrl, + MAILBOX: { + idFromName: (name: string) => base.MAILBOX.idFromName(name), + get: () => stub, + } as unknown as DurableObjectNamespace, + } +} + +async function seedVerifiedAccount(input: { email: string }) { + const username = `sender-${crypto.randomUUID().slice(0, 8)}` + const stableUserId = await createStableUserIdFromEmail(input.email) + await env.APP_DB.prepare( + `INSERT INTO users (username, email, password_hash, email_verified_at, plan, stable_user_id) + VALUES (?, ?, ?, ?, 'max', ?)`, + ) + .bind( + username, + input.email, + 'test-password-hash', + new Date().toISOString(), + stableUserId, + ) + .run() + return { username, from: `${username}@${platformDomain}` } +} + +test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) + const sendEnv = { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-sent' } + }, + }, + } + + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror sent', + text: 'Mirrored body', + attachments: [ + { + filename: 'note.pdf', + contentType: 'application/pdf', + contentBase64: bytesToBase64(pdfBytes), + }, + ], + }) + + expect(result.status).toBe('sent') + expect(result.message.threadId).toBeTruthy() + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'sent', + providerMessageId: 'provider-mailbox-sent', + subject: 'Mailbox mirror sent', + threadId: result.message.threadId, + }) + const mirroredThread = await mailbox.getThread({ + threadId: result.message.threadId!, + }) + expect(mirroredThread).toMatchObject({ + id: result.message.threadId, + subjectNormalized: 'mailbox mirror sent', + }) + const mirroredAttachments = await mailbox.listAttachmentsForMessage({ + messageId: result.message.id, + }) + expect(mirroredAttachments).toEqual([ + expect.objectContaining({ + filename: 'note.pdf', + contentType: 'application/pdf', + size: pdfBytes.byteLength, + storageKind: 'external', + }), + ]) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'send_requested', + 'sent', + ]) +}, 30_000) + +test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { + silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + + using _server = createMswNodeServer( + [ + http.post(cloudflareEmailApi, () => + HttpResponse.json( + { success: false, errors: [{ message: 'provider down' }] }, + { status: 500 }, + ), + ), + ], + { onUnhandledRequest: 'bypass' }, + ) + + const result = await sendOutboundEmail({ + env: { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: undefined as unknown as SendEmail, + CLOUDFLARE_ACCOUNT_ID: 'account-123', + CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', + CLOUDFLARE_API_TOKEN: 'token-123', + }, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror failed', + text: 'Body', + }) + + expect(result.status).toBe('failed') + expect(result.error).toBe('provider down') + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'failed', + error: 'provider down', + threadId: result.message.threadId, + }) + expect( + await mailbox.getThread({ threadId: result.message.threadId! }), + ).toMatchObject({ id: result.message.threadId }) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'failed', + 'send_requested', + ]) +}, 30_000) + +test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-live-mirror-message-graph-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + + { + const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'throw' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-err' } + }, + }, + } + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox throw ignored', + text: 'Body', + }) + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-err', + error: null, + }) + const stored = await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: result.message.id, + }) + expect(stored?.processingStatus).toBe('sent') + } + + { + const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'hang' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-hang' } + }, + }, + } + const startedAt = Date.now() + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox hang ignored', + text: 'Body', + }) + const elapsedMs = Date.now() - startedAt + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-hang', + error: null, + }) + // Message + send_requested + sent each race the mirror RPC bound. + expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) + } +}, 30_000) diff --git a/packages/worker/src/email/outbound.workers.test.ts b/packages/worker/src/email/outbound.workers.test.ts index a1be15b7f7..3f87a19883 100644 --- a/packages/worker/src/email/outbound.workers.test.ts +++ b/packages/worker/src/email/outbound.workers.test.ts @@ -6,8 +6,6 @@ import { bytesToBase64 } from '@kody-internal/shared/base64.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' import { ensureEmailTestSchema } from './test-schema.ts' -import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' -import { rpcFor } from './mailbox-test-helpers.ts' import { deleteEmailMessageById, getEmailMessageById, @@ -50,37 +48,6 @@ function createBindingSendEnv() { } } -/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ -function createMailboxStubEnv(input: { - mode: 'throw' | 'hang' - base?: typeof env -}) { - const base = input.base ?? env - const hang = () => new Promise(() => {}) - const fail = async () => { - throw new Error('simulated mailbox failure') - } - const method = input.mode === 'hang' ? hang : fail - const stub = { - mirrorMessage: method, - upsertDeliveryEvent: method, - touchThread: method, - updateMessageDelivery: method, - setMessageClassification: method, - deleteMessageMetadata: method, - deleteDeliveryEvent: method, - deleteThreadIfEmpty: method, - } - return { - ...base, - APP_BASE_URL: platformBaseUrl, - MAILBOX: { - idFromName: (name: string) => base.MAILBOX.idFromName(name), - get: () => stub, - } as unknown as DurableObjectNamespace, - } -} - async function seedVerifiedAccount(input: { email: string username?: string @@ -999,212 +966,3 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async maxPlanEmailLimits.email_sends_per_day, ) }, 30_000) - -test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { - silenceIncidentalRuntimeWarnings() - await ensureEmailTestSchema(env.APP_DB) - const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) - const sendEnv = { - ...env, - APP_BASE_URL: platformBaseUrl, - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-sent' } - }, - }, - } - - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox mirror sent', - text: 'Mirrored body', - attachments: [ - { - filename: 'note.pdf', - contentType: 'application/pdf', - contentBase64: bytesToBase64(pdfBytes), - }, - ], - }) - - expect(result.status).toBe('sent') - expect(result.message.threadId).toBeTruthy() - const mailbox = rpcFor(userId) - const mirroredMessage = await mailbox.getMessage({ - messageId: result.message.id, - }) - expect(mirroredMessage).toMatchObject({ - id: result.message.id, - direction: 'outbound', - processingStatus: 'sent', - providerMessageId: 'provider-mailbox-sent', - subject: 'Mailbox mirror sent', - threadId: result.message.threadId, - }) - const mirroredThread = await mailbox.getThread({ - threadId: result.message.threadId!, - }) - expect(mirroredThread).toMatchObject({ - id: result.message.threadId, - subjectNormalized: 'mailbox mirror sent', - }) - const mirroredAttachments = await mailbox.listAttachmentsForMessage({ - messageId: result.message.id, - }) - expect(mirroredAttachments).toEqual([ - expect.objectContaining({ - filename: 'note.pdf', - contentType: 'application/pdf', - size: pdfBytes.byteLength, - storageKind: 'external', - }), - ]) - const mirroredEvents = await mailbox.listDeliveryEvents({ - messageId: result.message.id, - limit: 10, - }) - expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ - 'send_requested', - 'sent', - ]) -}, 30_000) - -test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { - silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) - await ensureEmailTestSchema(env.APP_DB) - const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - - using _server = createMswNodeServer( - [ - http.post(cloudflareEmailApi, () => - HttpResponse.json( - { success: false, errors: [{ message: 'provider down' }] }, - { status: 500 }, - ), - ), - ], - { onUnhandledRequest: 'bypass' }, - ) - - const result = await sendOutboundEmail({ - env: { - ...env, - APP_BASE_URL: platformBaseUrl, - EMAIL: undefined as unknown as SendEmail, - CLOUDFLARE_ACCOUNT_ID: 'account-123', - CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', - CLOUDFLARE_API_TOKEN: 'token-123', - }, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox mirror failed', - text: 'Body', - }) - - expect(result.status).toBe('failed') - expect(result.error).toBe('provider down') - const mailbox = rpcFor(userId) - const mirroredMessage = await mailbox.getMessage({ - messageId: result.message.id, - }) - expect(mirroredMessage).toMatchObject({ - id: result.message.id, - direction: 'outbound', - processingStatus: 'failed', - error: 'provider down', - threadId: result.message.threadId, - }) - expect( - await mailbox.getThread({ threadId: result.message.threadId! }), - ).toMatchObject({ id: result.message.threadId }) - const mirroredEvents = await mailbox.listDeliveryEvents({ - messageId: result.message.id, - limit: 10, - }) - expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ - 'failed', - 'send_requested', - ]) -}, 30_000) - -test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { - silenceIncidentalRuntimeWarnings([ - 'mailbox-mirror-message-failed', - 'mailbox-mirror-delivery-event-failed', - 'mailbox-live-mirror-message-graph-failed', - ]) - await ensureEmailTestSchema(env.APP_DB) - - { - const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const sendEnv = { - ...createMailboxStubEnv({ mode: 'throw' }), - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-err' } - }, - }, - } - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox throw ignored', - text: 'Body', - }) - expect(result).toMatchObject({ - status: 'sent', - providerMessageId: 'provider-mailbox-err', - error: null, - }) - const stored = await getEmailMessageById({ - db: env.APP_DB, - userId, - messageId: result.message.id, - }) - expect(stored?.processingStatus).toBe('sent') - } - - { - const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const sendEnv = { - ...createMailboxStubEnv({ mode: 'hang' }), - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-hang' } - }, - }, - } - const startedAt = Date.now() - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox hang ignored', - text: 'Body', - }) - const elapsedMs = Date.now() - startedAt - expect(result).toMatchObject({ - status: 'sent', - providerMessageId: 'provider-mailbox-hang', - error: null, - }) - // Message + send_requested + sent each race the mirror RPC bound. - expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) - } -}, 30_000) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index c7fc822a29..0327befafc 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -855,11 +855,13 @@ export async function getEmailMessageById(input: { } /** - * Reclassify a stored inbound message (spam <-> not spam). Reclassifying to + * D1-only inbound classification update. Prefer + * `setEmailMessageClassification` in `service.ts`, which dual-writes the + * Mailbox graph mirror after a successful mutation. Reclassifying to * `accepted` does not retroactively dispatch package subscription events; the * receive-time classification decides dispatch exactly once. */ -export async function setEmailMessageClassification(input: { +export async function updateEmailMessageClassificationInD1(input: { db: D1Database userId: string messageId: string diff --git a/packages/worker/src/email/service-classification.node.test.ts b/packages/worker/src/email/service-classification.node.test.ts new file mode 100644 index 0000000000..bc53f84752 --- /dev/null +++ b/packages/worker/src/email/service-classification.node.test.ts @@ -0,0 +1,119 @@ +import { expect, test, vi } from 'vitest' +import type * as EmailRepo from './repo.ts' + +const mocks = vi.hoisted(() => ({ + updateEmailMessageClassificationInD1: vi.fn(), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + })), +})) + +vi.mock('./repo.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + updateEmailMessageClassificationInD1: + mocks.updateEmailMessageClassificationInD1, + } +}) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + +const { setEmailMessageClassification } = await import('./service.ts') + +function createEnv() { + return { + APP_DB: {} as D1Database, + } as Env +} + +test('setEmailMessageClassification mutates D1 then mirrors only on success', async () => { + const env = createEnv() + const callOrder: Array = [] + mocks.updateEmailMessageClassificationInD1.mockImplementation(async () => { + callOrder.push('d1') + return true + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + callOrder.push('mirror') + return { + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + } + }) + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + }), + ).resolves.toBe(true) + + expect(callOrder).toEqual(['d1', 'mirror']) + expect(mocks.updateEmailMessageClassificationInD1).toHaveBeenCalledWith({ + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + now: undefined, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) +}) + +test('setEmailMessageClassification skips mirror when D1 updates nothing', async () => { + const env = createEnv() + mocks.updateEmailMessageClassificationInD1.mockResolvedValueOnce(false) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'missing', + classification: 'accepted', + classificationReason: null, + }), + ).resolves.toBe(false) + + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() +}) + +test('setEmailMessageClassification still returns true when mirror reports error', async () => { + const env = createEnv() + mocks.updateEmailMessageClassificationInD1.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'message-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + }), + ).resolves.toBe(true) +}) diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index d10de66ed1..2dc1d41770 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -9,6 +9,10 @@ import { markInboundDeliveryReceived, type InboundDelivery, } from './inbound-delivery.ts' +import { + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' import { recordEmailReportingEvent, type EmailReportingEnv, @@ -26,6 +30,7 @@ import { insertEmailMessage, listEmailAttachmentsForMessage, touchEmailThread, + updateEmailMessageClassificationInD1, } from './repo.ts' import { evaluateEmailSenderRules } from './sender-rules.ts' import { @@ -702,6 +707,40 @@ export async function recordBoundedEmailRejectionEvent(input: { return rejectionsToday } +/** + * Reclassify a stored inbound message and, on D1 success only, best-effort + * mirror the full message graph into Mailbox. Returns the D1 mutation boolean; + * mirror failures never throw or change that response. + */ +export async function setEmailMessageClassification(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + messageId: string + classification: EmailClassification + classificationReason?: string | null + now?: string +}) { + const updated = await updateEmailMessageClassificationInD1({ + db: input.db, + userId: input.userId, + messageId: input.messageId, + classification: input.classification, + classificationReason: input.classificationReason, + now: input.now, + }) + if (!updated) { + return false + } + await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.db, + userId: input.userId, + messageId: input.messageId, + }) + return true +} + export async function recordProviderEmailDeliveryEvent(input: { db: D1Database reportingEnv?: EmailReportingEnv diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts index 35f2299626..6df5845f83 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts @@ -3,21 +3,12 @@ import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ setEmailMessageClassification: vi.fn(), - mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ - messageId: 'message-1', - message: { status: 'mirrored' }, - events: [], - })), })) -vi.mock('#worker/email/repo.ts', () => ({ +vi.mock('#worker/email/service.ts', () => ({ setEmailMessageClassification: mocks.setEmailMessageClassification, })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, -})) - const { emailMessageClassifyCapability } = await import('./email-message-classify.ts') @@ -59,21 +50,15 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'quarantined', }) expect(mocks.setEmailMessageClassification).toHaveBeenCalledWith({ + env: context.env, db: context.env.APP_DB, userId: 'user-1', messageId: 'message-1', classification: 'quarantined', classificationReason: 'Reclassified by user.', }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env: context.env, - db: context.env.APP_DB, - userId: 'user-1', - messageId: 'message-1', - }) mocks.setEmailMessageClassification.mockResolvedValueOnce(true) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'message-1', classification: 'accepted' }, @@ -84,45 +69,19 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'accepted', }) expect(mocks.setEmailMessageClassification).toHaveBeenLastCalledWith({ + env: context.env, db: context.env.APP_DB, userId: 'user-1', messageId: 'message-1', classification: 'accepted', classificationReason: null, }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env: context.env, - db: context.env.APP_DB, - userId: 'user-1', - messageId: 'message-1', - }) mocks.setEmailMessageClassification.mockResolvedValueOnce(false) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'missing', classification: 'quarantined' }, context, ), ).rejects.toThrow('Email message not found: missing') - expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() - - mocks.setEmailMessageClassification.mockResolvedValueOnce(true) - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'message-1', - message: { - status: 'error', - error: new Error('mailbox mirror timed out'), - }, - events: [], - }) - await expect( - emailMessageClassifyCapability.handler( - { message_id: 'message-1', classification: 'quarantined' }, - context, - ), - ).resolves.toEqual({ - message_id: 'message-1', - classification: 'quarantined', - }) }) diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts index 1a9562ddf9..08ec83f2bb 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts @@ -1,8 +1,7 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' -import { setEmailMessageClassification } from '#worker/email/repo.ts' +import { setEmailMessageClassification } from '#worker/email/service.ts' import { emailClassificationValues } from '#worker/email/types.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' @@ -36,6 +35,7 @@ export const emailMessageClassifyCapability = defineDomainCapability( const classificationReason = args.classification === 'quarantined' ? 'Reclassified by user.' : null const updated = await setEmailMessageClassification({ + env: ctx.env, db: ctx.env.APP_DB, userId: user.userId, messageId: args.message_id, @@ -45,12 +45,6 @@ export const emailMessageClassifyCapability = defineDomainCapability( if (!updated) { throw new Error(`Email message not found: ${args.message_id}`) } - await mirrorMailboxMessageGraphFromD1({ - env: ctx.env, - db: ctx.env.APP_DB, - userId: user.userId, - messageId: args.message_id, - }) return { message_id: args.message_id, classification: args.classification, From d747c2d4c06d3177fc476892d39dd710d2b1c71f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:05:16 +0000 Subject: [PATCH 09/20] fix(email): repair persisted provider mirror gaps Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 74 +++++---- .../src/email/delivery-queue.node.test.ts | 153 +++++++++++++++++- packages/worker/src/email/delivery-queue.ts | 28 ++-- .../email/mailbox-live-mirror.node.test.ts | 49 +++++- .../worker/src/email/mailbox-live-mirror.ts | 42 +++-- packages/worker/src/email/mailbox-types.ts | 2 +- .../outbound-mailbox-mirror.workers.test.ts | 2 + 7 files changed, 284 insertions(+), 66 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 3ec77dc79e..07765dacc0 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -219,11 +219,11 @@ Durable Object export behavior: delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires - non-inbound live dual-write for outbound terminals, provider-recorded delivery - events, and user classification (`service.ts#setEmailMessageClassification`) - without changing D1 authority — D1 `email_*` rows remain the live source of - truth and are still exported in the `d1` section. See - [Mailbox](#durable-objects-mailbox). + non-inbound live dual-write for outbound terminals, provider delivery-queue + graph repair, and user classification + (`service.ts#setEmailMessageClassification`) without changing D1 authority — + D1 `email_*` rows remain the live source of truth and are still exported in + the `d1` section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -764,21 +764,26 @@ owner-bound `upsertDeliveryEvents` batch RPC (not concurrent per-event RPCs to the same DO). Event load queries newest `max+1` rows from D1, restores chronological order, and when truncated keeps the newest `mailboxLiveMirrorMaxEvents` (`mailboxUpsertDeliveryEventsMax`, 100) — dropping -oldest overflow. The batch RPC shares one 1s timeout and one -`mailbox_mirror:upsert_delivery_event_batch` telemetry outcome (timeout/error -apply uniformly to per-event summary entries). Analytics Engine budget per graph -call is two writes (1 message outcome + 1 batch outcome). Never throws; returns -a bounded summary. **Live callers (non-inbound phase-2 slice):** +oldest overflow — with a stable warning (`mailbox-live-mirror-events-truncated`; +`userId`, `messageId`, `loaded`, `max`). The batch RPC shares one 1s timeout and +one `mailbox_mirror:upsert_delivery_event_batch` telemetry outcome +(timeout/error apply uniformly to per-event summary entries). Each graph attempt +emits at most two Analytics Engine writes (1 message outcome + 1 batch outcome). +Never throws; returns a bounded summary. **Live callers (non-inbound phase-2 +slice):** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full message/thread/attachment/event graph. Passes a just-created thread when present; otherwise the orchestrator loads it from D1. Failures never affect send/refund. -- **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue - ingestion persists a new provider event (`recorded`), schedules - `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not - blocked; full graph repair via message + batch event upsert. +- **Provider delivery queue** (`delivery-queue.ts`) — when queue ingestion + resolves to `recorded`, `duplicate`, or `stale` with a bound message, + schedules `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack + is not blocked; full graph repair via message + batch event upsert. `recorded` + and `duplicate` still run subscription dispatch first; `stale` skips + subscription dispatch (abuse pause only) but still schedules graph repair when + a message is present. - **User classification** (`service.ts#setEmailMessageClassification`) — transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate here for the D1 mutation + full graph mirror invariant (mirror only after a @@ -810,20 +815,22 @@ mail content. Account export pages Mailbox state through the `mailbox` section This is an expand/contract migration. **Phase 2 is partially wired (non-inbound slice):** live dual-write covers outbound terminal -message/thread/attachment/event graphs, provider-recorded delivery events (queue -`waitUntil` full graph repair), and user classification (full graph repair after -D1 update via `service.ts#setEmailMessageClassification`). Mirror helpers emit -automatic namespaced outcome telemetry (`missing` and `timeout` included). -Message and batch event repair each use one 1s-bounded RPC; graph event repair -batches up to 100 events with explicit truncation (newest retained, -chronological insertion restored) and one batch telemetry outcome. -**`system:email` stays in D1 only.** Inbound lifecycle dual-write, -retention/metadata deletes, and scheduled parity reconcile are **not** wired in -this slice. Live email read/write authority is unchanged — D1 `email_*` tables -remain authoritative for all live paths. D1 email rows and existing R2 inventory -deletion stay authoritative during expand. **Phase 2 completes only when every -user-mail mutation that writes D1 also writes the DO and parity/reconciliation -are wired** — inbound paths and parity reconcile remain. +message/thread/attachment/event graphs, provider delivery-queue graph repair +(`recorded` / `duplicate` / `stale` with a message, via `waitUntil`), and user +classification (full graph repair after D1 update via +`service.ts#setEmailMessageClassification`). Mirror helpers emit automatic +namespaced outcome telemetry (`missing` and `timeout` included). Message and +batch event repair each use one 1s-bounded RPC; each graph attempt emits at most +two AE writes; event repair batches up to 100 events with explicit truncation +(newest retained, chronological insertion restored, stable truncation warning) +and one batch telemetry outcome. **`system:email` stays in D1 only.** Inbound +lifecycle dual-write, retention/metadata deletes, and scheduled parity reconcile +are **not** wired in this slice. Live email read/write authority is unchanged — +D1 `email_*` tables remain authoritative for all live paths. D1 email rows and +existing R2 inventory deletion stay authoritative during expand. **Phase 2 +completes only when every user-mail mutation that writes D1 also writes the DO +and parity/reconciliation are wired** — inbound paths and parity reconcile +remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -839,13 +846,14 @@ are wired** — inbound paths and parity reconcile remain. `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, - provider `recorded` queue events (`waitUntil`), and user classification + provider delivery-queue outcomes with a message (`recorded`, `duplicate`, + `stale`; `waitUntil`), and user classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` - outcomes automatically (graph repair uses message RPC + one batch event RPC). - **Still pending for phase-2 completion:** inbound commit/touch/classify - dual-write, retention and metadata-delete mirrors, and scheduled parity - compare/reconcile. + outcomes automatically (graph repair uses message RPC + one batch event RPC; + at most two AE writes per attempt). **Still pending for phase-2 completion:** + inbound commit/touch/classify dual-write, retention and metadata-delete + mirrors, and scheduled parity compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata diff --git a/packages/worker/src/email/delivery-queue.node.test.ts b/packages/worker/src/email/delivery-queue.node.test.ts index 8030c12213..f7a81a109c 100644 --- a/packages/worker/src/email/delivery-queue.node.test.ts +++ b/packages/worker/src/email/delivery-queue.node.test.ts @@ -159,14 +159,26 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched waitUntil: expect.any(Function), }, ) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(3) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(1, { env: expect.anything(), db: expect.anything(), userId: 'user-1', messageId: 'message-1', }) - expect(waitUntilPromises).toHaveLength(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(2, { + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(3, { + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(waitUntilPromises).toHaveLength(3) expect(consoleWarn).toHaveBeenCalledWith('email-delivery-event-unmatched', { queueMessageId: 'queue-unmatched', providerMessageId: 'provider-1', @@ -290,3 +302,138 @@ test('mailbox graph mirror failures scheduled in waitUntil do not reject or alte 'mailbox rpc unavailable', ) }) + +test('stale inserted provider events schedule mailbox graph mirror without subscription dispatch', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + + const providerEvent = { + payload: { + eventId: 'event-stale-insert', + messageId: 'provider-stale', + delivery: { status: 'deferred' }, + }, + } + const storedMessage = { id: 'message-stale', userId: 'user-stale' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'stale', + providerEvent, + message: storedMessage, + }) + + const queueMessage = createQueueMessage('queue-stale-insert', { + kind: 'stale', + }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + await Promise.all(waitUntilPromises) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + expect(mocks.dispatchEmailDeliverySubscriptionEvents).not.toHaveBeenCalled() + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-stale', + messageId: 'message-stale', + }) + expect(waitUntilPromises).toHaveLength(1) +}) + +test('duplicate retry after recorded dispatch failure schedules mailbox graph mirror', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + + const providerEvent = { + payload: { + eventId: 'event-retry', + messageId: 'provider-retry', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-retry', userId: 'user-retry' } + mocks.processCloudflareEmailDeliveryEvent + .mockResolvedValueOnce({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-retry' }, + message: storedMessage, + }) + .mockResolvedValueOnce({ + outcome: 'duplicate', + providerEvent, + message: storedMessage, + }) + mocks.dispatchEmailDeliverySubscriptionEvents + .mockRejectedValueOnce( + new Error('artifact_preparation_failed: transient KV failure'), + ) + .mockResolvedValueOnce([]) + + const queueMessage = createQueueMessage('queue-retry', { kind: 'recorded' }) + const ctx = { + waitUntil(promise: Promise) { + void promise + }, + passThroughOnException() {}, + } as ExecutionContext + const batch = { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch + + await handleEmailDeliveryQueue(batch, { APP_DB: {} } as Env, ctx) + expect(queueMessage.ack).not.toHaveBeenCalled() + expect(queueMessage.retry).toHaveBeenCalledWith({ delaySeconds: 30 }) + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + const waitUntilPromises: Array> = [] + const retryCtx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue(batch, { APP_DB: {} } as Env, retryCtx) + await Promise.all(waitUntilPromises) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).toHaveBeenCalledTimes(1) + expect(mocks.dispatchEmailDeliverySubscriptionEvents).toHaveBeenCalledTimes(2) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-retry', + messageId: 'message-retry', + }) + expect(waitUntilPromises).toHaveLength(1) + expect(consoleError).toHaveBeenCalledWith( + 'email-delivery-event-processing-failed', + expect.objectContaining({ message: expect.stringContaining('KV failure') }), + ) +}) diff --git a/packages/worker/src/email/delivery-queue.ts b/packages/worker/src/email/delivery-queue.ts index bb843ae987..ce9f7b8ff4 100644 --- a/packages/worker/src/email/delivery-queue.ts +++ b/packages/worker/src/email/delivery-queue.ts @@ -7,6 +7,22 @@ import { type EmailReportingEnv } from './reporting-events.ts' const unmatchedRetryDelaySeconds = 30 export const emailDeliveryQueueName = 'kody-email-delivery' +function scheduleMailboxGraphMirror( + waitUntil: (promise: Promise) => void, + env: Env, + message: { userId: string; id: string } | null, +) { + if (message == null) return + waitUntil( + mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: message.userId, + messageId: message.id, + }), + ) +} + export async function handleEmailDeliveryQueue( batch: MessageBatch, env: Env, @@ -43,6 +59,7 @@ export async function handleEmailDeliveryQueue( deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: false, }) + scheduleMailboxGraphMirror(waitUntil, env, result.message) queueMessage.ack() break case 'duplicate': @@ -63,16 +80,7 @@ export async function handleEmailDeliveryQueue( providerEvent: result.providerEvent, waitUntil, }) - if (result.outcome === 'recorded' && result.message != null) { - waitUntil( - mirrorMailboxMessageGraphFromD1({ - env, - db: env.APP_DB, - userId: result.message.userId, - messageId: result.message.id, - }), - ) - } + scheduleMailboxGraphMirror(waitUntil, env, result.message) queueMessage.ack() break } diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts index 91be2c89f8..f9109c71df 100644 --- a/packages/worker/src/email/mailbox-live-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -136,9 +136,9 @@ async function seedMessageGraph(db: D1Database) { }) } -test('live-mirror AE write budget stays under Analytics Engine request cap', () => { - expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) +test('live-mirror AE write budget is at most two writes per graph attempt', () => { expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe(2) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) }) test('live-mirror graph mirrors message before one batch event RPC and uses createdAt for event updatedAt', async () => { @@ -316,6 +316,7 @@ test('live-mirror graph uses one batch RPC after message settles, not per-event }) test('live-mirror graph keeps newest events when truncated and sets eventsTruncated', async () => { + consoleWarn.mockImplementation(() => {}) const { db } = await createEmailDb() await seedMessage(db) @@ -369,7 +370,7 @@ test('live-mirror graph keeps newest events when truncated and sets eventsTrunca expect(batchEvents.at(-1)?.id).toBe( `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, ) - // 1 message + 1 batch AE write; never per-event fan-out. + // At most two AE writes: message + batch; never per-event fan-out. expect(writeDataPoint).toHaveBeenCalledTimes( mailboxLiveMirrorMaxAnalyticsWrites, ) @@ -381,6 +382,15 @@ test('live-mirror graph keeps newest events when truncated and sets eventsTrunca 'mailbox_mirror:upsert_delivery_event_batch', ), ).toBe(true) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-live-mirror-events-truncated', + { + userId: 'user-aaa', + messageId: 'msg-1', + loaded: totalEvents, + max: mailboxLiveMirrorMaxEvents, + }, + ) }) test('live-mirror graph maps batch timeout and error onto each event summary', async () => { @@ -476,7 +486,8 @@ test('live-mirror graph returns missing when D1 message is absent', async () => ok: true as const, accepted: true, })) - const { env } = fakeMailboxEnv({ mirrorMessage }) + const upsertDeliveryEvents = vi.fn() + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvents }) await expect( mirrorMailboxMessageGraphFromD1({ @@ -492,6 +503,36 @@ test('live-mirror graph returns missing when D1 message is absent', async () => eventsTruncated: false, }) expect(mirrorMessage).not.toHaveBeenCalled() + expect(upsertDeliveryEvents).not.toHaveBeenCalled() +}) + +test('live-mirror graph returns missing for cross-owner message without RPCs', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const upsertDeliveryEvents = vi.fn(async () => ({ + results: [], + })) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvents }) + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'other-user', + messageId: 'msg-1', + }), + ).resolves.toEqual({ + messageId: 'msg-1', + message: { status: 'missing' }, + events: [], + eventsTruncated: false, + }) + expect(mirrorMessage).not.toHaveBeenCalled() + expect(upsertDeliveryEvents).not.toHaveBeenCalled() }) test('live-mirror graph catches unexpected errors without throwing', async () => { diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index 284f20d7fc..493a6c0e66 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -29,8 +29,8 @@ import { type EmailThreadRecord } from './types.ts' * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. * * Graph event mirroring uses one batch RPC after the message settles (not - * Promise.all of per-event RPCs). Analytics Engine parity writes stay small: - * 1 message + 1 batch outcome. + * Promise.all of per-event RPCs). Analytics Engine parity writes are at most + * two per graph attempt: 1 message outcome + 1 batch outcome. */ /** @@ -39,7 +39,7 @@ import { type EmailThreadRecord } from './types.ts' */ export const mailboxLiveMirrorMaxEvents = mailboxUpsertDeliveryEventsMax -/** AE write budget: message outcome + one batch outcome. */ +/** AE write budget cap: at most two writes (message + batch). */ export const mailboxLiveMirrorMaxAnalyticsWrites = 2 export type MailboxLiveMirrorEnv = MailboxMirrorEnv @@ -143,19 +143,23 @@ export async function mirrorMailboxMessageGraphFromD1(input: { return emptyGraphSummary(input.messageId, { status: 'missing' }) } - const attachments = await listEmailAttachmentsForMessage({ - db: input.db, - messageId: input.messageId, - }) - - let thread = input.thread - if (thread === undefined && message.threadId) { - thread = await getEmailThreadById({ + // After the owner-scoped message load, fan out attachment + optional + // thread reads concurrently. + const [attachments, thread] = await Promise.all([ + listEmailAttachmentsForMessage({ db: input.db, - userId: input.userId, - threadId: message.threadId, - }) - } + messageId: input.messageId, + }), + input.thread !== undefined + ? Promise.resolve(input.thread) + : message.threadId + ? getEmailThreadById({ + db: input.db, + userId: input.userId, + threadId: message.threadId, + }) + : Promise.resolve(null), + ]) // Message settles before the event-batch RPC starts (ordering guarantee). const messageResult = await mirrorMailboxMessageSnapshot({ @@ -174,6 +178,14 @@ export async function mirrorMailboxMessageGraphFromD1(input: { limit: mailboxLiveMirrorMaxEvents + 1, }) const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents + if (eventsTruncated) { + console.warn('mailbox-live-mirror-events-truncated', { + userId: input.userId, + messageId: input.messageId, + loaded: loadedEvents.length, + max: mailboxLiveMirrorMaxEvents, + }) + } const eventInputs = eventsTruncated ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) : loadedEvents diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index f506665e2d..5999409de4 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -21,7 +21,7 @@ export const mailboxDeliveryEventRetentionDays = 90 export const mailboxDefaultPageSize = 100 export const mailboxMaxPageSize = 500 /** Max complete delivery-event snapshots per `upsertDeliveryEvents` RPC. */ -export const mailboxUpsertDeliveryEventsMax = mailboxDefaultPageSize +export const mailboxUpsertDeliveryEventsMax = 100 export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ diff --git a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts index c93610214b..c62fa0c27a 100644 --- a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts @@ -32,6 +32,7 @@ function createMailboxStubEnv(input: { const stub = { mirrorMessage: method, upsertDeliveryEvent: method, + upsertDeliveryEvents: method, touchThread: method, updateMessageDelivery: method, setMessageClassification: method, @@ -207,6 +208,7 @@ test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { silenceIncidentalRuntimeWarnings([ 'mailbox-mirror-message-failed', 'mailbox-mirror-delivery-event-failed', + 'mailbox-mirror-delivery-event-batch-failed', 'mailbox-live-mirror-message-graph-failed', ]) await ensureEmailTestSchema(env.APP_DB) From 0551015dbf6d8b5e8e6bb708f1154bd3149f7ad6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:45:40 +0000 Subject: [PATCH 10/20] feat(email): add Mailbox parity soak gate Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 157 +++- .../architecture/feature-flags.md | 32 + .../migrations/0125-mailbox-parity-state.sql | 34 + .../handlers/admin-feature-flags.node.test.ts | 66 +- .../app/handlers/session-handler.node.test.ts | 1 + .../worker/src/email/mailbox-parity-repo.ts | 372 +++++++++ .../email/mailbox-read-cutover.node.test.ts | 528 ++++++++++++ .../worker/src/email/mailbox-read-cutover.ts | 195 +++++ .../src/email/mailbox-reconcile.node.test.ts | 783 +++++++++++++++++ .../worker/src/email/mailbox-reconcile.ts | 787 ++++++++++++++++++ .../email/mailbox-reconcile.workers.test.ts | 331 ++++++++ .../src/feature-flags/registry.node.test.ts | 26 + packages/worker/src/feature-flags/registry.ts | 13 + .../src/feature-flags/service.node.test.ts | 21 +- .../capabilities/access-control.node.test.ts | 1 + ...min-feature-flag-capabilities.node.test.ts | 38 +- .../scheduled/scheduled-lanes.node.test.ts | 54 ++ .../worker/src/scheduled/scheduled-lanes.ts | 15 + packages/worker/src/users-test-schema.ts | 16 + 19 files changed, 3399 insertions(+), 71 deletions(-) create mode 100644 packages/worker/migrations/0125-mailbox-parity-state.sql create mode 100644 packages/worker/src/email/mailbox-parity-repo.ts create mode 100644 packages/worker/src/email/mailbox-read-cutover.node.test.ts create mode 100644 packages/worker/src/email/mailbox-read-cutover.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.workers.test.ts create mode 100644 packages/worker/src/feature-flags/registry.node.test.ts create mode 100644 packages/worker/src/scheduled/scheduled-lanes.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 07765dacc0..e8d9a38814 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -268,11 +268,19 @@ The schema is defined by migrations in `packages/worker/migrations/`: D1 payload storage-byte read, enforcement, and reconciliation. UserMeter `storage_bytes_state` (schema v4) and `package_service_states` (schema v5) are optional expand-phase shadows only — see - [Entitlements](./entitlements.md#usermeter-expand-phase). Inbound email - routing does not reverse-resolve stable ids at all — it uses the indexed - username lookup (`findPublicUserIdentityByUsername`). Contextless paths - resolve stable ids with one indexed point read on `users.stable_user_id` (for - example `findUserAccountByStableUserId`). + [Entitlements](./entitlements.md#usermeter-expand-phase). Mailbox expand-phase + parity/backfill state (migration `0125-mailbox-parity-state.sql`): + `mailbox_parity_checked_at`, `mailbox_parity_matching_since`, + `mailbox_parity_mismatch_count`, `mailbox_parity_last_error`, + `mailbox_parity_content_watermark_at`, message and orphan-event backfill + cursors/completion timestamps on the user row. Indexed by + `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in + the hourly `mailbox_parity` reconcile lane (see + [Mailbox](#durable-objects-mailbox)). Inbound email routing does not + reverse-resolve stable ids at all — it uses the indexed username lookup + (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids + with one indexed point read on `users.stable_user_id` (for example + `findUserAccountByStableUserId`). - `platform_feedback`: attributed, user-approved Kody feedback and admin triage state. Submitter identity remains on the row; optional reviewer attribution is cleared if that admin account is deleted. Open and triaged rows remain until @@ -382,8 +390,10 @@ delivery-event table because Wrangler's emulated dataset has no SQL API. with a separate row shape defined in `packages/worker/src/email/mailbox-parity-events.ts` (`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome -per attempted operation; scheduled parity compares are not wired yet. These rows -do not feed the admin insights charts today: +per attempted operation; the hourly `mailbox_parity` reconcile lane records one +outcome per count comparison (`compare_threads`, `compare_messages`, +`compare_attachments`, `compare_delivery_events`). These rows do not feed the +admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -402,9 +412,9 @@ Mirror operations: `mirror_message`, `upsert_delivery_event`, `stale`, `missing`, `timeout`, `skipped`, `error`. Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome -writes are automatic from live dual-write helpers; parity compare writers remain -for future reconciliation. Writes are best-effort and never throw into D1 -authority paths. +writes are automatic from live dual-write helpers; parity compare writes come +from the scheduled reconcile lane after owner-scoped D1 and Mailbox counts are +compared. Writes are best-effort and never throw into D1 authority paths. Two D1 reporting projections deliberately remain: @@ -790,8 +800,69 @@ slice):** successful D1 update; failures never change the mutation response). Inbound commit/touch/classify paths, retention deletes, and partial mutation -mirrors are **not** wired in this slice. Parity compare/reconcile cron remains -future work. +mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired +(see below); read cutover is prepared but not flipped. + +**Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated +scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, +`packages/worker/src/email/mailbox-reconcile.ts`). The five-minute Worker cron +dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); +production uses the same per-lane queue isolation as reconcile, retention, and +sibling lanes so a slow parity pass cannot consume another lane's budget. + +User discovery is **D1-only** — no Mailbox Durable Object enumeration. Each tick +selects up to eight non-deleting mail owners (`mailboxParityUserBatchSize`) +ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who +have at least one `email_messages` or `email_delivery_events` row. Per-user work +is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). + +Per user, the lane: + +1. **Content watermark baseline** — on the first backfill attempt, sets + `mailbox_parity_content_watermark_at` to the run's `now` and retains it + across incomplete/error ticks so updates during creation/orphan mirroring + stay inside the replay window. +2. **Bounded initial message backfill** — keyset-pages D1 `email_messages` by + `(created_at, id)` and mirrors each graph via + `mirrorMailboxMessageGraphFromD1` until complete or budget-exhausted. +3. **Bounded orphan-event backfill** — after messages complete, keyset-pages D1 + `email_delivery_events` with `message_id IS NULL` and mirrors each via + `mirrorMailboxOrphanDeliveryEventFromD1`. +4. **Content watermark replay** — after both initial backfills complete, + keyset-replays every owner message with `updated_at` in + `(watermark, current-now]` (classification-like updates that landed after a + graph mirror but before backfill finished). The watermark advances only when + the full window succeeds. +5. **Count compare + AE signals** — owner-scoped D1 counts vs + `Mailbox.countMailbox()` for threads, messages, attachments, and delivery + events. Each comparison emits one `mailbox_parity:` Analytics + Engine row with `double2 = d1Count - doCount`. + +Parity progress persists on `users` (migration `0125-mailbox-parity-state.sql`). +D1 remains mail authority — the lane never mutates `email_*` rows and does not +flip read authority. + +**`matching_since` semantics:** records the start of continuous **exact** +D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact +compare when null; preserved across successful content replay + exact compare. +Cleared on: initial backfill work in the same tick (`initialBackfilled > 0`), +incomplete/budget-exhausted ticks, mirror retryable failures, content-replay +failure/incomplete, compare mismatch (also increments consecutive +`mailbox_parity_mismatch_count` and reopens bounded creation/orphan backfill +while retaining the content watermark), and compare-path errors +(`rotateCheckedAt` also clears soak so a poison tick cannot preserve a false +window). Resets `mismatch_count` to `0` on exact match. + +**Phase 3 read cutover (prepared, not flipped):** +`packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off +`mailbox-read-cutover` feature flag plus `getOwnerEmailMessageById` / +`isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user +parity soak: `mailbox_parity_matching_since` at least 24h old, +`mailbox_parity_checked_at` fresh within 2h, +`mailbox_parity_mismatch_count === 0`, and exact `stable_user_id` match. D1 +errors, missing parity state, and an off flag fail closed to D1 reads. No live +reader calls this adapter yet; exposure recording is intentionally omitted until +a live gate site exists (see [Feature flags](./feature-flags.md)). **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -823,14 +894,16 @@ namespaced outcome telemetry (`missing` and `timeout` included). Message and batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) -and one batch telemetry outcome. **`system:email` stays in D1 only.** Inbound -lifecycle dual-write, retention/metadata deletes, and scheduled parity reconcile -are **not** wired in this slice. Live email read/write authority is unchanged — -D1 `email_*` tables remain authoritative for all live paths. D1 email rows and -existing R2 inventory deletion stay authoritative during expand. **Phase 2 -completes only when every user-mail mutation that writes D1 also writes the DO -and parity/reconciliation are wired** — inbound paths and parity reconcile -remain. +and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly +queue-isolated `mailbox_parity` lane backfills historical graphs, replays +content-watermark updates, compares owner-scoped D1 vs Mailbox counts, and +persists soak state on `users` (migration `0125`). Inbound lifecycle dual-write, +retention/metadata deletes, and live read cutover are **still not wired**. Live +email read/write authority is unchanged — D1 `email_*` tables remain +authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. **Phase 2 completes only when every +user-mail mutation that writes D1 also writes the DO** — inbound paths and +delete mirrors remain pending. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -845,25 +918,35 @@ remain. `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and - `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, + `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, and + `mailbox-reconcile.ts`. **Live today (partial):** outbound terminals, provider delivery-queue outcomes with a message (`recorded`, `duplicate`, `stale`; `waitUntil`), and user classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; - at most two AE writes per attempt). **Still pending for phase-2 completion:** - inbound commit/touch/classify dual-write, retention and metadata-delete - mirrors, and scheduled parity compare/reconcile. + at most two AE writes per attempt). The hourly `mailbox_parity` scheduled + lane backfills, content-watermark replays, count compares, and soak tracking + on `users` (migration `0125`). **Still pending for phase-2 completion:** + inbound commit/touch/classify dual-write and retention/metadata-delete + mirrors. 3. **Reads cut over after production soak** — user-mail reads move to the DO - only after production parity soak is verified. D1 writes continue. + only after production parity soak is verified + (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh + `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`) **and** + the default-off `mailbox-read-cutover` flag is enabled per user. D1 writes + continue. A prepared `getOwnerEmailMessageById` adapter exists but is **not + wired**; no live reader has flipped and no cutover exposures are recorded + yet. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. 5. **Later contract migrations** — drop retired D1 user-mail tables/columns only after verification. No premature schema deletion. -A cron sibling track owns scheduled-lane wiring (for example parity / -reconciliation ticks). This Mailbox track owns DO storage semantics, alarm +The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in +`scheduled-lanes.ts`) owns backfill, content-watermark replay, count parity, and +soak state on `users`. This Mailbox track owns DO storage semantics, alarm retention, and the inbound durability boundary below. ### What stays in D1 @@ -1156,15 +1239,17 @@ Artifacts tokens. Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session -cleanup, system-email retention, general retention, job retention, and hourly -usage-rollup aggregation. Each production queue message preserves the existing -`scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled -lane failure is acknowledged and retried by the next cron tick, matching the old -cron semantics. A failed enqueue is reported and runs through the inline -fallback after all sibling enqueue attempts finish; multiple failed enqueues -fall back sequentially to avoid D1 lock contention. Consumer transport failures -retain the configured retry/DLQ behavior. No failure can abort or mask a sibling -invocation. +cleanup, system-email retention, general retention, job retention, hourly +usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC +hour only; D1 user discovery, bounded backfill, content-watermark replay, and +count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue +message preserves the existing `scheduled_lane_failed` / D1 lock-contention log +and Sentry context. A handled lane failure is acknowledged and retried by the +next cron tick, matching the old cron semantics. A failed enqueue is reported +and runs through the inline fallback after all sibling enqueue attempts finish; +multiple failed enqueues fall back sequentially to avoid D1 lock contention. +Consumer transport failures retain the configured retry/DLQ behavior. No failure +can abort or mask a sibling invocation. Production note: diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index 28cf7fcef5..627b59b25e 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -69,6 +69,38 @@ The registry ships with one permanent flag, `demo-indicator`, which renders a small badge in the app chrome and exists so the system stays exercised end-to-end (`e2e/admin-feature-flags.spec.ts`). +### Mailbox read cutover (`mailbox-read-cutover`) + +Default-off migration flag for phase 3 of the Mailbox expand/contract track (see +[Data storage — Mailbox](./data-storage.md#durable-objects-mailbox)). It gates +whether owner-scoped user-mail **message reads** may come from the per-user +Mailbox Durable Object instead of D1. + +Evaluation alone is not enough. A live read path must also pass the per-user +parity soak on `users` (migration `0125-mailbox-parity-state.sql`): + +- `mailbox_parity_matching_since` at least **24 hours** old (continuous exact + D1↔Mailbox count parity), +- `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), +- `mailbox_parity_mismatch_count === 0`, +- exact `stable_user_id` match between the gate caller and the loaded row. + +D1 errors, missing/incomplete parity state, stale `checked_at`, non-zero +mismatch depth, and an off flag all **fail closed** to D1-authoritative reads. +When the gate is on, Mailbox DO read errors propagate with no D1 fallback. + +The prepared adapter lives in +`packages/worker/src/email/mailbox-read-cutover.ts` +(`isMailboxReadCutoverEnabled`, `getOwnerEmailMessageById`). **No live reader +calls it yet** — phase 3 is not flipped. Exposure recording is intentionally +omitted until a live gate site exists (the session/MCP evaluation caches, or an +explicit `recordFeatureFlagExposures` at the first switched call site). Until +then, the flag appears in admin/session evaluation maps but records no +`FLAG_EXPOSURES` rows. + +Success metric: `email_received` error rate should decrease after verified +parity soak and cutover (`successMetric` on the registry definition). + ## Success metrics Every flag exists to move something; the `successMetric` field on a registry diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql new file mode 100644 index 0000000000..3a2af01586 --- /dev/null +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -0,0 +1,34 @@ +-- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. +-- Low-write columns live on users (not per-message). Creation cursors are +-- keyset (created_at, id) so message and message_id-null orphan-event backfill +-- can resume across ticks without DO enumeration. content_watermark_at is set +-- to the run's now at the start of the first backfill attempt (retained across +-- incomplete/error ticks) and bounds a keyset replay of messages with +-- updated_at in (watermark, current-now] after both initial backfills complete +-- so classification-like updates — including those that land after a graph was +-- mirrored but before backfill finished — are remirrored even when counts stay +-- equal; the watermark advances only after the full window succeeds. +-- matching_since records the start of continuous exact D1 ↔Mailbox count +-- parity for soak tracking; it clears on initial backfill work, +-- incomplete/budget-exhausted ticks, mirror errors, content-replay +-- failure/incomplete, or compare mismatch — but may persist across a +-- successful content replay + exact compare. mismatch_count is consecutive +-- current mismatch depth and resets to 0 on exact match. + +ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_matching_since TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_mismatch_count INTEGER NOT NULL DEFAULT 0 + CHECK (mailbox_parity_mismatch_count >= 0); +ALTER TABLE users ADD COLUMN mailbox_parity_last_error TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_watermark_at TEXT; + +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_completed_at TEXT; + +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_completed_at TEXT; + +CREATE INDEX idx_users_mailbox_parity_checked + ON users(mailbox_parity_checked_at, stable_user_id); diff --git a/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts b/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts index 9f21d66586..5cd10a41c7 100644 --- a/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts +++ b/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts @@ -361,18 +361,30 @@ test('admin feature flags HTTP lifecycle: auth, list, set_global, and validation const listResponse = await handler.handler(createHandlerRequest()) expect(listResponse.status).toBe(200) - await expect(listResponse.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const listBody = (await listResponse.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(listBody).toMatchObject({ ok: true }) + expect(listBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', stale: false, defaultEnabled: false, global: null, overrides: [], - }, - ], - }) + }), + expect.objectContaining({ + key: 'mailbox-read-cutover', + stale: false, + defaultEnabled: false, + successMetric: expect.objectContaining({ + eventType: 'email_received', + }), + }), + ]), + ) const setGlobalResponse = await handler.handler( createHandlerRequest({ @@ -387,20 +399,24 @@ test('admin feature flags HTTP lifecycle: auth, list, set_global, and validation }), ) expect(setGlobalResponse.status).toBe(200) - await expect(setGlobalResponse.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const setGlobalBody = (await setGlobalResponse.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(setGlobalBody).toMatchObject({ ok: true }) + expect(setGlobalBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', - global: { + global: expect.objectContaining({ enabled: true, rolloutPercent: 25, note: 'canary', updatedByStableUserId: null, - }, - }, - ], - }) + }), + }), + ]), + ) expect(logAuditEventSpy).toHaveBeenCalledWith( expect.objectContaining({ category: 'admin', @@ -523,10 +539,14 @@ test('admin feature flags set_user_override validates user identity and existenc }), ) expect(byUsername.status).toBe(200) - await expect(byUsername.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const byUsernameBody = (await byUsername.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(byUsernameBody).toMatchObject({ ok: true }) + expect(byUsernameBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', overrides: [ expect.objectContaining({ @@ -535,7 +555,7 @@ test('admin feature flags set_user_override validates user identity and existenc enabled: true, }), ], - }, - ], - }) + }), + ]), + ) }) diff --git a/packages/worker/src/app/handlers/session-handler.node.test.ts b/packages/worker/src/app/handlers/session-handler.node.test.ts index 213f4b8e4d..a8af528add 100644 --- a/packages/worker/src/app/handlers/session-handler.node.test.ts +++ b/packages/worker/src/app/handlers/session-handler.node.test.ts @@ -167,6 +167,7 @@ test('session handler only renews remembered sessions after the renewal window', permissions: [], featureFlags: { 'demo-indicator': false, + 'mailbox-read-cutover': false, }, }, }) diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts new file mode 100644 index 0000000000..f41c43a582 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -0,0 +1,372 @@ +import { systemEmailOwnerId } from './email-owner.ts' +import { type MailboxCountResult } from './mailbox-types.ts' + +/** + * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. + * Creation backfill uses (created_at, id) keysets; content replay uses + * (updated_at, id) within (watermark, now]. + */ + +export type MailboxParityBackfillCursor = { + createdAt: string + id: string +} + +export type MailboxParityContentCursor = { + updatedAt: string + id: string +} + +export type MailboxParityUserState = { + userId: string + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursor: MailboxParityBackfillCursor | null + messagesCompletedAt: string | null + orphanEventCursor: MailboxParityBackfillCursor | null + orphanEventsCompletedAt: string | null +} + +export type MailboxParityCreatedKeysetRow = { + id: string + created_at: string +} + +export type MailboxParityUpdatedKeysetRow = { + id: string + updated_at: string +} + +export async function listUsersForMailboxParity(input: { + db: D1Database + limit: number +}): Promise> { + const result = await input.db + .prepare( + `SELECT u.stable_user_id AS userId + FROM users u + WHERE u.deleting_at IS NULL + AND u.stable_user_id != ? + AND ( + EXISTS ( + SELECT 1 FROM email_messages m + WHERE m.user_id = u.stable_user_id + ) + OR EXISTS ( + SELECT 1 FROM email_delivery_events e + WHERE e.user_id = u.stable_user_id + ) + ) + ORDER BY u.mailbox_parity_checked_at ASC, u.stable_user_id ASC + LIMIT ?`, + ) + .bind(systemEmailOwnerId, input.limit) + .all<{ userId: string }>() + return result.results ?? [] +} + +export async function countD1MailboxParity(input: { + db: D1Database + userId: string +}): Promise { + const [threads, messages, attachments, deliveryEvents] = await Promise.all([ + input.db + .prepare(`SELECT COUNT(*) AS n FROM email_threads WHERE user_id = ?`) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare(`SELECT COUNT(*) AS n FROM email_messages WHERE user_id = ?`) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare( + `SELECT COUNT(*) AS n + FROM email_attachments ea + JOIN email_messages em ON em.id = ea.message_id + WHERE em.user_id = ?`, + ) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare( + `SELECT COUNT(*) AS n FROM email_delivery_events WHERE user_id = ?`, + ) + .bind(input.userId) + .first<{ n: number }>(), + ]) + return { + threads: Number(threads?.n ?? 0) || 0, + messages: Number(messages?.n ?? 0) || 0, + attachments: Number(attachments?.n ?? 0) || 0, + deliveryEvents: Number(deliveryEvents?.n ?? 0) || 0, + } +} + +export async function loadUserParityState(input: { + db: D1Database + userId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT + stable_user_id AS userId, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + FROM users + WHERE stable_user_id = ? + AND deleting_at IS NULL + LIMIT 1`, + ) + .bind(input.userId) + .first<{ + userId: string + matchingSince: string | null + mismatchCount: number | null + contentWatermarkAt: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorCreatedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + }>() + if (!row) return null + const messageCursor = + row.messageCursorCreatedAt != null && row.messageCursorId != null + ? { createdAt: row.messageCursorCreatedAt, id: row.messageCursorId } + : null + const orphanEventCursor = + row.orphanCursorCreatedAt != null && row.orphanCursorId != null + ? { createdAt: row.orphanCursorCreatedAt, id: row.orphanCursorId } + : null + return { + userId: row.userId, + matchingSince: row.matchingSince, + mismatchCount: Number(row.mismatchCount ?? 0) || 0, + contentWatermarkAt: row.contentWatermarkAt, + messageCursor, + messagesCompletedAt: row.messagesCompletedAt, + orphanEventCursor, + orphanEventsCompletedAt: row.orphanEventsCompletedAt, + } +} + +export async function listMessageBackfillPage(input: { + db: D1Database + userId: string + cursor: MailboxParityBackfillCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, created_at + FROM email_messages + WHERE user_id = ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.userId, input.limit) + .all() + : await input.db + .prepare( + `SELECT id, created_at + FROM email_messages + WHERE user_id = ? + AND ( + created_at > ? + OR (created_at = ? AND id > ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.cursor.createdAt, + input.cursor.createdAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +/** + * Cohesive D1 loader for owner-scoped delivery events with `message_id` null + * (orphan / unbound ledger rows). Keyset-paged for bounded backfill. + */ +export async function listOrphanDeliveryEventBackfillPage(input: { + db: D1Database + userId: string + cursor: MailboxParityBackfillCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, created_at + FROM email_delivery_events + WHERE user_id = ? + AND message_id IS NULL + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.userId, input.limit) + .all() + : await input.db + .prepare( + `SELECT id, created_at + FROM email_delivery_events + WHERE user_id = ? + AND message_id IS NULL + AND ( + created_at > ? + OR (created_at = ? AND id > ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.cursor.createdAt, + input.cursor.createdAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +/** + * Keyset page of owner messages updated after `watermarkAt` and at or before + * `upperBoundAt` (run now, or wall time when completing in the baseline run). + * In-tick `cursor` continues within the same window without skipping equal + * `updated_at` rows. + */ +export async function listContentReplayPage(input: { + db: D1Database + userId: string + watermarkAt: string + upperBoundAt: string + cursor: MailboxParityContentCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, updated_at + FROM email_messages + WHERE user_id = ? + AND updated_at > ? + AND updated_at <= ? + ORDER BY updated_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.watermarkAt, + input.upperBoundAt, + input.limit, + ) + .all() + : await input.db + .prepare( + `SELECT id, updated_at + FROM email_messages + WHERE user_id = ? + AND updated_at <= ? + AND ( + updated_at > ? + OR (updated_at = ? AND id > ?) + ) + ORDER BY updated_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.upperBoundAt, + input.cursor.updatedAt, + input.cursor.updatedAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +export async function persistUserParityProgress(input: { + db: D1Database + userId: string + nowIso: string + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursor: MailboxParityBackfillCursor | null + messagesCompletedAt: string | null + orphanEventCursor: MailboxParityBackfillCursor | null + orphanEventsCompletedAt: string | null + lastError: string | null +}) { + await input.db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = ?, + mailbox_parity_matching_since = ?, + mailbox_parity_mismatch_count = ?, + mailbox_parity_last_error = ?, + mailbox_parity_content_watermark_at = ?, + mailbox_parity_message_backfill_cursor_created_at = ?, + mailbox_parity_message_backfill_cursor_id = ?, + mailbox_parity_message_backfill_completed_at = ?, + mailbox_parity_orphan_event_backfill_cursor_created_at = ?, + mailbox_parity_orphan_event_backfill_cursor_id = ?, + mailbox_parity_orphan_event_backfill_completed_at = ? + WHERE stable_user_id = ?`, + ) + .bind( + input.nowIso, + input.matchingSince, + input.mismatchCount, + input.lastError, + input.contentWatermarkAt, + input.messageCursor?.createdAt ?? null, + input.messageCursor?.id ?? null, + input.messagesCompletedAt, + input.orphanEventCursor?.createdAt ?? null, + input.orphanEventCursor?.id ?? null, + input.orphanEventsCompletedAt, + input.userId, + ) + .run() +} + +export async function rotateCheckedAt(input: { + db: D1Database + userId: string + nowIso: string + lastError: string | null +}) { + // Clears matching_since so a poison/error tick cannot preserve a false soak. + // Zero-row updates (deleted/deleting race) are harmless. + await input.db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = ?, + mailbox_parity_last_error = ?, + mailbox_parity_matching_since = NULL + WHERE stable_user_id = ?`, + ) + .bind(input.nowIso, input.lastError, input.userId) + .run() +} diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts new file mode 100644 index 0000000000..119318cdef --- /dev/null +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -0,0 +1,528 @@ +import { expect, test, vi } from 'vitest' +import { + getOwnerEmailMessageById, + isMailboxReadCutoverEnabled, + mailboxMessageToEmailMessageRecord, + mailboxReadCutoverCheckedAtMaxAgeMs, + mailboxReadCutoverFlagKey, + mailboxReadCutoverSoakMs, +} from './mailbox-read-cutover.ts' +import { type MailboxMessageRecord } from './mailbox-types.ts' +import { type EmailMessageRecord } from './types.ts' + +const { getEmailMessageByIdMock } = vi.hoisted(() => ({ + getEmailMessageByIdMock: vi.fn(), +})) + +vi.mock('./repo.ts', () => ({ + getEmailMessageById: (...args: Array) => + getEmailMessageByIdMock(...args), +})) + +const nowIso = '2026-08-01T12:00:00.000Z' +const nowMs = Date.parse(nowIso) + +function hoursAgoIso(hours: number) { + return new Date(nowMs - hours * 60 * 60 * 1000).toISOString() +} + +function hoursFromNowIso(hours: number) { + return new Date(nowMs + hours * 60 * 60 * 1000).toISOString() +} + +type ParityRow = { + stable_user_id: string + mailbox_parity_matching_since: string | null + mailbox_parity_checked_at: string | null + mailbox_parity_mismatch_count: number +} + +type FlagState = { + overrideEnabled: boolean | null + globalEnabled: boolean | null +} + +function createCutoverTestDb(input: { + parityByUserId?: Map + flags?: FlagState +}) { + const parityByUserId = input.parityByUserId ?? new Map() + const flags: FlagState = input.flags ?? { + overrideEnabled: null, + globalEnabled: null, + } + + function normalize(query: string) { + return query.replace(/\s+/g, ' ').trim().toLowerCase() + } + + function createStatement(query: string, params: Array = []) { + const normalized = normalize(query) + return { + bind(...nextParams: Array) { + return createStatement(query, nextParams) + }, + async first() { + if ( + normalized.includes('from feature_flag_user_overrides') && + normalized.includes('where flag_key = ? and user_id = ?') + ) { + expect(params[0]).toBe(mailboxReadCutoverFlagKey) + if (flags.overrideEnabled === null) return null + return { enabled: flags.overrideEnabled ? 1 : 0 } as T + } + if ( + normalized.includes('from feature_flags') && + normalized.includes('where key = ?') + ) { + expect(params[0]).toBe(mailboxReadCutoverFlagKey) + if (flags.globalEnabled === null) return null + return { + enabled: flags.globalEnabled ? 1 : 0, + rollout_percent: null, + } as T + } + if ( + normalized.includes('from users') && + normalized.includes('mailbox_parity_matching_since') + ) { + const userId = Number(params[0]) + const row = parityByUserId.get(userId) + if (row === 'error') throw new Error('d1 parity read failed') + if (row === undefined || row === null) return null + return row as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + } + } + + return { + prepare(query: string) { + return createStatement(query) + }, + } as unknown as D1Database +} + +function baseMailboxMessage( + overrides?: Partial, +): MailboxMessageRecord { + return { + id: 'msg-1', + direction: 'inbound', + inboxId: 'inbox-1', + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: 'sender@example.com', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: 'Hello', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: { 'x-test': '1' }, + authResults: null, + textBody: 'body', + htmlBody: null, + rawMimeKey: 'user-aaa/msg-1.eml', + rawSize: 32, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +function baseEmailMessage( + overrides?: Partial, +): EmailMessageRecord { + return { + ...mailboxMessageToEmailMessageRecord(baseMailboxMessage(), 'user-aaa'), + ...overrides, + } +} + +function soakedParity(stableUserId = 'user-aaa'): ParityRow { + return { + stable_user_id: stableUserId, + mailbox_parity_matching_since: hoursAgoIso(30), + mailbox_parity_checked_at: hoursAgoIso(1), + mailbox_parity_mismatch_count: 0, + } +} + +function fakeMailboxEnv(getMessage: ReturnType) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => ({ getMessage })) + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + }, + idFromName, + get, + } +} + +test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch parity', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + // Registry default: flag off → D1, even with soaked parity. + const defaultOffDb = createCutoverTestDb({ + flags: { overrideEnabled: null, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: defaultOffDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env: mailboxEnv, idFromName } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...mailboxEnv, APP_DB: defaultOffDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ + db: defaultOffDb, + userId: stableUserId, + messageId: 'msg-1', + }) + expect(getMessage).not.toHaveBeenCalled() + expect(idFromName).not.toHaveBeenCalled() + + // Override/global on but no parity → still D1. + for (const flags of [ + { overrideEnabled: true as boolean | null, globalEnabled: null }, + { overrideEnabled: null, globalEnabled: true as boolean | null }, + ]) { + getEmailMessageByIdMock.mockClear() + const noParityDb = createCutoverTestDb({ + flags, + parityByUserId: new Map([[dbUserId, null]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: noParityDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + await expect( + getOwnerEmailMessageById({ + env: { ...mailboxEnv, APP_DB: noParityDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalled() + } + + // Young matching_since (< 24h soak). + const youngDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_matching_since: hoursAgoIso(23), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: youngDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Stale checked_at (> 2h). + const staleDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_checked_at: hoursAgoIso(3), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: staleDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Mismatch count blocks cutover. + const mismatchedDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_mismatch_count: 1, + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: mismatchedDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Cross-user parity row denied (stable_user_id must match exactly). + const crossUserDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity('user-other')]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: crossUserDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Soaked + fresh + flag on → Mailbox DO; maps userId from the caller. + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(true) + + getEmailMessageByIdMock.mockClear() + const doMessage = baseMailboxMessage({ subject: 'From DO' }) + const doGetMessage = vi.fn(async () => doMessage) + const doEnv = fakeMailboxEnv(doGetMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...doEnv.env, APP_DB: readyDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual( + mailboxMessageToEmailMessageRecord(doMessage, stableUserId), + ) + expect(doEnv.idFromName).toHaveBeenCalledWith(stableUserId) + expect(doGetMessage).toHaveBeenCalledWith({ messageId: 'msg-1' }) + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() + + // Boundary: matching_since exactly soak age and checked_at exactly max age. + const boundaryDb = createCutoverTestDb({ + flags: { overrideEnabled: null, globalEnabled: true }, + parityByUserId: new Map([ + [ + dbUserId, + { + stable_user_id: stableUserId, + mailbox_parity_matching_since: new Date( + nowMs - mailboxReadCutoverSoakMs, + ).toISOString(), + mailbox_parity_checked_at: new Date( + nowMs - mailboxReadCutoverCheckedAtMaxAgeMs, + ).toISOString(), + mailbox_parity_mismatch_count: 0, + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: boundaryDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(true) +}) + +test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: 'not-a-timestamp', + }), + ).resolves.toBe(false) + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: new Date(Number.NaN), + }), + ).resolves.toBe(false) + + const futureCheckedDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_checked_at: hoursFromNowIso(1), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: futureCheckedDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const futureMatchingDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_matching_since: hoursFromNowIso(1), + mailbox_parity_checked_at: hoursAgoIso(1), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: futureMatchingDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) +}) + +test('mailbox-read-cutover fails closed on D1 parity errors and does not fall back from DO', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + const parityErrorDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, 'error']]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: parityErrorDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...env, APP_DB: parityErrorDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getMessage).not.toHaveBeenCalled() + expect(getEmailMessageByIdMock).toHaveBeenCalled() + + // When cutover is on, DO errors propagate — no silent D1 fallback. + getEmailMessageByIdMock.mockClear() + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + const failingGet = vi.fn(async () => { + throw new Error('mailbox unavailable') + }) + const failingEnv = fakeMailboxEnv(failingGet) + await expect( + getOwnerEmailMessageById({ + env: { ...failingEnv.env, APP_DB: readyDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).rejects.toThrow(/mailbox unavailable/) + expect(failingGet).toHaveBeenCalled() + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() +}) + +test('mailboxMessageToEmailMessageRecord maps DO row with supplied stable userId', () => { + const mapped = mailboxMessageToEmailMessageRecord( + baseMailboxMessage({ id: 'msg-map' }), + 'user-xyz', + ) + expect(mapped).toMatchObject({ + id: 'msg-map', + userId: 'user-xyz', + direction: 'inbound', + subject: 'Hello', + rawMimeKey: 'user-aaa/msg-1.eml', + }) + expect(mapped).not.toHaveProperty('ownerId') +}) diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts new file mode 100644 index 0000000000..fa3501ebce --- /dev/null +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -0,0 +1,195 @@ +/** + * Default-off, parity-gated Mailbox read cutover (prepared; not wired). + * + * Gate requires both: + * 1. feature flag `mailbox-read-cutover` enabled for the numeric `users.id` + * 2. per-user parity soak on `users`: `mailbox_parity_matching_since` at least + * {@link mailboxReadCutoverSoakMs} old, `mailbox_parity_checked_at` fresh + * within {@link mailboxReadCutoverCheckedAtMaxAgeMs}, + * `mailbox_parity_mismatch_count === 0`, and `stable_user_id` exact match + * + * D1 errors, missing/incomplete parity state, and an off flag all fail closed + * to D1-authoritative reads. When the gate is on, Mailbox DO errors propagate + * with no D1 fallback (fallback would hide parity bugs). + * + * Exposure recording is intentionally omitted here: no live caller uses this + * adapter yet. Future read-path wiring owns the exposure chokepoint (the + * existing session/MCP evaluation caches once a live gate evaluates the flag, + * or an explicit `recordFeatureFlagExposures` at the first switched call site). + */ + +import { isFeatureEnabled } from '#worker/feature-flags/service.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { type MailboxMessageRecord } from './mailbox-types.ts' +import { getEmailMessageById } from './repo.ts' +import { type EmailMessageRecord } from './types.ts' + +/** Continuous exact parity must hold at least this long before cutover. */ +export const mailboxReadCutoverSoakMs = 24 * 60 * 60 * 1000 + +/** Parity `checked_at` must be newer than this (hourly lane + slack). */ +export const mailboxReadCutoverCheckedAtMaxAgeMs = 2 * 60 * 60 * 1000 + +export const mailboxReadCutoverFlagKey = 'mailbox-read-cutover' as const + +type MailboxParityStateRow = { + stable_user_id: string + mailbox_parity_matching_since: string | null + mailbox_parity_checked_at: string | null + mailbox_parity_mismatch_count: number +} + +export type MailboxReadCutoverEnv = MailboxEnv & { + APP_DB: D1Database +} + +/** + * Resolve an optional clock override. `undefined` uses wall clock; invalid + * Date/string values return null so the gate fails closed (never Date.now()). + */ +function resolveNowMs(now?: Date | string): number | null { + if (now === undefined) return Date.now() + if (now instanceof Date) { + const ms = now.getTime() + return Number.isFinite(ms) ? ms : null + } + const parsed = Date.parse(now) + return Number.isFinite(parsed) ? parsed : null +} + +/** + * Fail-closed cutover eligibility for one account. + * Never throws: D1 / flag evaluation failures return false. + */ +export async function isMailboxReadCutoverEnabled(input: { + db: D1Database + dbUserId: number + stableUserId: string + now?: Date | string +}): Promise { + try { + const flagEnabled = await isFeatureEnabled( + input.db, + mailboxReadCutoverFlagKey, + input.dbUserId, + ) + if (!flagEnabled) return false + + const row = await input.db + .prepare( + `SELECT stable_user_id, + mailbox_parity_matching_since, + mailbox_parity_checked_at, + mailbox_parity_mismatch_count + FROM users + WHERE id = ?`, + ) + .bind(input.dbUserId) + .first() + + if (!row) return false + if (row.stable_user_id !== input.stableUserId) return false + if (row.mailbox_parity_mismatch_count !== 0) return false + if ( + row.mailbox_parity_matching_since == null || + row.mailbox_parity_checked_at == null + ) { + return false + } + + const matchingSinceMs = Date.parse(row.mailbox_parity_matching_since) + const checkedAtMs = Date.parse(row.mailbox_parity_checked_at) + if (!Number.isFinite(matchingSinceMs) || !Number.isFinite(checkedAtMs)) { + return false + } + + const nowMs = resolveNowMs(input.now) + if (nowMs === null) return false + // Future timestamps are not trustworthy (clock skew / bad writes). + if (matchingSinceMs > nowMs) return false + if (checkedAtMs > nowMs) return false + if (nowMs - matchingSinceMs < mailboxReadCutoverSoakMs) return false + if (nowMs - checkedAtMs > mailboxReadCutoverCheckedAtMaxAgeMs) return false + + return true + } catch { + return false + } +} + +/** Map a Mailbox DO message row back to the D1 `EmailMessageRecord` shape. */ +export function mailboxMessageToEmailMessageRecord( + message: MailboxMessageRecord, + userId: string, +): EmailMessageRecord { + return { + id: message.id, + direction: message.direction, + userId, + inboxId: message.inboxId, + threadId: message.threadId, + senderIdentityId: message.senderIdentityId, + fromAddress: message.fromAddress, + envelopeFrom: message.envelopeFrom, + toAddresses: message.toAddresses, + ccAddresses: message.ccAddresses, + bccAddresses: message.bccAddresses, + replyToAddresses: message.replyToAddresses, + subject: message.subject, + messageIdHeader: message.messageIdHeader, + inReplyToHeader: message.inReplyToHeader, + references: message.references, + headers: message.headers, + authResults: message.authResults, + textBody: message.textBody, + htmlBody: message.htmlBody, + rawMimeKey: message.rawMimeKey, + rawSize: message.rawSize, + processingStatus: message.processingStatus, + classification: message.classification, + classificationReason: message.classificationReason, + providerMessageId: message.providerMessageId, + deliveryStatus: message.deliveryStatus, + deliveryStatusAt: message.deliveryStatusAt, + error: message.error, + receivedAt: message.receivedAt, + sentAt: message.sentAt, + createdAt: message.createdAt, + updatedAt: message.updatedAt, + } +} + +/** + * Prepared owner-scoped message get. Selects D1 or Mailbox via the cutover + * gate. Not wired into existing readers in this PR. + */ +export async function getOwnerEmailMessageById(input: { + env: MailboxReadCutoverEnv + dbUserId: number + stableUserId: string + messageId: string + now?: Date | string +}): Promise { + const useMailbox = await isMailboxReadCutoverEnabled({ + db: input.env.APP_DB, + dbUserId: input.dbUserId, + stableUserId: input.stableUserId, + now: input.now, + }) + + if (!useMailbox) { + return getEmailMessageById({ + db: input.env.APP_DB, + userId: input.stableUserId, + messageId: input.messageId, + }) + } + + const message = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).getMessage({ messageId: input.messageId }) + + if (!message) return null + return mailboxMessageToEmailMessageRecord(message, input.stableUserId) +} diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts new file mode 100644 index 0000000000..37416204b4 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -0,0 +1,783 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxMessageGraphFromD1: vi.fn(), + mirrorMailboxDeliveryEventFromD1: vi.fn(), + awaitMailboxMirrorRpc: vi.fn(), + recordMailboxParityEvent: vi.fn(), + mailboxRpc: vi.fn(), +})) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./mailbox-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, + } +}) + +vi.mock('./mailbox-parity-events.ts', () => ({ + recordMailboxParityEvent: mocks.recordMailboxParityEvent, +})) + +vi.mock('./mailbox-client.ts', () => ({ + mailboxRpc: mocks.mailboxRpc, +})) + +const { + countD1MailboxParity, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, + mailboxParityMessagePageSize, + reconcileMailboxParity, +} = await import('./mailbox-reconcile.ts') + +async function createParityDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureUsersTestSchema({ db }) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +async function insertUser(input: { + db: D1Database + userId: string + email?: string + deletingAt?: string | null + checkedAt?: string | null +}) { + await input.db + .prepare( + `INSERT INTO users ( + username, email, password_hash, stable_user_id, deleting_at, + mailbox_parity_checked_at + ) VALUES (?, ?, 'hash', ?, ?, ?)`, + ) + .bind( + `u-${crypto.randomUUID().slice(0, 8)}`, + input.email ?? `${crypto.randomUUID()}@example.test`, + input.userId, + input.deletingAt ?? null, + input.checkedAt ?? null, + ) + .run() +} + +async function insertMessage(input: { + db: D1Database + userId: string + id: string + createdAt: string +}) { + await input.db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, + 'stored', ?, ?)`, + ) + .bind(input.id, input.userId, input.createdAt, input.createdAt) + .run() +} + +async function insertOrphanEvent(input: { + db: D1Database + userId: string + id: string + createdAt: string +}) { + await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind(input.id, input.userId, input.createdAt) + .run() +} + +async function parkUser(db: D1Database, userId: string) { + await db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +async function readParityRow(db: D1Database, userId: string) { + return db + .prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_last_error AS lastError, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + lastError: string | null + contentWatermarkAt: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorCreatedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + }>() +} + +function successGraph(messageId: string) { + return { + messageId, + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated: false, + } +} + +function mockDoCounts(counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number +}) { + mocks.awaitMailboxMirrorRpc.mockResolvedValue({ + ok: true, + value: counts, + }) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => counts), + })) +} + +test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const ownerA = 'a'.repeat(64) + const ownerB = 'b'.repeat(64) + const deleting = 'c'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + + await insertUser({ db, userId: ownerA, checkedAt: null }) + await insertUser({ + db, + userId: ownerB, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + await insertUser({ + db, + userId: deleting, + deletingAt: '2026-07-31T00:00:00.000Z', + checkedAt: null, + }) + + const messageIds = Array.from( + { length: mailboxParityMessagePageSize + 1 }, + (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, + ) + for (const [index, id] of messageIds.entries()) { + await insertMessage({ + db, + userId: ownerA, + id, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + await insertOrphanEvent({ + db, + userId: ownerA, + id: 'orphan-1', + createdAt: '2026-07-02T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: ownerB, + id: 'b-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: systemEmailOwnerId, + id: 'sys-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertOrphanEvent({ + db, + userId: deleting, + id: 'del-orphan', + createdAt: '2026-07-01T00:00:00.000Z', + }) + + // NULL checked_at sorts first under ORDER BY checked_at ASC (index-friendly). + const discovered = await listUsersForMailboxParity({ db, limit: 10 }) + expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) + expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) + expect(discovered.map((row) => row.userId)).not.toContain(deleting) + + const orphanPage = await listOrphanDeliveryEventBackfillPage({ + db, + userId: ownerA, + cursor: null, + limit: 10, + }) + expect(orphanPage).toEqual([ + { id: 'orphan-1', created_at: '2026-07-02T00:00:00.000Z' }, + ]) + + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, + ) + .bind( + ownerA, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + ) + .run() + await db + .prepare( + `INSERT INTO email_attachments ( + id, message_id, content_type, size, storage_kind, created_at + ) VALUES ('att-1', ?, 'text/plain', 1, 'unavailable', ?)`, + ) + .bind(messageIds[0]!, '2026-07-01T00:00:00.000Z') + .run() + + await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ + threads: 1, + messages: mailboxParityMessagePageSize + 1, + attachments: 1, + deliveryEvents: 1, + }) + + let graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + graphCalls += 1 + if (graphCalls === mailboxParityMessagePageSize) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + mocks.recordMailboxParityEvent.mockClear() + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + // Seed a false soak that must clear on mirror timeout / backfill work. + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', + mailbox_parity_mismatch_count = 2 + WHERE stable_user_id = ?`, + ) + .bind(ownerA) + .run() + + await parkUser(db, ownerB) + const first = await reconcileMailboxParity({ + env, + now, + batchSize: 1, + }) + expect(first).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize - 1, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-message-backfill-retryable', + ownerA, + 'error-or-timeout', + ) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.every( + (call) => call[0].userId === ownerA, + ), + ).toBe(true) + expect(mocks.mailboxRpc).not.toHaveBeenCalled() + + const afterFirst = await readParityRow(db, ownerA) + expect(afterFirst).toMatchObject({ + checkedAt: now.toISOString(), + messageCursorId: messageIds[mailboxParityMessagePageSize - 2], + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + // Baseline is persisted at first backfill attempt start and retained. + contentWatermarkAt: now.toISOString(), + matchingSince: null, + mismatchCount: 2, + lastError: 'message backfill mirror error or timeout', + }) + + // Finish backfill; compare mismatches → reopen completed_at, retain cursors, + // increment consecutive mismatch count, keep matching_since cleared. + graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.recordMailboxParityEvent.mockClear() + await parkUser(db, ownerB) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + const mismatch = await reconcileMailboxParity({ + env, + now: mismatchNow, + batchSize: 1, + }) + expect(mismatch).toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'orphan-1' }), + ) + expect(mocks.recordMailboxParityEvent).toHaveBeenCalledTimes(4) + + const afterMismatch = await readParityRow(db, ownerA) + expect(afterMismatch).toMatchObject({ + checkedAt: mismatchNow.toISOString(), + matchingSince: null, + mismatchCount: 3, + contentWatermarkAt: mismatchNow.toISOString(), + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + messageCursorId: messageIds[mailboxParityMessagePageSize], + orphanCursorId: 'orphan-1', + lastError: null, + }) + + // Live new row after retained cursor must be repaired on the reopen pass. + await insertMessage({ + db, + userId: ownerA, + id: 'msg-live-new', + createdAt: '2026-07-03T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mockDoCounts({ + threads: 1, + messages: mailboxParityMessagePageSize + 2, + attachments: 1, + deliveryEvents: 1, + }) + await parkUser(db, ownerB) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + const rematch = await reconcileMailboxParity({ + env, + now: rematchNow, + batchSize: 1, + }) + expect(rematch).toEqual({ + scanned: 1, + backfilled: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), + ) + const afterRematch = await readParityRow(db, ownerA) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: rematchNow.toISOString(), + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'msg-live-new', + }) + + // 24h soak preservation: exact completed compare with empty content window + // advances the watermark and keeps matching_since. + await parkUser(db, ownerB) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityRow(db, ownerA) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) + expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + + // Compare-path error rotates checked_at and clears matching_since. + consoleWarn.mockClear() + mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( + new Error('count boom\nwith\tcontrol'), + ) + await parkUser(db, ownerB) + const errorNow = new Date('2026-08-02T13:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterError = await readParityRow(db, ownerA) + expect(afterError?.checkedAt).toBe(errorNow.toISOString()) + expect(afterError?.lastError).toBe('count boom with control') + expect(afterError?.matchingSince).toBeNull() + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-reconcile-user-failed', + ownerA, + expect.any(Error), + ) + + // Discovery stays D1-only: mailboxRpc is never used to enumerate candidates. + const rpcCallsBefore = mocks.mailboxRpc.mock.calls.length + const discoveryOnly = await listUsersForMailboxParity({ db, limit: 5 }) + expect(discoveryOnly.length).toBeGreaterThan(0) + expect(mocks.mailboxRpc).toHaveBeenCalledTimes(rpcCallsBefore) +}) + +test('mailbox parity content watermark repairs equal-count updates and soak', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'd'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-a', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-b', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + // Baseline at first-attempt start; empty same-run window; then compare. + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterBaseline = await readParityRow(db, owner) + expect(afterBaseline).toMatchObject({ + matchingSince: now.toISOString(), + contentWatermarkAt: now.toISOString(), + mismatchCount: 0, + }) + + // Classification-like update: same counts, newer updated_at past watermark. + const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-a'`, + ) + .bind(sharedUpdatedAt) + .run() + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-b'`, + ) + .bind(sharedUpdatedAt) + .run() + + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const repairNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(repairNow) + await expect( + reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const replayedIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ) + expect(replayedIds).toEqual(['msg-a', 'msg-b']) + const afterRepair = await readParityRow(db, owner) + // Successful content sweep + exact compare keeps the soak window. + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.mismatchCount).toBe(0) + + // Equal-timestamp rows after the advanced watermark are both replayed. + const equalStamp = '2026-08-01T11:15:00.000Z' + await db + .prepare( + `UPDATE email_messages SET updated_at = ? WHERE id IN ('msg-a', 'msg-b')`, + ) + .bind(equalStamp) + .run() + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const equalNow = new Date('2026-08-01T12:00:00.000Z') + vi.setSystemTime(equalNow) + await expect( + reconcileMailboxParity({ env, now: equalNow, batchSize: 1 }), + ).resolves.toMatchObject({ + backfilled: 2, + compared: 1, + matched: 1, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-a', 'msg-b']) + expect((await readParityRow(db, owner))?.matchingSince).toBe( + now.toISOString(), + ) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + equalNow.toISOString(), + ) + + // Failure mid-window: watermark does not advance; matching_since clears. + await db + .prepare( + `UPDATE email_messages + SET updated_at = '2026-08-01T12:30:00.000Z' + WHERE id IN ('msg-a', 'msg-b')`, + ) + .run() + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = ? + WHERE stable_user_id = ?`, + ) + .bind(now.toISOString(), owner) + .run() + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === 1) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T13:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-content-replay-retryable', + owner, + 'error-or-timeout', + ) + const afterFail = await readParityRow(db, owner) + expect(afterFail).toMatchObject({ + contentWatermarkAt: equalNow.toISOString(), + matchingSince: null, + checkedAt: failNow.toISOString(), + lastError: 'content replay mirror error or timeout', + mismatchCount: 0, + }) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity replays message updated during initial backfill', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'e'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-early', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-late', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + if (input.messageId === 'msg-early') { + // Concurrent classification-like update after the graph mirror. + vi.setSystemTime(midUpdateAt) + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-early'`, + ) + .bind(midUpdateAt.toISOString()) + .run() + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + + const mirroredIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ) + expect(mirroredIds).toEqual(['msg-early', 'msg-late', 'msg-early']) + const after = await readParityRow(db, owner) + expect(after).toMatchObject({ + matchingSince: now.toISOString(), + contentWatermarkAt: midUpdateAt.toISOString(), + mismatchCount: 0, + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + }) + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts new file mode 100644 index 0000000000..a8d3dcc097 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -0,0 +1,787 @@ +import { truncateToUtf8Bytes } from '@kody-internal/shared/backup-restore-safety.ts' +import { mailboxRpc } from './mailbox-client.ts' +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, + type MailboxLiveMirrorGraphSummary, +} from './mailbox-live-mirror.ts' +import { + awaitMailboxMirrorRpc, + mailboxMirrorRpcTimeoutMs, + type MailboxMirrorResult, +} from './mailbox-mirror.ts' +import { + recordMailboxParityEvent, + type MailboxParityOperation, +} from './mailbox-parity-events.ts' +import { + countD1MailboxParity, + listContentReplayPage, + listMessageBackfillPage, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, + loadUserParityState, + persistUserParityProgress, + rotateCheckedAt, + type MailboxParityContentCursor, + type MailboxParityUserState, +} from './mailbox-parity-repo.ts' +import { type MailboxCountResult } from './mailbox-types.ts' + +/** + * Bounded hourly Mailbox backfill + count-parity reconciler. + * + * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO + * enumeration), mirrors message graphs then message_id-null orphan delivery + * events through existing live-mirror helpers, then keyset-replays messages + * updated after the content watermark before comparing owner-scoped D1 counts + * with `countMailbox`. D1 remains mail authority; this lane never mutates + * email_* rows and does not flip read authority. + */ + +export const mailboxParityUserBatchSize = 8 +export const mailboxParityMessagePageSize = 10 +export const mailboxParityOrphanEventPageSize = 25 +export const mailboxParityContentPageSize = 10 +export const mailboxParityTimeBudgetMs = 10_000 +export const mailboxParityLastErrorMaxBytes = 512 + +export type MailboxParityReconcileEnv = MailboxLiveMirrorEnv & { + APP_DB: D1Database +} + +export type MailboxParityReconcileMetrics = { + scanned: number + backfilled: number + compared: number + matched: number + mismatched: number + failed: number +} + +export { + countD1MailboxParity, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, +} + +function isRetryableMirrorResult(result: MailboxMirrorResult): boolean { + return result.status === 'error' || result.status === 'timeout' +} + +function graphMirrorSucceeded(summary: MailboxLiveMirrorGraphSummary): boolean { + if (isRetryableMirrorResult(summary.message)) return false + return !summary.events.some((event) => isRetryableMirrorResult(event.result)) +} + +function safeParityErrorText(error: unknown): string { + const raw = error instanceof Error ? error.message : String(error) + let cleaned = '' + for (const character of raw) { + const code = character.codePointAt(0) ?? 0 + cleaned += code < 0x20 || code === 0x7f ? ' ' : character + } + cleaned = cleaned.trim() + return truncateToUtf8Bytes( + cleaned.length > 0 ? cleaned : 'unknown error', + mailboxParityLastErrorMaxBytes, + ) +} + +/** + * Load one orphan delivery-event projection from D1 and mirror it. + * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. + */ +export async function mirrorMailboxOrphanDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string +}): Promise { + return mirrorMailboxDeliveryEventFromD1(input) +} + +function emitCountComparisons(input: { + env: MailboxParityReconcileEnv + userId: string + d1: MailboxCountResult + mailbox: MailboxCountResult + timestamp: string +}): { matched: boolean } { + const comparisons: Array<{ + operation: MailboxParityOperation + d1Count: number + doCount: number + }> = [ + { + operation: 'compare_threads', + d1Count: input.d1.threads, + doCount: input.mailbox.threads, + }, + { + operation: 'compare_messages', + d1Count: input.d1.messages, + doCount: input.mailbox.messages, + }, + { + operation: 'compare_attachments', + d1Count: input.d1.attachments, + doCount: input.mailbox.attachments, + }, + { + operation: 'compare_delivery_events', + d1Count: input.d1.deliveryEvents, + doCount: input.mailbox.deliveryEvents, + }, + ] + let matched = true + for (const comparison of comparisons) { + const outcome = + comparison.d1Count === comparison.doCount ? 'match' : 'mismatch' + if (outcome === 'mismatch') matched = false + recordMailboxParityEvent(input.env, { + userId: input.userId, + category: 'parity', + operation: comparison.operation, + outcome, + d1Count: comparison.d1Count, + doCount: comparison.doCount, + timestamp: input.timestamp, + }) + } + return { matched } +} + +async function backfillMessagesForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean +}> { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listMessageBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.messageCursor, + limit: mailboxParityMessagePageSize, + }) + if (page.length === 0) { + state = { + ...state, + messagesCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + let cursor = state.messageCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (!graphMirrorSucceeded(summary)) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, messageCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } +} + +async function backfillOrphanEventsForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean +}> { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt == null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + if (state.orphanEventsCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listOrphanDeliveryEventBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.orphanEventCursor, + limit: mailboxParityOrphanEventPageSize, + }) + if (page.length === 0) { + state = { + ...state, + orphanEventsCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + let cursor = state.orphanEventCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, orphanEventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } + } + const result = await mirrorMailboxOrphanDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + eventId: row.id, + }) + if (isRetryableMirrorResult(result)) { + state = { ...state, orphanEventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, orphanEventCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } +} + +/** + * Baseline is established before the first creation scan. When it still equals + * this run's nowIso, sample wall time so same-run updates after the baseline + * are included; otherwise the run's nowIso is the upper bound. + */ +function resolveContentReplayUpperBound(input: { + nowIso: string + watermarkAt: string +}): string { + let upperBoundAt = input.nowIso + if (input.watermarkAt === input.nowIso) { + const wallIso = new Date().toISOString() + if (wallIso > upperBoundAt) upperBoundAt = wallIso + } + if (upperBoundAt < input.watermarkAt) return input.watermarkAt + return upperBoundAt +} + +/** + * Persist the content watermark at the start of the first backfill attempt so + * updates during creation/orphan mirroring cannot fall before a completion-time + * baseline. Retained across incomplete/error ticks. + */ +async function ensureContentWatermarkBaseline(input: { + db: D1Database + state: MailboxParityUserState + nowIso: string +}): Promise { + if (input.state.contentWatermarkAt != null) return input.state + const state = { + ...input.state, + contentWatermarkAt: input.nowIso, + } + await persistUserParityProgress({ + db: input.db, + userId: state.userId, + nowIso: input.nowIso, + matchingSince: state.matchingSince, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: null, + }) + return state +} + +/** + * After initial creation/orphan backfill completes: keyset-replay every owner + * message with updated_at in (watermark, current-now]. Watermark advances only + * when the entire window succeeds; in-tick cursor avoids skipping equal + * timestamps. Always replay — never treat a fresh baseline as an empty skip. + */ +async function replayContentUpdatesForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + nowIso: string + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean + windowComplete: boolean +}> { + const state = input.state + const watermarkAt = state.contentWatermarkAt ?? input.nowIso + const upperBoundAt = resolveContentReplayUpperBound({ + nowIso: input.nowIso, + watermarkAt, + }) + let cursor: MailboxParityContentCursor | null = null + let backfilled = 0 + + while (Date.now() < input.deadlineMs) { + const page = await listContentReplayPage({ + db: input.env.APP_DB, + userId: state.userId, + watermarkAt, + upperBoundAt, + cursor, + limit: mailboxParityContentPageSize, + }) + if (page.length === 0) { + return { + state: { ...state, contentWatermarkAt: upperBoundAt }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + windowComplete: true, + } + } + + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + windowComplete: false, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (!graphMirrorSucceeded(summary)) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + windowComplete: false, + } + } + cursor = { updatedAt: row.updated_at, id: row.id } + backfilled += 1 + } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + windowComplete: false, + } +} + +async function compareParityForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + nowIso: string +}): Promise<{ + state: MailboxParityUserState + matched: boolean +}> { + const d1 = await countD1MailboxParity({ + db: input.env.APP_DB, + userId: input.state.userId, + }) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ env: input.env, userId: input.state.userId }).countMailbox(), + mailboxMirrorRpcTimeoutMs, + ) + if (!raced.ok) { + throw new Error('Mailbox countMailbox timed out') + } + const mailbox = raced.value + const { matched } = emitCountComparisons({ + env: input.env, + userId: input.state.userId, + d1, + mailbox, + timestamp: input.nowIso, + }) + if (matched) { + return { + state: { + ...input.state, + matchingSince: input.state.matchingSince ?? input.nowIso, + mismatchCount: 0, + }, + matched: true, + } + } + // Reopen bounded creation/orphan backfill after completed cursors so later + // live rows past the retained keyset are mirrored on the next ticks; keep + // the content watermark so classification updates remain covered. + return { + state: { + ...input.state, + matchingSince: null, + mismatchCount: input.state.mismatchCount + 1, + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + }, + matched: false, + } +} + +async function reconcileOneUser(input: { + env: MailboxParityReconcileEnv + userId: string + nowIso: string + deadlineMs: number +}): Promise<{ + backfilled: number + compared: boolean + matched: boolean + mismatched: boolean + failed: boolean + budgetExhausted: boolean +}> { + const loaded = await loadUserParityState({ + db: input.env.APP_DB, + userId: input.userId, + }) + if (!loaded) { + return { + backfilled: 0, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } + + let state = loaded + let backfilled = 0 + let initialBackfilled = 0 + + const persistIncomplete = async (inputPersist: { + lastError: string | null + }) => { + await persistUserParityProgress({ + db: input.env.APP_DB, + userId: state.userId, + nowIso: input.nowIso, + // Soak cannot survive incomplete/error/backfill ticks. + matchingSince: null, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: inputPersist.lastError, + }) + } + + // Baseline before scanning so mid-backfill updates stay inside the replay window. + state = await ensureContentWatermarkBaseline({ + db: input.env.APP_DB, + state, + nowIso: input.nowIso, + }) + + const messagePass = await backfillMessagesForUser({ + env: input.env, + state, + deadlineMs: input.deadlineMs, + }) + state = messagePass.state + initialBackfilled += messagePass.backfilled + backfilled += messagePass.backfilled + if (messagePass.retryableFailure) { + await persistIncomplete({ + lastError: 'message backfill mirror error or timeout', + }) + console.warn( + 'mailbox-parity-message-backfill-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (messagePass.budgetExhausted) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: true, + } + } + + const orphanPass = await backfillOrphanEventsForUser({ + env: input.env, + state, + deadlineMs: input.deadlineMs, + }) + state = orphanPass.state + initialBackfilled += orphanPass.backfilled + backfilled += orphanPass.backfilled + if (orphanPass.retryableFailure) { + await persistIncomplete({ + lastError: 'orphan-event backfill mirror error or timeout', + }) + console.warn( + 'mailbox-parity-orphan-event-backfill-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (orphanPass.budgetExhausted) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: true, + } + } + + if ( + state.messagesCompletedAt == null || + state.orphanEventsCompletedAt == null + ) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } + + // Initial creation/orphan work ends the continuous soak window. Successful + // content replay below may keep matching_since when the final compare matches. + if (initialBackfilled > 0) { + state = { ...state, matchingSince: null } + } + + const contentPass = await replayContentUpdatesForUser({ + env: input.env, + state, + nowIso: input.nowIso, + deadlineMs: input.deadlineMs, + }) + state = contentPass.state + backfilled += contentPass.backfilled + if (contentPass.retryableFailure) { + await persistIncomplete({ + lastError: 'content replay mirror error or timeout', + }) + console.warn( + 'mailbox-parity-content-replay-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (!contentPass.windowComplete) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: contentPass.budgetExhausted, + } + } + + const compared = await compareParityForUser({ + env: input.env, + state, + nowIso: input.nowIso, + }) + state = compared.state + await persistUserParityProgress({ + db: input.env.APP_DB, + userId: state.userId, + nowIso: input.nowIso, + matchingSince: state.matchingSince, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: null, + }) + return { + backfilled, + compared: true, + matched: compared.matched, + mismatched: !compared.matched, + failed: false, + budgetExhausted: false, + } +} + +export async function reconcileMailboxParity(input: { + env: MailboxParityReconcileEnv + now?: Date + batchSize?: number +}): Promise { + const now = input.now ?? new Date() + const nowIso = now.toISOString() + const deadlineMs = Date.now() + mailboxParityTimeBudgetMs + const batchSize = input.batchSize ?? mailboxParityUserBatchSize + const users = await listUsersForMailboxParity({ + db: input.env.APP_DB, + limit: batchSize, + }) + + const metrics: MailboxParityReconcileMetrics = { + scanned: 0, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + } + + for (const { userId } of users) { + if (Date.now() >= deadlineMs) break + metrics.scanned += 1 + try { + const result = await reconcileOneUser({ + env: input.env, + userId, + nowIso, + deadlineMs, + }) + metrics.backfilled += result.backfilled + if (result.compared) metrics.compared += 1 + if (result.matched) metrics.matched += 1 + if (result.mismatched) metrics.mismatched += 1 + if (result.failed) metrics.failed += 1 + if (result.budgetExhausted) break + } catch (error) { + metrics.failed += 1 + const lastError = safeParityErrorText(error) + await rotateCheckedAt({ + db: input.env.APP_DB, + userId, + nowIso, + lastError, + }).catch(() => undefined) + console.warn('mailbox-parity-reconcile-user-failed', userId, error) + } + } + + return metrics +} diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts new file mode 100644 index 0000000000..243d0b2086 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -0,0 +1,331 @@ +import { env } from 'cloudflare:workers' +import { expect, test, vi } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { + countD1MailboxParity, + reconcileMailboxParity, +} from './mailbox-reconcile.ts' +import { rpcFor } from './mailbox-test-helpers.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +async function seedParityUser(input: { + email: string + checkedAt?: string | null +}) { + const userId = await createStableUserIdFromEmail(input.email) + await env.APP_DB.prepare( + `INSERT INTO users ( + username, email, password_hash, plan, stable_user_id, + mailbox_parity_checked_at + ) VALUES (?, ?, 'test-password-hash', 'max', ?, ?)`, + ) + .bind( + `parity-${crypto.randomUUID().slice(0, 8)}`, + input.email, + userId, + input.checkedAt ?? null, + ) + .run() + return userId +} + +async function seedMessageGraph(input: { + userId: string + messageId: string + threadId: string + createdAt: string + withAttachment?: boolean +}) { + await env.APP_DB.prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES (?, ?, 'hello', ?, ?, ?)`, + ) + .bind( + input.threadId, + input.userId, + input.createdAt, + input.createdAt, + input.createdAt, + ) + .run() + await env.APP_DB.prepare( + `INSERT INTO email_messages ( + id, direction, user_id, thread_id, from_address, subject, text_body, + raw_size, processing_status, created_at, updated_at + ) VALUES (?, 'outbound', ?, ?, 'from@example.test', 'Hello', 'body', 4, + 'sent', ?, ?)`, + ) + .bind( + input.messageId, + input.userId, + input.threadId, + input.createdAt, + input.createdAt, + ) + .run() + if (input.withAttachment) { + await env.APP_DB.prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, created_at + ) VALUES (?, ?, 'note.txt', 'text/plain', 4, 'unavailable', ?)`, + ) + .bind(`att-${input.messageId}`, input.messageId, input.createdAt) + .run() + } + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, + ) + .bind( + `evt-${input.messageId}`, + input.messageId, + input.userId, + input.createdAt, + ) + .run() +} + +async function seedOrphanEvent(input: { + userId: string + eventId: string + createdAt: string +}) { + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind(input.eventId, input.userId, input.createdAt) + .run() +} + +async function readParityState(userId: string) { + return env.APP_DB.prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt, + mailbox_parity_last_error AS lastError + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + lastError: string | null + }>() +} + +async function parkOther(userId: string) { + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves soak', async () => { + silenceIncidentalRuntimeWarnings() + // Keep wall time aligned with each tick's nowIso so same-run content upper + // bounds stay deterministic (without freezing RPC timeout clocks). + vi.useFakeTimers({ shouldAdvanceTime: true }) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + try { + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const userId = await seedParityUser({ + email: `parity-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + const otherUserId = await seedParityUser({ + email: `other-${crypto.randomUUID()}@example.test`, + checkedAt: '9999-12-31T23:59:59.999Z', + }) + + await seedMessageGraph({ + userId, + messageId: 'parity-msg-1', + threadId: 'parity-thread-1', + createdAt: '2026-07-01T12:00:00.000Z', + withAttachment: true, + }) + await seedMessageGraph({ + userId, + messageId: 'parity-msg-2', + threadId: 'parity-thread-2', + createdAt: '2026-07-01T12:01:00.000Z', + }) + await seedOrphanEvent({ + userId, + eventId: 'parity-orphan-1', + createdAt: '2026-07-01T12:02:00.000Z', + }) + await seedMessageGraph({ + userId: otherUserId, + messageId: 'other-msg-1', + threadId: 'other-thread-1', + createdAt: '2026-07-01T12:00:00.000Z', + }) + + const mailbox = rpcFor(userId) + // Warm the DO so the first 1s-bounded mirror RPC is not cold-start timeout. + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const firstNow = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(firstNow) + await expect( + reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + + const d1Counts = await countD1MailboxParity({ db: env.APP_DB, userId }) + expect(await mailbox.countMailbox()).toEqual(d1Counts) + expect(await mailbox.getMessage({ messageId: 'other-msg-1' })).toBeNull() + + const afterMatch = await readParityState(userId) + expect(afterMatch).toMatchObject({ + checkedAt: firstNow.toISOString(), + matchingSince: firstNow.toISOString(), + contentWatermarkAt: expect.any(String), + mismatchCount: 0, + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'parity-msg-2', + orphanCursorId: 'parity-orphan-1', + lastError: null, + }) + if (!afterMatch?.contentWatermarkAt) { + throw new Error('Expected parity content watermark after match.') + } + expect(afterMatch.contentWatermarkAt >= firstNow.toISOString()).toBe(true) + + // Live D1 row after completed cursors: compare mismatches, reopens backfill + // while retaining cursors so the next tick repairs only the new row. + await seedMessageGraph({ + userId, + messageId: 'parity-msg-3', + threadId: 'parity-thread-3', + createdAt: '2026-07-01T12:03:00.000Z', + }) + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(mismatchNow) + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + const afterMismatch = await readParityState(userId) + expect(afterMismatch).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + messageCursorId: 'parity-msg-2', + orphanCursorId: 'parity-orphan-1', + checkedAt: mismatchNow.toISOString(), + }) + expect(afterMismatch?.contentWatermarkAt).not.toBeNull() + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + vi.setSystemTime(rematchNow) + await expect( + reconcileMailboxParity({ env, now: rematchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + await mailbox.getMessage({ messageId: 'parity-msg-3' }), + ).toMatchObject({ id: 'parity-msg-3' }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + const afterRematch = await readParityState(userId) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'parity-msg-3', + }) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + vi.setSystemTime(soakNow) + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityState(userId) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toEqual(expect.any(String)) + expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + } finally { + vi.useRealTimers() + } +}, 30_000) diff --git a/packages/worker/src/feature-flags/registry.node.test.ts b/packages/worker/src/feature-flags/registry.node.test.ts new file mode 100644 index 0000000000..5361d993d0 --- /dev/null +++ b/packages/worker/src/feature-flags/registry.node.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from 'vitest' +import { + featureFlagDefinitions, + getFeatureFlagDefinition, + isFeatureFlagKey, + measuredFeatureFlagKeys, +} from './registry.ts' + +test('mailbox-read-cutover is registered default-off with email_received success metric', () => { + expect(isFeatureFlagKey('mailbox-read-cutover')).toBe(true) + const definition = getFeatureFlagDefinition('mailbox-read-cutover') + expect(definition).toMatchObject({ + key: 'mailbox-read-cutover', + defaultEnabled: false, + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }, + }) + expect(definition.description.length).toBeGreaterThan(0) + expect(measuredFeatureFlagKeys.has('mailbox-read-cutover')).toBe(true) + expect( + featureFlagDefinitions.some((flag) => flag.key === 'mailbox-read-cutover'), + ).toBe(true) +}) diff --git a/packages/worker/src/feature-flags/registry.ts b/packages/worker/src/feature-flags/registry.ts index 563fe37da6..9900341d0b 100644 --- a/packages/worker/src/feature-flags/registry.ts +++ b/packages/worker/src/feature-flags/registry.ts @@ -49,6 +49,19 @@ export const featureFlagDefinitions = [ // exercise the flag system itself (including the "no success metric" // admin notice), not to move a product metric. }, + { + key: 'mailbox-read-cutover', + defaultEnabled: false, + description: + 'Serve owner-scoped user-mail message reads from the per-user Mailbox Durable Object when the account also has continuous D1↔Mailbox parity soak (matching_since ≥ 24h, fresh checked_at, zero mismatches). Default off; fail closed without verified parity. No live reader is switched until a later wiring PR.', + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + hypothesis: + 'After verified parity soak, reading owner mail from Mailbox should reduce read-path failures that currently surface alongside inbound mail handling.', + }, + }, ] as const satisfies ReadonlyArray export type FeatureFlagKey = (typeof featureFlagDefinitions)[number]['key'] diff --git a/packages/worker/src/feature-flags/service.node.test.ts b/packages/worker/src/feature-flags/service.node.test.ts index 9cf08d2814..68f259fe2f 100644 --- a/packages/worker/src/feature-flags/service.node.test.ts +++ b/packages/worker/src/feature-flags/service.node.test.ts @@ -268,6 +268,7 @@ test('isFeatureEnabled falls back to registry default when no DB state exists', ) await expect(getFeatureFlagsForUser(db, 1)).resolves.toEqual({ 'demo-indicator': false, + 'mailbox-read-cutover': false, }) }) @@ -434,6 +435,7 @@ test('user override wins over global off and global on; clear restores evaluatio await expect(isFeatureEnabled(db, 'demo-indicator', 8)).resolves.toBe(false) await expect(getFeatureFlagsForUser(db, 7)).resolves.toEqual({ 'demo-indicator': true, + 'mailbox-read-cutover': false, }) await setFeatureFlagGlobalState(db, { @@ -465,6 +467,7 @@ test('getFeatureFlagEvaluationsForUser reports assignment sources', async () => await expect(getFeatureFlagEvaluationsForUser(db, 7)).resolves.toEqual({ 'demo-indicator': { enabled: false, source: 'default' }, + 'mailbox-read-cutover': { enabled: false, source: 'default' }, }) await setFeatureFlagGlobalState(db, { @@ -550,7 +553,7 @@ test('listFeatureFlagsForAdmin includes registry flags and stale DB-only keys', }) const listed = await listFeatureFlagsForAdmin(db) - expect(listed).toHaveLength(3) + expect(listed).toHaveLength(4) const demo = listed.find((flag) => flag.key === 'demo-indicator') expect(demo).toMatchObject({ @@ -575,6 +578,22 @@ test('listFeatureFlagsForAdmin includes registry flags and stale DB-only keys', expect(demo?.description).toEqual(expect.any(String)) expect(demo?.description).not.toHaveLength(0) + const mailboxReadCutover = listed.find( + (flag) => flag.key === 'mailbox-read-cutover', + ) + expect(mailboxReadCutover).toMatchObject({ + key: 'mailbox-read-cutover', + stale: false, + defaultEnabled: false, + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }, + global: null, + overrides: [], + }) + const retired = listed.find((flag) => flag.key === 'retired-flag') expect(retired).toEqual({ key: 'retired-flag', diff --git a/packages/worker/src/mcp/capabilities/access-control.node.test.ts b/packages/worker/src/mcp/capabilities/access-control.node.test.ts index 402dc3bbd3..47faea4da3 100644 --- a/packages/worker/src/mcp/capabilities/access-control.node.test.ts +++ b/packages/worker/src/mcp/capabilities/access-control.node.test.ts @@ -12,6 +12,7 @@ import { type Capability } from './types.ts' function createFlagMap(enabled: boolean): CallerFeatureFlags { return { 'demo-indicator': enabled, + 'mailbox-read-cutover': false, } } diff --git a/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts index f1f9ab28b9..f7f2318749 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts @@ -314,17 +314,33 @@ test('admin feature flag MCP capabilities: list, set, override, and audit wiring const ctx = createAdminCapabilityContext(db) const listResult = await adminFeatureFlagListCapability.handler({}, ctx) - expect(listResult.flags).toEqual([ - expect.objectContaining({ - key: 'demo-indicator', - description: expect.any(String), - defaultEnabled: false, - stale: false, - global: null, - overrides: [], - }), - ]) - expect(listResult.flags[0]?.description).not.toHaveLength(0) + expect(listResult.flags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: 'demo-indicator', + description: expect.any(String), + defaultEnabled: false, + stale: false, + global: null, + overrides: [], + successMetric: null, + }), + expect.objectContaining({ + key: 'mailbox-read-cutover', + defaultEnabled: false, + stale: false, + successMetric: expect.objectContaining({ + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }), + }), + ]), + ) + const demoFlag = listResult.flags.find( + (flag) => flag.key === 'demo-indicator', + ) + expect(demoFlag?.description).not.toHaveLength(0) const setResult = await adminFeatureFlagSetCapability.handler( { diff --git a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts new file mode 100644 index 0000000000..dcd75153d4 --- /dev/null +++ b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts @@ -0,0 +1,54 @@ +import { expect, test, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + reconcileMailboxParity: vi.fn(), +})) + +vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ + reconcileMailboxParity: mocks.reconcileMailboxParity, +})) + +const { getScheduledLanes, runScheduledLane, shouldRunMailboxParityCron } = + await import('./scheduled-lanes.ts') + +test('shouldRunMailboxParityCron gates to the top of each UTC hour', () => { + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:00:30.000Z'))).toBe( + true, + ) + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:05:00.000Z'))).toBe( + false, + ) + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:30:00.000Z'))).toBe( + false, + ) +}) + +test('getScheduledLanes includes mailbox_parity only on the hourly gate', () => { + const env = {} as Env + + expect( + getScheduledLanes({ + env, + scheduledAt: new Date('2026-07-05T10:00:30.000Z'), + }), + ).toContain('mailbox_parity') + expect( + getScheduledLanes({ + env, + scheduledAt: new Date('2026-07-05T10:30:00.000Z'), + }), + ).not.toContain('mailbox_parity') +}) + +test('runScheduledLane dispatches mailbox_parity through reconcileMailboxParity', async () => { + const env = {} as Env + const scheduledAt = new Date('2026-07-05T10:00:00.000Z') + mocks.reconcileMailboxParity.mockResolvedValueOnce({ compared: 0 }) + + await runScheduledLane({ env, lane: 'mailbox_parity', scheduledAt }) + + expect(mocks.reconcileMailboxParity).toHaveBeenCalledWith({ + env, + now: scheduledAt, + }) +}) diff --git a/packages/worker/src/scheduled/scheduled-lanes.ts b/packages/worker/src/scheduled/scheduled-lanes.ts index a6922c14d4..1ff5b09dee 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.ts @@ -16,6 +16,7 @@ import { shouldRunDrExportCron, shouldRunDrExportWatchdogCron, } from '#worker/dr/exporter.ts' +import { reconcileMailboxParity } from '#worker/email/mailbox-reconcile.ts' import { sweepStaleInboundDeliveries } from '#worker/email/reconcile-inbound-deliveries.ts' import { pruneSystemEmailRetention } from '#worker/email/system-email.ts' import { reconcileD1StorageBytes } from '#worker/entitlements/d1-storage-reconciliation.ts' @@ -32,6 +33,11 @@ import { shouldRunUsageAggregationCron, } from '#worker/usage/aggregate-rollups.ts' +/** Worker cron fires every 5 minutes; run this lane on the top of each UTC hour. */ +export function shouldRunMailboxParityCron(now: Date) { + return now.getUTCMinutes() === 0 +} + export const scheduledLaneNames = [ 'reconcile_artifacts_pushes', 'repo_session_cleanup', @@ -48,6 +54,7 @@ export const scheduledLaneNames = [ 'dr_export', 'dr_export_watchdog', 'job_schedule_watchdog', + 'mailbox_parity', ] as const export type ScheduledLaneName = (typeof scheduledLaneNames)[number] @@ -107,6 +114,9 @@ export function getScheduledLanes(input: { if (shouldRunJobScheduleWatchdogCron(input.scheduledAt)) { lanes.push('job_schedule_watchdog') } + if (shouldRunMailboxParityCron(input.scheduledAt)) { + lanes.push('mailbox_parity') + } return lanes } @@ -183,6 +193,11 @@ export async function runScheduledLane(input: { env: input.env, now: input.scheduledAt, }) + case 'mailbox_parity': + return reconcileMailboxParity({ + env: input.env, + now: input.scheduledAt, + }) default: { const exhaustive: never = input.lane throw new Error(`Unhandled scheduled lane: ${String(exhaustive)}`) diff --git a/packages/worker/src/users-test-schema.ts b/packages/worker/src/users-test-schema.ts index c678e36fad..fd279cb1da 100644 --- a/packages/worker/src/users-test-schema.ts +++ b/packages/worker/src/users-test-schema.ts @@ -74,6 +74,22 @@ const alwaysAdditiveColumns: Record = { active_write_expires_at: { create: 'TEXT' }, d1_storage_bytes: { create: 'INTEGER NOT NULL DEFAULT 0' }, d1_storage_bytes_updated_at: { create: 'TEXT' }, + mailbox_parity_checked_at: { create: 'TEXT' }, + mailbox_parity_matching_since: { create: 'TEXT' }, + mailbox_parity_mismatch_count: { + create: + 'INTEGER NOT NULL DEFAULT 0 CHECK (mailbox_parity_mismatch_count >= 0)', + alter: + 'INTEGER NOT NULL DEFAULT 0 CHECK (mailbox_parity_mismatch_count >= 0)', + }, + mailbox_parity_last_error: { create: 'TEXT' }, + mailbox_parity_content_watermark_at: { create: 'TEXT' }, + mailbox_parity_message_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_message_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_message_backfill_completed_at: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_completed_at: { create: 'TEXT' }, } /** From 3492ad5187d1823e879d228f1144f6b85518383a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:51:20 +0000 Subject: [PATCH 11/20] test(email): register Mailbox parity migration Co-authored-by: Kent C. Dodds --- packages/worker/src/index.workers.test.ts | 2 +- tools/migration-ledger.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/index.workers.test.ts b/packages/worker/src/index.workers.test.ts index 2014fc5079..460f1f15bb 100644 --- a/packages/worker/src/index.workers.test.ts +++ b/packages/worker/src/index.workers.test.ts @@ -141,7 +141,7 @@ test('scheduled runs gated lanes and passes EMAIL_BLOBS to system-email retentio mocks.shouldRunAuthDenialAlertCron.mockReturnValueOnce(true) mocks.shouldRunEmailDeliveryAlertCron.mockReturnValueOnce(true) mocks.shouldRunJobScheduleWatchdogCron.mockReturnValueOnce(true) - const scheduledTime = Date.parse('2026-07-05T10:00:30.000Z') + const scheduledTime = Date.parse('2026-07-05T10:05:30.000Z') await worker.scheduled?.( createController(scheduledTime), diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 9febdba923..379b71d275 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -511,6 +511,10 @@ { "filename": "0124-entity-sources-external-reconcile-pending.sql", "sha256": "75b93b4c2fa14c6303834877d6a07a42bb054a33b16a55de23c6b56f4b31bd0d" + }, + { + "filename": "0125-mailbox-parity-state.sql", + "sha256": "d5a99f57d8db18943b7184c1c2cc2002c5ef6c20584def71b9860e823b0456cc" } ] } From f03a66f9f8f69f83f77ef81c5a9388803cec36c8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:06:59 +0000 Subject: [PATCH 12/20] fix(email): make Mailbox parity reconciliation resumable Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 142 ++-- .../architecture/feature-flags.md | 8 +- .../migrations/0125-mailbox-parity-state.sql | 44 +- .../worker/src/email/mailbox-parity-phases.ts | 522 ++++++++++++++ .../worker/src/email/mailbox-parity-repo.ts | 120 +++- .../email/mailbox-read-cutover.node.test.ts | 44 ++ .../worker/src/email/mailbox-read-cutover.ts | 4 +- .../src/email/mailbox-reconcile.node.test.ts | 674 ++++++++++++------ .../worker/src/email/mailbox-reconcile.ts | 512 ++++--------- .../email/mailbox-reconcile.workers.test.ts | 131 +++- packages/worker/src/users-test-schema.ts | 9 +- 11 files changed, 1455 insertions(+), 755 deletions(-) create mode 100644 packages/worker/src/email/mailbox-parity-phases.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e8d9a38814..e747ae096d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -272,12 +272,17 @@ The schema is defined by migrations in `packages/worker/migrations/`: parity/backfill state (migration `0125-mailbox-parity-state.sql`): `mailbox_parity_checked_at`, `mailbox_parity_matching_since`, `mailbox_parity_mismatch_count`, `mailbox_parity_last_error`, - `mailbox_parity_content_watermark_at`, message and orphan-event backfill - cursors/completion timestamps on the user row. Indexed by + `mailbox_parity_content_watermark_at`, durable content-replay window + (`mailbox_parity_content_replay_upper_at` plus `(updated_at, id)` cursor), + message and **all** owner delivery-event backfill cursors/completion + timestamps on the user row. Indexed by `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in the hourly `mailbox_parity` reconcile lane (see - [Mailbox](#durable-objects-mailbox)). Inbound email routing does not - reverse-resolve stable ids at all — it uses the indexed username lookup + [Mailbox](#durable-objects-mailbox)). The same migration adds keyset-friendly + composites on `email_messages(user_id, created_at, id)`, + `email_messages(user_id, updated_at, id)`, and + `email_delivery_events(user_id, created_at, id)`. Inbound email routing does + not reverse-resolve stable ids at all — it uses the indexed username lookup (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids with one indexed point read on `users.stable_user_id` (for example `findUserAccountByStableUserId`). @@ -805,7 +810,9 @@ mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired **Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, -`packages/worker/src/email/mailbox-reconcile.ts`). The five-minute Worker cron +`packages/worker/src/email/mailbox-reconcile.ts`, +`packages/worker/src/email/mailbox-parity-phases.ts`, +`packages/worker/src/email/mailbox-parity-repo.ts`). The five-minute Worker cron dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); production uses the same per-lane queue isolation as reconcile, retention, and sibling lanes so a slow parity pass cannot consume another lane's budget. @@ -816,27 +823,38 @@ ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who have at least one `email_messages` or `email_delivery_events` row. Per-user work is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). +**Account-deletion races:** `loadUserParityState`, `persistUserParityProgress`, +and `rotateCheckedAt` require `deleting_at IS NULL` (zero-row updates are +harmless). Before and after each mirror RPC the lane re-checks +`users.deleting_at`; if deletion started mid-tick it best-effort +`Mailbox.purge()`s and skips further progress for that user. + Per user, the lane: 1. **Content watermark baseline** — on the first backfill attempt, sets `mailbox_parity_content_watermark_at` to the run's `now` and retains it - across incomplete/error ticks so updates during creation/orphan mirroring - stay inside the replay window. + across incomplete/error ticks so updates during creation mirroring stay + inside the replay window. 2. **Bounded initial message backfill** — keyset-pages D1 `email_messages` by - `(created_at, id)` and mirrors each graph via - `mirrorMailboxMessageGraphFromD1` until complete or budget-exhausted. -3. **Bounded orphan-event backfill** — after messages complete, keyset-pages D1 - `email_delivery_events` with `message_id IS NULL` and mirrors each via - `mirrorMailboxOrphanDeliveryEventFromD1`. -4. **Content watermark replay** — after both initial backfills complete, - keyset-replays every owner message with `updated_at` in - `(watermark, current-now]` (classification-like updates that landed after a - graph mirror but before backfill finished). The watermark advances only when - the full window succeeds. -5. **Count compare + AE signals** — owner-scoped D1 counts vs - `Mailbox.countMailbox()` for threads, messages, attachments, and delivery - events. Each comparison emits one `mailbox_parity:` Analytics - Engine row with `double2 = d1Count - doCount`. + `(created_at, id)` (composite index from migration `0125`) and mirrors each + graph via `mirrorMailboxMessageGraphFromD1` until complete or + budget-exhausted. +3. **Bounded delivery-event backfill** — after messages complete, keyset-pages + **every** owner `email_delivery_events` row by `(created_at, id)` — not only + `message_id`-null orphans — and mirrors each via + `mirrorMailboxParityDeliveryEventFromD1`. This repairs events omitted when + per-message graph mirroring truncates the event batch. +4. **Durable content watermark replay** — after both creation phases complete, + opens a frozen window `(watermark, upper]` + (`mailbox_parity_content_replay_upper_at` set once when the window opens; + retained across ticks). Keyset-replays owner messages by `(updated_at, id)` + within that window; the `(updated_at, id)` cursor persists across incomplete + ticks. The watermark advances to `upper` only when the full window succeeds. +5. **Count compare + AE signals** — runs only after the event phase and a + completed content window. Owner-scoped D1 counts vs `Mailbox.countMailbox()` + for threads, messages, attachments, and delivery events. Each comparison + emits one `mailbox_parity:` Analytics Engine row with + `double2 = d1Count - doCount`. Parity progress persists on `users` (migration `0125-mailbox-parity-state.sql`). D1 remains mail authority — the lane never mutates `email_*` rows and does not @@ -845,13 +863,14 @@ flip read authority. **`matching_since` semantics:** records the start of continuous **exact** D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact compare when null; preserved across successful content replay + exact compare. -Cleared on: initial backfill work in the same tick (`initialBackfilled > 0`), +Cleared on: creation backfill work in the same tick (`creationBackfilled > 0`), incomplete/budget-exhausted ticks, mirror retryable failures, content-replay failure/incomplete, compare mismatch (also increments consecutive -`mailbox_parity_mismatch_count` and reopens bounded creation/orphan backfill -while retaining the content watermark), and compare-path errors -(`rotateCheckedAt` also clears soak so a poison tick cannot preserve a false -window). Resets `mismatch_count` to `0` on exact match. +`mailbox_parity_mismatch_count`, **resets both message and delivery-event +creation cursors and completion markers** for a full historical rescan while +retaining the content watermark and clearing any in-flight replay window), and +compare-path errors (`rotateCheckedAt` also clears soak so a poison tick cannot +preserve a false window). Resets `mismatch_count` to `0` on exact match. **Phase 3 read cutover (prepared, not flipped):** `packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off @@ -859,10 +878,11 @@ window). Resets `mismatch_count` to `0` on exact match. `isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user parity soak: `mailbox_parity_matching_since` at least 24h old, `mailbox_parity_checked_at` fresh within 2h, -`mailbox_parity_mismatch_count === 0`, and exact `stable_user_id` match. D1 -errors, missing parity state, and an off flag fail closed to D1 reads. No live -reader calls this adapter yet; exposure recording is intentionally omitted until -a live gate site exists (see [Feature flags](./feature-flags.md)). +`mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, and +`deleting_at IS NULL` (deleting accounts fail closed to D1). D1 errors, missing +parity state, and an off flag fail closed to D1 reads. No live reader calls this +adapter yet; exposure recording is intentionally omitted until a live gate site +exists (see [Feature flags](./feature-flags.md)). **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -895,9 +915,10 @@ batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly -queue-isolated `mailbox_parity` lane backfills historical graphs, replays -content-watermark updates, compares owner-scoped D1 vs Mailbox counts, and -persists soak state on `users` (migration `0125`). Inbound lifecycle dual-write, +queue-isolated `mailbox_parity` lane backfills all owner messages and delivery +events, durable content-watermark replays, compares owner-scoped D1 vs Mailbox +counts, persists soak state on `users` (migration `0125`), and re-purges the DO +when account deletion races the lane. Inbound lifecycle dual-write, retention/metadata deletes, and live read cutover are **still not wired**. Live email read/write authority is unchanged — D1 `email_*` tables remain authoritative for all live paths. D1 email rows and existing R2 inventory @@ -918,26 +939,26 @@ delete mirrors remain pending. `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and - `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, and - `mailbox-reconcile.ts`. **Live today (partial):** outbound terminals, - provider delivery-queue outcomes with a message (`recorded`, `duplicate`, - `stale`; `waitUntil`), and user classification - (`service.ts#setEmailMessageClassification`) call + `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, + `mailbox-parity-phases.ts`, and `mailbox-reconcile.ts`. **Live today + (partial):** outbound terminals, provider delivery-queue outcomes with a + message (`recorded`, `duplicate`, `stale`; `waitUntil`), and user + classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; at most two AE writes per attempt). The hourly `mailbox_parity` scheduled - lane backfills, content-watermark replays, count compares, and soak tracking - on `users` (migration `0125`). **Still pending for phase-2 completion:** - inbound commit/touch/classify dual-write and retention/metadata-delete - mirrors. + lane backfills all owner messages and delivery events, durable + content-watermark replays, count compares, and soak tracking on `users` + (migration `0125`). **Still pending for phase-2 completion:** inbound + commit/touch/classify dual-write and retention/metadata-delete mirrors. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh - `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`) **and** - the default-off `mailbox-read-cutover` flag is enabled per user. D1 writes - continue. A prepared `getOwnerEmailMessageById` adapter exists but is **not - wired**; no live reader has flipped and no cutover exposures are recorded - yet. + `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`, account + not marked for deletion) **and** the default-off `mailbox-read-cutover` flag + is enabled per user. D1 writes continue. A prepared + `getOwnerEmailMessageById` adapter exists but is **not wired**; no live + reader has flipped and no cutover exposures are recorded yet. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. @@ -945,9 +966,11 @@ delete mirrors remain pending. after verification. No premature schema deletion. The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in -`scheduled-lanes.ts`) owns backfill, content-watermark replay, count parity, and -soak state on `users`. This Mailbox track owns DO storage semantics, alarm -retention, and the inbound durability boundary below. +`scheduled-lanes.ts`) owns backfill of all owner messages and delivery events, +durable content-watermark replay, count parity, soak state on `users`, and +best-effort `Mailbox.purge()` when account deletion races the lane. This Mailbox +track owns DO storage semantics, alarm retention, and the inbound durability +boundary below. ### What stays in D1 @@ -1241,15 +1264,16 @@ Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session cleanup, system-email retention, general retention, job retention, hourly usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC -hour only; D1 user discovery, bounded backfill, content-watermark replay, and -count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue -message preserves the existing `scheduled_lane_failed` / D1 lock-contention log -and Sentry context. A handled lane failure is acknowledged and retried by the -next cron tick, matching the old cron semantics. A failed enqueue is reported -and runs through the inline fallback after all sibling enqueue attempts finish; -multiple failed enqueues fall back sequentially to avoid D1 lock contention. -Consumer transport failures retain the configured retry/DLQ behavior. No failure -can abort or mask a sibling invocation. +hour only; D1 user discovery, bounded message + full delivery-event backfill, +durable content-watermark replay, and count compare — see +[Mailbox](#durable-objects-mailbox)). Each production queue message preserves +`scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled +lane failure is acknowledged and retried by the next cron tick, matching the old +cron semantics. A failed enqueue is reported and runs through the inline +fallback after all sibling enqueue attempts finish; multiple failed enqueues +fall back sequentially to avoid D1 lock contention. Consumer transport failures +retain the configured retry/DLQ behavior. No failure can abort or mask a sibling +invocation. Production note: diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index 627b59b25e..b026c1f661 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -83,11 +83,13 @@ parity soak on `users` (migration `0125-mailbox-parity-state.sql`): D1↔Mailbox count parity), - `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), - `mailbox_parity_mismatch_count === 0`, -- exact `stable_user_id` match between the gate caller and the loaded row. +- exact `stable_user_id` match between the gate caller and the loaded row, +- `deleting_at IS NULL` (accounts marked for deletion fail closed to D1). D1 errors, missing/incomplete parity state, stale `checked_at`, non-zero -mismatch depth, and an off flag all **fail closed** to D1-authoritative reads. -When the gate is on, Mailbox DO read errors propagate with no D1 fallback. +mismatch depth, deleting accounts, and an off flag all **fail closed** to +D1-authoritative reads. When the gate is on, Mailbox DO read errors propagate +with no D1 fallback. The prepared adapter lives in `packages/worker/src/email/mailbox-read-cutover.ts` diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 3a2af01586..55a93e2459 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -1,19 +1,18 @@ -- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. -- Low-write columns live on users (not per-message). Creation cursors are --- keyset (created_at, id) so message and message_id-null orphan-event backfill --- can resume across ticks without DO enumeration. content_watermark_at is set --- to the run's now at the start of the first backfill attempt (retained across --- incomplete/error ticks) and bounds a keyset replay of messages with --- updated_at in (watermark, current-now] after both initial backfills complete --- so classification-like updates — including those that land after a graph was --- mirrored but before backfill finished — are remirrored even when counts stay --- equal; the watermark advances only after the full window succeeds. --- matching_since records the start of continuous exact D1 ↔Mailbox count --- parity for soak tracking; it clears on initial backfill work, --- incomplete/budget-exhausted ticks, mirror errors, content-replay --- failure/incomplete, or compare mismatch — but may persist across a --- successful content replay + exact compare. mismatch_count is consecutive --- current mismatch depth and resets to 0 on exact match. +-- keyset (created_at, id) for messages and ALL owner delivery events so backfill +-- can resume across ticks without DO enumeration. The event phase rescans every +-- email_delivery_events row (not only message_id-null) so per-message graph +-- truncation cannot leave DO events behind. content_watermark_at is set to the +-- run's now at the start of the first backfill attempt (retained across +-- incomplete/error ticks). After both creation phases complete, a durable +-- content-replay window (replay_upper_at + replay cursor) keyset-replays +-- messages with updated_at in (watermark, upper]; the watermark advances to +-- upper only when that window finishes. Count compare runs only after the full +-- event phase and a completed content window. Count mismatch clears soak +-- eligibility and resets both creation cursors and completion markers for a +-- full historical rescan (watermark preserved; in-flight replay window cleared). +-- matching_since may persist across a successful content replay + exact compare. ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_matching_since TEXT; @@ -21,14 +20,25 @@ ALTER TABLE users ADD COLUMN mailbox_parity_mismatch_count INTEGER NOT NULL DEFA CHECK (mailbox_parity_mismatch_count >= 0); ALTER TABLE users ADD COLUMN mailbox_parity_last_error TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_content_watermark_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_upper_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_cursor_updated_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_cursor_id TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_created_at TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_id TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_completed_at TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_created_at TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_id TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_completed_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_completed_at TEXT; CREATE INDEX idx_users_mailbox_parity_checked ON users(mailbox_parity_checked_at, stable_user_id); + +-- Keyset-friendly composites for creation scans and content replay. +CREATE INDEX IF NOT EXISTS idx_email_messages_user_created_at_id + ON email_messages(user_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_email_messages_user_updated_at_id + ON email_messages(user_id, updated_at, id); +CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_created_at_id + ON email_delivery_events(user_id, created_at, id); diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts new file mode 100644 index 0000000000..6a5306f98d --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -0,0 +1,522 @@ +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, + type MailboxLiveMirrorGraphSummary, +} from './mailbox-live-mirror.ts' +import { type MailboxMirrorResult } from './mailbox-mirror.ts' +import { + listContentReplayPage, + listEventBackfillPage, + listMessageBackfillPage, + type MailboxParityContentCursor, + type MailboxParityUserState, +} from './mailbox-parity-repo.ts' + +export const mailboxParityMessagePageSize = 10 +export const mailboxParityEventPageSize = 25 +export const mailboxParityContentPageSize = 10 + +export type MailboxParityPhaseEnv = MailboxLiveMirrorEnv & { + APP_DB: D1Database +} + +export type MailboxParityPhaseResult = { + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean + /** Set when a selected mirror outcome must not advance the cursor. */ + blockedReason: string | null +} + +/** + * Message graph cursor may advance only on mirrored/stale. missing / skipped / + * timeout / error must not advance (selected D1 row was expected). + * `eventsTruncated` is acceptable: the complete owner event phase remirrors + * every delivery event afterward. + */ +export function messageGraphAllowsCursorAdvance( + summary: MailboxLiveMirrorGraphSummary, +): boolean { + switch (summary.message.status) { + case 'mirrored': + case 'stale': + break + case 'missing': + case 'timeout': + case 'error': + case 'skipped': + return false + default: { + const exhaustive: never = summary.message + throw new Error( + `Unhandled graph message status: ${JSON.stringify(exhaustive)}`, + ) + } + } + return !summary.events.some((event) => { + switch (event.result.status) { + case 'mirrored': + case 'stale': + case 'missing': + return false + case 'timeout': + case 'error': + case 'skipped': + return true + default: { + const exhaustive: never = event.result + throw new Error( + `Unhandled graph event status: ${JSON.stringify(exhaustive)}`, + ) + } + } + }) +} + +/** + * Event-phase cursor: mirrored/stale advance; missing advances (D1 row may have + * been deleted concurrently after selection); skipped (incl. unconfigured), + * timeout, and error must not advance. + */ +export function eventMirrorAllowsCursorAdvance( + result: MailboxMirrorResult, +): boolean { + switch (result.status) { + case 'mirrored': + case 'stale': + case 'missing': + return true + case 'timeout': + case 'error': + case 'skipped': + return false + default: { + const exhaustive: never = result + throw new Error( + `Unhandled event mirror status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +/** + * Load one owner delivery-event projection from D1 and mirror it. + * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. + */ +export async function mirrorMailboxParityDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string +}): Promise { + return mirrorMailboxDeliveryEventFromD1(input) +} + +export function resolveContentReplayUpperBound(input: { + nowIso: string + watermarkAt: string +}): string { + let upperBoundAt = input.nowIso + if (input.watermarkAt === input.nowIso) { + const wallIso = new Date().toISOString() + if (wallIso > upperBoundAt) upperBoundAt = wallIso + } + if (upperBoundAt < input.watermarkAt) return input.watermarkAt + return upperBoundAt +} + +export async function backfillMessagesForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + deadlineMs: number + isDeleting: () => Promise +}): Promise { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listMessageBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.messageCursor, + limit: mailboxParityMessagePageSize, + }) + if (page.length === 0) { + state = { + ...state, + messagesCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + let cursor = state.messageCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (await input.isDeleting()) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (await input.isDeleting()) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + if (!messageGraphAllowsCursorAdvance(summary)) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: `message graph ${summary.message.status}`, + deletionStarted: false, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, messageCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } +} + +export async function backfillEventsForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + deadlineMs: number + isDeleting: () => Promise +}): Promise { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt == null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (state.eventsCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listEventBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.eventCursor, + limit: mailboxParityEventPageSize, + }) + if (page.length === 0) { + state = { + ...state, + eventsCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + let cursor = state.eventCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (await input.isDeleting()) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + const result = await mirrorMailboxParityDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + eventId: row.id, + }) + if (await input.isDeleting()) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + if (!eventMirrorAllowsCursorAdvance(result)) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: + result.status === 'skipped' + ? `event skipped:${result.reason}` + : `event ${result.status}`, + deletionStarted: false, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, eventCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } +} + +/** + * Durable content replay over (watermark, upper]. Freezes upper when opening a + * window; resumes cursor across ticks; advances watermark only on full success. + */ +export async function replayContentUpdatesForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + nowIso: string + deadlineMs: number + isDeleting: () => Promise +}): Promise< + MailboxParityPhaseResult & { + deletionStarted: boolean + windowComplete: boolean + openedWindow: boolean + } +> { + let state = input.state + const watermarkAt = state.contentWatermarkAt ?? input.nowIso + let openedWindow = false + + if (state.contentReplayUpperAt == null) { + const upperBoundAt = resolveContentReplayUpperBound({ + nowIso: input.nowIso, + watermarkAt, + }) + state = { + ...state, + contentWatermarkAt: watermarkAt, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: null, + } + openedWindow = true + } + + const upperBoundAt = state.contentReplayUpperAt! + let cursor: MailboxParityContentCursor | null = state.contentReplayCursor + let backfilled = 0 + + while (Date.now() < input.deadlineMs) { + const page = await listContentReplayPage({ + db: input.env.APP_DB, + userId: state.userId, + watermarkAt, + upperBoundAt, + cursor, + limit: mailboxParityContentPageSize, + }) + if (page.length === 0) { + return { + state: { + ...state, + contentWatermarkAt: upperBoundAt, + contentReplayUpperAt: null, + contentReplayCursor: null, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: true, + openedWindow, + } + } + + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: false, + openedWindow, + } + } + if (await input.isDeleting()) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + windowComplete: false, + openedWindow, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (await input.isDeleting()) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + windowComplete: false, + openedWindow, + } + } + if (!messageGraphAllowsCursorAdvance(summary)) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: `content replay ${summary.message.status}`, + deletionStarted: false, + windowComplete: false, + openedWindow, + } + } + cursor = { updatedAt: row.updated_at, id: row.id } + backfilled += 1 + state = { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + } + } + } + + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: false, + openedWindow, + } +} diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index f41c43a582..6c9c97ccf1 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -3,8 +3,9 @@ import { type MailboxCountResult } from './mailbox-types.ts' /** * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. - * Creation backfill uses (created_at, id) keysets; content replay uses - * (updated_at, id) within (watermark, now]. + * Creation backfill uses (created_at, id) keysets over messages and all owner + * delivery events; content replay uses a durable (watermark, upper] window with + * an (updated_at, id) cursor. */ export type MailboxParityBackfillCursor = { @@ -22,10 +23,12 @@ export type MailboxParityUserState = { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursor: MailboxParityContentCursor | null messageCursor: MailboxParityBackfillCursor | null messagesCompletedAt: string | null - orphanEventCursor: MailboxParityBackfillCursor | null - orphanEventsCompletedAt: string | null + eventCursor: MailboxParityBackfillCursor | null + eventsCompletedAt: string | null } export type MailboxParityCreatedKeysetRow = { @@ -66,6 +69,22 @@ export async function listUsersForMailboxParity(input: { return result.results ?? [] } +export async function isUserDeleting(input: { + db: D1Database + userId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT deleting_at AS deletingAt + FROM users + WHERE stable_user_id = ? + LIMIT 1`, + ) + .bind(input.userId) + .first<{ deletingAt: string | null }>() + return row == null || row.deletingAt != null +} + export async function countD1MailboxParity(input: { db: D1Database userId: string @@ -114,12 +133,15 @@ export async function loadUserParityState(input: { mailbox_parity_matching_since AS matchingSince, mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt FROM users WHERE stable_user_id = ? AND deleting_at IS NULL @@ -131,31 +153,44 @@ export async function loadUserParityState(input: { matchingSince: string | null mismatchCount: number | null contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null messageCursorCreatedAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorCreatedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null }>() if (!row) return null const messageCursor = row.messageCursorCreatedAt != null && row.messageCursorId != null ? { createdAt: row.messageCursorCreatedAt, id: row.messageCursorId } : null - const orphanEventCursor = - row.orphanCursorCreatedAt != null && row.orphanCursorId != null - ? { createdAt: row.orphanCursorCreatedAt, id: row.orphanCursorId } + const eventCursor = + row.eventCursorCreatedAt != null && row.eventCursorId != null + ? { createdAt: row.eventCursorCreatedAt, id: row.eventCursorId } + : null + const contentReplayCursor = + row.contentReplayCursorUpdatedAt != null && + row.contentReplayCursorId != null + ? { + updatedAt: row.contentReplayCursorUpdatedAt, + id: row.contentReplayCursorId, + } : null return { userId: row.userId, matchingSince: row.matchingSince, mismatchCount: Number(row.mismatchCount ?? 0) || 0, contentWatermarkAt: row.contentWatermarkAt, + contentReplayUpperAt: row.contentReplayUpperAt, + contentReplayCursor, messageCursor, messagesCompletedAt: row.messagesCompletedAt, - orphanEventCursor, - orphanEventsCompletedAt: row.orphanEventsCompletedAt, + eventCursor, + eventsCompletedAt: row.eventsCompletedAt, } } @@ -201,10 +236,10 @@ export async function listMessageBackfillPage(input: { } /** - * Cohesive D1 loader for owner-scoped delivery events with `message_id` null - * (orphan / unbound ledger rows). Keyset-paged for bounded backfill. + * Owner-scoped delivery-event creation keyset (all rows, including those bound + * to messages). Repairs events omitted by per-message graph truncation. */ -export async function listOrphanDeliveryEventBackfillPage(input: { +export async function listEventBackfillPage(input: { db: D1Database userId: string cursor: MailboxParityBackfillCursor | null @@ -217,7 +252,6 @@ export async function listOrphanDeliveryEventBackfillPage(input: { `SELECT id, created_at FROM email_delivery_events WHERE user_id = ? - AND message_id IS NULL ORDER BY created_at ASC, id ASC LIMIT ?`, ) @@ -228,7 +262,6 @@ export async function listOrphanDeliveryEventBackfillPage(input: { `SELECT id, created_at FROM email_delivery_events WHERE user_id = ? - AND message_id IS NULL AND ( created_at > ? OR (created_at = ? AND id > ?) @@ -248,10 +281,8 @@ export async function listOrphanDeliveryEventBackfillPage(input: { } /** - * Keyset page of owner messages updated after `watermarkAt` and at or before - * `upperBoundAt` (run now, or wall time when completing in the baseline run). - * In-tick `cursor` continues within the same window without skipping equal - * `updated_at` rows. + * Keyset page within a frozen content-replay window (watermark, upper]. + * In-tick / durable `cursor` continues without skipping equal `updated_at`. */ export async function listContentReplayPage(input: { db: D1Database @@ -312,13 +343,15 @@ export async function persistUserParityProgress(input: { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursor: MailboxParityContentCursor | null messageCursor: MailboxParityBackfillCursor | null messagesCompletedAt: string | null - orphanEventCursor: MailboxParityBackfillCursor | null - orphanEventsCompletedAt: string | null + eventCursor: MailboxParityBackfillCursor | null + eventsCompletedAt: string | null lastError: string | null -}) { - await input.db +}): Promise<{ wrote: boolean }> { + const result = await input.db .prepare( `UPDATE users SET mailbox_parity_checked_at = ?, @@ -326,13 +359,17 @@ export async function persistUserParityProgress(input: { mailbox_parity_mismatch_count = ?, mailbox_parity_last_error = ?, mailbox_parity_content_watermark_at = ?, + mailbox_parity_content_replay_upper_at = ?, + mailbox_parity_content_replay_cursor_updated_at = ?, + mailbox_parity_content_replay_cursor_id = ?, mailbox_parity_message_backfill_cursor_created_at = ?, mailbox_parity_message_backfill_cursor_id = ?, mailbox_parity_message_backfill_completed_at = ?, - mailbox_parity_orphan_event_backfill_cursor_created_at = ?, - mailbox_parity_orphan_event_backfill_cursor_id = ?, - mailbox_parity_orphan_event_backfill_completed_at = ? - WHERE stable_user_id = ?`, + mailbox_parity_event_backfill_cursor_created_at = ?, + mailbox_parity_event_backfill_cursor_id = ?, + mailbox_parity_event_backfill_completed_at = ? + WHERE stable_user_id = ? + AND deleting_at IS NULL`, ) .bind( input.nowIso, @@ -340,15 +377,19 @@ export async function persistUserParityProgress(input: { input.mismatchCount, input.lastError, input.contentWatermarkAt, + input.contentReplayUpperAt, + input.contentReplayCursor?.updatedAt ?? null, + input.contentReplayCursor?.id ?? null, input.messageCursor?.createdAt ?? null, input.messageCursor?.id ?? null, input.messagesCompletedAt, - input.orphanEventCursor?.createdAt ?? null, - input.orphanEventCursor?.id ?? null, - input.orphanEventsCompletedAt, + input.eventCursor?.createdAt ?? null, + input.eventCursor?.id ?? null, + input.eventsCompletedAt, input.userId, ) .run() + return { wrote: (result.meta?.changes ?? 0) > 0 } } export async function rotateCheckedAt(input: { @@ -356,17 +397,18 @@ export async function rotateCheckedAt(input: { userId: string nowIso: string lastError: string | null -}) { +}): Promise<{ wrote: boolean }> { // Clears matching_since so a poison/error tick cannot preserve a false soak. - // Zero-row updates (deleted/deleting race) are harmless. - await input.db + const result = await input.db .prepare( `UPDATE users SET mailbox_parity_checked_at = ?, mailbox_parity_last_error = ?, mailbox_parity_matching_since = NULL - WHERE stable_user_id = ?`, + WHERE stable_user_id = ? + AND deleting_at IS NULL`, ) .bind(input.nowIso, input.lastError, input.userId) .run() + return { wrote: (result.meta?.changes ?? 0) > 0 } } diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index 119318cdef..5888ef3d66 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -44,9 +44,11 @@ type FlagState = { function createCutoverTestDb(input: { parityByUserId?: Map + deletingUserIds?: ReadonlySet flags?: FlagState }) { const parityByUserId = input.parityByUserId ?? new Map() + const deletingUserIds = input.deletingUserIds ?? new Set() const flags: FlagState = input.flags ?? { overrideEnabled: null, globalEnabled: null, @@ -86,7 +88,9 @@ function createCutoverTestDb(input: { normalized.includes('from users') && normalized.includes('mailbox_parity_matching_since') ) { + expect(normalized).toContain('deleting_at is null') const userId = Number(params[0]) + if (deletingUserIds.has(userId)) return null const row = parityByUserId.get(userId) if (row === 'error') throw new Error('d1 parity read failed') if (row === undefined || row === null) return null @@ -388,6 +392,46 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch ).resolves.toBe(true) }) +test('mailbox-read-cutover keeps deleting accounts on D1 even when otherwise eligible', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + const deletingDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + deletingUserIds: new Set([dbUserId]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: deletingDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env, idFromName } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...env, APP_DB: deletingDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ + db: deletingDb, + userId: stableUserId, + messageId: 'msg-1', + }) + expect(getMessage).not.toHaveBeenCalled() + expect(idFromName).not.toHaveBeenCalled() +}) + test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { const dbUserId = 7 const stableUserId = 'user-aaa' diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index fa3501ebce..c6058323b8 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -82,11 +82,13 @@ export async function isMailboxReadCutoverEnabled(input: { mailbox_parity_checked_at, mailbox_parity_mismatch_count FROM users - WHERE id = ?`, + WHERE id = ? + AND deleting_at IS NULL`, ) .bind(input.dbUserId) .first() + // Missing row includes deleting accounts (`deleting_at IS NOT NULL`). if (!row) return false if (row.stable_user_id !== input.stableUserId) return false if (row.mailbox_parity_mismatch_count !== 0) return false diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts index 37416204b4..8b639c4787 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -5,6 +5,10 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' import { systemEmailOwnerId } from './email-owner.ts' import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { + mailboxParityContentPageSize, + mailboxParityMessagePageSize, +} from './mailbox-parity-phases.ts' import { ensureEmailTestSchema } from './test-schema.ts' const mocks = vi.hoisted(() => ({ @@ -38,9 +42,8 @@ vi.mock('./mailbox-client.ts', () => ({ const { countD1MailboxParity, - listOrphanDeliveryEventBackfillPage, + listEventBackfillPage, listUsersForMailboxParity, - mailboxParityMessagePageSize, reconcileMailboxParity, } = await import('./mailbox-reconcile.ts') @@ -81,6 +84,7 @@ async function insertMessage(input: { userId: string id: string createdAt: string + updatedAt?: string }) { await input.db .prepare( @@ -90,23 +94,34 @@ async function insertMessage(input: { ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, 'stored', ?, ?)`, ) - .bind(input.id, input.userId, input.createdAt, input.createdAt) + .bind( + input.id, + input.userId, + input.createdAt, + input.updatedAt ?? input.createdAt, + ) .run() } -async function insertOrphanEvent(input: { +async function insertEvent(input: { db: D1Database userId: string id: string createdAt: string + messageId?: string | null }) { await input.db .prepare( `INSERT INTO email_delivery_events ( id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind( + input.id, + input.messageId === undefined ? null : input.messageId, + input.userId, + input.createdAt, ) - .bind(input.id, input.userId, input.createdAt) .run() } @@ -130,12 +145,15 @@ async function readParityRow(db: D1Database, userId: string) { mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_last_error AS lastError, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt FROM users WHERE stable_user_id = ?`, ) @@ -146,21 +164,24 @@ async function readParityRow(db: D1Database, userId: string) { mismatchCount: number lastError: string | null contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null messageCursorCreatedAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorCreatedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null }>() } -function successGraph(messageId: string) { +function successGraph(messageId: string, eventsTruncated = false) { return { messageId, message: { status: 'mirrored' as const }, events: [], - eventsTruncated: false, + eventsTruncated, } } @@ -176,9 +197,20 @@ function mockDoCounts(counts: { }) mocks.mailboxRpc.mockImplementation(() => ({ countMailbox: vi.fn(async () => counts), + purge: vi.fn(async () => ({ ok: true as const })), })) } +function parityEnv(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } +} + test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { consoleWarn.mockImplementation(() => {}) const { db } = await createParityDb() @@ -212,12 +244,19 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, }) } - await insertOrphanEvent({ + await insertEvent({ db, userId: ownerA, - id: 'orphan-1', + id: 'evt-bound-1', + messageId: messageIds[0]!, createdAt: '2026-07-02T00:00:00.000Z', }) + await insertEvent({ + db, + userId: ownerA, + id: 'evt-orphan-1', + createdAt: '2026-07-02T00:00:01.000Z', + }) await insertMessage({ db, userId: ownerB, @@ -230,27 +269,27 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou id: 'sys-msg-1', createdAt: '2026-07-01T00:00:00.000Z', }) - await insertOrphanEvent({ + await insertEvent({ db, userId: deleting, - id: 'del-orphan', + id: 'del-evt', createdAt: '2026-07-01T00:00:00.000Z', }) - // NULL checked_at sorts first under ORDER BY checked_at ASC (index-friendly). const discovered = await listUsersForMailboxParity({ db, limit: 10 }) expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) expect(discovered.map((row) => row.userId)).not.toContain(deleting) - const orphanPage = await listOrphanDeliveryEventBackfillPage({ + const eventPage = await listEventBackfillPage({ db, userId: ownerA, cursor: null, limit: 10, }) - expect(orphanPage).toEqual([ - { id: 'orphan-1', created_at: '2026-07-02T00:00:00.000Z' }, + expect(eventPage.map((row) => row.id)).toEqual([ + 'evt-bound-1', + 'evt-orphan-1', ]) await db @@ -266,20 +305,12 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou '2026-07-01T00:00:00.000Z', ) .run() - await db - .prepare( - `INSERT INTO email_attachments ( - id, message_id, content_type, size, storage_kind, created_at - ) VALUES ('att-1', ?, 'text/plain', 1, 'unavailable', ?)`, - ) - .bind(messageIds[0]!, '2026-07-01T00:00:00.000Z') - .run() await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ threads: 1, messages: mailboxParityMessagePageSize + 1, - attachments: 1, - deliveryEvents: 1, + attachments: 0, + deliveryEvents: 2, }) let graphCalls = 0 @@ -306,17 +337,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou attachments: 0, deliveryEvents: 0, }) - mocks.recordMailboxParityEvent.mockClear() - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - - // Seed a false soak that must clear on mirror timeout / backfill work. + const env = parityEnv(db) await db .prepare( `UPDATE users @@ -344,30 +366,22 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(consoleWarn).toHaveBeenCalledWith( 'mailbox-parity-message-backfill-retryable', ownerA, - 'error-or-timeout', + 'message graph timeout', ) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.every( - (call) => call[0].userId === ownerA, - ), - ).toBe(true) - expect(mocks.mailboxRpc).not.toHaveBeenCalled() const afterFirst = await readParityRow(db, ownerA) expect(afterFirst).toMatchObject({ checkedAt: now.toISOString(), messageCursorId: messageIds[mailboxParityMessagePageSize - 2], messagesCompletedAt: null, - orphanEventsCompletedAt: null, - // Baseline is persisted at first backfill attempt start and retained. + eventsCompletedAt: null, contentWatermarkAt: now.toISOString(), matchingSince: null, mismatchCount: 2, - lastError: 'message backfill mirror error or timeout', + lastError: 'message backfill message graph timeout', }) - // Finish backfill; compare mismatches → reopen completed_at, retain cursors, - // increment consecutive mismatch count, keep matching_since cleared. + // Finish creation; count mismatch resets BOTH cursors + completion markers. graphCalls = 0 mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), @@ -384,16 +398,18 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou }) expect(mismatch).toEqual({ scanned: 1, - backfilled: 3, + backfilled: 4, compared: 1, matched: 0, mismatched: 1, failed: 0, }) expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'orphan-1' }), + expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), + ) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), ) - expect(mocks.recordMailboxParityEvent).toHaveBeenCalledTimes(4) const afterMismatch = await readParityRow(db, ownerA) expect(afterMismatch).toMatchObject({ @@ -401,14 +417,16 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou matchingSince: null, mismatchCount: 3, contentWatermarkAt: mismatchNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, messagesCompletedAt: null, - orphanEventsCompletedAt: null, - messageCursorId: messageIds[mailboxParityMessagePageSize], - orphanCursorId: 'orphan-1', + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, lastError: null, }) - // Live new row after retained cursor must be repaired on the reopen pass. + // Full historical rescan after cursor reset, including a live new row. await insertMessage({ db, userId: ownerA, @@ -420,8 +438,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou mockDoCounts({ threads: 1, messages: mailboxParityMessagePageSize + 2, - attachments: 1, - deliveryEvents: 1, + attachments: 0, + deliveryEvents: 2, }) await parkUser(db, ownerB) const rematchNow = new Date('2026-08-01T12:00:00.000Z') @@ -432,7 +450,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou }) expect(rematch).toEqual({ scanned: 1, - backfilled: 1, + backfilled: mailboxParityMessagePageSize + 4, compared: 1, matched: 1, mismatched: 0, @@ -447,12 +465,11 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou mismatchCount: 0, contentWatermarkAt: rematchNow.toISOString(), messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'msg-live-new', + eventCursorId: 'evt-orphan-1', }) - // 24h soak preservation: exact completed compare with empty content window - // advances the watermark and keeps matching_since. await parkUser(db, ownerB) const soakNow = new Date('2026-08-02T12:00:00.000Z') await expect( @@ -468,10 +485,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou const afterSoak = await readParityRow(db, ownerA) expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) - expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) expect(afterSoak?.mismatchCount).toBe(0) - // Compare-path error rotates checked_at and clears matching_since. consoleWarn.mockClear() mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( new Error('count boom\nwith\tcontrol'), @@ -485,17 +500,6 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(afterError?.checkedAt).toBe(errorNow.toISOString()) expect(afterError?.lastError).toBe('count boom with control') expect(afterError?.matchingSince).toBeNull() - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-reconcile-user-failed', - ownerA, - expect.any(Error), - ) - - // Discovery stays D1-only: mailboxRpc is never used to enumerate candidates. - const rpcCallsBefore = mocks.mailboxRpc.mock.calls.length - const discoveryOnly = await listUsersForMailboxParity({ db, limit: 5 }) - expect(discoveryOnly.length).toBeGreaterThan(0) - expect(mocks.mailboxRpc).toHaveBeenCalledTimes(rpcCallsBefore) }) test('mailbox parity content watermark repairs equal-count updates and soak', async () => { @@ -533,52 +537,24 @@ test('mailbox parity content watermark repairs equal-count updates and soak', as attachments: 0, deliveryEvents: 0, }) + const env = parityEnv(db) - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - - // Baseline at first-attempt start; empty same-run window; then compare. await expect( reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 2, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterBaseline = await readParityRow(db, owner) - expect(afterBaseline).toMatchObject({ - matchingSince: now.toISOString(), - contentWatermarkAt: now.toISOString(), - mismatchCount: 0, - }) + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + now.toISOString(), + ) - // Classification-like update: same counts, newer updated_at past watermark. const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' await db .prepare( `UPDATE email_messages SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-a'`, - ) - .bind(sharedUpdatedAt) - .run() - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-b'`, + WHERE id IN ('msg-a', 'msg-b')`, ) .bind(sharedUpdatedAt) .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() const repairNow = new Date('2026-08-01T11:00:00.000Z') vi.setSystemTime(repairNow) @@ -592,104 +568,15 @@ test('mailbox parity content watermark repairs equal-count updates and soak', as mismatched: 0, failed: 0, }) - const replayedIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ) - expect(replayedIds).toEqual(['msg-a', 'msg-b']) - const afterRepair = await readParityRow(db, owner) - // Successful content sweep + exact compare keeps the soak window. - expect(afterRepair?.matchingSince).toBe(now.toISOString()) - expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) - expect(afterRepair?.mismatchCount).toBe(0) - - // Equal-timestamp rows after the advanced watermark are both replayed. - const equalStamp = '2026-08-01T11:15:00.000Z' - await db - .prepare( - `UPDATE email_messages SET updated_at = ? WHERE id IN ('msg-a', 'msg-b')`, - ) - .bind(equalStamp) - .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - const equalNow = new Date('2026-08-01T12:00:00.000Z') - vi.setSystemTime(equalNow) - await expect( - reconcileMailboxParity({ env, now: equalNow, batchSize: 1 }), - ).resolves.toMatchObject({ - backfilled: 2, - compared: 1, - matched: 1, - failed: 0, - }) expect( mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( (call) => call[0].messageId as string, ), ).toEqual(['msg-a', 'msg-b']) - expect((await readParityRow(db, owner))?.matchingSince).toBe( - now.toISOString(), - ) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - equalNow.toISOString(), - ) - - // Failure mid-window: watermark does not advance; matching_since clears. - await db - .prepare( - `UPDATE email_messages - SET updated_at = '2026-08-01T12:30:00.000Z' - WHERE id IN ('msg-a', 'msg-b')`, - ) - .run() - await db - .prepare( - `UPDATE users - SET mailbox_parity_matching_since = ? - WHERE stable_user_id = ?`, - ) - .bind(now.toISOString(), owner) - .run() - let contentCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - contentCalls += 1 - if (contentCalls === 1) { - return { - messageId: input.messageId, - message: { status: 'timeout' as const }, - events: [], - eventsTruncated: false, - } - } - return successGraph(input.messageId) - }, - ) - consoleWarn.mockClear() - const failNow = new Date('2026-08-01T13:00:00.000Z') - vi.setSystemTime(failNow) - await expect( - reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 0, - matched: 0, - mismatched: 0, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-content-replay-retryable', - owner, - 'error-or-timeout', - ) - const afterFail = await readParityRow(db, owner) - expect(afterFail).toMatchObject({ - contentWatermarkAt: equalNow.toISOString(), - matchingSince: null, - checkedAt: failNow.toISOString(), - lastError: 'content replay mirror error or timeout', - mismatchCount: 0, - }) + const afterRepair = await readParityRow(db, owner) + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.contentReplayUpperAt).toBeNull() } finally { vi.useRealTimers() } @@ -718,11 +605,9 @@ test('mailbox parity replays message updated during initial backfill', async () id: 'msg-late', createdAt: '2026-07-01T00:00:01.000Z', }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => { if (input.messageId === 'msg-early') { - // Concurrent classification-like update after the graph mirror. vi.setSystemTime(midUpdateAt) await db .prepare( @@ -745,17 +630,12 @@ test('mailbox parity replays message updated during initial backfill', async () attachments: 0, deliveryEvents: 0, }) - - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), ).resolves.toEqual({ scanned: 1, backfilled: 3, @@ -764,20 +644,356 @@ test('mailbox parity replays message updated during initial backfill', async () mismatched: 0, failed: 0, }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-early', 'msg-late', 'msg-early']) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + midUpdateAt.toISOString(), + ) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'f'.repeat(64) + const baseline = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(baseline) - const mirroredIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, + try { + await insertUser({ db, userId: owner, checkedAt: null }) + const messageCount = mailboxParityContentPageSize + 2 + for (let index = 0; index < messageCount; index += 1) { + await insertMessage({ + db, + userId: owner, + id: `msg-${String(index).padStart(2, '0')}`, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), ) - expect(mirroredIds).toEqual(['msg-early', 'msg-late', 'msg-early']) - const after = await readParityRow(db, owner) - expect(after).toMatchObject({ - matchingSince: now.toISOString(), - contentWatermarkAt: midUpdateAt.toISOString(), - mismatchCount: 0, - messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: messageCount, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) + + const updateAt = '2026-08-01T10:30:00.000Z' + await db + .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) + .bind(updateAt, owner) + .run() + + const replayNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(replayNow) + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === mailboxParityContentPageSize) { + // Exhaust budget mid-window by freezing time past deadline. + vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) + } + return successGraph(input.messageId) + }, + ) + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 0, + failed: 0, + backfilled: mailboxParityContentPageSize, + }) + const mid = await readParityRow(db, owner) + expect(mid).toMatchObject({ + contentWatermarkAt: baseline.toISOString(), + contentReplayUpperAt: replayNow.toISOString(), + contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, + matchingSince: null, + }) + + vi.setSystemTime(replayNow) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 1, + backfilled: 2, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual([ + `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, + `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, + ]) + const done = await readParityRow(db, owner) + expect(done).toMatchObject({ + contentWatermarkAt: replayNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, + // Soak was cleared on the incomplete tick; exact compare starts a new window. + matchingSince: replayNow.toISOString(), }) } finally { vi.useRealTimers() } }) + +test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'g'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-2', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { status: 'missing' as const }, + events: [], + eventsTruncated: false, + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 1, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + messageCursorId: null, + messagesCompletedAt: null, + contentWatermarkAt: now.toISOString(), + lastError: 'message backfill message graph missing', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ + status: 'skipped', + reason: 'mailbox-unconfigured', + }) + await expect( + reconcileMailboxParity({ + env, + now: new Date('2026-08-01T11:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterUnconfigured = await readParityRow(db, owner) + expect(afterUnconfigured).toMatchObject({ + messagesCompletedAt: expect.any(String), + eventCursorId: null, + eventsCompletedAt: null, + lastError: 'event backfill event skipped:mailbox-unconfigured', + }) +}) + +test('mailbox parity eventsTruncated still completes via full event phase', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'h'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + for (let index = 0; index < 3; index += 1) { + await insertEvent({ + db, + userId: owner, + id: `evt-${index}`, + messageId: 'msg-1', + createdAt: `2026-07-01T00:01:0${index}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( + successGraph('msg-1', true), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 3, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) +}) + +test('mailbox parity event missing advances cursor while unconfigured does not', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'j'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-gone', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-ok', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1 + .mockResolvedValueOnce({ status: 'missing' }) + .mockResolvedValueOnce({ status: 'mirrored' }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 2, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toMatchObject({ matched: 1, failed: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + eventCursorId: 'evt-ok', + eventsCompletedAt: expect.any(String), + }) +}) + +test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'i'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + })), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + try { + return { ok: true as const, value: await promise } + } catch { + return { ok: false as const, result: { status: 'timeout' as const } } + } + }, + ) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + await db + .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) + .bind(now.toISOString(), owner) + .run() + return successGraph('msg-1') + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const row = await readParityRow(db, owner) + // Baseline may have been written before deletion; no completion/compare state. + expect(row?.messagesCompletedAt).toBeNull() + expect(row?.matchingSince).toBeNull() + expect(row?.eventsCompletedAt).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index a8d3dcc097..d2c9943930 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -1,30 +1,31 @@ import { truncateToUtf8Bytes } from '@kody-internal/shared/backup-restore-safety.ts' import { mailboxRpc } from './mailbox-client.ts' -import { - mirrorMailboxDeliveryEventFromD1, - mirrorMailboxMessageGraphFromD1, - type MailboxLiveMirrorEnv, - type MailboxLiveMirrorGraphSummary, -} from './mailbox-live-mirror.ts' +import { type MailboxLiveMirrorEnv } from './mailbox-live-mirror.ts' import { awaitMailboxMirrorRpc, mailboxMirrorRpcTimeoutMs, - type MailboxMirrorResult, } from './mailbox-mirror.ts' import { recordMailboxParityEvent, type MailboxParityOperation, } from './mailbox-parity-events.ts' +import { + backfillEventsForUser, + backfillMessagesForUser, + mailboxParityContentPageSize, + mailboxParityEventPageSize, + mailboxParityMessagePageSize, + mirrorMailboxParityDeliveryEventFromD1, + replayContentUpdatesForUser, +} from './mailbox-parity-phases.ts' import { countD1MailboxParity, - listContentReplayPage, - listMessageBackfillPage, - listOrphanDeliveryEventBackfillPage, + isUserDeleting, + listEventBackfillPage, listUsersForMailboxParity, loadUserParityState, persistUserParityProgress, rotateCheckedAt, - type MailboxParityContentCursor, type MailboxParityUserState, } from './mailbox-parity-repo.ts' import { type MailboxCountResult } from './mailbox-types.ts' @@ -33,20 +34,24 @@ import { type MailboxCountResult } from './mailbox-types.ts' * Bounded hourly Mailbox backfill + count-parity reconciler. * * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO - * enumeration), mirrors message graphs then message_id-null orphan delivery - * events through existing live-mirror helpers, then keyset-replays messages - * updated after the content watermark before comparing owner-scoped D1 counts - * with `countMailbox`. D1 remains mail authority; this lane never mutates - * email_* rows and does not flip read authority. + * enumeration), mirrors message graphs then every owner delivery event through + * existing live-mirror helpers (repairing graph truncation), durably + * keyset-replays messages updated after the content watermark, then compares + * owner-scoped D1 counts with `countMailbox`. D1 remains mail authority; this + * lane never mutates email_* rows and does not flip read authority. */ export const mailboxParityUserBatchSize = 8 -export const mailboxParityMessagePageSize = 10 -export const mailboxParityOrphanEventPageSize = 25 -export const mailboxParityContentPageSize = 10 export const mailboxParityTimeBudgetMs = 10_000 export const mailboxParityLastErrorMaxBytes = 512 +export { + mailboxParityContentPageSize, + mailboxParityEventPageSize, + mailboxParityMessagePageSize, + mirrorMailboxParityDeliveryEventFromD1, +} + export type MailboxParityReconcileEnv = MailboxLiveMirrorEnv & { APP_DB: D1Database } @@ -62,19 +67,10 @@ export type MailboxParityReconcileMetrics = { export { countD1MailboxParity, - listOrphanDeliveryEventBackfillPage, + listEventBackfillPage, listUsersForMailboxParity, } -function isRetryableMirrorResult(result: MailboxMirrorResult): boolean { - return result.status === 'error' || result.status === 'timeout' -} - -function graphMirrorSucceeded(summary: MailboxLiveMirrorGraphSummary): boolean { - if (isRetryableMirrorResult(summary.message)) return false - return !summary.events.some((event) => isRetryableMirrorResult(event.result)) -} - function safeParityErrorText(error: unknown): string { const raw = error instanceof Error ? error.message : String(error) let cleaned = '' @@ -89,17 +85,18 @@ function safeParityErrorText(error: unknown): string { ) } -/** - * Load one orphan delivery-event projection from D1 and mirror it. - * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. - */ -export async function mirrorMailboxOrphanDeliveryEventFromD1(input: { - env: MailboxLiveMirrorEnv - db: D1Database +async function purgeMailboxBestEffort(input: { + env: MailboxParityReconcileEnv userId: string - eventId: string -}): Promise { - return mirrorMailboxDeliveryEventFromD1(input) +}) { + try { + await awaitMailboxMirrorRpc( + mailboxRpc({ env: input.env, userId: input.userId }).purge(), + mailboxMirrorRpcTimeoutMs, + ) + } catch (error) { + console.warn('mailbox-parity-purge-on-delete-failed', input.userId, error) + } } function emitCountComparisons(input: { @@ -153,199 +150,6 @@ function emitCountComparisons(input: { return { matched } } -async function backfillMessagesForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean -}> { - let state = input.state - let backfilled = 0 - if (state.messagesCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - while (Date.now() < input.deadlineMs) { - const page = await listMessageBackfillPage({ - db: input.env.APP_DB, - userId: state.userId, - cursor: state.messageCursor, - limit: mailboxParityMessagePageSize, - }) - if (page.length === 0) { - state = { - ...state, - messagesCompletedAt: new Date().toISOString(), - } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - let cursor = state.messageCursor - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } - } - const summary = await mirrorMailboxMessageGraphFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - messageId: row.id, - }) - if (!graphMirrorSucceeded(summary)) { - state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - } - } - cursor = { createdAt: row.created_at, id: row.id } - backfilled += 1 - } - state = { ...state, messageCursor: cursor } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } -} - -async function backfillOrphanEventsForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean -}> { - let state = input.state - let backfilled = 0 - if (state.messagesCompletedAt == null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - if (state.orphanEventsCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - while (Date.now() < input.deadlineMs) { - const page = await listOrphanDeliveryEventBackfillPage({ - db: input.env.APP_DB, - userId: state.userId, - cursor: state.orphanEventCursor, - limit: mailboxParityOrphanEventPageSize, - }) - if (page.length === 0) { - state = { - ...state, - orphanEventsCompletedAt: new Date().toISOString(), - } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - let cursor = state.orphanEventCursor - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - state = { ...state, orphanEventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } - } - const result = await mirrorMailboxOrphanDeliveryEventFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - eventId: row.id, - }) - if (isRetryableMirrorResult(result)) { - state = { ...state, orphanEventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - } - } - cursor = { createdAt: row.created_at, id: row.id } - backfilled += 1 - } - state = { ...state, orphanEventCursor: cursor } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } -} - -/** - * Baseline is established before the first creation scan. When it still equals - * this run's nowIso, sample wall time so same-run updates after the baseline - * are included; otherwise the run's nowIso is the upper bound. - */ -function resolveContentReplayUpperBound(input: { - nowIso: string - watermarkAt: string -}): string { - let upperBoundAt = input.nowIso - if (input.watermarkAt === input.nowIso) { - const wallIso = new Date().toISOString() - if (wallIso > upperBoundAt) upperBoundAt = wallIso - } - if (upperBoundAt < input.watermarkAt) return input.watermarkAt - return upperBoundAt -} - -/** - * Persist the content watermark at the start of the first backfill attempt so - * updates during creation/orphan mirroring cannot fall before a completion-time - * baseline. Retained across incomplete/error ticks. - */ async function ensureContentWatermarkBaseline(input: { db: D1Database state: MailboxParityUserState @@ -363,100 +167,17 @@ async function ensureContentWatermarkBaseline(input: { matchingSince: state.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, + contentReplayUpperAt: state.contentReplayUpperAt, + contentReplayCursor: state.contentReplayCursor, messageCursor: state.messageCursor, messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, + eventCursor: state.eventCursor, + eventsCompletedAt: state.eventsCompletedAt, lastError: null, }) return state } -/** - * After initial creation/orphan backfill completes: keyset-replay every owner - * message with updated_at in (watermark, current-now]. Watermark advances only - * when the entire window succeeds; in-tick cursor avoids skipping equal - * timestamps. Always replay — never treat a fresh baseline as an empty skip. - */ -async function replayContentUpdatesForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - nowIso: string - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean - windowComplete: boolean -}> { - const state = input.state - const watermarkAt = state.contentWatermarkAt ?? input.nowIso - const upperBoundAt = resolveContentReplayUpperBound({ - nowIso: input.nowIso, - watermarkAt, - }) - let cursor: MailboxParityContentCursor | null = null - let backfilled = 0 - - while (Date.now() < input.deadlineMs) { - const page = await listContentReplayPage({ - db: input.env.APP_DB, - userId: state.userId, - watermarkAt, - upperBoundAt, - cursor, - limit: mailboxParityContentPageSize, - }) - if (page.length === 0) { - return { - state: { ...state, contentWatermarkAt: upperBoundAt }, - backfilled, - budgetExhausted: false, - retryableFailure: false, - windowComplete: true, - } - } - - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - windowComplete: false, - } - } - const summary = await mirrorMailboxMessageGraphFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - messageId: row.id, - }) - if (!graphMirrorSucceeded(summary)) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - windowComplete: false, - } - } - cursor = { updatedAt: row.updated_at, id: row.id } - backfilled += 1 - } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - windowComplete: false, - } -} - async function compareParityForUser(input: { env: MailboxParityReconcileEnv state: MailboxParityUserState @@ -494,16 +215,19 @@ async function compareParityForUser(input: { matched: true, } } - // Reopen bounded creation/orphan backfill after completed cursors so later - // live rows past the retained keyset are mirrored on the next ticks; keep - // the content watermark so classification updates remain covered. + // Full historical rescan: clear creation cursors + completion markers. + // Preserve content watermark; drop any in-flight replay window; clear soak. return { state: { ...input.state, matchingSince: null, mismatchCount: input.state.mismatchCount + 1, + messageCursor: null, messagesCompletedAt: null, - orphanEventsCompletedAt: null, + eventCursor: null, + eventsCompletedAt: null, + contentReplayUpperAt: null, + contentReplayCursor: null, }, matched: false, } @@ -539,25 +263,50 @@ async function reconcileOneUser(input: { let state = loaded let backfilled = 0 - let initialBackfilled = 0 + let creationBackfilled = 0 + + const isDeleting = () => + isUserDeleting({ db: input.env.APP_DB, userId: state.userId }) + + const stopForDeletion = async () => { + await purgeMailboxBestEffort({ + env: input.env, + userId: state.userId, + }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } - const persistIncomplete = async (inputPersist: { + const persistProgress = async (inputPersist: { + matchingSince: string | null lastError: string | null }) => { - await persistUserParityProgress({ + const { wrote } = await persistUserParityProgress({ db: input.env.APP_DB, userId: state.userId, nowIso: input.nowIso, - // Soak cannot survive incomplete/error/backfill ticks. - matchingSince: null, + matchingSince: inputPersist.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, + contentReplayUpperAt: state.contentReplayUpperAt, + contentReplayCursor: state.contentReplayCursor, messageCursor: state.messageCursor, messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, + eventCursor: state.eventCursor, + eventsCompletedAt: state.eventsCompletedAt, lastError: inputPersist.lastError, }) + return wrote + } + + if (await isDeleting()) { + return stopForDeletion() } // Baseline before scanning so mid-backfill updates stay inside the replay window. @@ -566,23 +315,33 @@ async function reconcileOneUser(input: { state, nowIso: input.nowIso, }) + if (await isDeleting()) { + return stopForDeletion() + } const messagePass = await backfillMessagesForUser({ env: input.env, state, deadlineMs: input.deadlineMs, + isDeleting, }) state = messagePass.state - initialBackfilled += messagePass.backfilled + creationBackfilled += messagePass.backfilled backfilled += messagePass.backfilled + if (messagePass.deletionStarted) { + return stopForDeletion() + } if (messagePass.retryableFailure) { - await persistIncomplete({ - lastError: 'message backfill mirror error or timeout', + await persistProgress({ + matchingSince: null, + lastError: messagePass.blockedReason + ? `message backfill ${messagePass.blockedReason}` + : 'message backfill mirror error or timeout', }) console.warn( 'mailbox-parity-message-backfill-retryable', state.userId, - 'error-or-timeout', + messagePass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -594,7 +353,7 @@ async function reconcileOneUser(input: { } } if (messagePass.budgetExhausted) { - await persistIncomplete({ lastError: null }) + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -605,22 +364,29 @@ async function reconcileOneUser(input: { } } - const orphanPass = await backfillOrphanEventsForUser({ + const eventPass = await backfillEventsForUser({ env: input.env, state, deadlineMs: input.deadlineMs, + isDeleting, }) - state = orphanPass.state - initialBackfilled += orphanPass.backfilled - backfilled += orphanPass.backfilled - if (orphanPass.retryableFailure) { - await persistIncomplete({ - lastError: 'orphan-event backfill mirror error or timeout', + state = eventPass.state + creationBackfilled += eventPass.backfilled + backfilled += eventPass.backfilled + if (eventPass.deletionStarted) { + return stopForDeletion() + } + if (eventPass.retryableFailure) { + await persistProgress({ + matchingSince: null, + lastError: eventPass.blockedReason + ? `event backfill ${eventPass.blockedReason}` + : 'event backfill mirror error or timeout', }) console.warn( - 'mailbox-parity-orphan-event-backfill-retryable', + 'mailbox-parity-event-backfill-retryable', state.userId, - 'error-or-timeout', + eventPass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -631,8 +397,8 @@ async function reconcileOneUser(input: { budgetExhausted: false, } } - if (orphanPass.budgetExhausted) { - await persistIncomplete({ lastError: null }) + if (eventPass.budgetExhausted) { + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -643,11 +409,8 @@ async function reconcileOneUser(input: { } } - if ( - state.messagesCompletedAt == null || - state.orphanEventsCompletedAt == null - ) { - await persistIncomplete({ lastError: null }) + if (state.messagesCompletedAt == null || state.eventsCompletedAt == null) { + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -658,9 +421,9 @@ async function reconcileOneUser(input: { } } - // Initial creation/orphan work ends the continuous soak window. Successful - // content replay below may keep matching_since when the final compare matches. - if (initialBackfilled > 0) { + // Creation work ends the continuous soak window. Successful content replay + // below may keep matching_since when the final compare matches. + if (creationBackfilled > 0) { state = { ...state, matchingSince: null } } @@ -669,17 +432,31 @@ async function reconcileOneUser(input: { state, nowIso: input.nowIso, deadlineMs: input.deadlineMs, + isDeleting, }) state = contentPass.state backfilled += contentPass.backfilled + if (contentPass.deletionStarted) { + return stopForDeletion() + } + // Durably freeze a newly opened window even when this tick cannot finish it. + if (contentPass.openedWindow && !contentPass.windowComplete) { + await persistProgress({ + matchingSince: null, + lastError: null, + }) + } if (contentPass.retryableFailure) { - await persistIncomplete({ - lastError: 'content replay mirror error or timeout', + await persistProgress({ + matchingSince: null, + lastError: contentPass.blockedReason + ? `content replay ${contentPass.blockedReason}` + : 'content replay mirror error or timeout', }) console.warn( 'mailbox-parity-content-replay-retryable', state.userId, - 'error-or-timeout', + contentPass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -691,7 +468,8 @@ async function reconcileOneUser(input: { } } if (!contentPass.windowComplete) { - await persistIncomplete({ lastError: null }) + // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -702,23 +480,21 @@ async function reconcileOneUser(input: { } } + if (await isDeleting()) { + return stopForDeletion() + } + const compared = await compareParityForUser({ env: input.env, state, nowIso: input.nowIso, }) state = compared.state - await persistUserParityProgress({ - db: input.env.APP_DB, - userId: state.userId, - nowIso: input.nowIso, + if (await isDeleting()) { + return stopForDeletion() + } + await persistProgress({ matchingSince: state.matchingSince, - mismatchCount: state.mismatchCount, - contentWatermarkAt: state.contentWatermarkAt, - messageCursor: state.messageCursor, - messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, lastError: null, }) return { diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index 243d0b2086..36d40d50b0 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -3,6 +3,7 @@ import { expect, test, vi } from 'vitest' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { mailboxLiveMirrorMaxEvents } from './mailbox-live-mirror.ts' import { countD1MailboxParity, reconcileMailboxParity, @@ -37,6 +38,7 @@ async function seedMessageGraph(input: { threadId: string createdAt: string withAttachment?: boolean + eventCount?: number }) { await env.APP_DB.prepare( `INSERT INTO email_threads ( @@ -75,18 +77,22 @@ async function seedMessageGraph(input: { .bind(`att-${input.messageId}`, input.messageId, input.createdAt) .run() } - await env.APP_DB.prepare( - `INSERT INTO email_delivery_events ( - id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, - ) - .bind( - `evt-${input.messageId}`, - input.messageId, - input.userId, - input.createdAt, + const eventCount = input.eventCount ?? 1 + const baseMs = Date.parse(input.createdAt) + for (let index = 0; index < eventCount; index += 1) { + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, ) - .run() + .bind( + `evt-${input.messageId}-${String(index).padStart(3, '0')}`, + input.messageId, + input.userId, + new Date(baseMs + index * 1000).toISOString(), + ) + .run() + } } async function seedOrphanEvent(input: { @@ -110,10 +116,11 @@ async function readParityState(userId: string) { mailbox_parity_matching_since AS matchingSince, mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt, mailbox_parity_last_error AS lastError FROM users WHERE stable_user_id = ?`, @@ -124,10 +131,11 @@ async function readParityState(userId: string) { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null lastError: string | null }>() } @@ -142,10 +150,8 @@ async function parkOther(userId: string) { .run() } -test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves soak', async () => { +test('reconcileMailboxParity mismatch full reset and soak', async () => { silenceIncidentalRuntimeWarnings() - // Keep wall time aligned with each tick's nowIso so same-run content upper - // bounds stay deterministic (without freezing RPC timeout clocks). vi.useFakeTimers({ shouldAdvanceTime: true }) await ensureUsersTestSchema({ db: env.APP_DB }) await ensureEmailTestSchema(env.APP_DB) @@ -191,16 +197,16 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves }) const mailbox = rpcFor(userId) - // Warm the DO so the first 1s-bounded mirror RPC is not cold-start timeout. await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) const firstNow = new Date('2026-08-01T10:00:00.000Z') vi.setSystemTime(firstNow) + // 2 messages + 2 bound events + 1 orphan await expect( reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), ).resolves.toEqual({ scanned: 1, - backfilled: 3, + backfilled: 5, compared: 1, matched: 1, mismatched: 0, @@ -216,20 +222,15 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves checkedAt: firstNow.toISOString(), matchingSince: firstNow.toISOString(), contentWatermarkAt: expect.any(String), + contentReplayUpperAt: null, mismatchCount: 0, messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'parity-msg-2', - orphanCursorId: 'parity-orphan-1', + eventCursorId: 'parity-orphan-1', lastError: null, }) - if (!afterMatch?.contentWatermarkAt) { - throw new Error('Expected parity content watermark after match.') - } - expect(afterMatch.contentWatermarkAt >= firstNow.toISOString()).toBe(true) - // Live D1 row after completed cursors: compare mismatches, reopens backfill - // while retaining cursors so the next tick repairs only the new row. await seedMessageGraph({ userId, messageId: 'parity-msg-3', @@ -259,11 +260,11 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves expect(afterMismatch).toMatchObject({ matchingSince: null, mismatchCount: 1, - contentWatermarkAt: expect.any(String), messagesCompletedAt: null, - orphanEventsCompletedAt: null, - messageCursorId: 'parity-msg-2', - orphanCursorId: 'parity-orphan-1', + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + contentReplayUpperAt: null, checkedAt: mismatchNow.toISOString(), }) expect(afterMismatch?.contentWatermarkAt).not.toBeNull() @@ -276,11 +277,12 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves await parkOther(otherUserId) const rematchNow = new Date('2026-08-01T12:00:00.000Z') vi.setSystemTime(rematchNow) + // Full historical rescan: 3 messages + 3 bound events + 1 orphan await expect( reconcileMailboxParity({ env, now: rematchNow, batchSize: 1 }), ).resolves.toEqual({ scanned: 1, - backfilled: 1, + backfilled: 7, compared: 1, matched: 1, mismatched: 0, @@ -296,9 +298,8 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves expect(afterRematch).toMatchObject({ matchingSince: rematchNow.toISOString(), mismatchCount: 0, - contentWatermarkAt: expect.any(String), messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'parity-msg-3', }) @@ -322,10 +323,68 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves }) const afterSoak = await readParityState(userId) expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) - expect(afterSoak?.contentWatermarkAt).toEqual(expect.any(String)) expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) expect(afterSoak?.mismatchCount).toBe(0) } finally { vi.useRealTimers() } }, 30_000) + +test('reconcileMailboxParity eventually repairs >100 bound delivery events', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-parity-reconcile-user-failed', + 'mailbox-parity-message-backfill-retryable', + 'mailbox-parity-event-backfill-retryable', + 'mailbox-parity-content-replay-retryable', + 'mailbox-live-mirror-events-truncated', + ]) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const heavyUserId = await seedParityUser({ + email: `heavy-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + const heavyEventCount = mailboxLiveMirrorMaxEvents + 5 + await seedMessageGraph({ + userId: heavyUserId, + messageId: 'heavy-msg-1', + threadId: 'heavy-thread-1', + createdAt: '2026-07-02T12:00:00.000Z', + eventCount: heavyEventCount, + }) + const heavyMailbox = rpcFor(heavyUserId) + await heavyMailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + let matched = 0 + let backfilled = 0 + for (let tick = 0; tick < 30; tick += 1) { + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(heavyUserId) + .run() + const metrics = await reconcileMailboxParity({ + env, + now: new Date(Date.UTC(2026, 7, 3, 10, tick)), + batchSize: 1, + }) + backfilled += metrics.backfilled + matched += metrics.matched + if (metrics.matched > 0) break + } + expect(matched).toBeGreaterThan(0) + expect(backfilled).toBeGreaterThan(mailboxLiveMirrorMaxEvents) + expect(await heavyMailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId: heavyUserId }), + ) + expect( + (await countD1MailboxParity({ db: env.APP_DB, userId: heavyUserId })) + .deliveryEvents, + ).toBe(heavyEventCount) +}, 60_000) diff --git a/packages/worker/src/users-test-schema.ts b/packages/worker/src/users-test-schema.ts index fd279cb1da..e2ae33bfef 100644 --- a/packages/worker/src/users-test-schema.ts +++ b/packages/worker/src/users-test-schema.ts @@ -84,12 +84,15 @@ const alwaysAdditiveColumns: Record = { }, mailbox_parity_last_error: { create: 'TEXT' }, mailbox_parity_content_watermark_at: { create: 'TEXT' }, + mailbox_parity_content_replay_upper_at: { create: 'TEXT' }, + mailbox_parity_content_replay_cursor_updated_at: { create: 'TEXT' }, + mailbox_parity_content_replay_cursor_id: { create: 'TEXT' }, mailbox_parity_message_backfill_cursor_created_at: { create: 'TEXT' }, mailbox_parity_message_backfill_cursor_id: { create: 'TEXT' }, mailbox_parity_message_backfill_completed_at: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_cursor_created_at: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_cursor_id: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_completed_at: { create: 'TEXT' }, + mailbox_parity_event_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_event_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_event_backfill_completed_at: { create: 'TEXT' }, } /** From 93aba3fedc6682564e3c23f90ce899282cdd2417 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:07:17 +0000 Subject: [PATCH 13/20] chore: refresh Mailbox parity migration hash Co-authored-by: Kent C. Dodds --- tools/migration-ledger.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 379b71d275..7dd7de2e83 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "d5a99f57d8db18943b7184c1c2cc2002c5ef6c20584def71b9860e823b0456cc" + "sha256": "ac22677b803abd75b66ec9ba331c2cff6694390409bbb1719c15ff0b9799aa89" } ] } From 9dc335e73267462295ed221629fe04889003ccda Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:22:32 +0000 Subject: [PATCH 14/20] fix(email): rebuild Mailbox on parity mismatch Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 91 ++++--- .../architecture/feature-flags.md | 3 +- .../migrations/0125-mailbox-parity-state.sql | 8 +- .../worker/src/email/mailbox-parity-repo.ts | 14 + .../email/mailbox-read-cutover.node.test.ts | 4 +- .../worker/src/email/mailbox-read-cutover.ts | 7 +- .../src/email/mailbox-reconcile.node.test.ts | 253 +++++++++++++++++- .../worker/src/email/mailbox-reconcile.ts | 81 +++++- .../email/mailbox-reconcile.workers.test.ts | 121 ++++++++- .../scheduled/scheduled-lanes.node.test.ts | 35 +-- .../worker/src/scheduled/scheduled-lanes.ts | 9 +- tools/migration-ledger.json | 2 +- 12 files changed, 522 insertions(+), 106 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e747ae096d..6b0231359d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -277,7 +277,7 @@ The schema is defined by migrations in `packages/worker/migrations/`: message and **all** owner delivery-event backfill cursors/completion timestamps on the user row. Indexed by `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in - the hourly `mailbox_parity` reconcile lane (see + the every-5-minute `mailbox_parity` reconcile lane (see [Mailbox](#durable-objects-mailbox)). The same migration adds keyset-friendly composites on `email_messages(user_id, created_at, id)`, `email_messages(user_id, updated_at, id)`, and @@ -395,10 +395,10 @@ delivery-event table because Wrangler's emulated dataset has no SQL API. with a separate row shape defined in `packages/worker/src/email/mailbox-parity-events.ts` (`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome -per attempted operation; the hourly `mailbox_parity` reconcile lane records one -outcome per count comparison (`compare_threads`, `compare_messages`, -`compare_attachments`, `compare_delivery_events`). These rows do not feed the -admin insights charts today: +per attempted operation; the `mailbox_parity` reconcile lane (every five-minute +cron tick) records one outcome per count comparison (`compare_threads`, +`compare_messages`, `compare_attachments`, `compare_delivery_events`). These +rows do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -809,19 +809,23 @@ mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired (see below); read cutover is prepared but not flipped. **Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated -scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, +scheduled lane on **every** five-minute Worker cron tick +(`packages/worker/src/scheduled/scheduled-lanes.ts`, `packages/worker/src/email/mailbox-reconcile.ts`, `packages/worker/src/email/mailbox-parity-phases.ts`, -`packages/worker/src/email/mailbox-parity-repo.ts`). The five-minute Worker cron -dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); -production uses the same per-lane queue isolation as reconcile, retention, and -sibling lanes so a slow parity pass cannot consume another lane's budget. +`packages/worker/src/email/mailbox-parity-repo.ts`). Production uses the same +per-lane queue isolation as reconcile, retention, and sibling lanes so a slow +parity pass cannot consume another lane's budget. User discovery is **D1-only** — no Mailbox Durable Object enumeration. Each tick -selects up to eight non-deleting mail owners (`mailboxParityUserBatchSize`) -ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who -have at least one `email_messages` or `email_delivery_events` row. Per-user work -is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). +selects up to sixteen non-deleting owners (`mailboxParityUserBatchSize`) ordered +by oldest `mailbox_parity_checked_at`, excluding `system:email`. Owners with D1 +mail (`email_messages` or `email_delivery_events`) are always eligible; +**previously tracked** owners (any non-null parity column on `users`) remain +discoverable even when D1 mail is empty so DO-only leftovers after the last-row +delete can be purged and reconciled. Never-tracked empty users stay out. +Per-user work shares a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`) +across the batch. **Account-deletion races:** `loadUserParityState`, `persistUserParityProgress`, and `rotateCheckedAt` require `deleting_at IS NULL` (zero-row updates are @@ -865,19 +869,25 @@ D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact compare when null; preserved across successful content replay + exact compare. Cleared on: creation backfill work in the same tick (`creationBackfilled > 0`), incomplete/budget-exhausted ticks, mirror retryable failures, content-replay -failure/incomplete, compare mismatch (also increments consecutive -`mailbox_parity_mismatch_count`, **resets both message and delivery-event -creation cursors and completion markers** for a full historical rescan while -retaining the content watermark and clearing any in-flight replay window), and -compare-path errors (`rotateCheckedAt` also clears soak so a poison tick cannot -preserve a false window). Resets `mismatch_count` to `0` on exact match. +failure/incomplete, and compare-path errors (`rotateCheckedAt` also clears soak +so a poison tick cannot preserve a false window). Resets `mismatch_count` to `0` +on exact match. + +**Count mismatch rebuild:** on mismatch the lane first **must** observe a +successful metadata-only `Mailbox.purge()` (DO SQLite / alarm state; no R2), +then resets **all** parity cursors, completion markers, content watermark, and +in-flight replay window for a full D1-authoritative rebuild on subsequent ticks +(increments consecutive `mailbox_parity_mismatch_count` and clears soak). If +purge fails (timeout/error), rebuild state is **not** reset — the user stays +ineligible for soak, `last_error` records the failure, and the next ticks retry +purge before advancing cursors. **Phase 3 read cutover (prepared, not flipped):** `packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off `mailbox-read-cutover` feature flag plus `getOwnerEmailMessageById` / `isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user parity soak: `mailbox_parity_matching_since` at least 24h old, -`mailbox_parity_checked_at` fresh within 2h, +`mailbox_parity_checked_at` fresh within 6h, `mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, and `deleting_at IS NULL` (deleting accounts fail closed to D1). D1 errors, missing parity state, and an off flag fail closed to D1 reads. No live reader calls this @@ -914,14 +924,14 @@ namespaced outcome telemetry (`missing` and `timeout` included). Message and batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) -and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly -queue-isolated `mailbox_parity` lane backfills all owner messages and delivery -events, durable content-watermark replays, compares owner-scoped D1 vs Mailbox -counts, persists soak state on `users` (migration `0125`), and re-purges the DO -when account deletion races the lane. Inbound lifecycle dual-write, -retention/metadata deletes, and live read cutover are **still not wired**. Live -email read/write authority is unchanged — D1 `email_*` tables remain -authoritative for all live paths. D1 email rows and existing R2 inventory +and one batch telemetry outcome. **`system:email` stays in D1 only.** The +every-5-minute queue-isolated `mailbox_parity` lane backfills all owner messages +and delivery events, durable content-watermark replays, compares owner-scoped D1 +vs Mailbox counts, persists soak state on `users` (migration `0125`), and +re-purges the DO when account deletion races the lane. Inbound lifecycle +dual-write, retention/metadata deletes, and live read cutover are **still not +wired**. Live email read/write authority is unchanged — D1 `email_*` tables +remain authoritative for all live paths. D1 email rows and existing R2 inventory deletion stay authoritative during expand. **Phase 2 completes only when every user-mail mutation that writes D1 also writes the DO** — inbound paths and delete mirrors remain pending. @@ -946,8 +956,8 @@ delete mirrors remain pending. classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; - at most two AE writes per attempt). The hourly `mailbox_parity` scheduled - lane backfills all owner messages and delivery events, durable + at most two AE writes per attempt). The every-5-minute `mailbox_parity` + scheduled lane backfills all owner messages and delivery events, durable content-watermark replays, count compares, and soak tracking on `users` (migration `0125`). **Still pending for phase-2 completion:** inbound commit/touch/classify dual-write and retention/metadata-delete mirrors. @@ -965,12 +975,13 @@ delete mirrors remain pending. 5. **Later contract migrations** — drop retired D1 user-mail tables/columns only after verification. No premature schema deletion. -The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in +The every-5-minute `mailbox_parity` scheduled lane (queue-isolated sibling in `scheduled-lanes.ts`) owns backfill of all owner messages and delivery events, -durable content-watermark replay, count parity, soak state on `users`, and -best-effort `Mailbox.purge()` when account deletion races the lane. This Mailbox -track owns DO storage semantics, alarm retention, and the inbound durability -boundary below. +durable content-watermark replay, count parity, soak state on `users`, +metadata-only DO purge + full rebuild on count mismatch, and best-effort +`Mailbox.purge()` when account deletion races the lane. This Mailbox track owns +DO storage semantics, alarm retention, and the inbound durability boundary +below. ### What stays in D1 @@ -1263,9 +1274,11 @@ Artifacts tokens. Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session cleanup, system-email retention, general retention, job retention, hourly -usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC -hour only; D1 user discovery, bounded message + full delivery-event backfill, -durable content-watermark replay, and count compare — see +usage-rollup aggregation, and the every-5-minute `mailbox_parity` lane +(queue-isolated; up to sixteen users per tick within a ~10s budget; D1 user +discovery including previously tracked empty-mail owners, bounded message + full +delivery-event backfill, durable content-watermark replay, metadata-only purge + +full rebuild on count mismatch, and count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue message preserves `scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled lane failure is acknowledged and retried by the next cron tick, matching the old diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index b026c1f661..f5e0b8a776 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -81,7 +81,8 @@ parity soak on `users` (migration `0125-mailbox-parity-state.sql`): - `mailbox_parity_matching_since` at least **24 hours** old (continuous exact D1↔Mailbox count parity), -- `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), +- `mailbox_parity_checked_at` fresh within **6 hours** (every-5m lane with + bounded batch scaling; not a strict 5m SLA), - `mailbox_parity_mismatch_count === 0`, - exact `stable_user_id` match between the gate caller and the loaded row, - `deleting_at IS NULL` (accounts marked for deletion fail closed to D1). diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 55a93e2459..937a5a76e2 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -9,9 +9,11 @@ -- content-replay window (replay_upper_at + replay cursor) keyset-replays -- messages with updated_at in (watermark, upper]; the watermark advances to -- upper only when that window finishes. Count compare runs only after the full --- event phase and a completed content window. Count mismatch clears soak --- eligibility and resets both creation cursors and completion markers for a --- full historical rescan (watermark preserved; in-flight replay window cleared). +-- event phase and a completed content window. Count mismatch best-effort purges +-- Mailbox SQLite metadata (no R2), then clears soak and resets creation cursors, +-- completion markers, and content watermark/window for an authoritative D1 +-- rebuild. Purge timeout/error clears soak, records failure, and does not reset +-- rebuild state. Previously tracked users stay eligible even at zero D1 rows. -- matching_since may persist across a successful content replay + exact compare. ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index 6c9c97ccf1..2ee1a86e06 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -45,6 +45,9 @@ export async function listUsersForMailboxParity(input: { db: D1Database limit: number }): Promise> { + // Include previously tracked owners (any parity column non-null) even when + // D1 mail is empty so DO-only leftovers after last-row deletion are purged. + // Never-tracked empty users, system email, and deleting users stay out. const result = await input.db .prepare( `SELECT u.stable_user_id AS userId @@ -60,6 +63,17 @@ export async function listUsersForMailboxParity(input: { SELECT 1 FROM email_delivery_events e WHERE e.user_id = u.stable_user_id ) + OR u.mailbox_parity_checked_at IS NOT NULL + OR u.mailbox_parity_matching_since IS NOT NULL + OR u.mailbox_parity_last_error IS NOT NULL + OR u.mailbox_parity_mismatch_count > 0 + OR u.mailbox_parity_content_watermark_at IS NOT NULL + OR u.mailbox_parity_content_replay_upper_at IS NOT NULL + OR u.mailbox_parity_content_replay_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_completed_at IS NOT NULL + OR u.mailbox_parity_event_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_event_backfill_completed_at IS NOT NULL ) ORDER BY u.mailbox_parity_checked_at ASC, u.stable_user_id ASC LIMIT ?`, diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index 5888ef3d66..ad36092808 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -272,7 +272,7 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch }), ).resolves.toBe(false) - // Stale checked_at (> 2h). + // Stale checked_at (> 6h freshness window). const staleDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([ @@ -280,7 +280,7 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch dbUserId, { ...soakedParity(), - mailbox_parity_checked_at: hoursAgoIso(3), + mailbox_parity_checked_at: hoursAgoIso(7), }, ], ]), diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index c6058323b8..40ad9fbb81 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -27,8 +27,11 @@ import { type EmailMessageRecord } from './types.ts' /** Continuous exact parity must hold at least this long before cutover. */ export const mailboxReadCutoverSoakMs = 24 * 60 * 60 * 1000 -/** Parity `checked_at` must be newer than this (hourly lane + slack). */ -export const mailboxReadCutoverCheckedAtMaxAgeMs = 2 * 60 * 60 * 1000 +/** + * Parity `checked_at` must be newer than this. Sized for the every-5m parity + * lane with bounded batch scaling (not a strict 5m SLA). + */ +export const mailboxReadCutoverCheckedAtMaxAgeMs = 6 * 60 * 60 * 1000 export const mailboxReadCutoverFlagKey = 'mailbox-read-cutover' as const diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts index 8b639c4787..f082b68e95 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -185,20 +185,27 @@ function successGraph(messageId: string, eventsTruncated = false) { } } -function mockDoCounts(counts: { - threads: number - messages: number - attachments: number - deliveryEvents: number -}) { - mocks.awaitMailboxMirrorRpc.mockResolvedValue({ - ok: true, - value: counts, - }) +function mockDoCounts( + counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number + }, + options?: { purge?: () => Promise<{ ok: true }> }, +) { + const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) mocks.mailboxRpc.mockImplementation(() => ({ countMailbox: vi.fn(async () => counts), - purge: vi.fn(async () => ({ ok: true as const })), + purge, })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => ({ + ok: true as const, + value: await promise, + }), + ) + return { purge } } function parityEnv(db: D1Database) { @@ -416,7 +423,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou checkedAt: mismatchNow.toISOString(), matchingSince: null, mismatchCount: 3, - contentWatermarkAt: mismatchNow.toISOString(), + contentWatermarkAt: null, contentReplayUpperAt: null, contentReplayCursorId: null, messagesCompletedAt: null, @@ -463,12 +470,16 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(afterRematch).toMatchObject({ matchingSince: rematchNow.toISOString(), mismatchCount: 0, - contentWatermarkAt: rematchNow.toISOString(), + contentWatermarkAt: expect.any(String), messagesCompletedAt: expect.any(String), eventsCompletedAt: expect.any(String), messageCursorId: 'msg-live-new', eventCursorId: 'evt-orphan-1', }) + if (!afterRematch?.contentWatermarkAt) { + throw new Error('Expected content watermark after parity rematch.') + } + expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) await parkUser(db, ownerB) const soakNow = new Date('2026-08-02T12:00:00.000Z') @@ -997,3 +1008,219 @@ test('mailbox parity deletion race purges Mailbox and skips parity writes', asyn expect(row?.matchingSince).toBeNull() expect(row?.eventsCompletedAt).toBeNull() }) + +test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'k'.repeat(64) + const neverTracked = 'l'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertUser({ db, userId: neverTracked, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + + // Simulate failed delete mirror: D1 cleared, DO still holds message/event. + await db + .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) + .bind(owner) + .run() + await db + .prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(owner) + .run() + + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).toEqual([owner]) + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).not.toContain(neverTracked) + + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const afterPurge = await readParityRow(db, owner) + expect(afterPurge).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterZero = await readParityRow(db, owner) + expect(afterZero).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + }) + if (!afterZero?.contentWatermarkAt) { + throw new Error('Expected content watermark after zero-count parity.') + } + expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) +}) + +test('mismatch purge failure clears soak and does not start rebuild reset', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'm'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + const afterMatch = await readParityRow(db, owner) + expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) + expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) + + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + })), + purge: vi.fn(async () => ({ ok: true as const })), + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + const settled = await promise + if (settled && typeof settled === 'object' && 'messages' in settled) { + return { ok: true as const, value: settled } + } + return { ok: false as const, timedOut: true as const } + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-mismatch-purge-failed', + owner, + 'timeout', + 'mailbox metadata purge timeout', + ) + const afterFail = await readParityRow(db, owner) + // Empty content window may advance the watermark before compare; purge + // failure must not clear creation completion/cursors (rebuild not started). + expect(afterFail).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + messagesCompletedAt: afterMatch?.messagesCompletedAt, + eventsCompletedAt: afterMatch?.eventsCompletedAt, + messageCursorId: afterMatch?.messageCursorId, + lastError: 'mailbox metadata purge timeout', + checkedAt: failNow.toISOString(), + }) + expect(afterFail?.contentWatermarkAt).not.toBeNull() + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index d2c9943930..c1cea9693b 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -41,7 +41,7 @@ import { type MailboxCountResult } from './mailbox-types.ts' * lane never mutates email_* rows and does not flip read authority. */ -export const mailboxParityUserBatchSize = 8 +export const mailboxParityUserBatchSize = 16 export const mailboxParityTimeBudgetMs = 10_000 export const mailboxParityLastErrorMaxBytes = 512 @@ -85,17 +85,49 @@ function safeParityErrorText(error: unknown): string { ) } -async function purgeMailboxBestEffort(input: { +/** + * Bounded Mailbox SQLite metadata purge (no R2). Used for account-deletion + * races (best-effort) and count-mismatch rebuild (must observe success). + */ +async function purgeMailboxMetadata(input: { env: MailboxParityReconcileEnv userId: string -}) { +}): Promise< + { ok: true } | { ok: false; reason: 'timeout' | 'error'; errorText: string } +> { try { - await awaitMailboxMirrorRpc( + const raced = await awaitMailboxMirrorRpc( mailboxRpc({ env: input.env, userId: input.userId }).purge(), mailboxMirrorRpcTimeoutMs, ) + if (!raced.ok) { + return { + ok: false, + reason: 'timeout', + errorText: 'mailbox metadata purge timeout', + } + } + return { ok: true } } catch (error) { - console.warn('mailbox-parity-purge-on-delete-failed', input.userId, error) + return { + ok: false, + reason: 'error', + errorText: safeParityErrorText(error), + } + } +} + +async function purgeMailboxBestEffort(input: { + env: MailboxParityReconcileEnv + userId: string +}) { + const result = await purgeMailboxMetadata(input) + if (!result.ok) { + console.warn( + 'mailbox-parity-purge-on-delete-failed', + input.userId, + result.errorText, + ) } } @@ -185,6 +217,8 @@ async function compareParityForUser(input: { }): Promise<{ state: MailboxParityUserState matched: boolean + purgeFailed: boolean + lastError: string | null }> { const d1 = await countD1MailboxParity({ db: input.env.APP_DB, @@ -213,10 +247,36 @@ async function compareParityForUser(input: { mismatchCount: 0, }, matched: true, + purgeFailed: false, + lastError: null, } } - // Full historical rescan: clear creation cursors + completion markers. - // Preserve content watermark; drop any in-flight replay window; clear soak. + + // DO may hold extras after D1 deletes. Purge Mailbox metadata (no R2) before + // resetting rebuild state so the next ticks authoritatively mirror D1. + const purged = await purgeMailboxMetadata({ + env: input.env, + userId: input.state.userId, + }) + if (!purged.ok) { + console.warn( + 'mailbox-parity-mismatch-purge-failed', + input.state.userId, + purged.reason, + purged.errorText, + ) + return { + state: { + ...input.state, + matchingSince: null, + mismatchCount: input.state.mismatchCount + 1, + }, + matched: false, + purgeFailed: true, + lastError: purged.errorText, + } + } + return { state: { ...input.state, @@ -226,10 +286,13 @@ async function compareParityForUser(input: { messagesCompletedAt: null, eventCursor: null, eventsCompletedAt: null, + contentWatermarkAt: null, contentReplayUpperAt: null, contentReplayCursor: null, }, matched: false, + purgeFailed: false, + lastError: null, } } @@ -495,14 +558,14 @@ async function reconcileOneUser(input: { } await persistProgress({ matchingSince: state.matchingSince, - lastError: null, + lastError: compared.lastError, }) return { backfilled, compared: true, matched: compared.matched, mismatched: !compared.matched, - failed: false, + failed: compared.purgeFailed, budgetExhausted: false, } } diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index 36d40d50b0..fd0350d209 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -6,9 +6,14 @@ import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' import { mailboxLiveMirrorMaxEvents } from './mailbox-live-mirror.ts' import { countD1MailboxParity, + listUsersForMailboxParity, reconcileMailboxParity, } from './mailbox-reconcile.ts' -import { rpcFor } from './mailbox-test-helpers.ts' +import { + baseDeliveryEvent, + baseMessage, + rpcFor, +} from './mailbox-test-helpers.ts' import { ensureEmailTestSchema } from './test-schema.ts' async function seedParityUser(input: { @@ -264,10 +269,10 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { eventsCompletedAt: null, messageCursorId: null, eventCursorId: null, + contentWatermarkAt: null, contentReplayUpperAt: null, checkedAt: mismatchNow.toISOString(), }) - expect(afterMismatch?.contentWatermarkAt).not.toBeNull() await env.APP_DB.prepare( `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, @@ -330,6 +335,118 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { } }, 30_000) +test('reconcileMailboxParity purges DO-only leftover for tracked empty owners', async () => { + silenceIncidentalRuntimeWarnings(['mailbox-live-mirror-events-truncated']) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const userId = await seedParityUser({ + email: `do-extra-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + await seedMessageGraph({ + userId, + messageId: 'do-extra-msg', + threadId: 'do-extra-thread', + createdAt: '2026-07-01T12:00:00.000Z', + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const firstNow = new Date('2026-08-01T10:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + + // Failed delete mirror: clear D1, leave Mailbox metadata populated. + await env.APP_DB.prepare( + `DELETE FROM email_delivery_events WHERE user_id = ?`, + ) + .bind(userId) + .run() + await env.APP_DB.prepare(`DELETE FROM email_attachments WHERE message_id = ?`) + .bind('do-extra-msg') + .run() + await env.APP_DB.prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(userId) + .run() + await env.APP_DB.prepare(`DELETE FROM email_threads WHERE user_id = ?`) + .bind(userId) + .run() + await mailbox.mirrorMessage({ + ownerId: userId, + message: baseMessage(userId, { id: 'do-extra-msg' }), + }) + await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + baseDeliveryEvent({ + id: 'do-extra-evt', + messageId: 'do-extra-msg', + }), + ], + }) + expect((await mailbox.countMailbox()).messages).toBeGreaterThan(0) + expect( + (await listUsersForMailboxParity({ db: env.APP_DB, limit: 10 })).some( + (row) => row.userId === userId, + ), + ).toBe(true) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(await readParityState(userId)).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + }) + expect(await mailbox.countMailbox()).toEqual({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, mismatched: 0 }) + expect(await readParityState(userId)).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) +}, 30_000) + test('reconcileMailboxParity eventually repairs >100 bound delivery events', async () => { silenceIncidentalRuntimeWarnings([ 'mailbox-parity-reconcile-user-failed', diff --git a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts index dcd75153d4..9bb5eacf74 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts @@ -8,36 +8,19 @@ vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ reconcileMailboxParity: mocks.reconcileMailboxParity, })) -const { getScheduledLanes, runScheduledLane, shouldRunMailboxParityCron } = +const { getScheduledLanes, runScheduledLane } = await import('./scheduled-lanes.ts') -test('shouldRunMailboxParityCron gates to the top of each UTC hour', () => { - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:00:30.000Z'))).toBe( - true, - ) - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:05:00.000Z'))).toBe( - false, - ) - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:30:00.000Z'))).toBe( - false, - ) -}) - -test('getScheduledLanes includes mailbox_parity only on the hourly gate', () => { +test('getScheduledLanes includes mailbox_parity on every cron tick', () => { const env = {} as Env - expect( - getScheduledLanes({ - env, - scheduledAt: new Date('2026-07-05T10:00:30.000Z'), - }), - ).toContain('mailbox_parity') - expect( - getScheduledLanes({ - env, - scheduledAt: new Date('2026-07-05T10:30:00.000Z'), - }), - ).not.toContain('mailbox_parity') + for (const scheduledAt of [ + new Date('2026-07-05T10:00:30.000Z'), + new Date('2026-07-05T10:05:00.000Z'), + new Date('2026-07-05T10:30:00.000Z'), + ]) { + expect(getScheduledLanes({ env, scheduledAt })).toContain('mailbox_parity') + } }) test('runScheduledLane dispatches mailbox_parity through reconcileMailboxParity', async () => { diff --git a/packages/worker/src/scheduled/scheduled-lanes.ts b/packages/worker/src/scheduled/scheduled-lanes.ts index 1ff5b09dee..34a143f539 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.ts @@ -33,11 +33,6 @@ import { shouldRunUsageAggregationCron, } from '#worker/usage/aggregate-rollups.ts' -/** Worker cron fires every 5 minutes; run this lane on the top of each UTC hour. */ -export function shouldRunMailboxParityCron(now: Date) { - return now.getUTCMinutes() === 0 -} - export const scheduledLaneNames = [ 'reconcile_artifacts_pushes', 'repo_session_cleanup', @@ -86,6 +81,7 @@ export function getScheduledLanes(input: { 'storage_bucket_estimate_backfill', 'd1_storage_reconciliation', 'oauth_purge_expired', + 'mailbox_parity', ] if (shouldRunRetentionCron(input.scheduledAt)) { lanes.push('retention', 'job_retention') @@ -114,9 +110,6 @@ export function getScheduledLanes(input: { if (shouldRunJobScheduleWatchdogCron(input.scheduledAt)) { lanes.push('job_schedule_watchdog') } - if (shouldRunMailboxParityCron(input.scheduledAt)) { - lanes.push('mailbox_parity') - } return lanes } diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 7dd7de2e83..7cd41d25a2 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "ac22677b803abd75b66ec9ba331c2cff6694390409bbb1719c15ff0b9799aa89" + "sha256": "a0cfd916aff30eecf74d315088c131a8fd9f800f44aa8ed4406a806cfa1f1cff" } ] } From 48d23b7833b94522cd62a6f553ade0762b21e446 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:23:37 +0000 Subject: [PATCH 15/20] test(email): isolate Mailbox parity lane Co-authored-by: Kent C. Dodds --- packages/worker/src/index.workers.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/worker/src/index.workers.test.ts b/packages/worker/src/index.workers.test.ts index 460f1f15bb..701688a659 100644 --- a/packages/worker/src/index.workers.test.ts +++ b/packages/worker/src/index.workers.test.ts @@ -57,6 +57,14 @@ const mocks = vi.hoisted(() => ({ updated: 0, failed: 0, })), + reconcileMailboxParity: vi.fn(async () => ({ + scanned: 0, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + })), })) vi.mock('./jobs/reconcile-artifacts-pushes.ts', () => ({ @@ -75,6 +83,10 @@ vi.mock('#worker/email/reconcile-inbound-deliveries.ts', () => ({ sweepStaleInboundDeliveries: mocks.sweepStaleInboundDeliveries, })) +vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ + reconcileMailboxParity: mocks.reconcileMailboxParity, +})) + vi.mock('#app/retention.ts', () => ({ pruneRetention: mocks.pruneRetention, shouldRunRetentionCron: mocks.shouldRunRetentionCron, From 4c73520485c9bd8f1b531a8348634a6d91ba1a85 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:35:25 +0000 Subject: [PATCH 16/20] refactor(email): clarify Mailbox parity phases Co-authored-by: Kent C. Dodds --- .../worker/src/email/mailbox-parity-phases.ts | 208 ++---- ...ncile-cursor-backfill-content.node.test.ts | 626 ++++++++++++++++++ ...ncile-mismatch-deletion-error.node.test.ts | 423 ++++++++++++ .../email/mailbox-reconcile-test-support.ts | 219 ++++++ .../worker/src/email/mailbox-reconcile.ts | 476 ++++++++----- 5 files changed, 1627 insertions(+), 325 deletions(-) create mode 100644 packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile-test-support.ts diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts index 6a5306f98d..a7daa3b7dd 100644 --- a/packages/worker/src/email/mailbox-parity-phases.ts +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -21,15 +21,41 @@ export type MailboxParityPhaseEnv = MailboxLiveMirrorEnv & { APP_DB: D1Database } -export type MailboxParityPhaseResult = { +type MailboxParityPhaseBase = { state: MailboxParityUserState backfilled: number - budgetExhausted: boolean - retryableFailure: boolean - /** Set when a selected mirror outcome must not advance the cursor. */ - blockedReason: string | null } +/** Creation (message/event) phase outcome. */ +export type MailboxParityCreationPhaseResult = + | (MailboxParityPhaseBase & { status: 'complete' }) + | (MailboxParityPhaseBase & { status: 'budget_exhausted' }) + | (MailboxParityPhaseBase & { + status: 'retryable_failure' + blockedReason: string + }) + | (MailboxParityPhaseBase & { status: 'deletion_started' }) + +/** Content-replay phase outcome (`complete` ≡ full window finished). */ +export type MailboxParityContentPhaseResult = + | (MailboxParityPhaseBase & { + status: 'complete' + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'budget_exhausted' + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'retryable_failure' + blockedReason: string + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'deletion_started' + openedWindow: boolean + }) + /** * Message graph cursor may advance only on mirrored/stale. missing / skipped / * timeout / error must not advance (selected D1 row was expected). @@ -132,18 +158,11 @@ export async function backfillMessagesForUser(input: { state: MailboxParityUserState deadlineMs: number isDeleting: () => Promise -}): Promise { +}): Promise { let state = input.state let backfilled = 0 if (state.messagesCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } while (Date.now() < input.deadlineMs) { @@ -158,39 +177,18 @@ export async function backfillMessagesForUser(input: { ...state, messagesCompletedAt: new Date().toISOString(), } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } let cursor = state.messageCursor for (const row of page) { if (Date.now() >= input.deadlineMs) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } if (await input.isDeleting()) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } const summary = await mirrorMailboxMessageGraphFromD1({ env: input.env, @@ -200,24 +198,15 @@ export async function backfillMessagesForUser(input: { }) if (await input.isDeleting()) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } if (!messageGraphAllowsCursorAdvance(summary)) { state = { ...state, messageCursor: cursor } return { + status: 'retryable_failure', state, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: `message graph ${summary.message.status}`, - deletionStarted: false, } } cursor = { createdAt: row.created_at, id: row.id } @@ -226,14 +215,7 @@ export async function backfillMessagesForUser(input: { state = { ...state, messageCursor: cursor } } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } export async function backfillEventsForUser(input: { @@ -241,28 +223,14 @@ export async function backfillEventsForUser(input: { state: MailboxParityUserState deadlineMs: number isDeleting: () => Promise -}): Promise { +}): Promise { let state = input.state let backfilled = 0 if (state.messagesCompletedAt == null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } if (state.eventsCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } while (Date.now() < input.deadlineMs) { @@ -277,39 +245,18 @@ export async function backfillEventsForUser(input: { ...state, eventsCompletedAt: new Date().toISOString(), } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } let cursor = state.eventCursor for (const row of page) { if (Date.now() >= input.deadlineMs) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } if (await input.isDeleting()) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } const result = await mirrorMailboxParityDeliveryEventFromD1({ env: input.env, @@ -319,27 +266,18 @@ export async function backfillEventsForUser(input: { }) if (await input.isDeleting()) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } if (!eventMirrorAllowsCursorAdvance(result)) { state = { ...state, eventCursor: cursor } return { + status: 'retryable_failure', state, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: result.status === 'skipped' ? `event skipped:${result.reason}` : `event ${result.status}`, - deletionStarted: false, } } cursor = { createdAt: row.created_at, id: row.id } @@ -348,14 +286,7 @@ export async function backfillEventsForUser(input: { state = { ...state, eventCursor: cursor } } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } /** @@ -368,13 +299,7 @@ export async function replayContentUpdatesForUser(input: { nowIso: string deadlineMs: number isDeleting: () => Promise -}): Promise< - MailboxParityPhaseResult & { - deletionStarted: boolean - windowComplete: boolean - openedWindow: boolean - } -> { +}): Promise { let state = input.state const watermarkAt = state.contentWatermarkAt ?? input.nowIso let openedWindow = false @@ -408,6 +333,7 @@ export async function replayContentUpdatesForUser(input: { }) if (page.length === 0) { return { + status: 'complete', state: { ...state, contentWatermarkAt: upperBoundAt, @@ -415,11 +341,6 @@ export async function replayContentUpdatesForUser(input: { contentReplayCursor: null, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: true, openedWindow, } } @@ -427,33 +348,25 @@ export async function replayContentUpdatesForUser(input: { for (const row of page) { if (Date.now() >= input.deadlineMs) { return { + status: 'budget_exhausted', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: false, openedWindow, } } if (await input.isDeleting()) { return { + status: 'deletion_started', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - windowComplete: false, openedWindow, } } @@ -465,33 +378,26 @@ export async function replayContentUpdatesForUser(input: { }) if (await input.isDeleting()) { return { + status: 'deletion_started', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - windowComplete: false, openedWindow, } } if (!messageGraphAllowsCursorAdvance(summary)) { return { + status: 'retryable_failure', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: `content replay ${summary.message.status}`, - deletionStarted: false, - windowComplete: false, openedWindow, } } @@ -506,17 +412,13 @@ export async function replayContentUpdatesForUser(input: { } return { + status: 'budget_exhausted', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: false, openedWindow, } } diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts new file mode 100644 index 0000000000..9e509c5547 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -0,0 +1,626 @@ +import { + countD1MailboxParity, + createParityDb, + getMailboxReconcileTestMocks, + insertEvent, + insertMessage, + insertUser, + listEventBackfillPage, + listUsersForMailboxParity, + mockDoCounts, + parkUser, + parityEnv, + readParityRow, + reconcileMailboxParity, + successGraph, +} from './mailbox-reconcile-test-support.ts' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { + mailboxParityContentPageSize, + mailboxParityMessagePageSize, +} from './mailbox-parity-phases.ts' + +const mocks = getMailboxReconcileTestMocks() + +test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const ownerA = 'a'.repeat(64) + const ownerB = 'b'.repeat(64) + const deleting = 'c'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + + await insertUser({ db, userId: ownerA, checkedAt: null }) + await insertUser({ + db, + userId: ownerB, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + await insertUser({ + db, + userId: deleting, + deletingAt: '2026-07-31T00:00:00.000Z', + checkedAt: null, + }) + + const messageIds = Array.from( + { length: mailboxParityMessagePageSize + 1 }, + (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, + ) + for (const [index, id] of messageIds.entries()) { + await insertMessage({ + db, + userId: ownerA, + id, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + await insertEvent({ + db, + userId: ownerA, + id: 'evt-bound-1', + messageId: messageIds[0]!, + createdAt: '2026-07-02T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: ownerA, + id: 'evt-orphan-1', + createdAt: '2026-07-02T00:00:01.000Z', + }) + await insertMessage({ + db, + userId: ownerB, + id: 'b-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: systemEmailOwnerId, + id: 'sys-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: deleting, + id: 'del-evt', + createdAt: '2026-07-01T00:00:00.000Z', + }) + + const discovered = await listUsersForMailboxParity({ db, limit: 10 }) + expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) + expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) + expect(discovered.map((row) => row.userId)).not.toContain(deleting) + + const eventPage = await listEventBackfillPage({ + db, + userId: ownerA, + cursor: null, + limit: 10, + }) + expect(eventPage.map((row) => row.id)).toEqual([ + 'evt-bound-1', + 'evt-orphan-1', + ]) + + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, + ) + .bind( + ownerA, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + ) + .run() + + await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ + threads: 1, + messages: mailboxParityMessagePageSize + 1, + attachments: 0, + deliveryEvents: 2, + }) + + let graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + graphCalls += 1 + if (graphCalls === mailboxParityMessagePageSize) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + + const env = parityEnv(db) + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', + mailbox_parity_mismatch_count = 2 + WHERE stable_user_id = ?`, + ) + .bind(ownerA) + .run() + + await parkUser(db, ownerB) + const first = await reconcileMailboxParity({ + env, + now, + batchSize: 1, + }) + expect(first).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize - 1, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-message-backfill-retryable', + ownerA, + 'message graph timeout', + ) + + const afterFirst = await readParityRow(db, ownerA) + expect(afterFirst).toMatchObject({ + checkedAt: now.toISOString(), + messageCursorId: messageIds[mailboxParityMessagePageSize - 2], + messagesCompletedAt: null, + eventsCompletedAt: null, + contentWatermarkAt: now.toISOString(), + matchingSince: null, + mismatchCount: 2, + lastError: 'message backfill message graph timeout', + }) + + // Finish creation; count mismatch resets BOTH cursors + completion markers. + graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.recordMailboxParityEvent.mockClear() + await parkUser(db, ownerB) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + const mismatch = await reconcileMailboxParity({ + env, + now: mismatchNow, + batchSize: 1, + }) + expect(mismatch).toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), + ) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), + ) + + const afterMismatch = await readParityRow(db, ownerA) + expect(afterMismatch).toMatchObject({ + checkedAt: mismatchNow.toISOString(), + matchingSince: null, + mismatchCount: 3, + contentWatermarkAt: null, + contentReplayUpperAt: null, + contentReplayCursorId: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + // Full historical rescan after cursor reset, including a live new row. + await insertMessage({ + db, + userId: ownerA, + id: 'msg-live-new', + createdAt: '2026-07-03T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mockDoCounts({ + threads: 1, + messages: mailboxParityMessagePageSize + 2, + attachments: 0, + deliveryEvents: 2, + }) + await parkUser(db, ownerB) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + const rematch = await reconcileMailboxParity({ + env, + now: rematchNow, + batchSize: 1, + }) + expect(rematch).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize + 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), + ) + const afterRematch = await readParityRow(db, ownerA) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), + messageCursorId: 'msg-live-new', + eventCursorId: 'evt-orphan-1', + }) + if (!afterRematch?.contentWatermarkAt) { + throw new Error('Expected content watermark after parity rematch.') + } + expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) + + await parkUser(db, ownerB) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityRow(db, ownerA) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + + consoleWarn.mockClear() + mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( + new Error('count boom\nwith\tcontrol'), + ) + await parkUser(db, ownerB) + const errorNow = new Date('2026-08-02T13:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterError = await readParityRow(db, ownerA) + expect(afterError?.checkedAt).toBe(errorNow.toISOString()) + expect(afterError?.lastError).toBe('count boom with control') + expect(afterError?.matchingSince).toBeNull() +}) + +test('mailbox parity content watermark repairs equal-count updates and soak', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'd'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-a', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-b', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + now.toISOString(), + ) + + const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id IN ('msg-a', 'msg-b')`, + ) + .bind(sharedUpdatedAt) + .run() + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const repairNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(repairNow) + await expect( + reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-a', 'msg-b']) + const afterRepair = await readParityRow(db, owner) + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.contentReplayUpperAt).toBeNull() + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity replays message updated during initial backfill', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'e'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-early', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-late', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + if (input.messageId === 'msg-early') { + vi.setSystemTime(midUpdateAt) + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-early'`, + ) + .bind(midUpdateAt.toISOString()) + .run() + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-early', 'msg-late', 'msg-early']) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + midUpdateAt.toISOString(), + ) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'f'.repeat(64) + const baseline = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(baseline) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + const messageCount = mailboxParityContentPageSize + 2 + for (let index = 0; index < messageCount; index += 1) { + await insertMessage({ + db, + userId: owner, + id: `msg-${String(index).padStart(2, '0')}`, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: messageCount, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) + + const updateAt = '2026-08-01T10:30:00.000Z' + await db + .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) + .bind(updateAt, owner) + .run() + + const replayNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(replayNow) + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === mailboxParityContentPageSize) { + // Exhaust budget mid-window by freezing time past deadline. + vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) + } + return successGraph(input.messageId) + }, + ) + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 0, + failed: 0, + backfilled: mailboxParityContentPageSize, + }) + const mid = await readParityRow(db, owner) + expect(mid).toMatchObject({ + contentWatermarkAt: baseline.toISOString(), + contentReplayUpperAt: replayNow.toISOString(), + contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, + matchingSince: null, + }) + + vi.setSystemTime(replayNow) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 1, + backfilled: 2, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual([ + `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, + `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, + ]) + const done = await readParityRow(db, owner) + expect(done).toMatchObject({ + contentWatermarkAt: replayNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, + // Soak was cleared on the incomplete tick; exact compare starts a new window. + matchingSince: replayNow.toISOString(), + }) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity eventsTruncated still completes via full event phase', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'h'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + for (let index = 0; index < 3; index += 1) { + await insertEvent({ + db, + userId: owner, + id: `evt-${index}`, + messageId: 'msg-1', + createdAt: `2026-07-01T00:01:0${index}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( + successGraph('msg-1', true), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 3, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) +}) diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts new file mode 100644 index 0000000000..6b8f9bd795 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -0,0 +1,423 @@ +import { + createParityDb, + getMailboxReconcileTestMocks, + insertEvent, + insertMessage, + insertUser, + listUsersForMailboxParity, + mockDoCounts, + parityEnv, + readParityRow, + reconcileMailboxParity, + successGraph, +} from './mailbox-reconcile-test-support.ts' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' + +const mocks = getMailboxReconcileTestMocks() + +test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'g'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-2', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { status: 'missing' as const }, + events: [], + eventsTruncated: false, + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 1, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + messageCursorId: null, + messagesCompletedAt: null, + contentWatermarkAt: now.toISOString(), + lastError: 'message backfill message graph missing', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ + status: 'skipped', + reason: 'mailbox-unconfigured', + }) + await expect( + reconcileMailboxParity({ + env, + now: new Date('2026-08-01T11:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterUnconfigured = await readParityRow(db, owner) + expect(afterUnconfigured).toMatchObject({ + messagesCompletedAt: expect.any(String), + eventCursorId: null, + eventsCompletedAt: null, + lastError: 'event backfill event skipped:mailbox-unconfigured', + }) +}) + +test('mailbox parity event missing advances cursor while unconfigured does not', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'j'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-gone', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-ok', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1 + .mockResolvedValueOnce({ status: 'missing' }) + .mockResolvedValueOnce({ status: 'mirrored' }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 2, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toMatchObject({ matched: 1, failed: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + eventCursorId: 'evt-ok', + eventsCompletedAt: expect.any(String), + }) +}) + +test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'i'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + })), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + try { + return { ok: true as const, value: await promise } + } catch { + return { ok: false as const, result: { status: 'timeout' as const } } + } + }, + ) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + await db + .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) + .bind(now.toISOString(), owner) + .run() + return successGraph('msg-1') + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const row = await readParityRow(db, owner) + // Baseline may have been written before deletion; no completion/compare state. + expect(row?.messagesCompletedAt).toBeNull() + expect(row?.matchingSince).toBeNull() + expect(row?.eventsCompletedAt).toBeNull() +}) + +test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'k'.repeat(64) + const neverTracked = 'l'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertUser({ db, userId: neverTracked, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + + // Simulate failed delete mirror: D1 cleared, DO still holds message/event. + await db + .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) + .bind(owner) + .run() + await db + .prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(owner) + .run() + + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).toEqual([owner]) + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).not.toContain(neverTracked) + + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const afterPurge = await readParityRow(db, owner) + expect(afterPurge).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterZero = await readParityRow(db, owner) + expect(afterZero).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + }) + if (!afterZero?.contentWatermarkAt) { + throw new Error('Expected content watermark after zero-count parity.') + } + expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) +}) + +test('mismatch purge failure clears soak and does not start rebuild reset', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'm'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + const afterMatch = await readParityRow(db, owner) + expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) + expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) + + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + })), + purge: vi.fn(async () => ({ ok: true as const })), + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + const settled = await promise + if (settled && typeof settled === 'object' && 'messages' in settled) { + return { ok: true as const, value: settled } + } + return { ok: false as const, timedOut: true as const } + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-mismatch-purge-failed', + owner, + 'timeout', + 'mailbox metadata purge timeout', + ) + const afterFail = await readParityRow(db, owner) + // Empty content window may advance the watermark before compare; purge + // failure must not clear creation completion/cursors (rebuild not started). + expect(afterFail).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + messagesCompletedAt: afterMatch?.messagesCompletedAt, + eventsCompletedAt: afterMatch?.eventsCompletedAt, + messageCursorId: afterMatch?.messageCursorId, + lastError: 'mailbox metadata purge timeout', + checkedAt: failNow.toISOString(), + }) + expect(afterFail?.contentWatermarkAt).not.toBeNull() + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile-test-support.ts b/packages/worker/src/email/mailbox-reconcile-test-support.ts new file mode 100644 index 0000000000..2baf8e1cff --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-test-support.ts @@ -0,0 +1,219 @@ +import { DatabaseSync } from 'node:sqlite' +import { vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxMessageGraphFromD1: vi.fn(), + mirrorMailboxDeliveryEventFromD1: vi.fn(), + awaitMailboxMirrorRpc: vi.fn(), + recordMailboxParityEvent: vi.fn(), + mailboxRpc: vi.fn(), +})) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./mailbox-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, + } +}) + +vi.mock('./mailbox-parity-events.ts', () => ({ + recordMailboxParityEvent: mocks.recordMailboxParityEvent, +})) + +vi.mock('./mailbox-client.ts', () => ({ + mailboxRpc: mocks.mailboxRpc, +})) + +export type MailboxReconcileTestMocks = typeof mocks + +export function getMailboxReconcileTestMocks(): MailboxReconcileTestMocks { + return mocks +} + +export const { + countD1MailboxParity, + listEventBackfillPage, + listUsersForMailboxParity, + reconcileMailboxParity, +} = await import('./mailbox-reconcile.ts') + +export async function createParityDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureUsersTestSchema({ db }) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +export async function insertUser(input: { + db: D1Database + userId: string + email?: string + deletingAt?: string | null + checkedAt?: string | null +}) { + await input.db + .prepare( + `INSERT INTO users ( + username, email, password_hash, stable_user_id, deleting_at, + mailbox_parity_checked_at + ) VALUES (?, ?, 'hash', ?, ?, ?)`, + ) + .bind( + `u-${crypto.randomUUID().slice(0, 8)}`, + input.email ?? `${crypto.randomUUID()}@example.test`, + input.userId, + input.deletingAt ?? null, + input.checkedAt ?? null, + ) + .run() +} + +export async function insertMessage(input: { + db: D1Database + userId: string + id: string + createdAt: string + updatedAt?: string +}) { + await input.db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, + 'stored', ?, ?)`, + ) + .bind( + input.id, + input.userId, + input.createdAt, + input.updatedAt ?? input.createdAt, + ) + .run() +} + +export async function insertEvent(input: { + db: D1Database + userId: string + id: string + createdAt: string + messageId?: string | null +}) { + await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind( + input.id, + input.messageId === undefined ? null : input.messageId, + input.userId, + input.createdAt, + ) + .run() +} + +export async function parkUser(db: D1Database, userId: string) { + await db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +export async function readParityRow(db: D1Database, userId: string) { + return db + .prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_last_error AS lastError, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + lastError: string | null + contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null + }>() +} + +export function successGraph(messageId: string, eventsTruncated = false) { + return { + messageId, + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated, + } +} + +export function mockDoCounts( + counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number + }, + options?: { purge?: () => Promise<{ ok: true }> }, +) { + const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => counts), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => ({ + ok: true as const, + value: await promise, + }), + ) + return { purge } +} + +export function parityEnv(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } +} diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index c1cea9693b..b2b1dec2dd 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -17,6 +17,8 @@ import { mailboxParityMessagePageSize, mirrorMailboxParityDeliveryEventFromD1, replayContentUpdatesForUser, + type MailboxParityContentPhaseResult, + type MailboxParityCreationPhaseResult, } from './mailbox-parity-phases.ts' import { countD1MailboxParity, @@ -71,6 +73,32 @@ export { listUsersForMailboxParity, } +type MailboxParityCompareResult = + | { status: 'matched'; state: MailboxParityUserState } + | { status: 'mismatched_reset'; state: MailboxParityUserState } + | { + status: 'mismatched_purge_failed' + state: MailboxParityUserState + lastError: string + } + +type MailboxParityUserTickResult = + | { status: 'skipped' } + | { status: 'deletion_stopped'; backfilled: number } + | { + status: 'phase_checkpoint' + backfilled: number + failed: boolean + budgetExhausted: boolean + } + | { + status: 'compared' + backfilled: number + matched: boolean + mismatched: boolean + failed: boolean + } + function safeParityErrorText(error: unknown): string { const raw = error instanceof Error ? error.message : String(error) let cleaned = '' @@ -214,12 +242,7 @@ async function compareParityForUser(input: { env: MailboxParityReconcileEnv state: MailboxParityUserState nowIso: string -}): Promise<{ - state: MailboxParityUserState - matched: boolean - purgeFailed: boolean - lastError: string | null -}> { +}): Promise { const d1 = await countD1MailboxParity({ db: input.env.APP_DB, userId: input.state.userId, @@ -241,14 +264,12 @@ async function compareParityForUser(input: { }) if (matched) { return { + status: 'matched', state: { ...input.state, matchingSince: input.state.matchingSince ?? input.nowIso, mismatchCount: 0, }, - matched: true, - purgeFailed: false, - lastError: null, } } @@ -266,18 +287,18 @@ async function compareParityForUser(input: { purged.errorText, ) return { + status: 'mismatched_purge_failed', state: { ...input.state, matchingSince: null, mismatchCount: input.state.mismatchCount + 1, }, - matched: false, - purgeFailed: true, lastError: purged.errorText, } } return { + status: 'mismatched_reset', state: { ...input.state, matchingSince: null, @@ -290,9 +311,85 @@ async function compareParityForUser(input: { contentReplayUpperAt: null, contentReplayCursor: null, }, - matched: false, - purgeFailed: false, - lastError: null, + } +} + +function creationPhaseCheckpoint(input: { + phase: 'message' | 'event' + result: MailboxParityCreationPhaseResult +}): { + kind: 'continue' | 'deletion' | 'failed' | 'budget' + lastError: string | null + warnDetail: string | null +} { + switch (input.result.status) { + case 'complete': + return { kind: 'continue', lastError: null, warnDetail: null } + case 'deletion_started': + return { kind: 'deletion', lastError: null, warnDetail: null } + case 'budget_exhausted': + return { kind: 'budget', lastError: null, warnDetail: null } + case 'retryable_failure': { + const prefix = + input.phase === 'message' ? 'message backfill' : 'event backfill' + return { + kind: 'failed', + lastError: `${prefix} ${input.result.blockedReason}`, + warnDetail: input.result.blockedReason, + } + } + default: { + const exhaustive: never = input.result + throw new Error( + `Unhandled creation phase status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function contentPhaseCheckpoint(input: { + result: MailboxParityContentPhaseResult +}): { + kind: 'continue' | 'deletion' | 'failed' | 'budget' + lastError: string | null + warnDetail: string | null + freezeOpenedWindow: boolean +} { + switch (input.result.status) { + case 'complete': + return { + kind: 'continue', + lastError: null, + warnDetail: null, + freezeOpenedWindow: false, + } + case 'deletion_started': + return { + kind: 'deletion', + lastError: null, + warnDetail: null, + freezeOpenedWindow: false, + } + case 'budget_exhausted': + return { + kind: 'budget', + lastError: null, + warnDetail: null, + freezeOpenedWindow: input.result.openedWindow, + } + case 'retryable_failure': + return { + kind: 'failed', + lastError: `content replay ${input.result.blockedReason}`, + warnDetail: input.result.blockedReason, + freezeOpenedWindow: input.result.openedWindow, + } + default: { + const exhaustive: never = input.result + throw new Error( + `Unhandled content phase status: ${JSON.stringify(exhaustive)}`, + ) + } } } @@ -301,27 +398,13 @@ async function reconcileOneUser(input: { userId: string nowIso: string deadlineMs: number -}): Promise<{ - backfilled: number - compared: boolean - matched: boolean - mismatched: boolean - failed: boolean - budgetExhausted: boolean -}> { +}): Promise { const loaded = await loadUserParityState({ db: input.env.APP_DB, userId: input.userId, }) if (!loaded) { - return { - backfilled: 0, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: false, - } + return { status: 'skipped' } } let state = loaded @@ -331,30 +414,23 @@ async function reconcileOneUser(input: { const isDeleting = () => isUserDeleting({ db: input.env.APP_DB, userId: state.userId }) - const stopForDeletion = async () => { + const stopForDeletion = async (): Promise => { await purgeMailboxBestEffort({ env: input.env, userId: state.userId, }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: false, - } + return { status: 'deletion_stopped', backfilled } } - const persistProgress = async (inputPersist: { + const persistCheckpoint = async (checkpoint: { matchingSince: string | null lastError: string | null }) => { - const { wrote } = await persistUserParityProgress({ + await persistUserParityProgress({ db: input.env.APP_DB, userId: state.userId, nowIso: input.nowIso, - matchingSince: inputPersist.matchingSince, + matchingSince: checkpoint.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, contentReplayUpperAt: state.contentReplayUpperAt, @@ -363,9 +439,8 @@ async function reconcileOneUser(input: { messagesCompletedAt: state.messagesCompletedAt, eventCursor: state.eventCursor, eventsCompletedAt: state.eventsCompletedAt, - lastError: inputPersist.lastError, + lastError: checkpoint.lastError, }) - return wrote } if (await isDeleting()) { @@ -391,39 +466,42 @@ async function reconcileOneUser(input: { state = messagePass.state creationBackfilled += messagePass.backfilled backfilled += messagePass.backfilled - if (messagePass.deletionStarted) { - return stopForDeletion() - } - if (messagePass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: messagePass.blockedReason - ? `message backfill ${messagePass.blockedReason}` - : 'message backfill mirror error or timeout', - }) - console.warn( - 'mailbox-parity-message-backfill-retryable', - state.userId, - messagePass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (messagePass.budgetExhausted) { - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: true, + const messageCheckpoint = creationPhaseCheckpoint({ + phase: 'message', + result: messagePass, + }) + switch (messageCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + await persistCheckpoint({ + matchingSince: null, + lastError: messageCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-message-backfill-retryable', + state.userId, + messageCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = messageCheckpoint.kind + throw new Error(`Unhandled message checkpoint: ${exhaustive}`) } } @@ -436,49 +514,50 @@ async function reconcileOneUser(input: { state = eventPass.state creationBackfilled += eventPass.backfilled backfilled += eventPass.backfilled - if (eventPass.deletionStarted) { - return stopForDeletion() - } - if (eventPass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: eventPass.blockedReason - ? `event backfill ${eventPass.blockedReason}` - : 'event backfill mirror error or timeout', - }) - console.warn( - 'mailbox-parity-event-backfill-retryable', - state.userId, - eventPass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (eventPass.budgetExhausted) { - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: true, + const eventCheckpoint = creationPhaseCheckpoint({ + phase: 'event', + result: eventPass, + }) + switch (eventCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + await persistCheckpoint({ + matchingSince: null, + lastError: eventCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-event-backfill-retryable', + state.userId, + eventCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = eventCheckpoint.kind + throw new Error(`Unhandled event checkpoint: ${exhaustive}`) } } if (state.messagesCompletedAt == null || state.eventsCompletedAt == null) { - await persistProgress({ matchingSince: null, lastError: null }) + await persistCheckpoint({ matchingSince: null, lastError: null }) return { + status: 'phase_checkpoint', backfilled, - compared: false, - matched: false, - mismatched: false, failed: false, budgetExhausted: false, } @@ -499,47 +578,47 @@ async function reconcileOneUser(input: { }) state = contentPass.state backfilled += contentPass.backfilled - if (contentPass.deletionStarted) { - return stopForDeletion() - } - // Durably freeze a newly opened window even when this tick cannot finish it. - if (contentPass.openedWindow && !contentPass.windowComplete) { - await persistProgress({ - matchingSince: null, - lastError: null, - }) - } - if (contentPass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: contentPass.blockedReason - ? `content replay ${contentPass.blockedReason}` - : 'content replay mirror error or timeout', - }) - console.warn( - 'mailbox-parity-content-replay-retryable', - state.userId, - contentPass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (!contentPass.windowComplete) { - // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: contentPass.budgetExhausted, + const contentCheckpoint = contentPhaseCheckpoint({ result: contentPass }) + switch (contentCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + // Durably freeze a newly opened window even when this tick cannot finish it. + if (contentCheckpoint.freezeOpenedWindow) { + await persistCheckpoint({ matchingSince: null, lastError: null }) + } + await persistCheckpoint({ + matchingSince: null, + lastError: contentCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-content-replay-retryable', + state.userId, + contentCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + if (contentCheckpoint.freezeOpenedWindow) { + await persistCheckpoint({ matchingSince: null, lastError: null }) + } + // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = contentCheckpoint.kind + throw new Error(`Unhandled content checkpoint: ${exhaustive}`) } } @@ -556,17 +635,48 @@ async function reconcileOneUser(input: { if (await isDeleting()) { return stopForDeletion() } - await persistProgress({ - matchingSince: state.matchingSince, - lastError: compared.lastError, - }) - return { - backfilled, - compared: true, - matched: compared.matched, - mismatched: !compared.matched, - failed: compared.purgeFailed, - budgetExhausted: false, + + switch (compared.status) { + case 'matched': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: null, + }) + return { + status: 'compared', + backfilled, + matched: true, + mismatched: false, + failed: false, + } + case 'mismatched_reset': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: null, + }) + return { + status: 'compared', + backfilled, + matched: false, + mismatched: true, + failed: false, + } + case 'mismatched_purge_failed': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: compared.lastError, + }) + return { + status: 'compared', + backfilled, + matched: false, + mismatched: true, + failed: true, + } + default: { + const exhaustive: never = compared + throw new Error(`Unhandled compare status: ${JSON.stringify(exhaustive)}`) + } } } @@ -603,12 +713,34 @@ export async function reconcileMailboxParity(input: { nowIso, deadlineMs, }) - metrics.backfilled += result.backfilled - if (result.compared) metrics.compared += 1 - if (result.matched) metrics.matched += 1 - if (result.mismatched) metrics.mismatched += 1 - if (result.failed) metrics.failed += 1 - if (result.budgetExhausted) break + switch (result.status) { + case 'skipped': + break + case 'deletion_stopped': + metrics.backfilled += result.backfilled + break + case 'phase_checkpoint': + metrics.backfilled += result.backfilled + if (result.failed) metrics.failed += 1 + if (result.budgetExhausted) { + // Stop scanning further users this tick. + return metrics + } + break + case 'compared': + metrics.backfilled += result.backfilled + metrics.compared += 1 + if (result.matched) metrics.matched += 1 + if (result.mismatched) metrics.mismatched += 1 + if (result.failed) metrics.failed += 1 + break + default: { + const exhaustive: never = result + throw new Error( + `Unhandled user tick status: ${JSON.stringify(exhaustive)}`, + ) + } + } } catch (error) { metrics.failed += 1 const lastError = safeParityErrorText(error) From a09e7c6e73a6017d47cb087e447a0e3b48bfe89e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:35:38 +0000 Subject: [PATCH 17/20] test(email): remove superseded parity suite Co-authored-by: Kent C. Dodds --- .../src/email/mailbox-reconcile.node.test.ts | 1226 ----------------- 1 file changed, 1226 deletions(-) delete mode 100644 packages/worker/src/email/mailbox-reconcile.node.test.ts diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts deleted file mode 100644 index f082b68e95..0000000000 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ /dev/null @@ -1,1226 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import { expect, test, vi } from 'vitest' -import { consoleWarn } from '#worker/test-support/console-spies.ts' -import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' -import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' -import { systemEmailOwnerId } from './email-owner.ts' -import type * as MailboxMirrorModule from './mailbox-mirror.ts' -import { - mailboxParityContentPageSize, - mailboxParityMessagePageSize, -} from './mailbox-parity-phases.ts' -import { ensureEmailTestSchema } from './test-schema.ts' - -const mocks = vi.hoisted(() => ({ - mirrorMailboxMessageGraphFromD1: vi.fn(), - mirrorMailboxDeliveryEventFromD1: vi.fn(), - awaitMailboxMirrorRpc: vi.fn(), - recordMailboxParityEvent: vi.fn(), - mailboxRpc: vi.fn(), -})) - -vi.mock('./mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, - mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, -})) - -vi.mock('./mailbox-mirror.ts', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, - } -}) - -vi.mock('./mailbox-parity-events.ts', () => ({ - recordMailboxParityEvent: mocks.recordMailboxParityEvent, -})) - -vi.mock('./mailbox-client.ts', () => ({ - mailboxRpc: mocks.mailboxRpc, -})) - -const { - countD1MailboxParity, - listEventBackfillPage, - listUsersForMailboxParity, - reconcileMailboxParity, -} = await import('./mailbox-reconcile.ts') - -async function createParityDb() { - const sqlite = new DatabaseSync(':memory:') - const db = createD1FromSqlite(sqlite) - await ensureUsersTestSchema({ db }) - await ensureEmailTestSchema(db) - return { sqlite, db } -} - -async function insertUser(input: { - db: D1Database - userId: string - email?: string - deletingAt?: string | null - checkedAt?: string | null -}) { - await input.db - .prepare( - `INSERT INTO users ( - username, email, password_hash, stable_user_id, deleting_at, - mailbox_parity_checked_at - ) VALUES (?, ?, 'hash', ?, ?, ?)`, - ) - .bind( - `u-${crypto.randomUUID().slice(0, 8)}`, - input.email ?? `${crypto.randomUUID()}@example.test`, - input.userId, - input.deletingAt ?? null, - input.checkedAt ?? null, - ) - .run() -} - -async function insertMessage(input: { - db: D1Database - userId: string - id: string - createdAt: string - updatedAt?: string -}) { - await input.db - .prepare( - `INSERT INTO email_messages ( - id, direction, user_id, from_address, subject, text_body, raw_size, - processing_status, created_at, updated_at - ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, - 'stored', ?, ?)`, - ) - .bind( - input.id, - input.userId, - input.createdAt, - input.updatedAt ?? input.createdAt, - ) - .run() -} - -async function insertEvent(input: { - db: D1Database - userId: string - id: string - createdAt: string - messageId?: string | null -}) { - await input.db - .prepare( - `INSERT INTO email_delivery_events ( - id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, - ) - .bind( - input.id, - input.messageId === undefined ? null : input.messageId, - input.userId, - input.createdAt, - ) - .run() -} - -async function parkUser(db: D1Database, userId: string) { - await db - .prepare( - `UPDATE users - SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' - WHERE stable_user_id = ?`, - ) - .bind(userId) - .run() -} - -async function readParityRow(db: D1Database, userId: string) { - return db - .prepare( - `SELECT - mailbox_parity_checked_at AS checkedAt, - mailbox_parity_matching_since AS matchingSince, - mailbox_parity_mismatch_count AS mismatchCount, - mailbox_parity_last_error AS lastError, - mailbox_parity_content_watermark_at AS contentWatermarkAt, - mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, - mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, - mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, - mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, - mailbox_parity_message_backfill_cursor_id AS messageCursorId, - mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, - mailbox_parity_event_backfill_cursor_id AS eventCursorId, - mailbox_parity_event_backfill_completed_at AS eventsCompletedAt - FROM users - WHERE stable_user_id = ?`, - ) - .bind(userId) - .first<{ - checkedAt: string | null - matchingSince: string | null - mismatchCount: number - lastError: string | null - contentWatermarkAt: string | null - contentReplayUpperAt: string | null - contentReplayCursorUpdatedAt: string | null - contentReplayCursorId: string | null - messageCursorCreatedAt: string | null - messageCursorId: string | null - messagesCompletedAt: string | null - eventCursorCreatedAt: string | null - eventCursorId: string | null - eventsCompletedAt: string | null - }>() -} - -function successGraph(messageId: string, eventsTruncated = false) { - return { - messageId, - message: { status: 'mirrored' as const }, - events: [], - eventsTruncated, - } -} - -function mockDoCounts( - counts: { - threads: number - messages: number - attachments: number - deliveryEvents: number - }, - options?: { purge?: () => Promise<{ ok: true }> }, -) { - const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => counts), - purge, - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => ({ - ok: true as const, - value: await promise, - }), - ) - return { purge } -} - -function parityEnv(db: D1Database) { - return { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } -} - -test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const ownerA = 'a'.repeat(64) - const ownerB = 'b'.repeat(64) - const deleting = 'c'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - - await insertUser({ db, userId: ownerA, checkedAt: null }) - await insertUser({ - db, - userId: ownerB, - checkedAt: '2026-07-01T00:00:00.000Z', - }) - await insertUser({ - db, - userId: deleting, - deletingAt: '2026-07-31T00:00:00.000Z', - checkedAt: null, - }) - - const messageIds = Array.from( - { length: mailboxParityMessagePageSize + 1 }, - (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, - ) - for (const [index, id] of messageIds.entries()) { - await insertMessage({ - db, - userId: ownerA, - id, - createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, - }) - } - await insertEvent({ - db, - userId: ownerA, - id: 'evt-bound-1', - messageId: messageIds[0]!, - createdAt: '2026-07-02T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: ownerA, - id: 'evt-orphan-1', - createdAt: '2026-07-02T00:00:01.000Z', - }) - await insertMessage({ - db, - userId: ownerB, - id: 'b-msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: systemEmailOwnerId, - id: 'sys-msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: deleting, - id: 'del-evt', - createdAt: '2026-07-01T00:00:00.000Z', - }) - - const discovered = await listUsersForMailboxParity({ db, limit: 10 }) - expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) - expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) - expect(discovered.map((row) => row.userId)).not.toContain(deleting) - - const eventPage = await listEventBackfillPage({ - db, - userId: ownerA, - cursor: null, - limit: 10, - }) - expect(eventPage.map((row) => row.id)).toEqual([ - 'evt-bound-1', - 'evt-orphan-1', - ]) - - await db - .prepare( - `INSERT INTO email_threads ( - id, user_id, subject_normalized, last_message_at, created_at, updated_at - ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, - ) - .bind( - ownerA, - '2026-07-01T00:00:00.000Z', - '2026-07-01T00:00:00.000Z', - '2026-07-01T00:00:00.000Z', - ) - .run() - - await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ - threads: 1, - messages: mailboxParityMessagePageSize + 1, - attachments: 0, - deliveryEvents: 2, - }) - - let graphCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - graphCalls += 1 - if (graphCalls === mailboxParityMessagePageSize) { - return { - messageId: input.messageId, - message: { status: 'timeout' as const }, - events: [], - eventsTruncated: false, - } - } - return successGraph(input.messageId) - }, - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - }) - - const env = parityEnv(db) - await db - .prepare( - `UPDATE users - SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', - mailbox_parity_mismatch_count = 2 - WHERE stable_user_id = ?`, - ) - .bind(ownerA) - .run() - - await parkUser(db, ownerB) - const first = await reconcileMailboxParity({ - env, - now, - batchSize: 1, - }) - expect(first).toEqual({ - scanned: 1, - backfilled: mailboxParityMessagePageSize - 1, - compared: 0, - matched: 0, - mismatched: 0, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-message-backfill-retryable', - ownerA, - 'message graph timeout', - ) - - const afterFirst = await readParityRow(db, ownerA) - expect(afterFirst).toMatchObject({ - checkedAt: now.toISOString(), - messageCursorId: messageIds[mailboxParityMessagePageSize - 2], - messagesCompletedAt: null, - eventsCompletedAt: null, - contentWatermarkAt: now.toISOString(), - matchingSince: null, - mismatchCount: 2, - lastError: 'message backfill message graph timeout', - }) - - // Finish creation; count mismatch resets BOTH cursors + completion markers. - graphCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() - mocks.recordMailboxParityEvent.mockClear() - await parkUser(db, ownerB) - const mismatchNow = new Date('2026-08-01T11:00:00.000Z') - const mismatch = await reconcileMailboxParity({ - env, - now: mismatchNow, - batchSize: 1, - }) - expect(mismatch).toEqual({ - scanned: 1, - backfilled: 4, - compared: 1, - matched: 0, - mismatched: 1, - failed: 0, - }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), - ) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), - ) - - const afterMismatch = await readParityRow(db, ownerA) - expect(afterMismatch).toMatchObject({ - checkedAt: mismatchNow.toISOString(), - matchingSince: null, - mismatchCount: 3, - contentWatermarkAt: null, - contentReplayUpperAt: null, - contentReplayCursorId: null, - messagesCompletedAt: null, - eventsCompletedAt: null, - messageCursorId: null, - eventCursorId: null, - lastError: null, - }) - - // Full historical rescan after cursor reset, including a live new row. - await insertMessage({ - db, - userId: ownerA, - id: 'msg-live-new', - createdAt: '2026-07-03T00:00:00.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() - mockDoCounts({ - threads: 1, - messages: mailboxParityMessagePageSize + 2, - attachments: 0, - deliveryEvents: 2, - }) - await parkUser(db, ownerB) - const rematchNow = new Date('2026-08-01T12:00:00.000Z') - const rematch = await reconcileMailboxParity({ - env, - now: rematchNow, - batchSize: 1, - }) - expect(rematch).toEqual({ - scanned: 1, - backfilled: mailboxParityMessagePageSize + 4, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), - ) - const afterRematch = await readParityRow(db, ownerA) - expect(afterRematch).toMatchObject({ - matchingSince: rematchNow.toISOString(), - mismatchCount: 0, - contentWatermarkAt: expect.any(String), - messagesCompletedAt: expect.any(String), - eventsCompletedAt: expect.any(String), - messageCursorId: 'msg-live-new', - eventCursorId: 'evt-orphan-1', - }) - if (!afterRematch?.contentWatermarkAt) { - throw new Error('Expected content watermark after parity rematch.') - } - expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) - - await parkUser(db, ownerB) - const soakNow = new Date('2026-08-02T12:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterSoak = await readParityRow(db, ownerA) - expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) - expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) - expect(afterSoak?.mismatchCount).toBe(0) - - consoleWarn.mockClear() - mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( - new Error('count boom\nwith\tcontrol'), - ) - await parkUser(db, ownerB) - const errorNow = new Date('2026-08-02T13:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), - ).resolves.toMatchObject({ failed: 1, compared: 0 }) - const afterError = await readParityRow(db, ownerA) - expect(afterError?.checkedAt).toBe(errorNow.toISOString()) - expect(afterError?.lastError).toBe('count boom with control') - expect(afterError?.matchingSince).toBeNull() -}) - -test('mailbox parity content watermark repairs equal-count updates and soak', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'd'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-a', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-b', - createdAt: '2026-07-01T00:00:01.000Z', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1, compared: 1 }) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - now.toISOString(), - ) - - const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id IN ('msg-a', 'msg-b')`, - ) - .bind(sharedUpdatedAt) - .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - const repairNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(repairNow) - await expect( - reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 2, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual(['msg-a', 'msg-b']) - const afterRepair = await readParityRow(db, owner) - expect(afterRepair?.matchingSince).toBe(now.toISOString()) - expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) - expect(afterRepair?.contentReplayUpperAt).toBeNull() - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity replays message updated during initial backfill', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'e'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-early', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-late', - createdAt: '2026-07-01T00:00:01.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - if (input.messageId === 'msg-early') { - vi.setSystemTime(midUpdateAt) - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-early'`, - ) - .bind(midUpdateAt.toISOString()) - .run() - } - return successGraph(input.messageId) - }, - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 3, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual(['msg-early', 'msg-late', 'msg-early']) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - midUpdateAt.toISOString(), - ) - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'f'.repeat(64) - const baseline = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(baseline) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - const messageCount = mailboxParityContentPageSize + 2 - for (let index = 0; index < messageCount; index += 1) { - await insertMessage({ - db, - userId: owner, - id: `msg-${String(index).padStart(2, '0')}`, - createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, - }) - } - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: messageCount, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) - - const updateAt = '2026-08-01T10:30:00.000Z' - await db - .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) - .bind(updateAt, owner) - .run() - - const replayNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(replayNow) - let contentCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - contentCalls += 1 - if (contentCalls === mailboxParityContentPageSize) { - // Exhaust budget mid-window by freezing time past deadline. - vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) - } - return successGraph(input.messageId) - }, - ) - await expect( - reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), - ).resolves.toMatchObject({ - compared: 0, - failed: 0, - backfilled: mailboxParityContentPageSize, - }) - const mid = await readParityRow(db, owner) - expect(mid).toMatchObject({ - contentWatermarkAt: baseline.toISOString(), - contentReplayUpperAt: replayNow.toISOString(), - contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, - matchingSince: null, - }) - - vi.setSystemTime(replayNow) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - await expect( - reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), - ).resolves.toMatchObject({ - compared: 1, - matched: 1, - backfilled: 2, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual([ - `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, - `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, - ]) - const done = await readParityRow(db, owner) - expect(done).toMatchObject({ - contentWatermarkAt: replayNow.toISOString(), - contentReplayUpperAt: null, - contentReplayCursorId: null, - // Soak was cleared on the incomplete tick; exact compare starts a new window. - matchingSince: replayNow.toISOString(), - }) - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'g'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-2', - createdAt: '2026-07-01T00:00:01.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-1', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:02.000Z', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'msg-1', - message: { status: 'missing' as const }, - events: [], - eventsTruncated: false, - }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 1, - }) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) - expect(await readParityRow(db, owner)).toMatchObject({ - messageCursorId: null, - messagesCompletedAt: null, - contentWatermarkAt: now.toISOString(), - lastError: 'message backfill message graph missing', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ - status: 'skipped', - reason: 'mailbox-unconfigured', - }) - await expect( - reconcileMailboxParity({ - env, - now: new Date('2026-08-01T11:00:00.000Z'), - batchSize: 1, - }), - ).resolves.toMatchObject({ failed: 1, compared: 0 }) - const afterUnconfigured = await readParityRow(db, owner) - expect(afterUnconfigured).toMatchObject({ - messagesCompletedAt: expect.any(String), - eventCursorId: null, - eventsCompletedAt: null, - lastError: 'event backfill event skipped:mailbox-unconfigured', - }) -}) - -test('mailbox parity eventsTruncated still completes via full event phase', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'h'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - for (let index = 0; index < 3; index += 1) { - await insertEvent({ - db, - userId: owner, - id: `evt-${index}`, - messageId: 'msg-1', - createdAt: `2026-07-01T00:01:0${index}.000Z`, - }) - } - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( - successGraph('msg-1', true), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 3, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 4, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) -}) - -test('mailbox parity event missing advances cursor while unconfigured does not', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'j'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-gone', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:01.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-ok', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:02.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1 - .mockResolvedValueOnce({ status: 'missing' }) - .mockResolvedValueOnce({ status: 'mirrored' }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 2, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toMatchObject({ matched: 1, failed: 0 }) - expect(await readParityRow(db, owner)).toMatchObject({ - eventCursorId: 'evt-ok', - eventsCompletedAt: expect.any(String), - }) -}) - -test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'i'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - const purge = vi.fn(async () => ({ ok: true as const })) - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => ({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - })), - purge, - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => { - try { - return { ok: true as const, value: await promise } - } catch { - return { ok: false as const, result: { status: 'timeout' as const } } - } - }, - ) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { - await db - .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) - .bind(now.toISOString(), owner) - .run() - return successGraph('msg-1') - }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 0, - matched: 0, - mismatched: 0, - failed: 0, - }) - expect(purge).toHaveBeenCalledTimes(1) - const row = await readParityRow(db, owner) - // Baseline may have been written before deletion; no completion/compare state. - expect(row?.messagesCompletedAt).toBeNull() - expect(row?.matchingSince).toBeNull() - expect(row?.eventsCompletedAt).toBeNull() -}) - -test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'k'.repeat(64) - const neverTracked = 'l'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertUser({ db, userId: neverTracked, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-1', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:01.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - const purge = vi.fn(async () => ({ ok: true as const })) - mockDoCounts( - { - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 1, - }, - { purge }, - ) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1, compared: 1 }) - - // Simulate failed delete mirror: D1 cleared, DO still holds message/event. - await db - .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) - .bind(owner) - .run() - await db - .prepare(`DELETE FROM email_messages WHERE user_id = ?`) - .bind(owner) - .run() - - expect( - (await listUsersForMailboxParity({ db, limit: 10 })).map( - (row) => row.userId, - ), - ).toEqual([owner]) - expect( - (await listUsersForMailboxParity({ db, limit: 10 })).map( - (row) => row.userId, - ), - ).not.toContain(neverTracked) - - mockDoCounts( - { - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 1, - }, - { purge }, - ) - const mismatchNow = new Date('2026-08-01T11:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 0, - mismatched: 1, - failed: 0, - }) - expect(purge).toHaveBeenCalledTimes(1) - const afterPurge = await readParityRow(db, owner) - expect(afterPurge).toMatchObject({ - matchingSince: null, - mismatchCount: 1, - contentWatermarkAt: null, - messagesCompletedAt: null, - eventsCompletedAt: null, - messageCursorId: null, - eventCursorId: null, - lastError: null, - }) - - mockDoCounts({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - }) - const zeroNow = new Date('2026-08-01T12:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterZero = await readParityRow(db, owner) - expect(afterZero).toMatchObject({ - matchingSince: zeroNow.toISOString(), - mismatchCount: 0, - contentWatermarkAt: expect.any(String), - }) - if (!afterZero?.contentWatermarkAt) { - throw new Error('Expected content watermark after zero-count parity.') - } - expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) -}) - -test('mismatch purge failure clears soak and does not start rebuild reset', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'm'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1 }) - const afterMatch = await readParityRow(db, owner) - expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) - expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) - - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => ({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - })), - purge: vi.fn(async () => ({ ok: true as const })), - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => { - const settled = await promise - if (settled && typeof settled === 'object' && 'messages' in settled) { - return { ok: true as const, value: settled } - } - return { ok: false as const, timedOut: true as const } - }, - ) - consoleWarn.mockClear() - const failNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(failNow) - await expect( - reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 0, - mismatched: 1, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-mismatch-purge-failed', - owner, - 'timeout', - 'mailbox metadata purge timeout', - ) - const afterFail = await readParityRow(db, owner) - // Empty content window may advance the watermark before compare; purge - // failure must not clear creation completion/cursors (rebuild not started). - expect(afterFail).toMatchObject({ - matchingSince: null, - mismatchCount: 1, - messagesCompletedAt: afterMatch?.messagesCompletedAt, - eventsCompletedAt: afterMatch?.eventsCompletedAt, - messageCursorId: afterMatch?.messageCursorId, - lastError: 'mailbox metadata purge timeout', - checkedAt: failNow.toISOString(), - }) - expect(afterFail?.contentWatermarkAt).not.toBeNull() - } finally { - vi.useRealTimers() - } -}) From c43af54bbe9b6c40a0d4f795c3817651b8b1dbe5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:42:49 +0000 Subject: [PATCH 18/20] test(email): exclude parity support from worker build Co-authored-by: Kent C. Dodds --- .../mailbox-reconcile-cursor-backfill-content.node.test.ts | 2 +- .../mailbox-reconcile-mismatch-deletion-error.node.test.ts | 2 +- ...e-test-support.ts => mailbox-reconcile.node.test-support.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/worker/src/email/{mailbox-reconcile-test-support.ts => mailbox-reconcile.node.test-support.ts} (100%) diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts index 9e509c5547..c41a0d8f43 100644 --- a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -13,7 +13,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile-test-support.ts' +} from './mailbox-reconcile.node.test-support.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts index 6b8f9bd795..67557aa658 100644 --- a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -10,7 +10,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile-test-support.ts' +} from './mailbox-reconcile.node.test-support.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-test-support.ts b/packages/worker/src/email/mailbox-reconcile.node.test-support.ts similarity index 100% rename from packages/worker/src/email/mailbox-reconcile-test-support.ts rename to packages/worker/src/email/mailbox-reconcile.node.test-support.ts From 669e93bcc71887dd728d7b636f2d2aa7f21fd6b8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:43:31 +0000 Subject: [PATCH 19/20] test(email): relocate parity test support Co-authored-by: Kent C. Dodds --- ...-reconcile-cursor-backfill-content.node.test.ts | 2 +- ...-reconcile-mismatch-deletion-error.node.test.ts | 2 +- .../mailbox-reconcile.ts} | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) rename packages/worker/src/{email/mailbox-reconcile.node.test-support.ts => test-support/mailbox-reconcile.ts} (92%) diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts index c41a0d8f43..f5188db190 100644 --- a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -13,7 +13,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile.node.test-support.ts' +} from '#worker/test-support/mailbox-reconcile.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts index 67557aa658..2f41201db7 100644 --- a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -10,7 +10,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile.node.test-support.ts' +} from '#worker/test-support/mailbox-reconcile.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' diff --git a/packages/worker/src/email/mailbox-reconcile.node.test-support.ts b/packages/worker/src/test-support/mailbox-reconcile.ts similarity index 92% rename from packages/worker/src/email/mailbox-reconcile.node.test-support.ts rename to packages/worker/src/test-support/mailbox-reconcile.ts index 2baf8e1cff..35ce0801c6 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test-support.ts +++ b/packages/worker/src/test-support/mailbox-reconcile.ts @@ -2,8 +2,8 @@ import { DatabaseSync } from 'node:sqlite' import { vi } from 'vitest' import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' -import type * as MailboxMirrorModule from './mailbox-mirror.ts' -import { ensureEmailTestSchema } from './test-schema.ts' +import type * as MailboxMirrorModule from '#worker/email/mailbox-mirror.ts' +import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' const mocks = vi.hoisted(() => ({ mirrorMailboxMessageGraphFromD1: vi.fn(), @@ -13,12 +13,12 @@ const mocks = vi.hoisted(() => ({ mailboxRpc: vi.fn(), })) -vi.mock('./mailbox-live-mirror.ts', () => ({ +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, })) -vi.mock('./mailbox-mirror.ts', async (importOriginal) => { +vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { const actual = await importOriginal() return { ...actual, @@ -26,11 +26,11 @@ vi.mock('./mailbox-mirror.ts', async (importOriginal) => { } }) -vi.mock('./mailbox-parity-events.ts', () => ({ +vi.mock('#worker/email/mailbox-parity-events.ts', () => ({ recordMailboxParityEvent: mocks.recordMailboxParityEvent, })) -vi.mock('./mailbox-client.ts', () => ({ +vi.mock('#worker/email/mailbox-client.ts', () => ({ mailboxRpc: mocks.mailboxRpc, })) @@ -45,7 +45,7 @@ export const { listEventBackfillPage, listUsersForMailboxParity, reconcileMailboxParity, -} = await import('./mailbox-reconcile.ts') +} = await import('#worker/email/mailbox-reconcile.ts') export async function createParityDb() { const sqlite = new DatabaseSync(':memory:') From 03aa7e9a5c8cc9f45cd5a53e68a108aba70a7467 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:06:03 +0000 Subject: [PATCH 20/20] fix(email): preserve parity test module exports Co-authored-by: Kent C. Dodds --- .../migrations/0125-mailbox-parity-state.sql | 2 +- .../worker/src/email/mailbox-parity-repo.ts | 2 +- .../worker/src/email/mailbox-reconcile.ts | 2 +- .../src/test-support/mailbox-reconcile.ts | 35 +++++++++++++------ tools/migration-ledger.json | 2 +- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 937a5a76e2..f0c29743bd 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -1,4 +1,4 @@ --- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. +-- Account-owned Mailbox parity / backfill state for the five-minute reconcile lane. -- Low-write columns live on users (not per-message). Creation cursors are -- keyset (created_at, id) for messages and ALL owner delivery events so backfill -- can resume across ticks without DO enumeration. The event phase rescans every diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index 2ee1a86e06..fe1f767cba 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -2,7 +2,7 @@ import { systemEmailOwnerId } from './email-owner.ts' import { type MailboxCountResult } from './mailbox-types.ts' /** - * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. + * D1 loaders/persisters for the five-minute Mailbox parity reconcile lane. * Creation backfill uses (created_at, id) keysets over messages and all owner * delivery events; content replay uses a durable (watermark, upper] window with * an (updated_at, id) cursor. diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index b2b1dec2dd..d2ee662310 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -33,7 +33,7 @@ import { import { type MailboxCountResult } from './mailbox-types.ts' /** - * Bounded hourly Mailbox backfill + count-parity reconciler. + * Bounded five-minute Mailbox backfill + count-parity reconciler. * * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO * enumeration), mirrors message graphs then every owner delivery event through diff --git a/packages/worker/src/test-support/mailbox-reconcile.ts b/packages/worker/src/test-support/mailbox-reconcile.ts index 35ce0801c6..0388df44bd 100644 --- a/packages/worker/src/test-support/mailbox-reconcile.ts +++ b/packages/worker/src/test-support/mailbox-reconcile.ts @@ -2,7 +2,10 @@ import { DatabaseSync } from 'node:sqlite' import { vi } from 'vitest' import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import type * as MailboxClientModule from '#worker/email/mailbox-client.ts' +import type * as MailboxLiveMirrorModule from '#worker/email/mailbox-live-mirror.ts' import type * as MailboxMirrorModule from '#worker/email/mailbox-mirror.ts' +import type * as MailboxParityEventsModule from '#worker/email/mailbox-parity-events.ts' import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' const mocks = vi.hoisted(() => ({ @@ -13,10 +16,14 @@ const mocks = vi.hoisted(() => ({ mailboxRpc: vi.fn(), })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, - mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, -})) +vi.mock('#worker/email/mailbox-live-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, + } +}) vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { const actual = await importOriginal() @@ -26,13 +33,21 @@ vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { } }) -vi.mock('#worker/email/mailbox-parity-events.ts', () => ({ - recordMailboxParityEvent: mocks.recordMailboxParityEvent, -})) +vi.mock('#worker/email/mailbox-parity-events.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + recordMailboxParityEvent: mocks.recordMailboxParityEvent, + } +}) -vi.mock('#worker/email/mailbox-client.ts', () => ({ - mailboxRpc: mocks.mailboxRpc, -})) +vi.mock('#worker/email/mailbox-client.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mailboxRpc: mocks.mailboxRpc, + } +}) export type MailboxReconcileTestMocks = typeof mocks diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 7cd41d25a2..070dc3f59f 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "a0cfd916aff30eecf74d315088c131a8fd9f800f44aa8ed4406a806cfa1f1cff" + "sha256": "6d2d54b8d8a2cc7399c4f0dbb8069cfa5d6686958d8fef1a3b874c472a564754" } ] }