From 47f3e14e0b3eaf747f98afa43729b7f9fedbc5a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 19:36:33 +0000 Subject: [PATCH 01/12] wip(admin-digest): per-recipient delivery engine and tests (local checkpoint; message rewritten before push) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- lib/admin/digest-delivery.ts | 559 +++++++++++++++++ tests/admin-digest-delivery-engine.test.ts | 582 ++++++++++++++++++ tests/helpers/digest-delivery-fakes.ts | 257 ++++++++ .../helpers/digest-delivery-store-contract.ts | 152 +++++ 4 files changed, 1550 insertions(+) create mode 100644 lib/admin/digest-delivery.ts create mode 100644 tests/admin-digest-delivery-engine.test.ts create mode 100644 tests/helpers/digest-delivery-fakes.ts create mode 100644 tests/helpers/digest-delivery-store-contract.ts diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts new file mode 100644 index 000000000..6e4108494 --- /dev/null +++ b/lib/admin/digest-delivery.ts @@ -0,0 +1,559 @@ +// Per-recipient delivery engine for the daily Super Admin digest. +// +// NOT WIRED INTO ANY ROUTE. /api/cron/admin-digest still sends the way it did; +// this module is the delivery half of the durable design, published on its own +// so its rules can be reviewed and proven before any storage or route change. +// docs/admin-digest-delivery-contract.md is the integration contract: what a +// caller passes in, what a database adapter must guarantee, and what a +// provider adapter must classify. +// +// The caller owns scheduling. It decides the occurrence (its stable id and its +// window), reads the recipients (failing closed when that read fails, as #685 +// does) and renders the payload. This engine takes those as a FROZEN plan and +// owns only delivery: +// +// - freeze the plan is stored, one row per recipient, with the recipient's +// stable idempotency key and the exact bytes that will be sent, +// BEFORE anything is sent. A plan already stored for the occurrence +// wins over a new one: a retry re-sends the original payload, never +// a recomputed one, so a reused key never meets a changed payload. +// - claim one atomic step per recipient, under a lease, with a fence that +// grows on every claim. Two engines never both hold a recipient. +// - beginSend a fenced mark, written before the provider call, so a crash can +// be told apart: no mark means nothing reached the provider. +// - complete the provider's answer, fenced: a worker whose lease was taken over +// writes nothing. +// +// What it guarantees, and no more: +// - a recipient whose delivery was CONFIRMED accepted (a success with a message +// id) is never sent again; +// - every retry of a recipient uses the same key and the same bytes; +// - an outcome that is unknown (a timeout, a lost receipt, a worker that died +// after it may have sent) is retried only while the provider still remembers +// the key; after that the recipient is parked for reconciliation, because a +// retry the provider no longer deduplicates could deliver twice; +// - a payload conflict (the key already used for other bytes) is never retried +// and never escaped by changing the key. +// It is NOT exactly-once. The provider's key deduplication is what makes a +// retry after an unknown outcome harmless, and only inside the retention window. +// Nor does an in-memory store prove anything about PostgreSQL durability; the +// adapter requirements in the contract are what a real store must meet. + +import { createHash } from 'node:crypto'; + +export const DIGEST_DELIVERY_ENGINE_VERSION = 1; + +/** + * How long Resend remembers an idempotency key: 24 hours (docs re-checked + * 2026-09-30, https://resend.com/docs/dashboard/emails/idempotency-keys). + * Re-verify before changing providers or this value. + */ +export const RESEND_KEY_RETENTION_MS = 24 * 60 * 60 * 1000; + +// ── Plan ──────────────────────────────────────────────────────────────────── + +/** The occurrence-wide part of the email. Non-secret by construction: no headers, no credentials. */ +export type DigestPayload = { from: string; subject: string; html: string }; +/** What one recipient is sent, byte for byte (see `payloadJson`). */ +export type DeliveryPayload = { from: string; to: string; subject: string; html: string }; + +export type OccurrencePlan = { + /** Chosen by the caller and stable across retries of the same occurrence, e.g. `admin-digest:2026-09-30T12:30:00.000Z`. */ + occurrenceId: string; + /** Recorded, not interpreted: which window of activity this digest covers. */ + window: { start: string; end: string }; + /** Frozen by the caller from a read that succeeded (#685 fails closed before this point). */ + recipients: readonly string[]; + payload: DigestPayload; +}; + +export type FrozenOccurrence = { + occurrenceId: string; + window: { start: string; end: string }; + /** Sorted. */ + recipientKeys: string[]; + /** sha256 of the canonical occurrence payload; a changed payload changes this. */ + payloadHash: string; + engineVersion: number; +}; + +// ── Delivery rows and their state machine ─────────────────────────────────── +// +// pending ──claim──▶ in_flight ──complete──▶ accepted (terminal) +// │ ▲ rejected (terminal) +// │ │ conflict (terminal) +// │ └──claim── failed (a definitive refusal; nothing was accepted) +// │ └──claim── unknown (might have been accepted; ambiguous) +// │ +// └─ lease expires: back to pending/failed (no beginSend mark) or unknown (marked) +// +// exhausted attempts used up, nothing ever possibly accepted (terminal) +// needs_reconciliation possibly accepted, and retrying is no longer safe (terminal) + +export type DeliveryStatus = + | 'pending' | 'in_flight' | 'failed' | 'unknown' + | 'accepted' | 'rejected' | 'conflict' | 'exhausted' | 'needs_reconciliation'; + +export const TERMINAL_STATUSES: readonly DeliveryStatus[] = ['accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation']; +export const isTerminal = (s: DeliveryStatus) => TERMINAL_STATUSES.includes(s); + +export type DeliveryRow = { + occurrenceId: string; + /** sha256 hex of the normalized address: stable, and free of the address itself. */ + recipientKey: string; + /** Derived from (occurrence, recipient) only. Never rotated. */ + idempotencyKey: string; + payload: DeliveryPayload; + /** The exact request body. Stored, and sent verbatim. */ + payloadJson: string; + /** sha256 hex of `payloadJson`. */ + payloadHash: string; + status: DeliveryStatus; + /** Claims so far. */ + attempts: number; + /** Grows by one on every claim; `beginSend` and `complete` must present the current value. */ + fence: number; + leaseOwner: string | null; + leaseExpiresAt: string | null; + /** Set by `beginSend` for the CURRENT claim: the provider may have been called. */ + sendStartedAt: string | null; + /** The first time any attempt may have reached the provider: the retention window's anchor. */ + firstSendAt: string | null; + /** Sticky: some attempt may have been accepted without a receipt. */ + ambiguous: boolean; + providerMessageId: string | null; + /** Classified and non-secret. */ + lastError: string | null; + updatedAt: string; +}; + +export type ClaimPolicy = { + leaseMs: number; + maxAttempts: number; + /** How long the provider remembers a key (RESEND_KEY_RETENTION_MS). */ + providerKeyRetentionMs: number; + /** Stop ambiguous retries this long before the provider can forget the key: covers clock skew and the send itself. */ + retentionSafetyMarginMs: number; +}; + +export type ClaimRefusal = 'leased' | 'not_found' | Exclude; + +export type ClaimDecision = + | { claimed: true; next: DeliveryRow } + /** `next` is set when the refusal itself is a transition the store must persist (a parked or exhausted row). */ + | { claimed: false; reason: ClaimRefusal; next: DeliveryRow | null }; + +const MAX_ERROR = 200; +const clip = (s: string) => s.slice(0, MAX_ERROR); +const plusMs = (iso: string, ms: number) => new Date(Date.parse(iso) + ms).toISOString(); + +/** + * The claim rule. A store MUST evaluate it and persist `next` in ONE atomic + * step, with its own clock as `now` (a database adapter: the database's now(), + * never a caller's timestamp). + */ +export function decideClaim(row: DeliveryRow, now: Date, owner: string, policy: ClaimPolicy): ClaimDecision { + if (isTerminal(row.status)) return { claimed: false, reason: row.status as ClaimRefusal, next: null }; + const nowIso = now.toISOString(); + const next: DeliveryRow = { ...row }; + if (next.status === 'in_flight') { + if (next.leaseExpiresAt !== null && Date.parse(next.leaseExpiresAt) > now.getTime()) { + return { claimed: false, reason: 'leased', next: null }; + } + // The worker died or stalled. With a beginSend mark it may have reached the + // provider; without one it cannot have, because the mark precedes the call. + if (next.sendStartedAt !== null) { + next.ambiguous = true; + next.status = 'unknown'; + next.lastError = 'lease_expired_after_send_started'; + } else { + next.status = next.ambiguous ? 'unknown' : next.attempts > 1 ? 'failed' : 'pending'; + next.lastError = 'lease_expired_before_send'; + } + next.leaseOwner = null; + next.leaseExpiresAt = null; + next.sendStartedAt = null; + } + const park = (status: 'exhausted' | 'needs_reconciliation'): ClaimDecision => { + next.status = status; + next.updatedAt = nowIso; + return { claimed: false, reason: status, next }; + }; + if (next.attempts >= policy.maxAttempts) return park(next.ambiguous ? 'needs_reconciliation' : 'exhausted'); + if (next.ambiguous) { + // Anchor on the FIRST attempt that may have reached the provider: its key + // window can have started no later than that. + // A row marked ambiguous with no anchor is inconsistent: nothing says it is still safe. + if (next.firstSendAt === null) return park('needs_reconciliation'); + const lastSafe = Date.parse(next.firstSendAt) + policy.providerKeyRetentionMs - policy.retentionSafetyMarginMs; + if (now.getTime() >= lastSafe) return park('needs_reconciliation'); + } + next.status = 'in_flight'; + next.attempts += 1; + next.fence += 1; + next.leaseOwner = owner; + next.leaseExpiresAt = plusMs(nowIso, policy.leaseMs); + next.sendStartedAt = null; + next.updatedAt = nowIso; + return { claimed: true, next }; +} + +/** The beginSend rule: fenced. `null` means the caller's claim is no longer current and it must not send. */ +export function decideBeginSend(row: DeliveryRow, fence: number, now: Date): DeliveryRow | null { + if (row.status !== 'in_flight' || row.fence !== fence) return null; + const nowIso = now.toISOString(); + return { ...row, sendStartedAt: nowIso, firstSendAt: row.firstSendAt ?? nowIso, updatedAt: nowIso }; +} + +// ── Provider ──────────────────────────────────────────────────────────────── + +/** What a provider adapter must answer. Only `accepted` (with a message id) proves delivery. */ +export type ProviderSendResult = + | { kind: 'accepted'; messageId: string } + /** The provider answered and did NOT accept. `retryable`: the same bytes may succeed later (a rate limit, a fixed configuration). */ + | { kind: 'rejected'; httpStatus: number; code: string; retryable: boolean } + /** The key was already used for different bytes. Never retried, never re-keyed. */ + | { kind: 'payload_conflict' } + /** A request under this key is still being processed. Its outcome is unknown. */ + | { kind: 'in_progress' } + /** No answer that settles it: the provider may or may not have accepted. */ + | { kind: 'unknown'; reason: 'timeout' | 'network' | 'unreadable_response' | 'server_error' }; + +export interface DigestEmailProvider { + send(request: { idempotencyKey: string; payload: DeliveryPayload; payloadJson: string; signal: AbortSignal }): Promise; +} + +/** + * Resend's answers, classified (docs re-checked 2026-09-30). The future HTTP + * adapter feeds its response here; #692's `sendEmail` answers only `{ ok }`, + * which cannot tell a payload conflict or a message id apart, so it is not + * enough on its own. See the contract. + */ +export function classifyResendResponse(status: number, body: unknown): ProviderSendResult { + const b = (body && typeof body === 'object' ? body : {}) as { id?: unknown; name?: unknown }; + const name = typeof b.name === 'string' ? b.name : ''; + if (status >= 200 && status < 300) { + return typeof b.id === 'string' && b.id.length > 0 + ? { kind: 'accepted', messageId: b.id } + : { kind: 'unknown', reason: 'unreadable_response' }; + } + if (status === 409 && name === 'invalid_idempotent_request') return { kind: 'payload_conflict' }; + if (status === 409 && name === 'concurrent_idempotent_requests') return { kind: 'in_progress' }; + if (status === 409) return { kind: 'unknown', reason: 'unreadable_response' }; + if (status === 429) return { kind: 'rejected', httpStatus: status, code: name || 'rate_limited', retryable: true }; + // A missing or revoked key, or an unverified domain: nothing was accepted, and an operator can fix it. + if (status === 401 || status === 403) return { kind: 'rejected', httpStatus: status, code: name || 'unauthorized', retryable: true }; + if (status >= 400 && status < 500) return { kind: 'rejected', httpStatus: status, code: name || 'invalid_request', retryable: false }; + // A 5xx may come after the provider accepted: it settles nothing. + if (status >= 500) return { kind: 'unknown', reason: 'server_error' }; + return { kind: 'unknown', reason: 'unreadable_response' }; +} + +/** The complete rule: fenced. `null` means the caller's claim is no longer current and nothing may be written. */ +export function decideCompletion(row: DeliveryRow, fence: number, result: ProviderSendResult, now: Date, maxAttempts: number): DeliveryRow | null { + if (row.status !== 'in_flight' || row.fence !== fence) return null; + const nowIso = now.toISOString(); + const next: DeliveryRow = { ...row, leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, updatedAt: nowIso }; + switch (result.kind) { + case 'accepted': + return { ...next, status: 'accepted', providerMessageId: result.messageId, lastError: null }; + case 'payload_conflict': + return { ...next, status: 'conflict', lastError: 'provider_payload_conflict' }; + case 'rejected': + next.lastError = clip(`provider_rejected:${result.httpStatus}:${result.code}`); + // A refusal cannot undo an earlier attempt that may have been accepted. + if (!result.retryable) { next.status = next.ambiguous ? 'needs_reconciliation' : 'rejected'; return next; } + next.status = 'failed'; + break; + case 'in_progress': + next.ambiguous = true; next.status = 'unknown'; next.lastError = 'provider_in_progress'; + break; + case 'unknown': + next.ambiguous = true; next.status = 'unknown'; next.lastError = clip(`outcome_unknown:${result.reason}`); + break; + } + if (next.attempts >= maxAttempts) next.status = next.ambiguous ? 'needs_reconciliation' : 'exhausted'; + return next; +} + +// ── Storage ───────────────────────────────────────────────────────────────── + +export type StoredOccurrence = { occurrence: FrozenOccurrence; deliveries: DeliveryRow[] }; + +/** + * What a store must provide. Each method is ONE atomic step against durable + * state, and every lease or retention decision uses the store's clock. + * docs/admin-digest-delivery-contract.md gives the exact database requirements. + */ +export interface DigestDeliveryStore { + /** Create the occurrence and all of its deliveries if absent, all or nothing. Return what is stored either way. */ + freeze(occurrence: FrozenOccurrence, deliveries: DeliveryRow[]): Promise; + load(occurrenceId: string): Promise; + /** Apply `decideClaim` atomically. */ + claim(occurrenceId: string, recipientKey: string, owner: string, policy: ClaimPolicy): Promise<{ claimed: true; row: DeliveryRow } | { claimed: false; reason: ClaimRefusal }>; + /** Apply `decideBeginSend` atomically. */ + beginSend(occurrenceId: string, recipientKey: string, fence: number): Promise<'ok' | 'fenced_out'>; + /** Apply `decideCompletion` atomically. */ + complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number): Promise<'ok' | 'fenced_out'>; +} + +// ── Keys and payloads ─────────────────────────────────────────────────────── + +const sha256 = (s: string) => createHash('sha256').update(s, 'utf8').digest('hex'); + +export const normalizeRecipient = (address: string) => address.trim().toLowerCase(); +export const recipientKeyOf = (address: string) => sha256(normalizeRecipient(address)); + +/** + * The stable key: a function of (occurrence, recipient) and nothing else, so no + * attempt count, clock or payload can change it. Hashes keep the address out of + * a header the provider stores. Fits #692's rule: printable ASCII, 1–256. + */ +export function idempotencyKeyFor(occurrenceId: string, recipientKey: string): string { + return `bubaly/admin-digest/v${DIGEST_DELIVERY_ENGINE_VERSION}/${sha256(occurrenceId).slice(0, 32)}/${recipientKey.slice(0, 32)}`; +} + +/** Fixed key order, so the same payload is always the same bytes. */ +export const payloadJsonOf = (p: DeliveryPayload) => JSON.stringify({ from: p.from, to: p.to, subject: p.subject, html: p.html }); +const occurrencePayloadHash = (p: DigestPayload) => sha256(JSON.stringify({ from: p.from, subject: p.subject, html: p.html })); + +// ── Validation (before any storage call or send) ──────────────────────────── + +const MAX_RECIPIENTS = 200; +const MAX_HTML_BYTES = 512 * 1024; +const EMAIL = /^[^\s@<>()",;:\\[\]]+@[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$/; +const PRINTABLE = /^[\x21-\x7e](?:[\x20-\x7e]*[\x21-\x7e])?$/; +const LINE_BREAK = /[\r\n\u2028\u2029]/; + +function planError(msg: string): never { throw new TypeError(`digest-delivery: ${msg}`); } + +export function validatePlan(plan: OccurrencePlan): void { + if (typeof plan.occurrenceId !== 'string' || plan.occurrenceId.length > 200 || !PRINTABLE.test(plan.occurrenceId)) planError('occurrenceId must be 1–200 printable ASCII characters'); + const start = Date.parse(plan.window?.start); const end = Date.parse(plan.window?.end); + if (!Number.isFinite(start) || !Number.isFinite(end) || start >= end) planError('window must be two ISO instants, start before end'); + if (!Array.isArray(plan.recipients) || plan.recipients.length === 0) planError('recipients must be a non-empty frozen list; an empty or unreadable list is the caller\'s to refuse'); + if (plan.recipients.length > MAX_RECIPIENTS) planError(`at most ${MAX_RECIPIENTS} recipients`); + const seen = new Set(); + for (const r of plan.recipients) { + if (typeof r !== 'string' || !EMAIL.test(normalizeRecipient(r))) planError('every recipient must be one plain email address'); + const k = normalizeRecipient(r); + if (seen.has(k)) planError('recipients must be distinct'); + seen.add(k); + } + const p = plan.payload as Record; + const keys = p && typeof p === 'object' ? Object.keys(p).sort() : []; + if (keys.join(',') !== 'from,html,subject') planError('payload must be exactly { from, subject, html }: nothing else is stored or sent'); + if (typeof p.from !== 'string' || !p.from.trim() || LINE_BREAK.test(p.from)) planError('payload.from must be one line'); + if (typeof p.subject !== 'string' || !p.subject.trim() || LINE_BREAK.test(p.subject)) planError('payload.subject must be one non-empty line'); + if (typeof p.html !== 'string' || !p.html.trim() || Buffer.byteLength(p.html, 'utf8') > MAX_HTML_BYTES) planError('payload.html must be non-empty and at most 512 KiB'); +} + +export type EngineConfig = ClaimPolicy & { sendTimeoutMs: number }; + +export function validateConfig(c: EngineConfig): void { + const pos = (n: number) => Number.isFinite(n) && n > 0; + if (!pos(c.leaseMs) || !pos(c.sendTimeoutMs) || !pos(c.providerKeyRetentionMs) || !pos(c.retentionSafetyMarginMs)) planError('durations must be positive'); + if (!Number.isInteger(c.maxAttempts) || c.maxAttempts < 1) planError('maxAttempts must be a positive integer'); + // A lease that can lapse mid-send lets a second worker claim while the first is still sending. + if (c.leaseMs <= c.sendTimeoutMs) planError('leaseMs must exceed sendTimeoutMs'); + // The last safe retry must reach the provider before it forgets the key. + if (c.retentionSafetyMarginMs <= c.sendTimeoutMs) planError('retentionSafetyMarginMs must exceed sendTimeoutMs'); + if (c.retentionSafetyMarginMs >= c.providerKeyRetentionMs) planError('retentionSafetyMarginMs must be shorter than the retention window'); +} + +/** Frozen form of a validated plan: what `freeze` stores. Pure. */ +export function freezePlan(plan: OccurrencePlan, now: Date): { occurrence: FrozenOccurrence; deliveries: DeliveryRow[] } { + validatePlan(plan); + const nowIso = now.toISOString(); + const deliveries = plan.recipients.map(normalizeRecipient).sort().map((to): DeliveryRow => { + const recipientKey = recipientKeyOf(to); + const payload: DeliveryPayload = { from: plan.payload.from, to, subject: plan.payload.subject, html: plan.payload.html }; + const payloadJson = payloadJsonOf(payload); + return { + occurrenceId: plan.occurrenceId, recipientKey, idempotencyKey: idempotencyKeyFor(plan.occurrenceId, recipientKey), + payload, payloadJson, payloadHash: sha256(payloadJson), status: 'pending', attempts: 0, fence: 0, + leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, firstSendAt: null, ambiguous: false, + providerMessageId: null, lastError: null, updatedAt: nowIso, + }; + }).sort((a, b) => (a.recipientKey < b.recipientKey ? -1 : 1)); + return { + occurrence: { + occurrenceId: plan.occurrenceId, window: { start: new Date(plan.window.start).toISOString(), end: new Date(plan.window.end).toISOString() }, + recipientKeys: deliveries.map((d) => d.recipientKey), payloadHash: occurrencePayloadHash(plan.payload), engineVersion: DIGEST_DELIVERY_ENGINE_VERSION, + }, + deliveries, + }; +} + +// ── Engine ────────────────────────────────────────────────────────────────── + +export type EngineDeps = { + store: DigestDeliveryStore; + provider: DigestEmailProvider; + /** Unique per engine instance or run; informational (the fence is what protects). */ + owner: string; + config: EngineConfig; + /** For the report and the freeze timestamp only. Leases and retention use the store's clock. */ + now: () => Date; +}; + +export type AttemptReport = { + recipientKey: string; + fence: number; + result: ProviderSendResult['kind'] | 'not_sent'; + /** Whether the outcome is durable. `receipt_write_failed`: the provider answered but the store did not record it. */ + recorded: 'ok' | 'fenced_out' | 'receipt_write_failed' | 'not_sent'; + detail?: string; +}; + +export type OccurrenceReport = { + occurrenceId: string; + /** This call stored the plan. False: a plan was already stored, and IT was used. */ + created: boolean; + /** The caller's plan differs from the stored one; the stored one was used and the difference is reported, not merged. */ + planMismatch: null | { recipientsAdded: number; recipientsRemoved: number; payloadChanged: boolean }; + attempts: AttemptReport[]; + refused: { recipientKey: string; reason: ClaimRefusal }[]; + storageErrors: { stage: 'freeze' | 'load' | 'claim' | 'beginSend' | 'complete' | 'final_load'; recipientKey?: string }[]; + /** From a final read; null when that read failed. */ + statuses: Record | null; + /** Every recipient confirmed accepted. */ + complete: boolean; + /** Parked rows a person must look at. */ + needsAttention: { recipientKey: string; status: DeliveryStatus }[]; +}; + +const ATTENTION: readonly DeliveryStatus[] = ['rejected', 'conflict', 'exhausted', 'needs_reconciliation']; + +/** Freeze (or load) the occurrence, then deliver to every recipient still owed. */ +export async function deliverDigestOccurrence(plan: OccurrencePlan, deps: EngineDeps): Promise { + validateConfig(deps.config); + const frozen = freezePlan(plan, deps.now()); + const report = emptyReport(plan.occurrenceId); + let stored: StoredOccurrence & { created: boolean }; + try { + stored = await deps.store.freeze(frozen.occurrence, frozen.deliveries); + } catch { + report.storageErrors.push({ stage: 'freeze' }); + return report; + } + report.created = stored.created; + report.planMismatch = mismatch(frozen.occurrence, stored.occurrence); + return run(stored, deps, report); +} + +/** Continue an occurrence from saved state alone: no plan is recomputed. */ +export async function resumeDigestOccurrence(occurrenceId: string, deps: EngineDeps): Promise { + validateConfig(deps.config); + const report = emptyReport(occurrenceId); + let stored: StoredOccurrence | null; + try { + stored = await deps.store.load(occurrenceId); + } catch { + report.storageErrors.push({ stage: 'load' }); + return report; + } + if (!stored) { report.refused.push({ recipientKey: '*', reason: 'not_found' }); return report; } + return run(stored, deps, report); +} + +function emptyReport(occurrenceId: string): OccurrenceReport { + return { occurrenceId, created: false, planMismatch: null, attempts: [], refused: [], storageErrors: [], statuses: null, complete: false, needsAttention: [] }; +} + +function mismatch(wanted: FrozenOccurrence, stored: FrozenOccurrence): OccurrenceReport['planMismatch'] { + const have = new Set(stored.recipientKeys); + const want = new Set(wanted.recipientKeys); + const recipientsAdded = [...want].filter((k) => !have.has(k)).length; + const recipientsRemoved = [...have].filter((k) => !want.has(k)).length; + const payloadChanged = wanted.payloadHash !== stored.payloadHash; + return recipientsAdded || recipientsRemoved || payloadChanged ? { recipientsAdded, recipientsRemoved, payloadChanged } : null; +} + +async function run(stored: StoredOccurrence, deps: EngineDeps, report: OccurrenceReport): Promise { + const { store, config } = deps; + const id = stored.occurrence.occurrenceId; + for (const delivery of [...stored.deliveries].sort((a, b) => (a.recipientKey < b.recipientKey ? -1 : 1))) { + if (isTerminal(delivery.status)) continue; + const key = delivery.recipientKey; + let claim: Awaited>; + try { + claim = await store.claim(id, key, deps.owner, config); + } catch { + report.storageErrors.push({ stage: 'claim', recipientKey: key }); + continue; + } + if (!claim.claimed) { report.refused.push({ recipientKey: key, reason: claim.reason }); continue; } + const row = claim.row; + // Send only what the store holds, and only if it is still what was frozen. + if (sha256(row.payloadJson) !== row.payloadHash || payloadJsonOf(row.payload) !== row.payloadJson + || row.idempotencyKey !== idempotencyKeyFor(id, key) || recipientKeyOf(row.payload.to) !== key) { + // Not sent. Every later claim meets the same mismatch, so the row runs out of + // attempts (exhausted: nothing reached the provider) unless a person repairs it. + report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'not_sent', detail: 'stored_payload_integrity_failed' }); + continue; + } + let marked: 'ok' | 'fenced_out'; + try { + marked = await store.beginSend(id, key, row.fence); + } catch { + report.storageErrors.push({ stage: 'beginSend', recipientKey: key }); + continue; + } + if (marked !== 'ok') { + report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'fenced_out' }); + continue; + } + const result = await sendWithDeadline(deps.provider, row, config.sendTimeoutMs); + let recorded: 'ok' | 'fenced_out'; + try { + recorded = await store.complete(id, key, row.fence, result, config.maxAttempts); + } catch { + // The provider answered, but nothing durable says so. The row stays in flight + // with its mark; after the lease it is retried under the same key and bytes. + report.storageErrors.push({ stage: 'complete', recipientKey: key }); + report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded: 'receipt_write_failed' }); + continue; + } + report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded }); + } + try { + const final = await store.load(id); + if (final) { + report.statuses = Object.fromEntries(final.deliveries.map((d) => [d.recipientKey, d.status])); + report.complete = final.deliveries.length > 0 && final.deliveries.every((d) => d.status === 'accepted'); + report.needsAttention = final.deliveries.filter((d) => ATTENTION.includes(d.status)).map((d) => ({ recipientKey: d.recipientKey, status: d.status })); + } + } catch { + report.storageErrors.push({ stage: 'final_load' }); + } + return report; +} + +async function sendWithDeadline(provider: DigestEmailProvider, row: DeliveryRow, ms: number): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => { controller.abort(); resolve({ kind: 'unknown', reason: 'timeout' }); }, ms); + }); + // A provider that throws, even synchronously, settles nothing: the request may have left. + const attempt = Promise.resolve() + .then(() => provider.send({ idempotencyKey: row.idempotencyKey, payload: { ...row.payload }, payloadJson: row.payloadJson, signal: controller.signal })) + .then(checkResult, (): ProviderSendResult => ({ kind: 'unknown', reason: 'network' })); + try { + return await Promise.race([attempt, deadline]); + } finally { + clearTimeout(timer); + } +} + +/** Only a success WITH a message id is an accepted receipt; anything malformed settles nothing. */ +function checkResult(r: ProviderSendResult): ProviderSendResult { + if (!r || typeof r !== 'object') return { kind: 'unknown', reason: 'unreadable_response' }; + switch (r.kind) { + case 'accepted': return typeof r.messageId === 'string' && r.messageId.length > 0 ? r : { kind: 'unknown', reason: 'unreadable_response' }; + case 'rejected': return Number.isInteger(r.httpStatus) && typeof r.retryable === 'boolean' ? { ...r, code: String(r.code ?? '') } : { kind: 'unknown', reason: 'unreadable_response' }; + case 'payload_conflict': case 'in_progress': case 'unknown': return r; + default: return { kind: 'unknown', reason: 'unreadable_response' }; + } +} diff --git a/tests/admin-digest-delivery-engine.test.ts b/tests/admin-digest-delivery-engine.test.ts new file mode 100644 index 000000000..feb172095 --- /dev/null +++ b/tests/admin-digest-delivery-engine.test.ts @@ -0,0 +1,582 @@ +// lib/admin/digest-delivery.ts: the per-recipient replay engine for the admin +// digest, driven through its public API with an in-memory store and a fake +// provider that follows Resend's documented idempotency semantics. Synthetic +// data only; no database, no network, no email. +// +// What these tests prove: the engine's rules and its use of the storage and +// provider contracts, under the interleavings, crashes and failures below. +// What they do NOT prove: anything about PostgreSQL. The in-memory store is +// atomic because each step is one synchronous critical section; a database +// adapter must earn the same properties (docs/admin-digest-delivery-contract.md) +// and pass tests/helpers/digest-delivery-store-contract.ts against a real +// database. +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + RESEND_KEY_RETENTION_MS, classifyResendResponse, deliverDigestOccurrence, freezePlan, idempotencyKeyFor, + payloadJsonOf, recipientKeyOf, resumeDigestOccurrence, + type EngineConfig, type EngineDeps, type OccurrencePlan, +} from '@/lib/admin/digest-delivery'; +import { + FakeClock, FakeResendProvider, HOUR, MINUTE, MemoryDigestDeliveryStore, deferred, never, type StoreHook, +} from './helpers/digest-delivery-fakes'; +import { contractPlan, describeDigestDeliveryStoreContract } from './helpers/digest-delivery-store-contract'; + +describeDigestDeliveryStoreContract('in-memory store', (clock) => new MemoryDigestDeliveryStore(clock.now)); + +const ONE = 'admin-one@example.test'; +const TWO = 'admin-two@example.test'; +const K1 = recipientKeyOf(ONE); +const K2 = recipientKeyOf(TWO); +const T0 = '2026-09-30T12:31:00.000Z'; +const OCC = 'admin-digest:2026-09-30T12:30:00.000Z'; +/** The engine visits recipients in key order; some crash tests need to know which comes second. */ +const [FIRST, SECOND] = [{ to: ONE, key: K1 }, { to: TWO, key: K2 }].sort((a, b) => (a.key < b.key ? -1 : 1)); + +const CONFIG: EngineConfig = { + leaseMs: 5 * MINUTE, // on the store's (fake) clock + sendTimeoutMs: 150, // real milliseconds: only the timeout tests wait for it + maxAttempts: 5, + providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, + retentionSafetyMarginMs: HOUR, +}; + +/** Resend's own rule for a key, as #692's sendEmail enforces it. */ +const KEY_RULE = /^[\x21-\x7e](?:[\x20-\x7e]{0,254}[\x21-\x7e])?$/; + +function world(start = T0) { + const clock = new FakeClock(start); + const store = new MemoryDigestDeliveryStore(clock.now); + const provider = new FakeResendProvider(clock.now); + const engine = (owner: string, over: Partial = {}): EngineDeps => ({ + store, provider, owner, config: CONFIG, now: clock.now, ...over, + }); + return { clock, store, provider, engine }; +} + +const plan = (over: Partial = {}) => contractPlan(over); +const hookOn = (method: string, phase: 'before' | 'after', key: string | null, act: () => void | Promise): StoreHook => + (m, p, k) => (m === method && p === phase && (key === null || k === key) ? act() : undefined); +const fail = () => { throw new Error('synthetic storage failure'); }; + +afterEach(() => { vi.useRealTimers(); }); + +// ── Inputs ───────────────────────────────────────────────────────────────── + +describe('inputs are checked before anything is stored or sent', () => { + const cases: [string, Partial][] = [ + ['an empty recipient list (the caller must refuse it, as #685 does)', { recipients: [] }], + ['a duplicate recipient, however it is cased', { recipients: [ONE, ' Admin-One@Example.test '] }], + ['a recipient that is not one plain address', { recipients: ['Admin '] }], + ['a payload field beyond { from, subject, html }', { payload: { ...plan().payload, headers: { Authorization: 'x' } } as never }], + ['a subject with a line break', { payload: { ...plan().payload, subject: 'a\r\nBcc: x@example.test' } }], + ['an empty body', { payload: { ...plan().payload, html: ' ' } }], + ['a window that ends before it starts', { window: { start: '2026-09-30T12:30:00Z', end: '2026-09-29T12:30:00Z' } }], + ['an occurrence id with a line break', { occurrenceId: 'admin-digest:\n1' }], + ]; + it.each(cases)('%s', async (_name, over) => { + const { store, provider, engine } = world(); + await expect(deliverDigestOccurrence(plan(over), engine('a'))).rejects.toThrow(TypeError); + expect(store.calls).toEqual([]); + expect(provider.requests).toEqual([]); + }); + + it.each<[string, Partial]>([ + ['a lease no longer than the send deadline', { leaseMs: 150 }], + ['a retention margin no longer than the send deadline', { retentionSafetyMarginMs: 150 }], + ['a margin as long as the retention window', { retentionSafetyMarginMs: RESEND_KEY_RETENTION_MS }], + ['zero attempts', { maxAttempts: 0 }], + ])('a config with %s is refused', async (_name, over) => { + const { store, provider, engine } = world(); + await expect(deliverDigestOccurrence(plan(), engine('a', { config: { ...CONFIG, ...over } }))).rejects.toThrow(TypeError); + expect(store.calls).toEqual([]); + expect(provider.requests).toEqual([]); + }); +}); + +describe('keys and bytes', () => { + it('each recipient has one stable key: a function of occurrence and recipient only, free of the address, within Resend\'s rule', () => { + const a = freezePlan(plan(), new Date(T0)); + const b = freezePlan(plan({ payload: { ...plan().payload, html: '

other

' }, recipients: [TWO.toUpperCase(), ONE] }), new Date('2027-01-01T00:00:00Z')); + expect(a.deliveries.map((d) => d.idempotencyKey)).toEqual(b.deliveries.map((d) => d.idempotencyKey)); + for (const d of a.deliveries) { + expect(d.idempotencyKey).toMatch(KEY_RULE); + expect(d.idempotencyKey).not.toMatch(/admin-one|admin-two|example/); + expect(d.idempotencyKey).toBe(idempotencyKeyFor(OCC, d.recipientKey)); + } + expect(new Set(a.deliveries.map((d) => d.idempotencyKey)).size).toBe(2); + expect(idempotencyKeyFor('admin-digest:2026-10-01T12:30:00.000Z', K1)).not.toBe(idempotencyKeyFor(OCC, K1)); + }); + + it('the stored bytes are the body sendEmail would build for the same message (from, to, subject, html; no reply_to)', () => { + const d = freezePlan(plan(), new Date(T0)).deliveries[0]; + const { from, to, subject, html } = d.payload; + expect(d.payloadJson).toBe(JSON.stringify({ from, to, subject, html, reply_to: undefined })); + expect(d.payloadJson).toBe(payloadJsonOf(d.payload)); + }); +}); + +// ── The contract #677 asked for, at the engine ───────────────────────────── + +describe('one occurrence, each recipient once', () => { + it('delivers once to each recipient and reports the occurrence complete', async () => { + const { provider, engine } = world(); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r).toMatchObject({ created: true, planMismatch: null, complete: true, needsAttention: [], storageErrors: [] }); + expect(r.statuses).toEqual({ [K1]: 'accepted', [K2]: 'accepted' }); + expect(provider.deliveredTo(ONE)).toBe(1); + expect(provider.deliveredTo(TWO)).toBe(1); + }); + + it('persists the key and the exact bytes, and marks the attempt, BEFORE the provider sees the request', async () => { + const { store, provider, engine } = world(); + const seen: string[] = []; + provider.onArrive = (req) => { + const row = store.row(OCC, recipientKeyOf(req.to))!; + expect(row.idempotencyKey).toBe(req.key); + expect(row.payloadJson).toBe(req.payloadJson); + expect(row.status).toBe('in_flight'); + expect(row.sendStartedAt).not.toBeNull(); + seen.push(req.to); + }; + await deliverDigestOccurrence(plan(), engine('a')); + expect(seen.sort()).toEqual([ONE, TWO]); + }); + + it('C1: a second call for the same occurrence sends nothing (sequential repeat)', async () => { + const { provider, engine } = world(); + await deliverDigestOccurrence(plan(), engine('a')); + const again = await deliverDigestOccurrence(plan(), engine('b')); + expect(again.created).toBe(false); + expect(again.attempts).toEqual([]); + expect(provider.requests).toHaveLength(2); + expect(provider.inbox).toHaveLength(2); + }); + + it('C2: two engine instances at once, interleaved at the provider: each recipient once', async () => { + const { store, provider, engine } = world(); + const gate = deferred(); + provider.script = () => ({ do: 'delay_before_arrival', until: gate.promise, then: { do: 'accept' } }); + const a = deliverDigestOccurrence(plan(), engine('instance-a')); + const b = deliverDigestOccurrence(plan(), engine('instance-b')); + // Both recipients are on the wire at once, one from each instance. + await vi.waitFor(() => expect(provider.requests).toHaveLength(2)); + expect(store.calls.filter((c) => c.method === 'claim').length).toBeGreaterThanOrEqual(3); + gate.resolve(); + const [ra, rb] = await Promise.all([a, b]); + expect(provider.inbox.map((d) => d.to).sort()).toEqual([ONE, TWO]); + expect(provider.requests).toHaveLength(2); + expect(ra.attempts).toHaveLength(1); + expect(rb.attempts).toHaveLength(1); + // Whoever lost a claim was told why, and sent nothing for it. + for (const x of [...ra.refused, ...rb.refused]) expect(['leased', 'accepted']).toContain(x.reason); + }); + + it('C3: a retry after one recipient failed attempts ONLY that recipient', async () => { + const { provider, engine } = world(); + provider.script = ({ to, n }) => (to === TWO && n === 1 ? { do: 'reject', status: 429, code: 'rate_limit_exceeded', retryable: true } : { do: 'accept' }); + const first = await deliverDigestOccurrence(plan(), engine('a')); + expect(first.complete).toBe(false); + expect(first.statuses).toEqual({ [K1]: 'accepted', [K2]: 'failed' }); + const retry = await deliverDigestOccurrence(plan(), engine('b')); + expect(retry.attempts.map((x) => x.recipientKey)).toEqual([K2]); + expect(retry.complete).toBe(true); + expect(provider.requests.filter((r) => r.to === ONE)).toHaveLength(1); + expect(provider.requests.filter((r) => r.to === TWO)).toHaveLength(2); + expect(provider.deliveredTo(ONE)).toBe(1); + expect(provider.deliveredTo(TWO)).toBe(1); + }); + + it('C4: an ambiguous send (accepted, answer lost) is retried with the SAME key and bytes; nobody else is re-sent', async () => { + const { clock, provider, engine } = world(); + provider.script = ({ to, n }) => (to === ONE && n === 1 ? { do: 'accept_then_lose_answer', reason: 'timeout' } : { do: 'accept' }); + const first = await deliverDigestOccurrence(plan(), engine('a')); + expect(first.statuses).toEqual({ [K1]: 'unknown', [K2]: 'accepted' }); + clock.advance(10 * MINUTE); + const retry = await deliverDigestOccurrence(plan(), engine('b')); + expect(retry.statuses).toEqual({ [K1]: 'accepted', [K2]: 'accepted' }); + const toOne = provider.requests.filter((r) => r.to === ONE); + expect(toOne).toHaveLength(2); + expect(new Set(toOne.map((r) => r.key)).size).toBe(1); + expect(new Set(toOne.map((r) => r.payloadJson)).size).toBe(1); + expect(provider.deliveredTo(ONE)).toBe(1); // the provider answered the retry with the original message + expect(provider.requests.filter((r) => r.to === TWO)).toHaveLength(1); + }); +}); + +// ── Concurrency and leases ───────────────────────────────────────────────── + +describe('concurrent claims and stale leases', () => { + it('eight engines racing for one occurrence: one request per recipient in total', async () => { + const { store, provider, engine } = world(); + const gate = deferred(); + provider.script = () => ({ do: 'delay_before_arrival', until: gate.promise, then: { do: 'accept' } }); + const runs = Array.from({ length: 8 }, (_, i) => deliverDigestOccurrence(plan(), engine(`e${i}`))); + await vi.waitFor(() => expect(provider.requests).toHaveLength(2)); + gate.resolve(); + const reports = await Promise.all(runs); + expect(provider.requests).toHaveLength(2); + expect(provider.inbox).toHaveLength(2); + expect(reports.filter((r) => r.created)).toHaveLength(1); + }); + + it('a worker that stalls past its lease: the next one takes over; the stale request is answered from the key; the stale receipt is fenced out', async () => { + const { clock, store, provider, engine } = world(); + const stall = deferred(); + provider.script = ({ to, n }) => (to === ONE && n === 1 ? { do: 'delay_before_arrival', until: stall.promise, then: { do: 'accept' } } : { do: 'accept' }); + const a = deliverDigestOccurrence(plan(), engine('slow', { config: { ...CONFIG, sendTimeoutMs: 60_000 } })); + await vi.waitFor(() => expect(provider.requests.some((q) => q.to === ONE)).toBe(true)); + clock.advance(5 * MINUTE); // A's lease lapses while its request is still on the wire + const b = await deliverDigestOccurrence(plan(), engine('fresh')); + expect(b.statuses).toEqual({ [K1]: 'accepted', [K2]: 'accepted' }); + const messageId = provider.inbox.find((d) => d.to === ONE)!.messageId; + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', fence: 2, providerMessageId: messageId }); + stall.resolve(); // A's request finally arrives: same key, same bytes + const ra = await a; + expect(ra.attempts.find((x) => x.recipientKey === K1)).toMatchObject({ fence: 1, result: 'accepted', recorded: 'fenced_out' }); + expect(provider.deliveredTo(ONE)).toBe(1); + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', fence: 2, providerMessageId: messageId }); + }); + + it('while a stale request is still being processed, the new one is told it is in progress, stays unknown, and a later run settles it', async () => { + const { clock, store, provider, engine } = world(); + const held = deferred(); + provider.script = ({ to, n }) => (to === ONE && n === 1 ? { do: 'hold_at_provider', until: held.promise, then: { do: 'accept' } } : { do: 'accept' }); + const a = deliverDigestOccurrence(plan(), engine('slow', { config: { ...CONFIG, sendTimeoutMs: 60_000 } })); + await vi.waitFor(() => expect(provider.requests.some((q) => q.to === ONE)).toBe(true)); + clock.advance(5 * MINUTE); + const b = await deliverDigestOccurrence(plan(), engine('fresh')); + expect(b.attempts.find((x) => x.recipientKey === K1)).toMatchObject({ result: 'in_progress', recorded: 'ok' }); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true }); + held.resolve(); + await a; + clock.advance(10 * MINUTE); + const c = await deliverDigestOccurrence(plan(), engine('later')); + expect(c.statuses![K1]).toBe('accepted'); + expect(provider.deliveredTo(ONE)).toBe(1); + }); +}); + +// ── Crashes and restarts ─────────────────────────────────────────────────── + +describe('crashes and restarts from saved state', () => { + it('restart: a new process with only the saved state resumes, never re-sends the accepted, and sends the rest with the frozen bytes', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('claim', 'before', SECOND.key, never)); // the first process dies before its second claim + void deliverDigestOccurrence(plan(), engine('first')); + await vi.waitFor(() => expect(store.row(OCC, FIRST.key)?.status).toBe('accepted')); + const saved = store.snapshot(); + const restarted = MemoryDigestDeliveryStore.restore(saved, clock.now); + const r = await resumeDigestOccurrence(OCC, { store: restarted, provider, owner: 'second', config: CONFIG, now: clock.now }); + expect(r.attempts.map((x) => x.recipientKey)).toEqual([SECOND.key]); + expect(r.complete).toBe(true); + expect(provider.inbox.find((d) => d.to === SECOND.to)!.payloadJson).toBe(freezePlan(plan(), new Date(T0)).deliveries.find((d) => d.recipientKey === SECOND.key)!.payloadJson); + expect(provider.inbox).toHaveLength(2); + }); + + it('crash right after the claim, before the mark: nothing can have been sent, so it is retried even days later', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('claim', 'after', K1, never)); + void deliverDigestOccurrence(plan(), engine('dies')); + await vi.waitFor(() => expect(store.row(OCC, K1)?.status).toBe('in_flight')); + store.hooks = []; + clock.advance(3 * 24 * HOUR); + const r = await deliverDigestOccurrence(plan(), engine('next')); + expect(r.statuses).toEqual({ [K1]: 'accepted', [K2]: 'accepted' }); + expect(store.row(OCC, K1)).toMatchObject({ ambiguous: false, attempts: 2 }); + expect(provider.deliveredTo(ONE)).toBe(1); + }); + + it('crash after the mark, before the provider call: treated as possibly sent, retried with the same key inside the window', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('beginSend', 'after', K1, never)); + void deliverDigestOccurrence(plan(), engine('dies')); + await vi.waitFor(() => expect(typeof store.row(OCC, K1)?.sendStartedAt).toBe('string')); + store.hooks = []; + clock.advance(6 * MINUTE); + const r = await deliverDigestOccurrence(plan(), engine('next')); + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', ambiguous: true }); + expect(r.complete).toBe(true); + expect(provider.requests.filter((q) => q.to === ONE)).toHaveLength(1); + expect(provider.deliveredTo(ONE)).toBe(1); + }); + + it('crash after the provider accepted, before the receipt: the retry reuses the key and gets the original message back', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('complete', 'before', K1, never)); + void deliverDigestOccurrence(plan(), engine('dies')); + await vi.waitFor(() => expect(provider.deliveredTo(ONE)).toBe(1)); + store.hooks = []; + clock.advance(6 * MINUTE); + const r = await deliverDigestOccurrence(plan(), engine('next')); + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', providerMessageId: provider.inbox.find((d) => d.to === ONE)!.messageId }); + expect(r.complete).toBe(true); + const toOne = provider.requests.filter((q) => q.to === ONE); + expect(toOne).toHaveLength(2); + expect(new Set(toOne.map((q) => `${q.key}|${q.payloadJson}`)).size).toBe(1); + expect(provider.deliveredTo(ONE)).toBe(1); + }); +}); + +// ── Partial failure ──────────────────────────────────────────────────────── + +describe('partial failure', () => { + it('a permanent refusal parks that recipient as rejected; the other is delivered; nothing is retried for it', async () => { + const { provider, engine } = world(); + provider.script = ({ to }) => (to === TWO ? { do: 'reject', status: 422, code: 'validation_error', retryable: false } : { do: 'accept' }); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r).toMatchObject({ complete: false, needsAttention: [{ recipientKey: K2, status: 'rejected' }] }); + const again = await deliverDigestOccurrence(plan(), engine('b')); + expect(again.attempts).toEqual([]); + expect(provider.requests.filter((q) => q.to === TWO)).toHaveLength(1); + }); + + it('a refusal after an earlier ambiguous attempt is NOT reported as undelivered: it needs reconciliation', async () => { + const { clock, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'lose_before_arrival', reason: 'timeout' } : { do: 'reject', status: 422, code: 'validation_error', retryable: false }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + clock.advance(10 * MINUTE); + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + }); + + it('retryable refusals are bounded: after maxAttempts the recipient is exhausted (nothing was ever possibly accepted)', async () => { + const { clock, provider, engine } = world(); + provider.script = () => ({ do: 'reject', status: 429, code: 'rate_limit_exceeded', retryable: true }); + for (let i = 0; i < 7; i += 1) { await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine(`r${i}`)); clock.advance(MINUTE); } + expect(provider.requests).toHaveLength(CONFIG.maxAttempts); + const r = await resumeDigestOccurrence(OCC, engine('last')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'exhausted' }]); + }); + + it('ambiguous outcomes are bounded too: after maxAttempts the recipient needs reconciliation, never "exhausted"', async () => { + const { clock, provider, engine } = world(); + provider.script = () => ({ do: 'lose_before_arrival', reason: 'network' }); + for (let i = 0; i < 7; i += 1) { await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine(`r${i}`)); clock.advance(MINUTE); } + expect(provider.requests).toHaveLength(CONFIG.maxAttempts); + expect(new Set(provider.requests.map((q) => q.key)).size).toBe(1); + const r = await resumeDigestOccurrence(OCC, engine('last')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + }); +}); + +// ── Storage failures ─────────────────────────────────────────────────────── + +describe('storage failures', () => { + it('freeze fails: nothing is sent', async () => { + const { store, provider, engine } = world(); + store.hooks.push(hookOn('freeze', 'before', null, fail)); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r.storageErrors).toEqual([{ stage: 'freeze' }]); + expect(provider.requests).toEqual([]); + }); + + it('freeze commits but its answer is lost: nothing is sent now; the next run finds the plan and delivers once', async () => { + const { store, provider, engine } = world(); + store.hooks.push(hookOn('freeze', 'after', null, fail)); + await deliverDigestOccurrence(plan(), engine('a')); + expect(provider.requests).toEqual([]); + store.hooks = []; + const r = await deliverDigestOccurrence(plan(), engine('b')); + expect(r).toMatchObject({ created: false, complete: true }); + expect(provider.inbox).toHaveLength(2); + }); + + it('resume cannot load: nothing is sent', async () => { + const { store, provider, engine } = world(); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + store.hooks.push(hookOn('load', 'before', null, fail)); + const r = await resumeDigestOccurrence(OCC, engine('b')); + expect(r.storageErrors).toEqual([{ stage: 'load' }]); + expect(provider.requests).toHaveLength(1); + }); + + it('a claim fails (before or after it committed): that recipient is not sent this run; the other still is; a later run completes once', async () => { + for (const phase of ['before', 'after'] as const) { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('claim', phase, K1, fail)); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r.storageErrors).toEqual([{ stage: 'claim', recipientKey: K1 }]); + expect(provider.requests.map((q) => q.to)).toEqual([TWO]); + store.hooks = []; + clock.advance(6 * MINUTE); + expect((await deliverDigestOccurrence(plan(), engine('b'))).complete).toBe(true); + expect(provider.inbox).toHaveLength(2); + } + }); + + it('the mark fails (before or after it committed): nothing is sent for that recipient this run', async () => { + for (const phase of ['before', 'after'] as const) { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('beginSend', phase, K1, fail)); + await deliverDigestOccurrence(plan(), engine('a')); + expect(provider.requests.map((q) => q.to)).toEqual([TWO]); + store.hooks = []; + clock.advance(6 * MINUTE); + expect((await deliverDigestOccurrence(plan(), engine('b'))).complete).toBe(true); + expect(store.row(OCC, K1)!.ambiguous).toBe(phase === 'after'); // a committed mark is honestly ambiguous + expect(provider.deliveredTo(ONE)).toBe(1); + } + }); + + it('the receipt write fails after the provider accepted: reported as receipt_write_failed, and the retry is answered from the key', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('complete', 'before', K1, fail)); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r.attempts.find((x) => x.recipientKey === K1)).toEqual({ recipientKey: K1, fence: 1, result: 'accepted', recorded: 'receipt_write_failed' }); + expect(r.complete).toBe(false); + expect(store.row(OCC, K1)!.status).toBe('in_flight'); + store.hooks = []; + expect((await deliverDigestOccurrence(plan(), engine('b'))).attempts).toEqual([]); // the lease still holds + clock.advance(5 * MINUTE); + const later = await deliverDigestOccurrence(plan(), engine('c')); + expect(later.complete).toBe(true); + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', providerMessageId: provider.inbox.find((d) => d.to === ONE)!.messageId }); + expect(provider.deliveredTo(ONE)).toBe(1); + }); + + it('the receipt write committed but its answer was lost: the next run sees it accepted and sends nothing', async () => { + const { store, provider, engine } = world(); + store.hooks.push(hookOn('complete', 'after', K1, fail)); + await deliverDigestOccurrence(plan(), engine('a')); + store.hooks = []; + const r = await deliverDigestOccurrence(plan(), engine('b')); + expect(r.attempts).toEqual([]); + expect(provider.requests).toHaveLength(2); + }); +}); + +// ── Payload changes and conflicts ────────────────────────────────────────── + +describe('changed payloads and provider conflicts', () => { + it('a retry whose recomputed digest differs sends the FROZEN bytes, and says the plan differed', async () => { + const { clock, provider, engine } = world(); + provider.script = ({ to, n }) => (to === TWO && n === 1 ? { do: 'reject', status: 429, code: 'rate_limit_exceeded', retryable: true } : { do: 'accept' }); + await deliverDigestOccurrence(plan(), engine('a')); + clock.advance(30 * MINUTE); + const changed = plan({ + recipients: [ONE, TWO, 'admin-three@example.test'], + payload: { ...plan().payload, html: '

3 new families (a row committed late).

' }, + }); + const r = await deliverDigestOccurrence(changed, engine('b')); + expect(r.planMismatch).toEqual({ recipientsAdded: 1, recipientsRemoved: 0, payloadChanged: true }); + const toTwo = provider.requests.filter((q) => q.to === TWO); + expect(toTwo).toHaveLength(2); + expect(new Set(toTwo.map((q) => q.payloadJson)).size).toBe(1); + expect(toTwo[1].payloadJson).toContain('2 new families.'); + expect(provider.requests.some((q) => q.to === 'admin-three@example.test')).toBe(false); + expect(r.complete).toBe(true); + }); + + it('409 invalid_idempotent_request: a conflict, parked, never retried and never re-keyed', async () => { + const { clock, provider, engine } = world(); + const row = freezePlan(plan(), new Date(T0)).deliveries.find((d) => d.recipientKey === K1)!; + provider.preuse(row.idempotencyKey, JSON.stringify({ ...row.payload, html: '

what the old sender sent

' })); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'conflict' }]); + expect(r.attempts.find((x) => x.recipientKey === K1)!.result).toBe('payload_conflict'); + for (let i = 0; i < 3; i += 1) { clock.advance(HOUR); await deliverDigestOccurrence(plan(), engine(`later-${i}`)); } + const toOne = provider.requests.filter((q) => q.to === ONE); + expect(toOne).toHaveLength(1); + expect(toOne[0].key).toBe(row.idempotencyKey); + expect(provider.deliveredTo(ONE)).toBe(0); + }); +}); + +// ── The provider's retention window ─────────────────────────────────────── + +describe('the provider\'s key retention window', () => { + it('an unknown outcome is retried under the same key only while the provider still remembers it; then parked for reconciliation', async () => { + const { clock, provider, engine } = world(); + provider.script = () => ({ do: 'lose_before_arrival', reason: 'timeout' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); // first beginSend at T0 + clock.set('2026-10-01T11:30:59.999Z'); // T0 + 23 h − 1 ms: still inside (24 h − 1 h margin) + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(provider.requests).toHaveLength(2); + clock.set('2026-10-01T11:31:00.000Z'); // T0 + 23 h: no longer safe + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('c')); + expect(r.refused).toEqual([{ recipientKey: K1, reason: 'needs_reconciliation' }]); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + clock.set('2026-10-03T00:00:00.000Z'); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('d')); + expect(provider.requests).toHaveLength(2); + expect(new Set(provider.requests.map((q) => q.key)).size).toBe(1); + }); + + it('why: the provider forgets keys after 24 h, so a retry after that would deliver a second copy of an accepted message', async () => { + const { clock, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'accept_then_lose_answer', reason: 'timeout' } : { do: 'accept' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(provider.deliveredTo(ONE)).toBe(1); + clock.set('2026-10-01T13:00:00.000Z'); // T0 + 24 h 29 min: the fake provider no longer knows the key + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + expect(provider.requests).toHaveLength(1); + expect(provider.deliveredTo(ONE)).toBe(1); + }); + + it('a delivery that was only ever definitively refused is safe to retry at any age', async () => { + const { clock, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'reject', status: 401, code: 'missing_api_key', retryable: true } : { do: 'accept' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + clock.set('2026-10-02T18:00:00.000Z'); // T0 + 53 h + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(r.complete).toBe(true); + expect(provider.deliveredTo(ONE)).toBe(1); + }); +}); + +// ── What counts as an answer ─────────────────────────────────────────────── + +describe('what the provider said, taken at its word and no further', () => { + it('a 2xx without a message id is not a receipt', async () => { + const { store, provider, engine } = world(); + provider.script = () => ({ do: 'accept_without_id' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true, providerMessageId: null }); + }); + + it('a provider that throws settles nothing', async () => { + const { store, provider, engine } = world(); + provider.script = () => ({ do: 'throw' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true, lastError: 'outcome_unknown:network' }); + }); + + it('the engine\'s own deadline: the request is aborted and the outcome is unknown, not failed', async () => { + const { store, provider, engine } = world(); + let aborted = false; + provider.send = async (req) => { req.signal.addEventListener('abort', () => { aborted = true; }); return never(); }; + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(aborted).toBe(true); + expect(r.attempts[0]).toMatchObject({ result: 'unknown', recorded: 'ok' }); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true, lastError: 'outcome_unknown:timeout' }); + }); + + it('a stored payload that no longer matches its hash is never sent', async () => { + const { store, provider, engine } = world(); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), { ...engine('a'), provider: { send: async () => ({ kind: 'unknown', reason: 'timeout' }) } }); + store.tamper(OCC, K1, (row) => { row.payloadJson = row.payloadJson.replace('2 new families', '9 new families'); row.status = 'failed'; row.ambiguous = false; }); + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(r.attempts[0]).toMatchObject({ result: 'not_sent', detail: 'stored_payload_integrity_failed' }); + expect(provider.requests).toEqual([]); + }); + + it.each<[number, unknown, string]>([ + [200, { id: 'e1' }, 'accepted'], + [200, {}, 'unknown'], + [409, { name: 'invalid_idempotent_request' }, 'payload_conflict'], + [409, { name: 'concurrent_idempotent_requests' }, 'in_progress'], + [409, {}, 'unknown'], + [429, { name: 'rate_limit_exceeded' }, 'rejected:retryable'], + [401, { name: 'missing_api_key' }, 'rejected:retryable'], + [403, { name: 'validation_error' }, 'rejected:retryable'], + [400, { name: 'invalid_idempotency_key' }, 'rejected:final'], + [422, { name: 'validation_error' }, 'rejected:final'], + [500, { name: 'internal_server_error' }, 'unknown'], + [503, null, 'unknown'], + ])('Resend %i %j → %s', (status, body, expected) => { + const r = classifyResendResponse(status, body); + const got = r.kind === 'rejected' ? `rejected:${r.retryable ? 'retryable' : 'final'}` : r.kind; + expect(got).toBe(expected); + }); +}); diff --git a/tests/helpers/digest-delivery-fakes.ts b/tests/helpers/digest-delivery-fakes.ts new file mode 100644 index 000000000..6ada5b2b7 --- /dev/null +++ b/tests/helpers/digest-delivery-fakes.ts @@ -0,0 +1,257 @@ +// Test doubles for lib/admin/digest-delivery.ts. Synthetic data only; nothing +// here talks to a database or a mail provider. +// +// MemoryDigestDeliveryStore applies the engine's own transition rules, each in +// one synchronous critical section, which is how it is atomic. It proves the +// RULES and the engine's use of them. It proves nothing about PostgreSQL: +// durability, isolation and the database clock are the adapter's to meet +// (docs/admin-digest-delivery-contract.md), and the same contract suite +// (tests/helpers/digest-delivery-store-contract.ts) must pass against it. +import { createHash } from 'node:crypto'; +import { + decideBeginSend, decideClaim, decideCompletion, + type ClaimPolicy, type ClaimRefusal, type DeliveryRow, type DigestDeliveryStore, type DigestEmailProvider, + type FrozenOccurrence, type ProviderSendResult, type StoredOccurrence, +} from '@/lib/admin/digest-delivery'; + +// ── A clock the test moves ───────────────────────────────────────────────── + +export class FakeClock { + private t: number; + constructor(iso: string) { this.t = Date.parse(iso); } + now = (): Date => new Date(this.t); + set(iso: string) { this.t = Date.parse(iso); } + advance(ms: number) { this.t += ms; } +} + +export const HOUR = 60 * 60 * 1000; +export const MINUTE = 60 * 1000; + +/** A promise that never settles: a process that stopped at this line. */ +export const never = () => new Promise(() => {}); + +export function deferred() { + let resolve!: () => void; + const promise = new Promise((r) => { resolve = r; }); + return { promise, resolve }; +} + +// ── Store ────────────────────────────────────────────────────────────────── + +type Method = 'freeze' | 'load' | 'claim' | 'beginSend' | 'complete'; +/** Runs before a method's critical section ('before') or after it committed ('after'). Throw to fail; hang to crash. */ +export type StoreHook = (method: Method, phase: 'before' | 'after', recipientKey: string | null) => void | Promise; + +type Snapshot = { occurrences: [string, FrozenOccurrence][]; rows: [string, DeliveryRow][] }; + +const rowKey = (occurrenceId: string, recipientKey: string) => `${occurrenceId}\u0000${recipientKey}`; +const copy = (v: T): T => structuredClone(v); + +export class MemoryDigestDeliveryStore implements DigestDeliveryStore { + private occurrences = new Map(); + private rows = new Map(); + hooks: StoreHook[] = []; + readonly calls: { method: Method; recipientKey: string | null }[] = []; + + constructor(private readonly clock: () => Date) {} + + /** Saved state, as text: what a restarted process would find. */ + snapshot(): string { + return JSON.stringify({ occurrences: [...this.occurrences], rows: [...this.rows] } satisfies Snapshot); + } + + static restore(text: string, clock: () => Date): MemoryDigestDeliveryStore { + const s = JSON.parse(text) as Snapshot; + const store = new MemoryDigestDeliveryStore(clock); + store.occurrences = new Map(s.occurrences); + store.rows = new Map(s.rows); + return store; + } + + /** Direct view for assertions; never used by the engine. */ + row(occurrenceId: string, recipientKey: string): DeliveryRow | undefined { + const r = this.rows.get(rowKey(occurrenceId, recipientKey)); + return r && copy(r); + } + + /** Test-only corruption, to prove the engine checks what it is about to send. */ + tamper(occurrenceId: string, recipientKey: string, change: (r: DeliveryRow) => void) { + const r = this.rows.get(rowKey(occurrenceId, recipientKey)); + if (r) change(r); + } + + private async hook(method: Method, phase: 'before' | 'after', recipientKey: string | null) { + if (phase === 'before') this.calls.push({ method, recipientKey }); + for (const h of this.hooks) await h(method, phase, recipientKey); + } + + private stored(occurrenceId: string): StoredOccurrence | null { + const occurrence = this.occurrences.get(occurrenceId); + if (!occurrence) return null; + const deliveries = occurrence.recipientKeys.map((k) => this.rows.get(rowKey(occurrenceId, k))!).filter(Boolean); + return copy({ occurrence, deliveries }); + } + + async freeze(occurrence: FrozenOccurrence, deliveries: DeliveryRow[]) { + await this.hook('freeze', 'before', null); + // ── critical section ── + const id = occurrence.occurrenceId; + let created = false; + if (!this.occurrences.has(id)) { + const keys = new Set(occurrence.recipientKeys); + if (deliveries.length !== keys.size || deliveries.some((d) => d.occurrenceId !== id || !keys.has(d.recipientKey))) { + throw new Error('freeze: deliveries do not match the occurrence'); + } + this.occurrences.set(id, copy(occurrence)); + for (const d of deliveries) this.rows.set(rowKey(id, d.recipientKey), copy(d)); + created = true; + } + const out = { ...this.stored(id)!, created }; + // ── end ── + await this.hook('freeze', 'after', null); + return out; + } + + async load(occurrenceId: string) { + await this.hook('load', 'before', null); + const out = this.stored(occurrenceId); + await this.hook('load', 'after', null); + return out; + } + + async claim(occurrenceId: string, recipientKey: string, owner: string, policy: ClaimPolicy) { + await this.hook('claim', 'before', recipientKey); + // ── critical section ── + const k = rowKey(occurrenceId, recipientKey); + const row = this.rows.get(k); + let out: { claimed: true; row: DeliveryRow } | { claimed: false; reason: ClaimRefusal }; + if (!row) out = { claimed: false, reason: 'not_found' }; + else { + const d = decideClaim(copy(row), this.clock(), owner, policy); + if (d.next) this.rows.set(k, copy(d.next)); + out = d.claimed ? { claimed: true, row: copy(d.next) } : { claimed: false, reason: d.reason }; + } + // ── end ── + await this.hook('claim', 'after', recipientKey); + return out; + } + + async beginSend(occurrenceId: string, recipientKey: string, fence: number) { + await this.hook('beginSend', 'before', recipientKey); + const k = rowKey(occurrenceId, recipientKey); + const row = this.rows.get(k); + const next = row ? decideBeginSend(copy(row), fence, this.clock()) : null; + if (next) this.rows.set(k, next); + await this.hook('beginSend', 'after', recipientKey); + return next ? 'ok' as const : 'fenced_out' as const; + } + + async complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number) { + await this.hook('complete', 'before', recipientKey); + const k = rowKey(occurrenceId, recipientKey); + const row = this.rows.get(k); + const next = row ? decideCompletion(copy(row), fence, result, this.clock(), maxAttempts) : null; + if (next) this.rows.set(k, next); + await this.hook('complete', 'after', recipientKey); + return next ? 'ok' as const : 'fenced_out' as const; + } +} + +// ── Provider: Resend's documented idempotency semantics, and nothing else ── +// +// https://resend.com/docs/dashboard/emails/idempotency-keys (re-checked 2026-09-30): +// - the same key with the same payload returns the ORIGINAL response, and +// nothing new is sent; +// - the same key with a different payload is 409 invalid_idempotent_request; +// - the same key while the first request is still being processed is 409 +// concurrent_idempotent_requests; +// - keys are kept for 24 hours; after that the key is new again. +// What happens to a key on a refused request is undocumented; this fake keeps +// keys for ACCEPTED requests only. + +export type Behaviour = + | { do: 'accept' } + | { do: 'reject'; status: number; code: string; retryable: boolean } + /** Accepted and delivered, but the answer never reaches the sender (a timeout after acceptance). */ + | { do: 'accept_then_lose_answer'; reason?: 'timeout' | 'network' | 'server_error' } + /** Lost before the provider saw it. */ + | { do: 'lose_before_arrival'; reason?: 'timeout' | 'network' } + /** Arrives (the key is now in flight at the provider) and waits for `until` before being processed as `then`. */ + | { do: 'hold_at_provider'; until: Promise; then: Behaviour } + /** Waits for `until` before it reaches the provider at all: a slow network. */ + | { do: 'delay_before_arrival'; until: Promise; then: Behaviour } + /** A 2xx without a message id. */ + | { do: 'accept_without_id' } + | { do: 'throw' }; + +export type Delivered = { to: string; key: string; payloadJson: string; messageId: string; at: string }; + +const hash = (s: string) => createHash('sha256').update(s).digest('hex'); + +export class FakeResendProvider implements DigestEmailProvider { + /** Messages that reached an inbox. */ + readonly inbox: Delivered[] = []; + /** Every request that left the sender, whatever became of it. */ + readonly requests: { to: string; key: string; payloadJson: string }[] = []; + private keys = new Map(); + private inFlight = new Set(); + /** Behaviour per request; the default is to accept. */ + script: (req: { to: string; key: string; n: number }) => Behaviour = () => ({ do: 'accept' }); + /** Runs when a request arrives, before it is processed: assertions about what was persisted first. */ + onArrive: ((req: { to: string; key: string; payloadJson: string }) => void) | null = null; + + constructor(private readonly clock: () => Date, private readonly retentionMs = 24 * HOUR) {} + + async send(req: { idempotencyKey: string; payload: { to: string }; payloadJson: string; signal: AbortSignal }): Promise { + const to = req.payload.to; + const n = this.requests.filter((r) => r.to === to).length + 1; + this.requests.push({ to, key: req.idempotencyKey, payloadJson: req.payloadJson }); + return this.act(this.script({ to, key: req.idempotencyKey, n }), req.idempotencyKey, to, req.payloadJson); + } + + private async act(b: Behaviour, key: string, to: string, payloadJson: string): Promise { + switch (b.do) { + case 'lose_before_arrival': return { kind: 'unknown', reason: b.reason ?? 'network' }; + case 'throw': throw new Error('synthetic transport failure'); + case 'delay_before_arrival': await b.until; return this.act(b.then, key, to, payloadJson); + case 'hold_at_provider': { + this.onArrive?.({ to, key, payloadJson }); + if (this.inFlight.has(key)) return { kind: 'in_progress' }; + this.inFlight.add(key); + try { await b.until; } finally { this.inFlight.delete(key); } + return this.process(b.then, key, to, payloadJson, true); + } + default: + this.onArrive?.({ to, key, payloadJson }); + if (this.inFlight.has(key)) return { kind: 'in_progress' }; + return this.process(b, key, to, payloadJson, false); + } + } + + private process(b: Behaviour, key: string, to: string, payloadJson: string, arrived: boolean): ProviderSendResult { + void arrived; + const now = this.clock().getTime(); + const known = this.keys.get(key); + if (known && now - known.storedAt >= this.retentionMs) this.keys.delete(key); + const live = this.keys.get(key); + if (live) return live.hash === hash(payloadJson) ? { kind: 'accepted', messageId: live.messageId } : { kind: 'payload_conflict' }; + switch (b.do) { + case 'reject': return { kind: 'rejected', httpStatus: b.status, code: b.code, retryable: b.retryable }; + case 'accept_without_id': return { kind: 'accepted', messageId: '' }; + case 'accept': case 'accept_then_lose_answer': { + const messageId = `msg-${this.inbox.length + 1}`; + this.keys.set(key, { hash: hash(payloadJson), messageId, storedAt: now }); + this.inbox.push({ to, key, payloadJson, messageId, at: this.clock().toISOString() }); + return b.do === 'accept' ? { kind: 'accepted', messageId } : { kind: 'unknown', reason: b.reason ?? 'timeout' }; + } + default: return this.process({ do: 'accept' }, key, to, payloadJson, arrived); + } + } + + /** Test-only: a key used earlier for other bytes (e.g. by the sender this engine replaces). */ + preuse(key: string, payloadJson: string) { + this.keys.set(key, { hash: hash(payloadJson), messageId: 'msg-earlier', storedAt: this.clock().getTime() }); + } + + deliveredTo(to: string) { return this.inbox.filter((d) => d.to === to).length; } +} diff --git a/tests/helpers/digest-delivery-store-contract.ts b/tests/helpers/digest-delivery-store-contract.ts new file mode 100644 index 000000000..7b7f2a151 --- /dev/null +++ b/tests/helpers/digest-delivery-store-contract.ts @@ -0,0 +1,152 @@ +// The executable half of the storage contract for lib/admin/digest-delivery.ts. +// Any DigestDeliveryStore must pass it: today the in-memory store; a PostgreSQL +// adapter later, run against the local stack. Passing it with the in-memory +// store proves the rules, not database durability or isolation. +import { describe, expect, it } from 'vitest'; +import { + freezePlan, recipientKeyOf, RESEND_KEY_RETENTION_MS, + type ClaimPolicy, type DigestDeliveryStore, type OccurrencePlan, +} from '@/lib/admin/digest-delivery'; +import { FakeClock, HOUR, MINUTE } from './digest-delivery-fakes'; + +export const CONTRACT_POLICY: ClaimPolicy = { + leaseMs: 5 * MINUTE, maxAttempts: 3, providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, retentionSafetyMarginMs: HOUR, +}; + +export const contractPlan = (over: Partial = {}): OccurrencePlan => ({ + occurrenceId: 'admin-digest:2026-09-30T12:30:00.000Z', + window: { start: '2026-09-29T12:30:00.000Z', end: '2026-09-30T12:30:00.000Z' }, + recipients: ['admin-one@example.test', 'admin-two@example.test'], + payload: { from: 'Bubaly ', subject: '[Bubaly] Daily digest · synthetic', html: '

2 new families.

' }, + ...over, +}); + +export function describeDigestDeliveryStoreContract(label: string, make: (clock: FakeClock) => DigestDeliveryStore) { + describe(`${label}: DigestDeliveryStore contract`, () => { + const start = '2026-09-30T12:31:00.000Z'; + const setup = async () => { + const clock = new FakeClock(start); + const store = make(clock); + const frozen = freezePlan(contractPlan(), clock.now()); + await store.freeze(frozen.occurrence, frozen.deliveries); + const id = frozen.occurrence.occurrenceId; + const key = recipientKeyOf('admin-one@example.test'); + return { clock, store, frozen, id, key }; + }; + + it('freeze stores the occurrence and every delivery, all pending, and returns what is stored', async () => { + const clock = new FakeClock(start); + const store = make(clock); + const frozen = freezePlan(contractPlan(), clock.now()); + const out = await store.freeze(frozen.occurrence, frozen.deliveries); + expect(out.created).toBe(true); + expect(out.occurrence).toEqual(frozen.occurrence); + expect(out.deliveries).toEqual(frozen.deliveries); + expect(await store.load(frozen.occurrence.occurrenceId)).toEqual({ occurrence: frozen.occurrence, deliveries: frozen.deliveries }); + }); + + it('a second freeze of a DIFFERENT plan for the same occurrence changes nothing and returns the first', async () => { + const { clock, store, frozen, id } = await setup(); + const other = freezePlan(contractPlan({ recipients: ['admin-three@example.test'], payload: { ...contractPlan().payload, html: '

changed

' } }), clock.now()); + const out = await store.freeze(other.occurrence, other.deliveries); + expect(out.created).toBe(false); + expect(out.occurrence).toEqual(frozen.occurrence); + expect(out.deliveries.map((d) => d.payloadJson)).toEqual(frozen.deliveries.map((d) => d.payloadJson)); + expect((await store.load(id))!.deliveries).toHaveLength(2); + }); + + it('concurrent freezes of two different plans: exactly one is stored, and both callers are told the same thing', async () => { + const clock = new FakeClock(start); + const store = make(clock); + const a = freezePlan(contractPlan(), clock.now()); + const b = freezePlan(contractPlan({ payload: { ...contractPlan().payload, html: '

b

' } }), clock.now()); + const [ra, rb] = await Promise.all([store.freeze(a.occurrence, a.deliveries), store.freeze(b.occurrence, b.deliveries)]); + expect([ra.created, rb.created].filter(Boolean)).toHaveLength(1); + expect(ra.occurrence).toEqual(rb.occurrence); + expect(ra.deliveries).toEqual(rb.deliveries); + }); + + it('load hands out copies: changing a returned row changes nothing stored', async () => { + const { store, id } = await setup(); + const first = (await store.load(id))!; + first.deliveries[0].payload.html = 'mutated'; + first.deliveries[0].status = 'accepted'; + const again = (await store.load(id))!; + expect(again.deliveries[0].payload.html).toBe('

2 new families.

'); + expect(again.deliveries[0].status).toBe('pending'); + }); + + it('claim is atomic: twelve concurrent claims of one delivery, exactly one wins, with fence 1 and attempt 1', async () => { + const { store, id, key } = await setup(); + const results = await Promise.all(Array.from({ length: 12 }, (_, i) => store.claim(id, key, `worker-${i}`, CONTRACT_POLICY))); + const won = results.filter((r) => r.claimed); + expect(won).toHaveLength(1); + expect(results.filter((r) => !r.claimed && r.reason === 'leased')).toHaveLength(11); + const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; + expect(row).toMatchObject({ status: 'in_flight', fence: 1, attempts: 1 }); + }); + + it('a live lease refuses; it expires by the STORE\'s clock, and the next claim gets a new fence', async () => { + const { clock, store, id, key } = await setup(); + expect((await store.claim(id, key, 'a', CONTRACT_POLICY)).claimed).toBe(true); + clock.advance(5 * MINUTE - 1); + expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'leased' }); + clock.advance(1); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + expect(b.claimed && b.row.fence).toBe(2); + }); + + it('beginSend and complete are fenced: a superseded claim writes nothing', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + clock.advance(5 * MINUTE); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!a.claimed || !b.claimed) throw new Error('setup'); + expect(await store.beginSend(id, key, a.row.fence)).toBe('fenced_out'); + expect(await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'stale' }, 3)).toBe('fenced_out'); + expect(await store.beginSend(id, key, b.row.fence)).toBe('ok'); + expect(await store.complete(id, key, b.row.fence, { kind: 'accepted', messageId: 'm-b' }, 3)).toBe('ok'); + expect(await store.complete(id, key, b.row.fence, { kind: 'rejected', httpStatus: 422, code: 'x', retryable: false }, 3)).toBe('fenced_out'); + const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; + expect(row).toMatchObject({ status: 'accepted', providerMessageId: 'm-b', fence: 2 }); + }); + + it('an accepted delivery is never claimed again', async () => { + const { store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence); + await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'm-1' }, 3); + expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'accepted' }); + }); + + it('a refusal that parks a row is itself persisted', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence); + await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); + clock.advance(23 * HOUR); // 24 h retention − 1 h margin, from the first beginSend + expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'needs_reconciliation' }); + const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; + expect(row.status).toBe('needs_reconciliation'); + }); + + it('nothing a claim, mark or completion does changes the recipient, key or bytes', async () => { + const { store, frozen, id, key } = await setup(); + const before = frozen.deliveries.find((d) => d.recipientKey === key)!; + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence); + await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'm-1' }, 3); + const after = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; + for (const f of ['recipientKey', 'idempotencyKey', 'payload', 'payloadJson', 'payloadHash', 'occurrenceId'] as const) expect(after[f]).toEqual(before[f]); + }); + + it('an unknown occurrence or recipient is not found, not created', async () => { + const { store, id } = await setup(); + expect(await store.claim(id, 'f'.repeat(64), 'a', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'not_found' }); + expect(await store.load('admin-digest:never')).toBeNull(); + }); + }); +} From 2286029b623e3c9e19990c7a3da4c377f4161337 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 19:54:06 +0000 Subject: [PATCH 02/12] docs(admin-digest): per-recipient delivery contract, adapter requirements and proposed schema This describes lib/admin/digest-delivery.ts, whose code and tests are in the previous (checkpoint) commit: - the caller's inputs (occurrence id, window, frozen recipients and payload); - the per-recipient state machine; - the exact requirements a PostgreSQL store adapter must meet (one transaction per step, the database clock, a fence counter, immutable key and bytes, RLS and grants); - the Resend classification a provider adapter must apply; - a PROPOSED schema. No migration number is taken until the coordinator reserves one; 0465 and NWV's locale work need coordinating first. No route is wired and no migration is added. The in-memory store proves the rules, not PostgreSQL durability. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 153 +++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/admin-digest-delivery-contract.md diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md new file mode 100644 index 000000000..90460c8ae --- /dev/null +++ b/docs/admin-digest-delivery-contract.md @@ -0,0 +1,153 @@ +# Admin digest: per-recipient delivery contract + +Status on 2026-09-30: +- `lib/admin/digest-delivery.ts` exists, and its tests pass against an in-memory store and a fake provider. +- **No route uses it.** `/api/cron/admin-digest` still sends the way it did. +- **Nothing is stored in a database.** No migration exists or is proposed as a file (the proposed schema is §5). +- Duplicate admin digests are **not** fixed in production. + +This document is what the next two pieces must meet: a PostgreSQL store adapter and the route wiring. + +## 1. What the caller owns, and what it passes in + +The engine does not schedule. The caller (the cron route, later) decides and passes: + +| Input | Meaning | Caller's duty | +|---|---|---| +| `occurrenceId` | Stable for every retry of one digest, e.g. `admin-digest:` | The same occurrence always gets the same id. The window policy (#677 C5: no activity row in two digests) is the caller's, not the engine's. | +| `window` | `{ start, end }`, recorded only | Chosen by the caller's scheduler. | +| `recipients` | Frozen list of addresses | Read with `readSuperAdminRecipients` (#685), which fails closed. An unreadable or empty list never reaches the engine; the engine also refuses an empty list. | +| `payload` | Exactly `{ from, subject, html }` | Rendered once. It is non-secret by construction: no headers and no credentials, and any other field is refused. | + +`deliverDigestOccurrence(plan)` stores the plan the first time it sees the occurrence. On every later call the **stored** plan is used. A recomputed plan with a different body or different recipients is reported in `planMismatch` and is never merged or sent. `resumeDigestOccurrence(id)` continues from stored state alone. + +## 2. Per-recipient states + +| Status | Meaning | Next | +|---|---|---| +| `pending` | Frozen, never claimed | Claim | +| `in_flight` | Claimed under a lease, fence *n* | `complete`, or lease expiry (below) | +| `failed` | The provider answered and did not accept; the answer can be retried (429, 401/403) | Claim, any time, same key and bytes | +| `unknown` | Might have been accepted: timeout, network error, 5xx, 409 `concurrent_idempotent_requests`, 2xx without an id, lost receipt | Claim, same key and bytes, **only while** `now < firstSendAt + retention − margin` | +| `accepted` | A 2xx **with** a message id | Terminal. Never sent again. | +| `rejected` | Final refusal (400, 422), nothing possibly accepted before it | Terminal, needs attention | +| `conflict` | 409 `invalid_idempotent_request`: the key was used for other bytes | Terminal, needs attention. The key is never changed. | +| `exhausted` | `maxAttempts` used, nothing ever possibly accepted | Terminal, needs attention | +| `needs_reconciliation` | Possibly accepted, and a retry is no longer safe (retention passed, attempts used up, or a final refusal after an ambiguous attempt) | Terminal. A person checks the provider by key. | + +**Lease expiry.** When a lease expires, the row's `sendStartedAt` mark decides what happened: +- **Mark present.** The worker may have reached the provider, so the row is `unknown`, ambiguous and sticky. +- **Mark absent.** It cannot have reached the provider, so the row is retried as not sent. + +**Keys.** A key is `bubaly/admin-digest/v1//`: +- It is a function of occurrence and recipient only. +- It contains no address. +- It fits Resend's rule (and #692's check): printable ASCII, 1–256 characters. + +**Bytes.** The stored `payloadJson` is `{"from","to","subject","html"}` in that order. This is byte-identical to the body `sendEmail` builds when there is no `reply_to`. + +## 3. Database adapter requirements (exact) + +A PostgreSQL `DigestDeliveryStore` must meet all of these. It must also pass `tests/helpers/digest-delivery-store-contract.ts` against the local Supabase stack. The in-memory store passing that suite proves only the **rules**, not durability or isolation. + +1. **`freeze` is one transaction.** Insert the occurrence and every delivery row, or nothing. Use `INSERT … ON CONFLICT (occurrence_id) DO NOTHING` on the occurrence and insert deliveries only if that insert won. In the same transaction, return what is stored. A concurrent loser must get the winner's plan, never a mix of the two. +2. **`claim` is one transaction on one row.** Take `SELECT … FOR UPDATE` on `(occurrence_id, recipient_key)`, evaluate exactly `decideClaim`, write `next`, and commit. A refusal that parks a row (`exhausted`, `needs_reconciliation`) is also written. `READ COMMITTED` plus the row lock is enough; no advisory locks and no `app_settings`. +3. **The database clock decides.** Lease expiry (`lease_expires_at > now()`), the new lease (`now() + lease`) and the retention cut-off (`first_send_at + retention − margin`) all use the database's `now()`. They never use a timestamp the caller sends. Retention, margin, lease and max attempts are passed as parameters, not hard-coded. +4. **The fence is a counter.** `fence` increments on every successful claim. + - `beginSend` and `complete` are each one `UPDATE … WHERE status = 'in_flight' AND fence = $fence RETURNING`. + - Zero rows means `fenced_out`, and nothing is written. + - Owner equality alone is not enough: the same owner can re-claim after its own lease lapsed. +5. **`beginSend` commits before the provider call.** It sets `send_started_at`, and sets `first_send_at` only if it is null. The adapter returns only after the commit (`synchronous_commit` on, the default). +6. **Frozen means immutable.** `recipient_key`, `idempotency_key`, `payload_json` and `payload_hash` never change after insert. Enforce this with a trigger, not only in code. `idempotency_key` is unique across the table. `payload_hash = sha256(payload_json)` is checked on insert. +7. **Row invariants are checks.** `status = 'in_flight'` if and only if there is a lease. `status = 'accepted'` if and only if `provider_message_id` is set. `send_started_at` is set only while `in_flight`. `ambiguous` is never reset from true to false. +8. **Access.** RLS on and no policies. Revoke all from `anon` and `authenticated`. Functions are `security definer`, with `search_path = public` and execute revoked from `public`, `anon` and `authenticated`. Only the service role calls them. +9. **Errors surface.** Any error or timeout is thrown to the engine, never swallowed as "not found". The engine treats each failure per step: it sends nothing without a durable claim and mark, and reports a lost receipt. + +What the contract suite **cannot** show with the in-memory store: crash durability, isolation under real concurrent transactions, and clock behaviour. Those need the PostgreSQL run in requirement 3 plus a kill-mid-transaction test on the local stack. + +## 4. Provider adapter requirements + +The engine takes a `DigestEmailProvider` whose `send` returns one classified outcome. For Resend, `classifyResendResponse` (tested) maps HTTP status and body: + +| Resend answer | Outcome | Engine | +|---|---|---| +| 2xx with `id` | `accepted` | Receipt; never resent | +| 2xx without `id` | `unknown` | Ambiguous | +| 409 `invalid_idempotent_request` | `payload_conflict` | `conflict`; never re-keyed | +| 409 `concurrent_idempotent_requests`, or another 409 | `in_progress` / `unknown` | Ambiguous | +| 429 | `rejected`, retryable | `failed` | +| 401, 403 | `rejected`, retryable (an operator can fix the key or the domain) | `failed`, bounded by `maxAttempts` | +| Other 4xx (400 `invalid_idempotency_key`, 422) | `rejected`, final | `rejected`, or `needs_reconciliation` if an earlier attempt was ambiguous | +| 5xx, timeout, network error, a throw | `unknown` | Ambiguous | + +**#692's `sendEmail` (merged 2026-09-30) is not enough on its own.** It sends the `Idempotency-Key` header, but answers only `{ ok }`, which cannot carry a message id or tell a conflict from a refusal. The adapter must read the status and body (bounded), then classify them here. It must: +- send `payloadJson` verbatim as the body; +- send the row's key unchanged; +- honour the abort signal. + +**Retention.** `RESEND_KEY_RETENTION_MS` is 24 hours, from Resend's documentation re-checked on 2026-09-30. Recommended margin: 1 hour, which must exceed the send deadline plus clock skew. Re-verify before changing provider or value. What Resend does with a key after a refused request is undocumented. The engine never relies on it: a refusal is retried with the same key, and the 24-hour window is anchored on the first send of any kind. + +## 5. Proposed schema (proposal only, no migration number) + +**No migration will be added** until the coordinator reserves a number. Main ends at `0464`. `0465` and NWV's locale work must be coordinated first. This also cannot reach production before the PROD-DB-0177 migration-ledger remedy. + +```sql +create table public.admin_digest_occurrences ( + occurrence_id text primary key check (occurrence_id ~ '^[\x21-\x7e]([\x20-\x7e]*[\x21-\x7e])?$' and length(occurrence_id) <= 200), + window_start timestamptz not null, + window_end timestamptz not null check (window_end > window_start), + recipient_keys text[] not null check (cardinality(recipient_keys) between 1 and 200), + payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), + engine_version integer not null, + frozen_at timestamptz not null default now() +); + +create table public.admin_digest_deliveries ( + occurrence_id text not null references public.admin_digest_occurrences (occurrence_id) on delete restrict, + recipient_key text not null check (recipient_key ~ '^[0-9a-f]{64}$'), + idempotency_key text not null unique check (length(idempotency_key) between 1 and 256), + payload_json text not null, + payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), + status text not null check (status in ('pending','in_flight','failed','unknown','accepted','rejected','conflict','exhausted','needs_reconciliation')), + attempts integer not null default 0 check (attempts >= 0), + fence bigint not null default 0 check (fence >= 0), + lease_owner text, + lease_expires_at timestamptz, + send_started_at timestamptz, + first_send_at timestamptz, + ambiguous boolean not null default false, + provider_message_id text, + last_error text check (last_error is null or length(last_error) <= 200), + updated_at timestamptz not null default now(), + primary key (occurrence_id, recipient_key), + check ((status = 'in_flight') = (lease_owner is not null and lease_expires_at is not null)), + check ((status = 'accepted') = (provider_message_id is not null)), + check (send_started_at is null or status = 'in_flight'), + check (not ambiguous or first_send_at is not null) +); +-- + trigger: payload_json, payload_hash, idempotency_key, recipient_key, occurrence_id are immutable; +-- payload_hash = encode(sha256(convert_to(payload_json, 'UTF8')), 'hex') on insert; ambiguous never true → false. +-- + RLS on, no policies; revoke all from anon, authenticated. +-- + security-definer functions (execute revoked from public, anon, authenticated), each mirroring one rule: +-- admin_digest_freeze(p_occurrence jsonb, p_deliveries jsonb) returns jsonb +-- admin_digest_claim(p_occurrence_id text, p_recipient_key text, p_owner text, p_lease_ms int, +-- p_max_attempts int, p_retention_ms bigint, p_margin_ms bigint) returns jsonb -- decideClaim +-- admin_digest_begin_send(p_occurrence_id text, p_recipient_key text, p_fence bigint) returns boolean -- decideBeginSend +-- admin_digest_complete(p_occurrence_id text, p_recipient_key text, p_fence bigint, +-- p_result jsonb, p_max_attempts int) returns boolean -- decideCompletion +``` + +## 6. What is and is not claimed + +- **Claimed and tested:** + - A confirmed accepted recipient is never sent again. + - Every retry uses the same key and the same bytes. + - Ambiguous outcomes stop at retention − margin. + - Conflicts are parked and never re-keyed. + - Stale workers are fenced out. + - Crashes before the mark are retried as not sent; crashes after it are retried as ambiguous. + - Storage failures send nothing without a durable claim and mark. +- **Not claimed:** + - Exactly-once delivery. The provider's key deduplication, inside its window, is what makes an ambiguous retry harmless. + - Anything about PostgreSQL durability or isolation. + - Anything about production. It is not wired, and production cron is not running (see the scheduler report). From 5c58ae4fc8c75d51b7fe514821b19708bf158376 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 20:03:27 +0000 Subject: [PATCH 03/12] fix(admin-digest delivery): a worker that stalls after its claim cannot send late; review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the first slice (5 angles, findings reproduced): - HIGH: beginSend checked only the fence. A worker that stalled between its claim and its mark could still send after its lease lapsed, or after an ambiguous row passed the retention cut-off, where a resend is no longer deduplicated. decideBeginSend now refuses in three cases: - `lease_expired`: no more than the send deadline of lease is left; - `retention_passed`: an ambiguous row passed the cut-off; the row is parked; - `fenced_out`: another claim superseded this one. In every case the engine sends nothing. The store interface passes the policy to beginSend. - A retryable refusal after an ambiguous attempt stays `unknown`, not `failed`, so retention still bounds it. - The completion rule refuses an `accepted` without a message id. The engine treats a provider "rejected" outside 4xx as unknown. - The report is more honest: - `receipt_write_failed` is renamed `receipt_write_unconfirmed`, because the write may have committed; - accepted message ids are reported; - `planMismatch.windowChanged` is added. - Config refuses a retention longer than the verified 24 h. New tests (engine 67 → 72, store contract 11 → 18; 90 in total): - engine: stalled worker fenced out, stalled with no takeover, marked after the cut-off, attempts bounded when nothing completes, a stale receipt fenced out while in flight under a newer claim, a failing final read, key and recipient integrity, window mismatch, adapter misclassification, ambiguous then retryable refusal; - storage-failure tests now inject on the recipient visited FIRST; - contract suite pins what beginSend persists, the sticky anchor and ambiguity, the attempt limit, conflict terminality, independent occurrences and the empty-id rule. The contract doc is updated to match. It now names clock_timestamp() read after the row lock, through one function the test harness can pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 35 +++- lib/admin/digest-delivery.ts | 94 +++++++-- tests/admin-digest-delivery-engine.test.ts | 184 +++++++++++++++++- tests/helpers/digest-delivery-fakes.ts | 16 +- .../helpers/digest-delivery-store-contract.ts | 133 ++++++++++++- 5 files changed, 413 insertions(+), 49 deletions(-) diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index 90460c8ae..ee825d066 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -27,8 +27,8 @@ The engine does not schedule. The caller (the cron route, later) decides and pas |---|---|---| | `pending` | Frozen, never claimed | Claim | | `in_flight` | Claimed under a lease, fence *n* | `complete`, or lease expiry (below) | -| `failed` | The provider answered and did not accept; the answer can be retried (429, 401/403) | Claim, any time, same key and bytes | -| `unknown` | Might have been accepted: timeout, network error, 5xx, 409 `concurrent_idempotent_requests`, 2xx without an id, lost receipt | Claim, same key and bytes, **only while** `now < firstSendAt + retention − margin` | +| `failed` | The provider answered and did not accept; the answer can be retried (429, 401/403); and **no** earlier attempt was ambiguous | Claim, any time, same key and bytes | +| `unknown` | Might have been accepted: timeout, network error, 5xx, 409 `concurrent_idempotent_requests`, 2xx without an id, lost receipt; or a retryable refusal after an earlier ambiguous attempt | Claim, same key and bytes, **only while** `now < firstSendAt + retention − margin` | | `accepted` | A 2xx **with** a message id | Terminal. Never sent again. | | `rejected` | Final refusal (400, 422), nothing possibly accepted before it | Terminal, needs attention | | `conflict` | 409 `invalid_idempotent_request`: the key was used for other bytes | Terminal, needs attention. The key is never changed. | @@ -39,6 +39,13 @@ The engine does not schedule. The caller (the cron route, later) decides and pas - **Mark present.** The worker may have reached the provider, so the row is `unknown`, ambiguous and sticky. - **Mark absent.** It cannot have reached the provider, so the row is retried as not sent. +**The mark is checked at the last moment.** `beginSend` is refused in three cases, and a refused `beginSend` sends nothing: +- **`lease_expired`:** the lease has no more than the send deadline left, so another worker could claim mid-send. +- **`retention_passed`:** an ambiguous row reached the cut-off after it was claimed. It is parked. +- **`fenced_out`:** another claim superseded this one. + +Together these stop a worker that stalled after its claim from sending late. + **Keys.** A key is `bubaly/admin-digest/v1//`: - It is a function of occurrence and recipient only. - It contains no address. @@ -52,16 +59,26 @@ A PostgreSQL `DigestDeliveryStore` must meet all of these. It must also pass `te 1. **`freeze` is one transaction.** Insert the occurrence and every delivery row, or nothing. Use `INSERT … ON CONFLICT (occurrence_id) DO NOTHING` on the occurrence and insert deliveries only if that insert won. In the same transaction, return what is stored. A concurrent loser must get the winner's plan, never a mix of the two. 2. **`claim` is one transaction on one row.** Take `SELECT … FOR UPDATE` on `(occurrence_id, recipient_key)`, evaluate exactly `decideClaim`, write `next`, and commit. A refusal that parks a row (`exhausted`, `needs_reconciliation`) is also written. `READ COMMITTED` plus the row lock is enough; no advisory locks and no `app_settings`. -3. **The database clock decides.** Lease expiry (`lease_expires_at > now()`), the new lease (`now() + lease`) and the retention cut-off (`first_send_at + retention − margin`) all use the database's `now()`. They never use a timestamp the caller sends. Retention, margin, lease and max attempts are passed as parameters, not hard-coded. +3. **The database clock decides.** Lease expiry, the new lease and the retention cut-off are all evaluated with the database's clock. + - **Which clock:** `clock_timestamp()` read **after** the row lock is taken. `now()` is the transaction's start and can lag behind a lock wait. + - **One function:** read it through a single function, e.g. `admin_digest_now()`, so the local test harness can pin it for the contract suite. + - **Never the caller's time:** no timestamp the caller sends is used. + - **Parameters:** retention, margin, lease, send deadline and max attempts are passed in, not hard-coded. The engine refuses a retention longer than the verified 24 hours. 4. **The fence is a counter.** `fence` increments on every successful claim. - `beginSend` and `complete` are each one `UPDATE … WHERE status = 'in_flight' AND fence = $fence RETURNING`. - Zero rows means `fenced_out`, and nothing is written. - Owner equality alone is not enough: the same owner can re-claim after its own lease lapsed. -5. **`beginSend` commits before the provider call.** It sets `send_started_at`, and sets `first_send_at` only if it is null. The adapter returns only after the commit (`synchronous_commit` on, the default). +5. **`beginSend` commits before the provider call.** In one step it: + - evaluates `decideBeginSend`: fence, remaining lease against the send deadline, and the retention cut-off for ambiguous rows; + - sets `send_started_at`; + - sets `first_send_at` only if it is null. + + A refusal that parks a row (`retention_passed`) is written in that same step. The adapter returns only after the commit (`synchronous_commit` on, the default). 6. **Frozen means immutable.** `recipient_key`, `idempotency_key`, `payload_json` and `payload_hash` never change after insert. Enforce this with a trigger, not only in code. `idempotency_key` is unique across the table. `payload_hash = sha256(payload_json)` is checked on insert. -7. **Row invariants are checks.** `status = 'in_flight'` if and only if there is a lease. `status = 'accepted'` if and only if `provider_message_id` is set. `send_started_at` is set only while `in_flight`. `ambiguous` is never reset from true to false. -8. **Access.** RLS on and no policies. Revoke all from `anon` and `authenticated`. Functions are `security definer`, with `search_path = public` and execute revoked from `public`, `anon` and `authenticated`. Only the service role calls them. -9. **Errors surface.** Any error or timeout is thrown to the engine, never swallowed as "not found". The engine treats each failure per step: it sends nothing without a durable claim and mark, and reports a lost receipt. +7. **Every predicate names the occurrence.** Each read and write is keyed by `(occurrence_id, recipient_key)`. The same recipient in two occurrences is two independent rows; the contract suite checks this. +8. **Row invariants are checks.** `status = 'in_flight'` if and only if there is a lease. `status = 'accepted'` if and only if `provider_message_id` is set. `send_started_at` is set only while `in_flight`. `ambiguous` is never reset from true to false. +9. **Access.** RLS on and no policies. Revoke all from `anon` and `authenticated`. Functions are `security definer`, with `search_path = public` and execute revoked from `public`, `anon` and `authenticated`. Only the service role calls them. +10. **Errors surface.** Any error or timeout is thrown to the engine, never swallowed as "not found". The engine treats each failure per step: it sends nothing without a durable claim and mark, and reports a receipt it could not confirm as `receipt_write_unconfirmed`, with the provider's message id. What the contract suite **cannot** show with the in-memory store: crash durability, isolation under real concurrent transactions, and clock behaviour. Those need the PostgreSQL run in requirement 3 plus a kill-mid-transaction test on the local stack. @@ -132,7 +149,9 @@ create table public.admin_digest_deliveries ( -- admin_digest_freeze(p_occurrence jsonb, p_deliveries jsonb) returns jsonb -- admin_digest_claim(p_occurrence_id text, p_recipient_key text, p_owner text, p_lease_ms int, -- p_max_attempts int, p_retention_ms bigint, p_margin_ms bigint) returns jsonb -- decideClaim --- admin_digest_begin_send(p_occurrence_id text, p_recipient_key text, p_fence bigint) returns boolean -- decideBeginSend +-- admin_digest_begin_send(p_occurrence_id text, p_recipient_key text, p_fence bigint, p_min_lease_ms int, +-- p_retention_ms bigint, p_margin_ms bigint) returns text -- decideBeginSend: ok | fenced_out | lease_expired | retention_passed +-- admin_digest_now() returns timestamptz -- clock_timestamp(); the one clock the local test harness pins -- admin_digest_complete(p_occurrence_id text, p_recipient_key text, p_fence bigint, -- p_result jsonb, p_max_attempts int) returns boolean -- decideCompletion ``` diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts index 6e4108494..fd3b30ab4 100644 --- a/lib/admin/digest-delivery.ts +++ b/lib/admin/digest-delivery.ts @@ -20,7 +20,10 @@ // - claim one atomic step per recipient, under a lease, with a fence that // grows on every claim. Two engines never both hold a recipient. // - beginSend a fenced mark, written before the provider call, so a crash can -// be told apart: no mark means nothing reached the provider. +// be told apart: no mark means nothing reached the provider. It is +// refused once the lease has too little time left for the send, or +// once an ambiguous row is past the retention cut-off, so a worker +// that stalled after its claim cannot send late. // - complete the provider's answer, fenced: a worker whose lease was taken over // writes nothing. // @@ -82,7 +85,7 @@ export type FrozenOccurrence = { // pending ──claim──▶ in_flight ──complete──▶ accepted (terminal) // │ ▲ rejected (terminal) // │ │ conflict (terminal) -// │ └──claim── failed (a definitive refusal; nothing was accepted) +// │ └──claim── failed (a definitive refusal, and nothing was ever possibly accepted) // │ └──claim── unknown (might have been accepted; ambiguous) // │ // └─ lease expires: back to pending/failed (no beginSend mark) or unknown (marked) @@ -198,11 +201,41 @@ export function decideClaim(row: DeliveryRow, now: Date, owner: string, policy: return { claimed: true, next }; } -/** The beginSend rule: fenced. `null` means the caller's claim is no longer current and it must not send. */ -export function decideBeginSend(row: DeliveryRow, fence: number, now: Date): DeliveryRow | null { - if (row.status !== 'in_flight' || row.fence !== fence) return null; +export type BeginSendPolicy = Pick & { + /** The send may start only while at least this much lease is left: the engine's send deadline. */ + minLeaseRemainingMs: number; +}; + +export type BeginSendDecision = + | { ok: true; next: DeliveryRow } + /** + * `fenced_out`: another claim superseded this one. `lease_expired`: too little lease left to finish + * the send before another worker could claim; the row is left for the next claim. `retention_passed`: + * an ambiguous row reached the cut-off after it was claimed; it is parked in `next`. + */ + | { ok: false; reason: 'fenced_out' | 'lease_expired' | 'retention_passed'; next: DeliveryRow | null }; + +/** + * The beginSend rule: fenced, and re-checked at the last moment before the + * provider call. A store MUST evaluate it and persist `next` in one atomic + * step, with its own clock. Only `ok` permits a send. + */ +export function decideBeginSend(row: DeliveryRow, fence: number, now: Date, policy: BeginSendPolicy): BeginSendDecision { + if (row.status !== 'in_flight' || row.fence !== fence) return { ok: false, reason: 'fenced_out', next: null }; const nowIso = now.toISOString(); - return { ...row, sendStartedAt: nowIso, firstSendAt: row.firstSendAt ?? nowIso, updatedAt: nowIso }; + if (row.leaseExpiresAt === null || Date.parse(row.leaseExpiresAt) - now.getTime() <= policy.minLeaseRemainingMs) { + return { ok: false, reason: 'lease_expired', next: null }; + } + if (row.ambiguous) { + const lastSafe = row.firstSendAt === null ? -Infinity : Date.parse(row.firstSendAt) + policy.providerKeyRetentionMs - policy.retentionSafetyMarginMs; + if (now.getTime() >= lastSafe) { + return { + ok: false, reason: 'retention_passed', + next: { ...row, status: 'needs_reconciliation', leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, updatedAt: nowIso }, + }; + } + } + return { ok: true, next: { ...row, sendStartedAt: nowIso, firstSendAt: row.firstSendAt ?? nowIso, updatedAt: nowIso } }; } // ── Provider ──────────────────────────────────────────────────────────────── @@ -256,6 +289,11 @@ export function decideCompletion(row: DeliveryRow, fence: number, result: Provid const next: DeliveryRow = { ...row, leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, updatedAt: nowIso }; switch (result.kind) { case 'accepted': + // A receipt needs a message id; without one nothing is settled. + if (typeof result.messageId !== 'string' || result.messageId.length === 0) { + next.ambiguous = true; next.status = 'unknown'; next.lastError = 'outcome_unknown:unreadable_response'; + break; + } return { ...next, status: 'accepted', providerMessageId: result.messageId, lastError: null }; case 'payload_conflict': return { ...next, status: 'conflict', lastError: 'provider_payload_conflict' }; @@ -263,7 +301,8 @@ export function decideCompletion(row: DeliveryRow, fence: number, result: Provid next.lastError = clip(`provider_rejected:${result.httpStatus}:${result.code}`); // A refusal cannot undo an earlier attempt that may have been accepted. if (!result.retryable) { next.status = next.ambiguous ? 'needs_reconciliation' : 'rejected'; return next; } - next.status = 'failed'; + // Still retryable, but an earlier attempt may have been accepted: it stays unknown, not "failed". + next.status = next.ambiguous ? 'unknown' : 'failed'; break; case 'in_progress': next.ambiguous = true; next.status = 'unknown'; next.lastError = 'provider_in_progress'; @@ -291,8 +330,8 @@ export interface DigestDeliveryStore { load(occurrenceId: string): Promise; /** Apply `decideClaim` atomically. */ claim(occurrenceId: string, recipientKey: string, owner: string, policy: ClaimPolicy): Promise<{ claimed: true; row: DeliveryRow } | { claimed: false; reason: ClaimRefusal }>; - /** Apply `decideBeginSend` atomically. */ - beginSend(occurrenceId: string, recipientKey: string, fence: number): Promise<'ok' | 'fenced_out'>; + /** Apply `decideBeginSend` atomically (persisting `next` when set). */ + beginSend(occurrenceId: string, recipientKey: string, fence: number, policy: BeginSendPolicy): Promise<'ok' | 'fenced_out' | 'lease_expired' | 'retention_passed'>; /** Apply `decideCompletion` atomically. */ complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number): Promise<'ok' | 'fenced_out'>; } @@ -359,6 +398,8 @@ export function validateConfig(c: EngineConfig): void { // The last safe retry must reach the provider before it forgets the key. if (c.retentionSafetyMarginMs <= c.sendTimeoutMs) planError('retentionSafetyMarginMs must exceed sendTimeoutMs'); if (c.retentionSafetyMarginMs >= c.providerKeyRetentionMs) planError('retentionSafetyMarginMs must be shorter than the retention window'); + // Never assume the provider remembers a key longer than its documentation says. + if (c.providerKeyRetentionMs > RESEND_KEY_RETENTION_MS) planError('providerKeyRetentionMs cannot exceed the verified 24 h'); } /** Frozen form of a validated plan: what `freeze` stores. Pure. */ @@ -401,8 +442,13 @@ export type AttemptReport = { recipientKey: string; fence: number; result: ProviderSendResult['kind'] | 'not_sent'; - /** Whether the outcome is durable. `receipt_write_failed`: the provider answered but the store did not record it. */ - recorded: 'ok' | 'fenced_out' | 'receipt_write_failed' | 'not_sent'; + /** + * Whether the outcome is durable. `receipt_write_unconfirmed`: the provider answered but the store + * did not confirm the write (it may or may not have committed; the next run finds out). + */ + recorded: 'ok' | 'fenced_out' | 'receipt_write_unconfirmed' | 'not_sent'; + /** The provider's message id, whenever it accepted, even if the receipt could not be recorded. */ + messageId?: string; detail?: string; }; @@ -411,7 +457,7 @@ export type OccurrenceReport = { /** This call stored the plan. False: a plan was already stored, and IT was used. */ created: boolean; /** The caller's plan differs from the stored one; the stored one was used and the difference is reported, not merged. */ - planMismatch: null | { recipientsAdded: number; recipientsRemoved: number; payloadChanged: boolean }; + planMismatch: null | { recipientsAdded: number; recipientsRemoved: number; payloadChanged: boolean; windowChanged: boolean }; attempts: AttemptReport[]; refused: { recipientKey: string; reason: ClaimRefusal }[]; storageErrors: { stage: 'freeze' | 'load' | 'claim' | 'beginSend' | 'complete' | 'final_load'; recipientKey?: string }[]; @@ -467,7 +513,9 @@ function mismatch(wanted: FrozenOccurrence, stored: FrozenOccurrence): Occurrenc const recipientsAdded = [...want].filter((k) => !have.has(k)).length; const recipientsRemoved = [...have].filter((k) => !want.has(k)).length; const payloadChanged = wanted.payloadHash !== stored.payloadHash; - return recipientsAdded || recipientsRemoved || payloadChanged ? { recipientsAdded, recipientsRemoved, payloadChanged } : null; + const windowChanged = wanted.window.start !== stored.window.start || wanted.window.end !== stored.window.end; + return recipientsAdded || recipientsRemoved || payloadChanged || windowChanged + ? { recipientsAdded, recipientsRemoved, payloadChanged, windowChanged } : null; } async function run(stored: StoredOccurrence, deps: EngineDeps, report: OccurrenceReport): Promise { @@ -493,15 +541,19 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'not_sent', detail: 'stored_payload_integrity_failed' }); continue; } - let marked: 'ok' | 'fenced_out'; + let marked: Awaited>; try { - marked = await store.beginSend(id, key, row.fence); + marked = await store.beginSend(id, key, row.fence, { + minLeaseRemainingMs: config.sendTimeoutMs, + providerKeyRetentionMs: config.providerKeyRetentionMs, + retentionSafetyMarginMs: config.retentionSafetyMarginMs, + }); } catch { report.storageErrors.push({ stage: 'beginSend', recipientKey: key }); continue; } if (marked !== 'ok') { - report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'fenced_out' }); + report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: marked === 'fenced_out' ? 'fenced_out' : 'not_sent', detail: marked }); continue; } const result = await sendWithDeadline(deps.provider, row, config.sendTimeoutMs); @@ -512,10 +564,10 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc // The provider answered, but nothing durable says so. The row stays in flight // with its mark; after the lease it is retried under the same key and bytes. report.storageErrors.push({ stage: 'complete', recipientKey: key }); - report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded: 'receipt_write_failed' }); + report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded: 'receipt_write_unconfirmed', ...messageIdOf(result) }); continue; } - report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded }); + report.attempts.push({ recipientKey: key, fence: row.fence, result: result.kind, recorded, ...messageIdOf(result) }); } try { const final = await store.load(id); @@ -547,12 +599,16 @@ async function sendWithDeadline(provider: DigestEmailProvider, row: DeliveryRow, } } +const messageIdOf = (r: ProviderSendResult) => (r.kind === 'accepted' ? { messageId: r.messageId } : {}); + /** Only a success WITH a message id is an accepted receipt; anything malformed settles nothing. */ function checkResult(r: ProviderSendResult): ProviderSendResult { if (!r || typeof r !== 'object') return { kind: 'unknown', reason: 'unreadable_response' }; switch (r.kind) { case 'accepted': return typeof r.messageId === 'string' && r.messageId.length > 0 ? r : { kind: 'unknown', reason: 'unreadable_response' }; - case 'rejected': return Number.isInteger(r.httpStatus) && typeof r.retryable === 'boolean' ? { ...r, code: String(r.code ?? '') } : { kind: 'unknown', reason: 'unreadable_response' }; + // A refusal is a 4xx: an adapter that calls a 5xx or a 2xx "rejected" has settled nothing. + case 'rejected': return Number.isInteger(r.httpStatus) && r.httpStatus >= 400 && r.httpStatus < 500 && typeof r.retryable === 'boolean' + ? { ...r, code: String(r.code ?? '') } : { kind: 'unknown', reason: 'unreadable_response' }; case 'payload_conflict': case 'in_progress': case 'unknown': return r; default: return { kind: 'unknown', reason: 'unreadable_response' }; } diff --git a/tests/admin-digest-delivery-engine.test.ts b/tests/admin-digest-delivery-engine.test.ts index feb172095..3bb47a898 100644 --- a/tests/admin-digest-delivery-engine.test.ts +++ b/tests/admin-digest-delivery-engine.test.ts @@ -10,6 +10,7 @@ // adapter must earn the same properties (docs/admin-digest-delivery-contract.md) // and pass tests/helpers/digest-delivery-store-contract.ts against a real // database. +import { createHash } from 'node:crypto'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { RESEND_KEY_RETENTION_MS, classifyResendResponse, deliverDigestOccurrence, freezePlan, idempotencyKeyFor, @@ -85,6 +86,7 @@ describe('inputs are checked before anything is stored or sent', () => { ['a retention margin no longer than the send deadline', { retentionSafetyMarginMs: 150 }], ['a margin as long as the retention window', { retentionSafetyMarginMs: RESEND_KEY_RETENTION_MS }], ['zero attempts', { maxAttempts: 0 }], + ['a retention longer than the provider\'s verified 24 h', { providerKeyRetentionMs: 48 * HOUR }], ])('a config with %s is refused', async (_name, over) => { const { store, provider, engine } = world(); await expect(deliverDigestOccurrence(plan(), engine('a', { config: { ...CONFIG, ...over } }))).rejects.toThrow(TypeError); @@ -256,6 +258,104 @@ describe('concurrent claims and stale leases', () => { }); }); +describe('a worker that stalls between its claim and its mark', () => { + it('is fenced out if another worker took over: it sends nothing', async () => { + const { clock, store, provider, engine } = world(); + const stalled = deferred(); + let first = true; + store.hooks.push(hookOn('beginSend', 'before', K1, async () => { if (first) { first = false; await stalled.promise; } })); + const held = deferred(); + provider.script = ({ to }) => (to === ONE ? { do: 'hold_at_provider', until: held.promise, then: { do: 'accept' } } : { do: 'accept' }); + const a = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('stalled')); + await vi.waitFor(() => expect(store.row(OCC, K1)?.status).toBe('in_flight')); + clock.advance(5 * MINUTE); + const b = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('takeover')); + await vi.waitFor(() => expect(provider.requests).toHaveLength(1)); // B is at the provider, fence 2, in flight + stalled.resolve(); + const ra = await a; + expect(ra.attempts).toEqual([{ recipientKey: K1, fence: 1, result: 'not_sent', recorded: 'fenced_out', detail: 'fenced_out' }]); + held.resolve(); + await b; + expect(provider.requests).toHaveLength(1); + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', fence: 2 }); + }); + + it('with no takeover and its lease gone, it still sends nothing; the next run sends once', async () => { + const { clock, store, provider, engine } = world(); + const stalled = deferred(); + store.hooks.push(hookOn('beginSend', 'before', K1, () => stalled.promise)); + const a = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('stalled')); + await vi.waitFor(() => expect(store.row(OCC, K1)?.status).toBe('in_flight')); + clock.advance(5 * MINUTE); + store.hooks = []; + stalled.resolve(); + expect((await a).attempts).toEqual([{ recipientKey: K1, fence: 1, result: 'not_sent', recorded: 'not_sent', detail: 'lease_expired' }]); + expect(provider.requests).toEqual([]); + const next = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('next')); + expect(next.complete).toBe(true); + expect(provider.requests).toHaveLength(1); + }); + + it('an ambiguous row claimed just inside the window but marked after the cut-off is parked, not sent', async () => { + const { clock, store, provider, engine } = world(); + const long = { ...CONFIG, leaseMs: 2 * HOUR }; + provider.script = () => ({ do: 'lose_before_arrival', reason: 'timeout' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a', { config: long })); // first mark at T0 + clock.set('2026-10-01T11:30:59.000Z'); // T0 + 23 h − 1 s: the claim is still allowed + const stalled = deferred(); + store.hooks.push(hookOn('beginSend', 'before', K1, () => stalled.promise)); + const b = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b', { config: long })); + await vi.waitFor(() => expect(store.row(OCC, K1)?.status).toBe('in_flight')); + clock.set('2026-10-01T11:31:00.000Z'); // the cut-off passes while it stalls + store.hooks = []; + stalled.resolve(); + expect((await b).attempts).toEqual([{ recipientKey: K1, fence: 2, result: 'not_sent', recorded: 'not_sent', detail: 'retention_passed' }]); + expect(provider.requests).toHaveLength(1); + expect(store.row(OCC, K1)!.status).toBe('needs_reconciliation'); + }); +}); + +describe('attempts are bounded even when no attempt ever completes', () => { + it('five claims that each die before the mark: the sixth parks the recipient as exhausted, and nothing was sent', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('claim', 'after', K1, never)); + for (let i = 1; i <= CONFIG.maxAttempts; i += 1) { + void deliverDigestOccurrence(plan({ recipients: [ONE] }), engine(`dies-${i}`)); + await vi.waitFor(() => expect(store.row(OCC, K1)?.attempts).toBe(i)); + clock.advance(5 * MINUTE); + } + store.hooks = []; + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('last')); + expect(r.refused).toEqual([{ recipientKey: K1, reason: 'exhausted' }]); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'exhausted' }]); + expect(provider.requests).toEqual([]); + }); +}); + +describe('a stale receipt while the row is in flight under a newer claim', () => { + it('is fenced out: it cannot overwrite the newer claim, which then settles on its own', async () => { + const { clock, store, provider, engine } = world(); + const aWire = deferred(); + const bHeld = deferred(); + provider.script = ({ n }) => (n === 1 + ? { do: 'delay_before_arrival', until: aWire.promise, then: { do: 'accept' } } + : { do: 'hold_at_provider', until: bHeld.promise, then: { do: 'accept' } }); + const a = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('slow', { config: { ...CONFIG, sendTimeoutMs: 60_000 } })); + await vi.waitFor(() => expect(provider.requests).toHaveLength(1)); + clock.advance(5 * MINUTE); + const b = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('fresh', { config: { ...CONFIG, sendTimeoutMs: 60_000 } })); + await vi.waitFor(() => expect(provider.requests).toHaveLength(2)); // B holds the key at the provider, row in flight at fence 2 + aWire.resolve(); // A's request arrives while B's is in progress: 409 concurrent, and A's receipt meets fence 2 + const ra = await a; + expect(ra.attempts).toEqual([{ recipientKey: K1, fence: 1, result: 'in_progress', recorded: 'fenced_out' }]); + expect(store.row(OCC, K1)).toMatchObject({ status: 'in_flight', fence: 2 }); + bHeld.resolve(); + await b; + expect(store.row(OCC, K1)).toMatchObject({ status: 'accepted', fence: 2 }); + expect(provider.deliveredTo(ONE)).toBe(1); + }); +}); + // ── Crashes and restarts ─────────────────────────────────────────────────── describe('crashes and restarts from saved state', () => { @@ -390,13 +490,14 @@ describe('storage failures', () => { expect(provider.requests).toHaveLength(1); }); + // Failures are injected on the recipient the engine visits FIRST, so "the other still is" is shown, not assumed. it('a claim fails (before or after it committed): that recipient is not sent this run; the other still is; a later run completes once', async () => { for (const phase of ['before', 'after'] as const) { const { clock, store, provider, engine } = world(); - store.hooks.push(hookOn('claim', phase, K1, fail)); + store.hooks.push(hookOn('claim', phase, FIRST.key, fail)); const r = await deliverDigestOccurrence(plan(), engine('a')); - expect(r.storageErrors).toEqual([{ stage: 'claim', recipientKey: K1 }]); - expect(provider.requests.map((q) => q.to)).toEqual([TWO]); + expect(r.storageErrors).toEqual([{ stage: 'claim', recipientKey: FIRST.key }]); + expect(provider.requests.map((q) => q.to)).toEqual([SECOND.to]); store.hooks = []; clock.advance(6 * MINUTE); expect((await deliverDigestOccurrence(plan(), engine('b'))).complete).toBe(true); @@ -407,22 +508,24 @@ describe('storage failures', () => { it('the mark fails (before or after it committed): nothing is sent for that recipient this run', async () => { for (const phase of ['before', 'after'] as const) { const { clock, store, provider, engine } = world(); - store.hooks.push(hookOn('beginSend', phase, K1, fail)); + store.hooks.push(hookOn('beginSend', phase, FIRST.key, fail)); await deliverDigestOccurrence(plan(), engine('a')); - expect(provider.requests.map((q) => q.to)).toEqual([TWO]); + expect(provider.requests.map((q) => q.to)).toEqual([SECOND.to]); store.hooks = []; clock.advance(6 * MINUTE); expect((await deliverDigestOccurrence(plan(), engine('b'))).complete).toBe(true); - expect(store.row(OCC, K1)!.ambiguous).toBe(phase === 'after'); // a committed mark is honestly ambiguous - expect(provider.deliveredTo(ONE)).toBe(1); + expect(store.row(OCC, FIRST.key)!.ambiguous).toBe(phase === 'after'); // a committed mark is honestly ambiguous + expect(provider.deliveredTo(FIRST.to)).toBe(1); } }); - it('the receipt write fails after the provider accepted: reported as receipt_write_failed, and the retry is answered from the key', async () => { + it('the receipt write fails after the provider accepted: reported as unconfirmed with the message id, and the retry is answered from the key', async () => { const { clock, store, provider, engine } = world(); store.hooks.push(hookOn('complete', 'before', K1, fail)); const r = await deliverDigestOccurrence(plan(), engine('a')); - expect(r.attempts.find((x) => x.recipientKey === K1)).toEqual({ recipientKey: K1, fence: 1, result: 'accepted', recorded: 'receipt_write_failed' }); + expect(r.attempts.find((x) => x.recipientKey === K1)).toEqual({ + recipientKey: K1, fence: 1, result: 'accepted', recorded: 'receipt_write_unconfirmed', messageId: provider.inbox.find((d) => d.to === ONE)!.messageId, + }); expect(r.complete).toBe(false); expect(store.row(OCC, K1)!.status).toBe('in_flight'); store.hooks = []; @@ -458,7 +561,7 @@ describe('changed payloads and provider conflicts', () => { payload: { ...plan().payload, html: '

3 new families (a row committed late).

' }, }); const r = await deliverDigestOccurrence(changed, engine('b')); - expect(r.planMismatch).toEqual({ recipientsAdded: 1, recipientsRemoved: 0, payloadChanged: true }); + expect(r.planMismatch).toEqual({ recipientsAdded: 1, recipientsRemoved: 0, payloadChanged: true, windowChanged: false }); const toTwo = provider.requests.filter((q) => q.to === TWO); expect(toTwo).toHaveLength(2); expect(new Set(toTwo.map((q) => q.payloadJson)).size).toBe(1); @@ -467,6 +570,14 @@ describe('changed payloads and provider conflicts', () => { expect(r.complete).toBe(true); }); + it('a retry that names a different window for the same occurrence is reported, and the stored window stands', async () => { + const { engine } = world(); + await deliverDigestOccurrence(plan(), engine('a')); + const r = await deliverDigestOccurrence(plan({ window: { start: '2026-09-29T12:33:00.000Z', end: '2026-09-30T12:33:00.000Z' } }), engine('b')); + expect(r.planMismatch).toEqual({ recipientsAdded: 0, recipientsRemoved: 0, payloadChanged: false, windowChanged: true }); + expect(r.attempts).toEqual([]); + }); + it('409 invalid_idempotent_request: a conflict, parked, never retried and never re-keyed', async () => { const { clock, provider, engine } = world(); const row = freezePlan(plan(), new Date(T0)).deliveries.find((d) => d.recipientKey === K1)!; @@ -561,6 +672,59 @@ describe('what the provider said, taken at its word and no further', () => { expect(provider.requests).toEqual([]); }); + it('an adapter that calls a 5xx "rejected" has settled nothing: the row is unknown, not failed', async () => { + const { store, engine } = world(); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), { + ...engine('a'), provider: { send: async () => ({ kind: 'rejected', httpStatus: 503, code: 'x', retryable: true }) }, + }); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true }); + }); + + it('after an ambiguous attempt, a retryable refusal leaves the row unknown (still bounded by retention), not "failed"', async () => { + const { clock, store, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'lose_before_arrival', reason: 'timeout' } : { do: 'reject', status: 429, code: 'rate_limit_exceeded', retryable: true }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + clock.advance(HOUR); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true }); + clock.set('2026-10-01T11:31:00.000Z'); + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('c')); + expect(r.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + expect(provider.requests).toHaveLength(2); + }); + + it('a stored key or recipient that no longer matches what was frozen is never sent', async () => { + for (const change of ['key', 'recipient'] as const) { + const { store, provider, engine } = world(); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), { ...engine('a'), provider: { send: async () => ({ kind: 'rejected', httpStatus: 429, code: 'x', retryable: true }) } }); + store.tamper(OCC, K1, (row) => { + if (change === 'key') row.idempotencyKey = 'bubaly/admin-digest/v1/rotated'; + else { + row.payload = { ...row.payload, to: 'someone-else@example.test' }; + row.payloadJson = payloadJsonOf(row.payload); + row.payloadHash = createHash('sha256').update(row.payloadJson).digest('hex'); + } + }); + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b')); + expect(r.attempts[0]).toMatchObject({ result: 'not_sent', detail: 'stored_payload_integrity_failed' }); + expect(provider.requests).toEqual([]); + } + }); + + it('when the final read fails the run is not reported complete, even though it sent', async () => { + const { store, provider, engine } = world(); + store.hooks.push(hookOn('load', 'before', null, fail)); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(provider.inbox).toHaveLength(2); + expect(r).toMatchObject({ complete: false, statuses: null, storageErrors: [{ stage: 'final_load' }] }); + }); + + it('every accepted attempt carries its message id in the report', async () => { + const { provider, engine } = world(); + const r = await deliverDigestOccurrence(plan(), engine('a')); + expect(r.attempts.map((x) => x.messageId).sort()).toEqual(provider.inbox.map((d) => d.messageId).sort()); + }); + it.each<[number, unknown, string]>([ [200, { id: 'e1' }, 'accepted'], [200, {}, 'unknown'], diff --git a/tests/helpers/digest-delivery-fakes.ts b/tests/helpers/digest-delivery-fakes.ts index 6ada5b2b7..7dab80da2 100644 --- a/tests/helpers/digest-delivery-fakes.ts +++ b/tests/helpers/digest-delivery-fakes.ts @@ -6,11 +6,13 @@ // RULES and the engine's use of them. It proves nothing about PostgreSQL: // durability, isolation and the database clock are the adapter's to meet // (docs/admin-digest-delivery-contract.md), and the same contract suite -// (tests/helpers/digest-delivery-store-contract.ts) must pass against it. +// (tests/helpers/digest-delivery-store-contract.ts) must pass against it, with +// the database's clock made to follow the test's clock (for example: the SQL +// reads time only through one function the local test harness can pin). import { createHash } from 'node:crypto'; import { decideBeginSend, decideClaim, decideCompletion, - type ClaimPolicy, type ClaimRefusal, type DeliveryRow, type DigestDeliveryStore, type DigestEmailProvider, + type BeginSendPolicy, type ClaimPolicy, type ClaimRefusal, type DeliveryRow, type DigestDeliveryStore, type DigestEmailProvider, type FrozenOccurrence, type ProviderSendResult, type StoredOccurrence, } from '@/lib/admin/digest-delivery'; @@ -136,14 +138,16 @@ export class MemoryDigestDeliveryStore implements DigestDeliveryStore { return out; } - async beginSend(occurrenceId: string, recipientKey: string, fence: number) { + async beginSend(occurrenceId: string, recipientKey: string, fence: number, policy: BeginSendPolicy) { await this.hook('beginSend', 'before', recipientKey); + // ── critical section ── const k = rowKey(occurrenceId, recipientKey); const row = this.rows.get(k); - const next = row ? decideBeginSend(copy(row), fence, this.clock()) : null; - if (next) this.rows.set(k, next); + const d = row ? decideBeginSend(copy(row), fence, this.clock(), policy) : { ok: false as const, reason: 'fenced_out' as const, next: null }; + if (d.next) this.rows.set(k, copy(d.next)); + // ── end ── await this.hook('beginSend', 'after', recipientKey); - return next ? 'ok' as const : 'fenced_out' as const; + return d.ok ? 'ok' as const : d.reason; } async complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number) { diff --git a/tests/helpers/digest-delivery-store-contract.ts b/tests/helpers/digest-delivery-store-contract.ts index 7b7f2a151..e755e0a60 100644 --- a/tests/helpers/digest-delivery-store-contract.ts +++ b/tests/helpers/digest-delivery-store-contract.ts @@ -2,10 +2,15 @@ // Any DigestDeliveryStore must pass it: today the in-memory store; a PostgreSQL // adapter later, run against the local stack. Passing it with the in-memory // store proves the rules, not database durability or isolation. +// +// Time: every lease and retention decision is the STORE's, on the store's +// clock. `make(clock)` must return a store whose clock follows `clock`; for a +// database adapter that means the harness pins the database's notion of now +// (the adapter's SQL reads time through one function the harness can replace). import { describe, expect, it } from 'vitest'; import { freezePlan, recipientKeyOf, RESEND_KEY_RETENTION_MS, - type ClaimPolicy, type DigestDeliveryStore, type OccurrencePlan, + type BeginSendPolicy, type ClaimPolicy, type DigestDeliveryStore, type OccurrencePlan, } from '@/lib/admin/digest-delivery'; import { FakeClock, HOUR, MINUTE } from './digest-delivery-fakes'; @@ -13,6 +18,10 @@ export const CONTRACT_POLICY: ClaimPolicy = { leaseMs: 5 * MINUTE, maxAttempts: 3, providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, retentionSafetyMarginMs: HOUR, }; +export const CONTRACT_BEGIN: BeginSendPolicy = { + minLeaseRemainingMs: 1_000, providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, retentionSafetyMarginMs: HOUR, +}; + export const contractPlan = (over: Partial = {}): OccurrencePlan => ({ occurrenceId: 'admin-digest:2026-09-30T12:30:00.000Z', window: { start: '2026-09-29T12:30:00.000Z', end: '2026-09-30T12:30:00.000Z' }, @@ -102,9 +111,9 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: clock.advance(5 * MINUTE); const b = await store.claim(id, key, 'b', CONTRACT_POLICY); if (!a.claimed || !b.claimed) throw new Error('setup'); - expect(await store.beginSend(id, key, a.row.fence)).toBe('fenced_out'); + expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('fenced_out'); expect(await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'stale' }, 3)).toBe('fenced_out'); - expect(await store.beginSend(id, key, b.row.fence)).toBe('ok'); + expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toBe('ok'); expect(await store.complete(id, key, b.row.fence, { kind: 'accepted', messageId: 'm-b' }, 3)).toBe('ok'); expect(await store.complete(id, key, b.row.fence, { kind: 'rejected', httpStatus: 422, code: 'x', retryable: false }, 3)).toBe('fenced_out'); const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; @@ -115,7 +124,7 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: const { store, id, key } = await setup(); const a = await store.claim(id, key, 'a', CONTRACT_POLICY); if (!a.claimed) throw new Error('setup'); - await store.beginSend(id, key, a.row.fence); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'm-1' }, 3); expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'accepted' }); }); @@ -124,7 +133,7 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: const { clock, store, id, key } = await setup(); const a = await store.claim(id, key, 'a', CONTRACT_POLICY); if (!a.claimed) throw new Error('setup'); - await store.beginSend(id, key, a.row.fence); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); clock.advance(23 * HOUR); // 24 h retention − 1 h margin, from the first beginSend expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'needs_reconciliation' }); @@ -137,12 +146,124 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: const before = frozen.deliveries.find((d) => d.recipientKey === key)!; const a = await store.claim(id, key, 'a', CONTRACT_POLICY); if (!a.claimed) throw new Error('setup'); - await store.beginSend(id, key, a.row.fence); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'm-1' }, 3); const after = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; for (const f of ['recipientKey', 'idempotencyKey', 'payload', 'payloadJson', 'payloadHash', 'occurrenceId'] as const) expect(after[f]).toEqual(before[f]); }); + it('a lease that lapsed after the mark makes the row ambiguous; one that lapsed before it does not', async () => { + const { clock, store, id, key } = await setup(); + const other = recipientKeyOf('admin-two@example.test'); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + const b = await store.claim(id, other, 'a', CONTRACT_POLICY); + if (!a.claimed || !b.claimed) throw new Error('setup'); + expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('ok'); // marked; `other` is not + clock.advance(5 * MINUTE); + const again = await store.claim(id, key, 'c', CONTRACT_POLICY); + const againOther = await store.claim(id, other, 'c', CONTRACT_POLICY); + expect(again.claimed && again.row).toMatchObject({ ambiguous: true, fence: 2, attempts: 2, firstSendAt: start }); + expect(againOther.claimed && againOther.row).toMatchObject({ ambiguous: false, fence: 2, attempts: 2, firstSendAt: null }); + }); + + it('the retention anchor is the FIRST mark; later marks do not move it', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); + clock.advance(HOUR); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!b.claimed) throw new Error('setup'); + await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN); + const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; + expect(row.firstSendAt).toBe(start); + }); + + it('a claim at the attempt limit parks the row instead of claiming it', async () => { + const { clock, store, id, key } = await setup(); + for (let i = 0; i < 3; i += 1) { expect((await store.claim(id, key, `w${i}`, CONTRACT_POLICY)).claimed).toBe(true); clock.advance(5 * MINUTE); } + expect(await store.claim(id, key, 'w4', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'exhausted' }); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!.status).toBe('exhausted'); + }); + + it('beginSend is refused when too little lease is left, and parks an ambiguous row that reached the cut-off', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + clock.advance(5 * MINUTE - 1_000); + expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('lease_expired'); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'in_flight', sendStartedAt: null }); + // An ambiguous row claimed just inside the window, marked just after it: + clock.advance(1_000); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!b.claimed) throw new Error('setup'); + expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toBe('ok'); + await store.complete(id, key, b.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); + const firstMark = Date.parse((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!.firstSendAt!); + clock.set(new Date(firstMark + 23 * HOUR - 1).toISOString()); + const c = await store.claim(id, key, 'c', { ...CONTRACT_POLICY, leaseMs: 2 * HOUR }); + if (!c.claimed) throw new Error('setup'); + clock.advance(1); + expect(await store.beginSend(id, key, c.row.fence, CONTRACT_BEGIN)).toBe('retention_passed'); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!.status).toBe('needs_reconciliation'); + }); + + it('ambiguity is sticky: a later retryable refusal leaves the row unknown and ambiguous', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); + clock.advance(MINUTE); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!b.claimed) throw new Error('setup'); + await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, b.row.fence, { kind: 'rejected', httpStatus: 429, code: 'rate_limit_exceeded', retryable: true }, 3); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'unknown', ambiguous: true }); + }); + + it('a conflict is terminal, and the completion that uses the last attempt parks the row', async () => { + const { clock, store, id, key } = await setup(); + const other = recipientKeyOf('admin-two@example.test'); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, a.row.fence, { kind: 'payload_conflict' }, 3); + expect(await store.claim(id, key, 'b', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'conflict' }); + for (let i = 1; i <= 3; i += 1) { + const c = await store.claim(id, other, `w${i}`, CONTRACT_POLICY); + if (!c.claimed) throw new Error(`claim ${i}`); + await store.beginSend(id, other, c.row.fence, CONTRACT_BEGIN); + await store.complete(id, other, c.row.fence, { kind: 'unknown', reason: 'network' }, 3); + clock.advance(MINUTE); + } + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === other)!.status).toBe('needs_reconciliation'); + }); + + it('occurrences are independent: the same recipient in another occurrence is untouched', async () => { + const { clock, store, id, key } = await setup(); + const second = freezePlan(contractPlan({ occurrenceId: 'admin-digest:2026-10-01T12:30:00.000Z' }), clock.now()); + await store.freeze(second.occurrence, second.deliveries); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'm-1' }, 3); + const untouched = (await store.load(second.occurrence.occurrenceId))!.deliveries.find((d) => d.recipientKey === key)!; + expect(untouched).toMatchObject({ status: 'pending', fence: 0, attempts: 0, providerMessageId: null }); + const b = await store.claim(second.occurrence.occurrenceId, key, 'b', CONTRACT_POLICY); + expect(b.claimed && b.row.fence).toBe(1); + }); + + it('the completion rule itself refuses an "accepted" without a message id', async () => { + const { store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN); + await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: '' }, 3); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'unknown', ambiguous: true, providerMessageId: null }); + }); + it('an unknown occurrence or recipient is not found, not created', async () => { const { store, id } = await setup(); expect(await store.claim(id, 'f'.repeat(64), 'a', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'not_found' }); From c9daac74a43f6ca6fad10f971ca551eb7a115342 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 20:25:23 +0000 Subject: [PATCH 04/12] feat(admin-digest delivery): 0471 delivery store, PostgreSQL adapter, disposable-database proofs Migration 0471, reserved by the coordinator on #699, and not applied to production: - Tables: - admin_digest_occurrences holds the frozen plan. - admin_digest_deliveries holds one row per recipient: stable key, exact bytes, status, attempts, fence, lease, send mark, retention anchor and provider message id. - Checks mirror the engine's invariants. - Five security-definer functions (freeze, load, claim, begin_send, complete), each the SQL twin of one pure rule. Each takes the row lock first, then reads admin_digest_now() = clock_timestamp(). now() would be the transaction start, which is stale after a lock wait. - Guard triggers: - on insert, the hash, recipient key and idempotency key must agree with the bytes; - bytes, key and recipient never change, and rows are never deleted; - a settled row stays settled; - attempts, fence, ambiguity and the anchor only move forward; - occurrences never change. - Access: RLS on with no policies. Every table privilege is revoked from anon, authenticated AND service_role. EXECUTE on the five functions goes to service_role only. lib/admin/digest-delivery-store.ts is the adapter: a strict mapping, with an injected transport (supabaseRpc for later). A malformed answer throws; it is never read as a claim or a receipt. Rule change, in both TypeScript and SQL: a completion needs the claim's mark. The differential test found the TypeScript rule letting an unmarked claim become ambiguous with no retention anchor, which the database refused. Proof: - docs/audit/an-admin-digest-reaches-each-admin-once-check.sql (CI, full replay): access, freeze-once, the fence, a completion needing a mark, retention parking, frozen identity. - tests/admin-digest-delivery-postgres.test.ts runs on a disposable database (opt-in: DIGEST_DELIVERY_PG=1) with Supabase-style default grants. It covers: - the store contract suite on PostgreSQL; - a differential check: 3 seeds x 160 random steps agree step for step with the TypeScript rules; - 40 concurrent claims from 40 connections; - clock-after-lock, with its now() counterfactual; - backend death mid-claim and mid-receipt; - an immediate-mode restart; - two engines at once, and an engine crash then restart; - access and immutability. The adapter's strictness tests always run. No route or scheduler wiring, provider call, live email, production migration, credential or deployment setting. docs/PENDING_PROD_MIGRATIONS.md is shared: its 0471 entry is proposed to the coordinator, not written here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 84 ++-- ...n-digest-reaches-each-admin-once-check.sql | 164 +++++++ lib/admin/digest-delivery-store.ts | 154 +++++++ lib/admin/digest-delivery.ts | 7 +- ...n_admin_digest_reaches_each_admin_once.sql | 419 ++++++++++++++++++ tests/admin-digest-delivery-postgres.test.ts | 342 ++++++++++++++ tests/helpers/digest-delivery-postgres.ts | 119 +++++ .../helpers/digest-delivery-store-contract.ts | 8 + 8 files changed, 1240 insertions(+), 57 deletions(-) create mode 100644 docs/audit/an-admin-digest-reaches-each-admin-once-check.sql create mode 100644 lib/admin/digest-delivery-store.ts create mode 100644 supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql create mode 100644 tests/admin-digest-delivery-postgres.test.ts create mode 100644 tests/helpers/digest-delivery-postgres.ts diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index ee825d066..378868f59 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -2,8 +2,8 @@ Status on 2026-09-30: - `lib/admin/digest-delivery.ts` exists, and its tests pass against an in-memory store and a fake provider. +- **The PostgreSQL store exists, but is not applied to production.** It is migration `0471` plus `lib/admin/digest-delivery-store.ts`, tested on disposable local databases and in CI's replay. - **No route uses it.** `/api/cron/admin-digest` still sends the way it did. -- **Nothing is stored in a database.** No migration exists or is proposed as a file (the proposed schema is §5). - Duplicate admin digests are **not** fixed in production. This document is what the next two pieces must meet: a PostgreSQL store adapter and the route wiring. @@ -64,7 +64,7 @@ A PostgreSQL `DigestDeliveryStore` must meet all of these. It must also pass `te - **One function:** read it through a single function, e.g. `admin_digest_now()`, so the local test harness can pin it for the contract suite. - **Never the caller's time:** no timestamp the caller sends is used. - **Parameters:** retention, margin, lease, send deadline and max attempts are passed in, not hard-coded. The engine refuses a retention longer than the verified 24 hours. -4. **The fence is a counter.** `fence` increments on every successful claim. +4. **The fence is a counter.** `fence` increments on every successful claim. `complete` also requires the claim's mark: a claim that never marked a send owns no provider answer. - `beginSend` and `complete` are each one `UPDATE … WHERE status = 'in_flight' AND fence = $fence RETURNING`. - Zero rows means `fenced_out`, and nothing is written. - Owner equality alone is not enough: the same owner can re-claim after its own lease lapsed. @@ -104,57 +104,23 @@ The engine takes a `DigestEmailProvider` whose `send` returns one classified out **Retention.** `RESEND_KEY_RETENTION_MS` is 24 hours, from Resend's documentation re-checked on 2026-09-30. Recommended margin: 1 hour, which must exceed the send deadline plus clock skew. Re-verify before changing provider or value. What Resend does with a key after a refused request is undocumented. The engine never relies on it: a refusal is retried with the same key, and the 24-hour window is anchored on the first send of any kind. -## 5. Proposed schema (proposal only, no migration number) - -**No migration will be added** until the coordinator reserves a number. Main ends at `0464`. `0465` and NWV's locale work must be coordinated first. This also cannot reach production before the PROD-DB-0177 migration-ledger remedy. - -```sql -create table public.admin_digest_occurrences ( - occurrence_id text primary key check (occurrence_id ~ '^[\x21-\x7e]([\x20-\x7e]*[\x21-\x7e])?$' and length(occurrence_id) <= 200), - window_start timestamptz not null, - window_end timestamptz not null check (window_end > window_start), - recipient_keys text[] not null check (cardinality(recipient_keys) between 1 and 200), - payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), - engine_version integer not null, - frozen_at timestamptz not null default now() -); - -create table public.admin_digest_deliveries ( - occurrence_id text not null references public.admin_digest_occurrences (occurrence_id) on delete restrict, - recipient_key text not null check (recipient_key ~ '^[0-9a-f]{64}$'), - idempotency_key text not null unique check (length(idempotency_key) between 1 and 256), - payload_json text not null, - payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), - status text not null check (status in ('pending','in_flight','failed','unknown','accepted','rejected','conflict','exhausted','needs_reconciliation')), - attempts integer not null default 0 check (attempts >= 0), - fence bigint not null default 0 check (fence >= 0), - lease_owner text, - lease_expires_at timestamptz, - send_started_at timestamptz, - first_send_at timestamptz, - ambiguous boolean not null default false, - provider_message_id text, - last_error text check (last_error is null or length(last_error) <= 200), - updated_at timestamptz not null default now(), - primary key (occurrence_id, recipient_key), - check ((status = 'in_flight') = (lease_owner is not null and lease_expires_at is not null)), - check ((status = 'accepted') = (provider_message_id is not null)), - check (send_started_at is null or status = 'in_flight'), - check (not ambiguous or first_send_at is not null) -); --- + trigger: payload_json, payload_hash, idempotency_key, recipient_key, occurrence_id are immutable; --- payload_hash = encode(sha256(convert_to(payload_json, 'UTF8')), 'hex') on insert; ambiguous never true → false. --- + RLS on, no policies; revoke all from anon, authenticated. --- + security-definer functions (execute revoked from public, anon, authenticated), each mirroring one rule: --- admin_digest_freeze(p_occurrence jsonb, p_deliveries jsonb) returns jsonb --- admin_digest_claim(p_occurrence_id text, p_recipient_key text, p_owner text, p_lease_ms int, --- p_max_attempts int, p_retention_ms bigint, p_margin_ms bigint) returns jsonb -- decideClaim --- admin_digest_begin_send(p_occurrence_id text, p_recipient_key text, p_fence bigint, p_min_lease_ms int, --- p_retention_ms bigint, p_margin_ms bigint) returns text -- decideBeginSend: ok | fenced_out | lease_expired | retention_passed --- admin_digest_now() returns timestamptz -- clock_timestamp(); the one clock the local test harness pins --- admin_digest_complete(p_occurrence_id text, p_recipient_key text, p_fence bigint, --- p_result jsonb, p_max_attempts int) returns boolean -- decideCompletion -``` +## 5. Schema: migration `0471` (reserved by the coordinator on #699) + +`supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql` implements §3. Its tables, guard triggers, grants and the five functions (`admin_digest_freeze`, `_load`, `_claim`, `_begin_send`, `_complete`) are the SQL twins of the pure rules. `lib/admin/digest-delivery-store.ts` is the adapter. + +**It is not applied to production.** PROD-DB-0177 blocks every migration there. The shared `docs/PENDING_PROD_MIGRATIONS.md` entry is proposed to the coordinator for reconciliation, not written here. + +**Evidence:** +- **CI.** `docs/audit/an-admin-digest-reaches-each-admin-once-check.sql` runs on the full migration replay: access refusals, freeze-once, the fence, a completion needing a mark, retention parking and frozen identity. +- **Disposable database.** `tests/admin-digest-delivery-postgres.test.ts` needs `DIGEST_DELIVERY_PG=1` and a local cluster; the `docs/audit/verify-pg.sh` harness is the default. It runs: + - the store contract suite against PostgreSQL; + - a differential check in which random steps, applied to the TypeScript rules and to the SQL, must agree after every step; + - 40 concurrent claims from 40 connections; + - a clock-after-lock test, with its `now()` counterfactual; + - backend death mid-claim and mid-receipt; + - an immediate-mode server restart, which needs `DIGEST_DELIVERY_PG_RESTART_CMD`; + - two engines at once, and an engine crash then restart; + - access and immutability. ## 6. What is and is not claimed @@ -166,7 +132,15 @@ create table public.admin_digest_deliveries ( - Stale workers are fenced out. - Crashes before the mark are retried as not sent; crashes after it are retried as ambiguous. - Storage failures send nothing without a durable claim and mark. + - A worker that stalls between its claim and its mark cannot send after its lease, or after the retention cut-off. +- **Tested on local PostgreSQL 16:** + - atomic claims under 40 concurrent connections; + - a claim decided on the clock read after its row lock; + - no partial state when a backend dies mid-claim or mid-receipt; + - a committed receipt surviving an immediate-mode restart; + - the SQL rules agreeing with the TypeScript rules step for step; + - the access refusals and frozen identity above. - **Not claimed:** - Exactly-once delivery. The provider's key deduplication, inside its window, is what makes an ambiguous retry harmless. - - Anything about PostgreSQL durability or isolation. - - Anything about production. It is not wired, and production cron is not running (see the scheduler report). + - Any behaviour of the production database. 0471 is not applied there. + - Anything about production at all. It is not wired, and production cron is not running (see the scheduler report). diff --git a/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql new file mode 100644 index 000000000..90599e693 --- /dev/null +++ b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql @@ -0,0 +1,164 @@ +-- An admin digest reaches each admin once: the 0471 delivery store. (API-96040DFB5635) +-- +-- On the fully replayed schema, as the roles Supabase uses: +-- refused anon and authenticated: every read, write and call; +-- service_role: every direct read or write of the two tables, and +-- the internal clock (it reaches the rows through the functions only); +-- anyone: changing a delivery's bytes or key, deleting a delivery, +-- changing or deleting an occurrence; +-- rules freeze stores a plan once and never replaces it; one claim wins; +-- a superseded claim cannot mark or complete; a completion needs a +-- mark; a lease that lapses after the mark makes the row ambiguous; +-- an ambiguous row is parked at retention minus margin, at claim and +-- at mark; too little lease left refuses the mark; a success without +-- a message id is not a receipt; a settled row stays settled. +-- Time is pinned for the transaction by replacing admin_digest_now() with a +-- version that reads a local setting; the replacement is rolled back with +-- everything else. The disposable-database suite +-- (tests/admin-digest-delivery-postgres.test.ts) adds real concurrency, crash +-- and restart evidence that one transaction cannot give. +-- +-- Rolled back: nothing here outlives the assertion. +\set ON_ERROR_STOP on +set client_min_messages = warning; + +begin; + +create or replace function public.admin_digest_now() +returns timestamptz language sql volatile set search_path = public, pg_temp +as $$ select coalesce(nullif(current_setting('admin_digest.test_now', true), '')::timestamptz, clock_timestamp()) $$; + +-- One synthetic delivery, built exactly as the engine builds it (hash, recipient key, idempotency key). +create function pg_temp.delivery(occ text, addr text, html text) returns jsonb language sql as $$ + select jsonb_build_object( + 'occurrenceId', occ, 'recipientKey', rk, + 'idempotencyKey', 'bubaly/admin-digest/v1/' || left(encode(sha256(convert_to(occ, 'UTF8')), 'hex'), 32) || '/' || left(rk, 32), + 'payloadJson', pj, 'payloadHash', encode(sha256(convert_to(pj, 'UTF8')), 'hex')) + from (select encode(sha256(convert_to(addr, 'UTF8')), 'hex') as rk, + json_build_object('from', 'Bubaly ', 'to', addr, 'subject', 'Probe digest', 'html', html)::text as pj) x +$$; +create function pg_temp.occurrence(occ text, deliveries jsonb) returns jsonb language sql as $$ + select jsonb_build_object('occurrenceId', occ, + 'window', jsonb_build_object('start', '2026-09-29T12:30:00Z', 'end', '2026-09-30T12:30:00Z'), + 'recipientKeys', (select jsonb_agg(d ->> 'recipientKey' order by d ->> 'recipientKey') from jsonb_array_elements(deliveries) d), + 'payloadHash', repeat('a', 64), 'engineVersion', 1) +$$; + +do $probe$ +declare + occ text := 'admin-digest:probe-2026-09-30T12:30:00.000Z'; + one jsonb := pg_temp.delivery(occ, 'probe-one@example.test', '

2 new families.

'); + two jsonb := pg_temp.delivery(occ, 'probe-two@example.test', '

2 new families.

'); + k1 text := one ->> 'recipientKey'; + k2 text := two ->> 'recipientKey'; + ret bigint := 86400000; + mar bigint := 3600000; + res jsonb; + txt text; + refused boolean; + failures int := 0; + pin timestamptz := '2026-09-30T12:31:00Z'; +begin + perform set_config('admin_digest.test_now', pin::text, true); + + -- ── anon and authenticated: nothing at all ───────────────────────────────── + foreach txt in array array['anon', 'authenticated'] loop + execute format('set local role %I', txt); + refused := false; + begin perform 1 from public.admin_digest_deliveries; exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'LEAK: % reads admin_digest_deliveries', txt; failures := failures + 1; end if; + refused := false; + begin perform 1 from public.admin_digest_occurrences; exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'LEAK: % reads admin_digest_occurrences', txt; failures := failures + 1; end if; + refused := false; + begin perform public.admin_digest_load(occ); exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'LEAK: % calls admin_digest_load', txt; failures := failures + 1; end if; + refused := false; + begin perform public.admin_digest_claim(occ, k1, 'x', 1000, 1, ret, mar); exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'LEAK: % calls admin_digest_claim', txt; failures := failures + 1; end if; + refused := false; + begin perform public.admin_digest_freeze('{}'::jsonb, '[]'::jsonb); exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'LEAK: % calls admin_digest_freeze', txt; failures := failures + 1; end if; + reset role; + end loop; + + -- ── the service role: the functions, and nothing direct ──────────────────── + set local role service_role; + refused := false; + begin perform 1 from public.admin_digest_deliveries; exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'WIDE: service_role reads admin_digest_deliveries directly'; failures := failures + 1; end if; + refused := false; + begin update public.admin_digest_deliveries set status = 'accepted'; exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'WIDE: service_role writes admin_digest_deliveries directly'; failures := failures + 1; end if; + refused := false; + begin perform public.admin_digest_now(); exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'WIDE: service_role calls the internal clock'; failures := failures + 1; end if; + + res := public.admin_digest_freeze(pg_temp.occurrence(occ, jsonb_build_array(one, two)), jsonb_build_array(one, two)); + if not (res ->> 'created')::boolean or jsonb_array_length(res -> 'deliveries') <> 2 then + raise warning 'freeze did not store the plan: %', res; failures := failures + 1; + end if; + -- A second, DIFFERENT plan for the same occurrence changes nothing. + res := public.admin_digest_freeze(pg_temp.occurrence(occ, jsonb_build_array(pg_temp.delivery(occ, 'probe-one@example.test', '

9 new families.

'))), + jsonb_build_array(pg_temp.delivery(occ, 'probe-one@example.test', '

9 new families.

'))); + if (res ->> 'created')::boolean or jsonb_array_length(res -> 'deliveries') <> 2 + or (res -> 'deliveries' -> 0 ->> 'payloadJson') like '%9 new%' or (res -> 'deliveries' -> 1 ->> 'payloadJson') like '%9 new%' then + raise warning 'a second freeze replaced the stored plan: %', res; failures := failures + 1; + end if; + + -- One claim wins; a superseded claim cannot mark or complete; a completion needs a mark. + res := public.admin_digest_claim(occ, k1, 'a', 300000, 3, ret, mar); + if not (res ->> 'claimed')::boolean or (res #>> '{row,fence}')::int <> 1 then raise warning 'first claim: %', res; failures := failures + 1; end if; + res := public.admin_digest_claim(occ, k1, 'b', 300000, 3, ret, mar); + if res ->> 'reason' is distinct from 'leased' then raise warning 'a live lease did not refuse: %', res; failures := failures + 1; end if; + if public.admin_digest_complete(occ, k1, 1, '{"kind":"unknown","reason":"timeout"}', 3) <> 'fenced_out' then + raise warning 'a completion without a mark was accepted'; failures := failures + 1; + end if; + if public.admin_digest_begin_send(occ, k1, 0, 150, ret, mar) <> 'fenced_out' then raise warning 'a stale fence marked'; failures := failures + 1; end if; + if public.admin_digest_begin_send(occ, k1, 1, 150, ret, mar) <> 'ok' then raise warning 'the current fence could not mark'; failures := failures + 1; end if; + if public.admin_digest_complete(occ, k1, 1, '{"kind":"accepted","messageId":""}', 3) <> 'ok' then raise warning 'complete refused'; failures := failures + 1; end if; + res := public.admin_digest_load(occ); + if (res -> 'deliveries') @> jsonb_build_array(jsonb_build_object('recipientKey', k1, 'status', 'accepted')) then + raise warning 'a success without a message id became a receipt'; failures := failures + 1; + end if; + + -- The retention cut-off (24 h − 1 h after the first mark) parks an ambiguous row at claim. + perform set_config('admin_digest.test_now', (pin + interval '23 hours')::text, true); + res := public.admin_digest_claim(occ, k1, 'c', 300000, 3, ret, mar); + if res ->> 'reason' is distinct from 'needs_reconciliation' then raise warning 'retention did not park: %', res; failures := failures + 1; end if; + + -- A lease that lapses after a mark makes the row ambiguous; too little lease refuses the mark. + res := public.admin_digest_claim(occ, k2, 'a', 300000, 3, ret, mar); + if public.admin_digest_begin_send(occ, k2, 1, 150, ret, mar) <> 'ok' then raise warning 'k2 mark'; failures := failures + 1; end if; + perform set_config('admin_digest.test_now', (pin + interval '23 hours 5 minutes 1 second')::text, true); + res := public.admin_digest_claim(occ, k2, 'b', 300000, 3, ret, mar); + if not (res #>> '{row,ambiguous}')::boolean or (res #>> '{row,fence}')::int <> 2 then raise warning 'a lapsed marked lease was not ambiguous: %', res; failures := failures + 1; end if; + perform set_config('admin_digest.test_now', (pin + interval '23 hours 10 minutes 0.9 seconds')::text, true); + if public.admin_digest_begin_send(occ, k2, 2, 150, ret, mar) <> 'lease_expired' then raise warning 'a nearly lapsed lease still marked'; failures := failures + 1; end if; + reset role; + + -- ── frozen means frozen, for everyone (these are the owner's own writes) ──── + refused := false; + begin update public.admin_digest_deliveries set payload_json = replace(payload_json, '2 new', '9 new') where occurrence_id = occ; + exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'a delivery''s bytes changed after freeze'; failures := failures + 1; end if; + refused := false; + begin delete from public.admin_digest_deliveries where occurrence_id = occ; exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'a delivery was deleted'; failures := failures + 1; end if; + refused := false; + begin update public.admin_digest_occurrences set window_end = window_end + interval '1 day' where occurrence_id = occ; + exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'an occurrence changed after freeze'; failures := failures + 1; end if; + refused := false; + begin update public.admin_digest_deliveries set status = 'pending' where occurrence_id = occ and recipient_key = k1; + exception when insufficient_privilege then refused := true; end; + if not refused then raise warning 'a settled delivery was reopened'; failures := failures + 1; end if; + + if failures > 0 then + raise exception 'the admin digest delivery store is not what 0471 promises: % finding(s)', failures; + end if; + raise notice 'OK: anon and authenticated reach nothing; service_role reaches the rows only through the functions; a plan is frozen once; one claim wins; stale fences, unmarked completions and short leases are refused; ambiguity parks at the retention cut-off; bytes, keys, occurrences and settled rows never change.'; +end +$probe$; + +rollback; diff --git a/lib/admin/digest-delivery-store.ts b/lib/admin/digest-delivery-store.ts new file mode 100644 index 000000000..31ca0145d --- /dev/null +++ b/lib/admin/digest-delivery-store.ts @@ -0,0 +1,154 @@ +// The PostgreSQL DigestDeliveryStore: a thin, strict mapping onto the five +// functions of migration 0471 (admin_digest_freeze, _load, _claim, _begin_send, +// _complete). Every rule lives in the database, evaluated under a row lock with +// the database clock; this file only translates arguments and checks the shape +// of what comes back, so a malformed answer fails loudly instead of being read +// as a claim or a receipt. +// +// NOT WIRED INTO ANY ROUTE. The transport is injected: `supabaseRpc(client)` +// for a service-role Supabase client (later), or the psql transport the +// disposable-database tests use (tests/helpers/digest-delivery-postgres.ts). +import { + payloadJsonOf, + type BeginSendPolicy, type ClaimPolicy, type ClaimRefusal, type DeliveryPayload, type DeliveryRow, type DeliveryStatus, + type DigestDeliveryStore, type FrozenOccurrence, type ProviderSendResult, type StoredOccurrence, +} from '@/lib/admin/digest-delivery'; + +/** Calls one database function with named arguments and resolves to its JSON result; rejects on any error. */ +export type RpcCall = (fn: string, args: Record) => Promise; + +type RpcClient = { rpc(fn: string, args: Record): PromiseLike<{ data: unknown; error: unknown }> }; + +/** The production transport: a SERVICE-ROLE client (the functions are not executable by anon or authenticated). */ +export function supabaseRpc(client: RpcClient): RpcCall { + return async (fn, args) => { + const { data, error } = await client.rpc(fn, args); + if (error) throw new Error(`digest-delivery store: ${fn} failed`, { cause: error }); + return data; + }; +} + +const STATUSES: readonly DeliveryStatus[] = ['pending', 'in_flight', 'failed', 'unknown', 'accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation']; +const REFUSALS: readonly string[] = ['leased', 'not_found', 'accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation']; +const BEGIN_ANSWERS = ['ok', 'fenced_out', 'lease_expired', 'retention_passed'] as const; + +function bad(what: string): never { throw new Error(`digest-delivery store: malformed ${what} from the database`); } +const obj = (v: unknown, what: string) => (v && typeof v === 'object' && !Array.isArray(v) ? v as Record : bad(what)); +const str = (v: unknown, what: string) => (typeof v === 'string' ? v : bad(what)); +const int = (v: unknown, what: string) => (typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 ? v : bad(what)); +/** Database timestamps come back as `2026-09-30T12:31:00.123456+00:00`; the engine speaks `toISOString()`. */ +const iso = (v: unknown, what: string) => { + const t = Date.parse(str(v, what)); + return Number.isFinite(t) ? new Date(t).toISOString() : bad(what); +}; +const isoOrNull = (v: unknown, what: string) => (v === null ? null : iso(v, what)); +const strOrNull = (v: unknown, what: string) => (v === null ? null : str(v, what)); + +function parsePayload(json: string): DeliveryPayload { + let p: unknown; + try { p = JSON.parse(json); } catch { return bad('payload_json'); } + const o = obj(p, 'payload'); + const payload = { from: str(o.from, 'payload.from'), to: str(o.to, 'payload.to'), subject: str(o.subject, 'payload.subject'), html: str(o.html, 'payload.html') }; + // The stored text must be exactly the engine's serialization of the same fields. + return payloadJsonOf(payload) === json ? payload : bad('payload_json (not canonical)'); +} + +export function parseDeliveryRow(v: unknown): DeliveryRow { + const r = obj(v, 'delivery'); + const status = str(r.status, 'status') as DeliveryStatus; + if (!STATUSES.includes(status)) bad('status'); + if (typeof r.ambiguous !== 'boolean') bad('ambiguous'); + const payloadJson = str(r.payloadJson, 'payloadJson'); + return { + occurrenceId: str(r.occurrenceId, 'occurrenceId'), + recipientKey: str(r.recipientKey, 'recipientKey'), + idempotencyKey: str(r.idempotencyKey, 'idempotencyKey'), + payload: parsePayload(payloadJson), + payloadJson, + payloadHash: str(r.payloadHash, 'payloadHash'), + status, + attempts: int(r.attempts, 'attempts'), + fence: int(r.fence, 'fence'), + leaseOwner: strOrNull(r.leaseOwner, 'leaseOwner'), + leaseExpiresAt: isoOrNull(r.leaseExpiresAt, 'leaseExpiresAt'), + sendStartedAt: isoOrNull(r.sendStartedAt, 'sendStartedAt'), + firstSendAt: isoOrNull(r.firstSendAt, 'firstSendAt'), + ambiguous: r.ambiguous, + providerMessageId: strOrNull(r.providerMessageId, 'providerMessageId'), + lastError: strOrNull(r.lastError, 'lastError'), + updatedAt: iso(r.updatedAt, 'updatedAt'), + }; +} + +function parseStored(v: unknown): StoredOccurrence { + const s = obj(v, 'occurrence'); + const o = obj(s.occurrence, 'occurrence'); + const w = obj(o.window, 'window'); + const keys = Array.isArray(o.recipientKeys) ? o.recipientKeys.map((k) => str(k, 'recipientKeys')) : bad('recipientKeys'); + const occurrence: FrozenOccurrence = { + occurrenceId: str(o.occurrenceId, 'occurrenceId'), + window: { start: iso(w.start, 'window.start'), end: iso(w.end, 'window.end') }, + recipientKeys: keys, + payloadHash: str(o.payloadHash, 'payloadHash'), + engineVersion: int(o.engineVersion, 'engineVersion'), + }; + const deliveries = Array.isArray(s.deliveries) ? s.deliveries.map(parseDeliveryRow) : bad('deliveries'); + if (deliveries.length !== keys.length || deliveries.some((d, i) => d.recipientKey !== keys[i] || d.occurrenceId !== occurrence.occurrenceId)) bad('deliveries (do not match the occurrence)'); + return { occurrence, deliveries }; +} + +const ms = (n: number, what: string) => (Number.isSafeInteger(Math.round(n)) && n >= 0 ? Math.round(n) : bad(what)); + +export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryStore { + return { + async freeze(occurrence, deliveries) { + const out = obj(await rpc('admin_digest_freeze', { + p_occurrence: occurrence, + // Only identity and bytes travel: the database starts every row pending, on its own clock. + p_deliveries: deliveries.map((d) => ({ + occurrenceId: d.occurrenceId, recipientKey: d.recipientKey, idempotencyKey: d.idempotencyKey, + payloadJson: d.payloadJson, payloadHash: d.payloadHash, + })), + }), 'freeze'); + if (typeof out.created !== 'boolean') bad('freeze.created'); + return { ...parseStored(out), created: out.created }; + }, + + async load(occurrenceId) { + const out = await rpc('admin_digest_load', { p_occurrence_id: occurrenceId }); + return out === null ? null : parseStored(out); + }, + + async claim(occurrenceId, recipientKey, owner, policy: ClaimPolicy) { + const out = obj(await rpc('admin_digest_claim', { + p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_owner: owner, + p_lease_ms: ms(policy.leaseMs, 'leaseMs'), p_max_attempts: policy.maxAttempts, + p_retention_ms: ms(policy.providerKeyRetentionMs, 'retention'), p_margin_ms: ms(policy.retentionSafetyMarginMs, 'margin'), + }), 'claim'); + if (out.claimed === true) { + const row = parseDeliveryRow(out.row); + if (row.status !== 'in_flight' || row.leaseOwner !== owner) bad('claim.row'); + return { claimed: true, row }; + } + const reason = str(out.reason, 'claim.reason'); + if (out.claimed !== false || !REFUSALS.includes(reason)) bad('claim'); + return { claimed: false, reason: reason as ClaimRefusal }; + }, + + async beginSend(occurrenceId, recipientKey, fence, policy: BeginSendPolicy) { + const out = await rpc('admin_digest_begin_send', { + p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fence, + p_min_lease_ms: ms(policy.minLeaseRemainingMs, 'minLease'), + p_retention_ms: ms(policy.providerKeyRetentionMs, 'retention'), p_margin_ms: ms(policy.retentionSafetyMarginMs, 'margin'), + }); + return (BEGIN_ANSWERS as readonly unknown[]).includes(out) ? out as typeof BEGIN_ANSWERS[number] : bad('begin_send'); + }, + + async complete(occurrenceId, recipientKey, fence, result: ProviderSendResult, maxAttempts) { + const out = await rpc('admin_digest_complete', { + p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fence, p_result: result, p_max_attempts: maxAttempts, + }); + return out === 'ok' || out === 'fenced_out' ? out : bad('complete'); + }, + }; +} diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts index fd3b30ab4..ca71bca90 100644 --- a/lib/admin/digest-delivery.ts +++ b/lib/admin/digest-delivery.ts @@ -282,9 +282,12 @@ export function classifyResendResponse(status: number, body: unknown): ProviderS return { kind: 'unknown', reason: 'unreadable_response' }; } -/** The complete rule: fenced. `null` means the caller's claim is no longer current and nothing may be written. */ +/** + * The complete rule: fenced, and only after a mark. `null` means nothing may be written: the claim is + * no longer current, or it never marked a send (so no provider answer can belong to it). + */ export function decideCompletion(row: DeliveryRow, fence: number, result: ProviderSendResult, now: Date, maxAttempts: number): DeliveryRow | null { - if (row.status !== 'in_flight' || row.fence !== fence) return null; + if (row.status !== 'in_flight' || row.fence !== fence || row.sendStartedAt === null) return null; const nowIso = now.toISOString(); const next: DeliveryRow = { ...row, leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, updatedAt: nowIso }; switch (result.kind) { diff --git a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql new file mode 100644 index 000000000..28a026901 --- /dev/null +++ b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql @@ -0,0 +1,419 @@ +-- Bubaly :: 0471 - an admin digest reaches each admin once +-- (API-96040DFB5635, JOB-EF2453D9F633; number reserved on #699) +-- ---------------------------------------------------------------------------- +-- The delivery store for lib/admin/digest-delivery.ts, the per-recipient +-- engine. NOTHING CALLS IT YET: no route or scheduler is wired to the engine, +-- and this migration is not applied to production (docs/PENDING_PROD_MIGRATIONS.md; +-- production's ledger stops at 0177, PROD-DB-0177). +-- +-- Why a ledger at all. /api/cron/admin-digest writes nothing, so it cannot tell +-- a first delivery from a repeat: two ticks, or a retry after one admin failed, +-- send everyone the digest again (tests/admin-digest-replay-contract.test.ts, +-- #677). The engine fixes that with one row per (occurrence, recipient) that +-- holds the recipient's stable idempotency key and the EXACT bytes that will be +-- sent, written before anything is sent, and a small state machine that is +-- advanced only by the four functions below. +-- +-- Each function is the SQL twin of one pure rule in lib/admin/digest-delivery.ts +-- and must stay equal to it (docs/admin-digest-delivery-contract.md §3; the +-- store contract suite runs against both): +-- +-- admin_digest_freeze store the plan once; a later plan never replaces it +-- admin_digest_claim decideClaim (row lock, lease, fence, retention) +-- admin_digest_begin_send decideBeginSend (fence, remaining lease, retention) +-- admin_digest_complete decideCompletion (fenced; only a message id is a receipt) +-- +-- Time. Every lease and retention decision reads admin_digest_now() AFTER the +-- row lock is taken. It is clock_timestamp(), not now(): now() is the +-- transaction's start and can be stale after a lock wait. One function so the +-- disposable test database can pin it; nothing in production redefines it. +-- +-- Frozen means frozen, and the database says so, not only the code: +-- - a delivery's key, bytes, hash and recipient never change after insert, +-- and they must agree with each other on insert (the hash is the sha256 of +-- the bytes, the recipient key is the sha256 of the bytes' `to`, the key is +-- the engine's formula); +-- - a terminal row (accepted, rejected, conflict, exhausted, +-- needs_reconciliation) never leaves its state, attempts and the fence +-- never go down, and "ambiguous" is never cleared; +-- - rows are never deleted: a deleted receipt is a resend. +-- +-- Access: the service role, through the functions only. RLS on with no +-- policies; every table privilege revoked from anon, authenticated AND +-- service_role; EXECUTE on the five API functions granted to service_role +-- alone. No client ever reads a digest's recipients or body, and no direct +-- write can skip a rule. +-- +-- Probe: docs/audit/an-admin-digest-reaches-each-admin-once-check.sql. + +create table if not exists public.admin_digest_occurrences ( + occurrence_id text primary key check (length(occurrence_id) between 1 and 200 and occurrence_id ~ '^[!-~]([ -~]*[!-~])?$'), + window_start timestamptz not null, + window_end timestamptz not null, + recipient_keys text[] not null check (cardinality(recipient_keys) between 1 and 200), + payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), + engine_version integer not null check (engine_version >= 1), + frozen_at timestamptz not null, + check (window_end > window_start) +); + +create table if not exists public.admin_digest_deliveries ( + occurrence_id text not null references public.admin_digest_occurrences (occurrence_id) on delete restrict, + recipient_key text not null check (recipient_key ~ '^[0-9a-f]{64}$'), + idempotency_key text not null unique check (length(idempotency_key) between 1 and 256 and idempotency_key ~ '^[!-~]([ -~]*[!-~])?$'), + payload_json text not null check (length(payload_json) between 2 and 600000), + payload_hash text not null check (payload_hash ~ '^[0-9a-f]{64}$'), + status text not null check (status in ('pending', 'in_flight', 'failed', 'unknown', 'accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation')), + attempts integer not null default 0 check (attempts >= 0), + fence bigint not null default 0 check (fence >= 0), + lease_owner text check (lease_owner is null or length(lease_owner) between 1 and 200), + lease_expires_at timestamptz, + send_started_at timestamptz, + first_send_at timestamptz, + ambiguous boolean not null default false, + provider_message_id text check (provider_message_id is null or length(provider_message_id) between 1 and 200), + last_error text check (last_error is null or length(last_error) <= 200), + updated_at timestamptz not null, + primary key (occurrence_id, recipient_key), + check ((status = 'in_flight') = (lease_owner is not null and lease_expires_at is not null)), + check ((status = 'accepted') = (provider_message_id is not null)), + check (send_started_at is null or status = 'in_flight'), + check (not ambiguous or first_send_at is not null) +); + +alter table public.admin_digest_occurrences enable row level security; +alter table public.admin_digest_deliveries enable row level security; +-- Narrow even for the service role: it reaches these rows only through the four +-- functions below, so no direct write can skip a rule. +revoke all on public.admin_digest_occurrences from anon, authenticated, service_role; +revoke all on public.admin_digest_deliveries from anon, authenticated, service_role; + +-- ── the one clock ──────────────────────────────────────────────────────────── +create or replace function public.admin_digest_now() +returns timestamptz +language sql +volatile +set search_path = public, pg_temp +as $$ select clock_timestamp() $$; + +-- ── frozen identity and a transition-only ledger ──────────────────────────── +create or replace function public.admin_digest_delivery_guard() +returns trigger +language plpgsql +set search_path = public, pg_temp +as $$ +declare + to_addr text; +begin + if tg_op = 'DELETE' then + raise exception 'admin_digest_deliveries rows are never deleted: a deleted receipt is a resend' using errcode = '42501'; + end if; + if tg_op = 'INSERT' then + begin + to_addr := (new.payload_json::jsonb) ->> 'to'; + exception when others then + raise exception 'admin_digest_deliveries.payload_json must be a JSON object' using errcode = '22023'; + end; + if new.payload_hash <> encode(sha256(convert_to(new.payload_json, 'UTF8')), 'hex') then + raise exception 'admin_digest_deliveries.payload_hash is not the sha256 of payload_json' using errcode = '22023'; + end if; + if to_addr is null or new.recipient_key <> encode(sha256(convert_to(to_addr, 'UTF8')), 'hex') then + raise exception 'admin_digest_deliveries.recipient_key is not the sha256 of the payload''s recipient' using errcode = '22023'; + end if; + if new.idempotency_key <> 'bubaly/admin-digest/v1/' || left(encode(sha256(convert_to(new.occurrence_id, 'UTF8')), 'hex'), 32) || '/' || left(new.recipient_key, 32) then + raise exception 'admin_digest_deliveries.idempotency_key is not the key for (occurrence, recipient)' using errcode = '22023'; + end if; + if new.status <> 'pending' or new.attempts <> 0 or new.fence <> 0 or new.ambiguous or new.first_send_at is not null + or new.provider_message_id is not null then + raise exception 'an admin digest delivery starts pending, unattempted' using errcode = '22023'; + end if; + return new; + end if; + -- UPDATE + if new.occurrence_id is distinct from old.occurrence_id or new.recipient_key is distinct from old.recipient_key + or new.idempotency_key is distinct from old.idempotency_key or new.payload_json is distinct from old.payload_json + or new.payload_hash is distinct from old.payload_hash then + raise exception 'a frozen admin digest delivery keeps its recipient, key and bytes' using errcode = '42501'; + end if; + if old.status in ('accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation') and new.status is distinct from old.status then + raise exception 'a settled admin digest delivery (%) stays settled', old.status using errcode = '42501'; + end if; + if new.attempts < old.attempts or new.fence < old.fence or (old.ambiguous and not new.ambiguous) + or (old.first_send_at is not null and new.first_send_at is distinct from old.first_send_at) then + raise exception 'attempts, fence, ambiguity and the retention anchor only move forward' using errcode = '42501'; + end if; + return new; +end $$; + +drop trigger if exists admin_digest_delivery_guard on public.admin_digest_deliveries; +create trigger admin_digest_delivery_guard + before insert or update or delete on public.admin_digest_deliveries + for each row execute function public.admin_digest_delivery_guard(); + +create or replace function public.admin_digest_occurrence_guard() +returns trigger +language plpgsql +set search_path = public, pg_temp +as $$ +begin + raise exception 'a frozen admin digest occurrence is never changed or deleted' using errcode = '42501'; +end $$; + +drop trigger if exists admin_digest_occurrence_guard on public.admin_digest_occurrences; +create trigger admin_digest_occurrence_guard + before update or delete on public.admin_digest_occurrences + for each row execute function public.admin_digest_occurrence_guard(); + +-- ── JSON shapes the adapter reads (camelCase, as lib/admin/digest-delivery.ts names them) ── +create or replace function public.admin_digest_delivery_json(r public.admin_digest_deliveries) +returns jsonb +language sql +stable +set search_path = public, pg_temp +as $$ + select jsonb_build_object( + 'occurrenceId', r.occurrence_id, 'recipientKey', r.recipient_key, 'idempotencyKey', r.idempotency_key, + 'payloadJson', r.payload_json, 'payloadHash', r.payload_hash, 'status', r.status, 'attempts', r.attempts, + 'fence', r.fence, 'leaseOwner', r.lease_owner, 'leaseExpiresAt', r.lease_expires_at, + 'sendStartedAt', r.send_started_at, 'firstSendAt', r.first_send_at, 'ambiguous', r.ambiguous, + 'providerMessageId', r.provider_message_id, 'lastError', r.last_error, 'updatedAt', r.updated_at) +$$; + +create or replace function public.admin_digest_load(p_occurrence_id text) +returns jsonb +language sql +stable +security definer +set search_path = public, pg_temp +as $$ + select jsonb_build_object( + 'occurrence', jsonb_build_object( + 'occurrenceId', o.occurrence_id, + 'window', jsonb_build_object('start', o.window_start, 'end', o.window_end), + 'recipientKeys', to_jsonb(o.recipient_keys), 'payloadHash', o.payload_hash, 'engineVersion', o.engine_version), + 'deliveries', coalesce(( + select jsonb_agg(public.admin_digest_delivery_json(d) order by d.recipient_key) + from public.admin_digest_deliveries d where d.occurrence_id = o.occurrence_id), '[]'::jsonb)) + from public.admin_digest_occurrences o + where o.occurrence_id = p_occurrence_id +$$; + +-- ── freeze: the plan is stored once; a later plan never replaces it ───────── +create or replace function public.admin_digest_freeze(p_occurrence jsonb, p_deliveries jsonb) +returns jsonb +language plpgsql +volatile +security definer +set search_path = public, pg_temp +as $$ +declare + v_id text := p_occurrence ->> 'occurrenceId'; + v_now timestamptz := public.admin_digest_now(); + v_keys text[]; + won int; + stored jsonb; +begin + if v_id is null or jsonb_typeof(p_deliveries) <> 'array' then + raise exception 'admin_digest_freeze: an occurrence id and a deliveries array are required' using errcode = '22023'; + end if; + select coalesce(array_agg(k order by k), '{}') into v_keys from jsonb_array_elements_text(p_occurrence -> 'recipientKeys') k; + insert into public.admin_digest_occurrences (occurrence_id, window_start, window_end, recipient_keys, payload_hash, engine_version, frozen_at) + values (v_id, (p_occurrence #>> '{window,start}')::timestamptz, (p_occurrence #>> '{window,end}')::timestamptz, + v_keys, p_occurrence ->> 'payloadHash', (p_occurrence ->> 'engineVersion')::int, v_now) + on conflict (occurrence_id) do nothing; + get diagnostics won = row_count; + if won = 1 then + insert into public.admin_digest_deliveries (occurrence_id, recipient_key, idempotency_key, payload_json, payload_hash, status, updated_at) + select v_id, d ->> 'recipientKey', d ->> 'idempotencyKey', d ->> 'payloadJson', d ->> 'payloadHash', 'pending', v_now + from jsonb_array_elements(p_deliveries) d + where d ->> 'occurrenceId' = v_id; + -- All or nothing: exactly one delivery per frozen recipient key, for this occurrence. + if (select count(*) from public.admin_digest_deliveries where occurrence_id = v_id) <> cardinality(v_keys) + or jsonb_array_length(p_deliveries) <> cardinality(v_keys) + or exists (select 1 from public.admin_digest_deliveries d where d.occurrence_id = v_id and not d.recipient_key = any (v_keys)) then + raise exception 'admin_digest_freeze: deliveries do not match the occurrence' using errcode = '22023'; + end if; + end if; + stored := public.admin_digest_load(v_id); + return stored || jsonb_build_object('created', won = 1); +end $$; + +-- ── claim: decideClaim, evaluated after the row lock with the database clock ── +create or replace function public.admin_digest_claim( + p_occurrence_id text, p_recipient_key text, p_owner text, + p_lease_ms integer, p_max_attempts integer, p_retention_ms bigint, p_margin_ms bigint +) returns jsonb +language plpgsql +volatile +security definer +set search_path = public, pg_temp +as $$ +declare + r public.admin_digest_deliveries%rowtype; + v_now timestamptz; +begin + if p_owner is null or length(p_owner) not between 1 and 200 or p_lease_ms is null or p_lease_ms < 1 + or p_max_attempts is null or p_max_attempts < 1 or p_retention_ms is null or p_margin_ms is null + or p_margin_ms < 1 or p_margin_ms >= p_retention_ms then + raise exception 'admin_digest_claim: bad owner, lease, attempts or retention' using errcode = '22023'; + end if; + select * into r from public.admin_digest_deliveries + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key + for update; + if not found then + return jsonb_build_object('claimed', false, 'reason', 'not_found'); + end if; + v_now := public.admin_digest_now(); -- after the lock: a lock wait must not age the decision + if r.status in ('accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation') then + return jsonb_build_object('claimed', false, 'reason', r.status); + end if; + if r.status = 'in_flight' then + if r.lease_expires_at is not null and r.lease_expires_at > v_now then + return jsonb_build_object('claimed', false, 'reason', 'leased'); + end if; + -- The worker died or stalled. With a beginSend mark it may have reached the provider. + if r.send_started_at is not null then + r.ambiguous := true; + r.status := 'unknown'; + r.last_error := 'lease_expired_after_send_started'; + else + r.status := case when r.ambiguous then 'unknown' when r.attempts > 1 then 'failed' else 'pending' end; + r.last_error := 'lease_expired_before_send'; + end if; + r.lease_owner := null; + r.lease_expires_at := null; + r.send_started_at := null; + end if; + if r.attempts >= p_max_attempts + or (r.ambiguous and (r.first_send_at is null + or v_now >= r.first_send_at + make_interval(secs => (p_retention_ms - p_margin_ms) / 1000.0))) then + -- Parked: attempts used up, or an ambiguous row past its retention cut-off. + r.status := case when r.ambiguous then 'needs_reconciliation' else 'exhausted' end; + update public.admin_digest_deliveries set + status = r.status, lease_owner = null, lease_expires_at = null, send_started_at = null, + ambiguous = r.ambiguous, last_error = r.last_error, updated_at = v_now + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; + return jsonb_build_object('claimed', false, 'reason', r.status); + end if; + update public.admin_digest_deliveries set + status = 'in_flight', attempts = r.attempts + 1, fence = r.fence + 1, lease_owner = p_owner, + lease_expires_at = v_now + make_interval(secs => p_lease_ms / 1000.0), send_started_at = null, + ambiguous = r.ambiguous, last_error = r.last_error, updated_at = v_now + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key + returning * into r; + return jsonb_build_object('claimed', true, 'row', public.admin_digest_delivery_json(r)); +end $$; + +-- ── beginSend: decideBeginSend, the last check before the provider call ───── +create or replace function public.admin_digest_begin_send( + p_occurrence_id text, p_recipient_key text, p_fence bigint, + p_min_lease_ms integer, p_retention_ms bigint, p_margin_ms bigint +) returns text +language plpgsql +volatile +security definer +set search_path = public, pg_temp +as $$ +declare + r public.admin_digest_deliveries%rowtype; + v_now timestamptz; +begin + if p_min_lease_ms is null or p_min_lease_ms < 0 or p_retention_ms is null or p_margin_ms is null + or p_margin_ms < 1 or p_margin_ms >= p_retention_ms then + raise exception 'admin_digest_begin_send: bad lease or retention' using errcode = '22023'; + end if; + select * into r from public.admin_digest_deliveries + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key + for update; + if not found or r.status <> 'in_flight' or r.fence <> p_fence then + return 'fenced_out'; + end if; + v_now := public.admin_digest_now(); + if r.lease_expires_at is null or r.lease_expires_at - v_now <= make_interval(secs => p_min_lease_ms / 1000.0) then + return 'lease_expired'; -- left for the next claim; nothing is written + end if; + if r.ambiguous and (r.first_send_at is null + or v_now >= r.first_send_at + make_interval(secs => (p_retention_ms - p_margin_ms) / 1000.0)) then + update public.admin_digest_deliveries set + status = 'needs_reconciliation', lease_owner = null, lease_expires_at = null, send_started_at = null, updated_at = v_now + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; + return 'retention_passed'; + end if; + update public.admin_digest_deliveries set + send_started_at = v_now, first_send_at = coalesce(r.first_send_at, v_now), updated_at = v_now + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; + return 'ok'; +end $$; + +-- ── complete: decideCompletion, fenced, after a mark; only a message id is a receipt ── +create or replace function public.admin_digest_complete( + p_occurrence_id text, p_recipient_key text, p_fence bigint, p_result jsonb, p_max_attempts integer +) returns text +language plpgsql +volatile +security definer +set search_path = public, pg_temp +as $$ +declare + r public.admin_digest_deliveries%rowtype; + v_now timestamptz; + v_kind text := p_result ->> 'kind'; + v_mid text := p_result ->> 'messageId'; +begin + if p_max_attempts is null or p_max_attempts < 1 + or v_kind is null or v_kind not in ('accepted', 'rejected', 'payload_conflict', 'in_progress', 'unknown') then + raise exception 'admin_digest_complete: bad result or attempts' using errcode = '22023'; + end if; + select * into r from public.admin_digest_deliveries + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key + for update; + -- Fenced, and only after a mark: a claim that never marked a send owns no provider answer. + if not found or r.status <> 'in_flight' or r.fence <> p_fence or r.send_started_at is null then + return 'fenced_out'; + end if; + v_now := public.admin_digest_now(); + r.lease_owner := null; + r.lease_expires_at := null; + r.send_started_at := null; + if v_kind = 'accepted' and coalesce(v_mid, '') <> '' then + r.status := 'accepted'; r.provider_message_id := left(v_mid, 200); r.last_error := null; + elsif v_kind = 'accepted' then + r.ambiguous := true; r.status := 'unknown'; r.last_error := 'outcome_unknown:unreadable_response'; + elsif v_kind = 'payload_conflict' then + r.status := 'conflict'; r.last_error := 'provider_payload_conflict'; + elsif v_kind = 'rejected' then + r.last_error := left('provider_rejected:' || coalesce(p_result ->> 'httpStatus', '') || ':' || coalesce(p_result ->> 'code', ''), 200); + if coalesce((p_result ->> 'retryable')::boolean, false) then + r.status := case when r.ambiguous then 'unknown' else 'failed' end; + else + r.status := case when r.ambiguous then 'needs_reconciliation' else 'rejected' end; + end if; + elsif v_kind = 'in_progress' then + r.ambiguous := true; r.status := 'unknown'; r.last_error := 'provider_in_progress'; + else + r.ambiguous := true; r.status := 'unknown'; r.last_error := left('outcome_unknown:' || coalesce(p_result ->> 'reason', ''), 200); + end if; + if r.status in ('failed', 'unknown') and r.attempts >= p_max_attempts then + r.status := case when r.ambiguous then 'needs_reconciliation' else 'exhausted' end; + end if; + update public.admin_digest_deliveries set + status = r.status, lease_owner = null, lease_expires_at = null, send_started_at = null, + ambiguous = r.ambiguous, provider_message_id = r.provider_message_id, last_error = r.last_error, updated_at = v_now + where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; + return 'ok'; +end $$; + +revoke all on function public.admin_digest_now() from public, anon, authenticated, service_role; +revoke all on function public.admin_digest_delivery_json(public.admin_digest_deliveries) from public, anon, authenticated, service_role; +revoke all on function public.admin_digest_load(text) from public, anon, authenticated; +revoke all on function public.admin_digest_freeze(jsonb, jsonb) from public, anon, authenticated; +revoke all on function public.admin_digest_claim(text, text, text, integer, integer, bigint, bigint) from public, anon, authenticated; +revoke all on function public.admin_digest_begin_send(text, text, bigint, integer, bigint, bigint) from public, anon, authenticated; +revoke all on function public.admin_digest_complete(text, text, bigint, jsonb, integer) from public, anon, authenticated; +revoke all on function public.admin_digest_delivery_guard() from public, anon, authenticated, service_role; +revoke all on function public.admin_digest_occurrence_guard() from public, anon, authenticated, service_role; +grant execute on function public.admin_digest_load(text) to service_role; +grant execute on function public.admin_digest_freeze(jsonb, jsonb) to service_role; +grant execute on function public.admin_digest_claim(text, text, text, integer, integer, bigint, bigint) to service_role; +grant execute on function public.admin_digest_begin_send(text, text, bigint, integer, bigint, bigint) to service_role; +grant execute on function public.admin_digest_complete(text, text, bigint, jsonb, integer) to service_role; diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts new file mode 100644 index 000000000..46fa3e1fb --- /dev/null +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -0,0 +1,342 @@ +// Migration 0471 and the PostgreSQL DigestDeliveryStore. +// +// The first block always runs: the adapter's strictness, with a fake transport. +// Everything else needs a real database. It is opt-in (DIGEST_DELIVERY_PG=1, +// see tests/helpers/digest-delivery-postgres.ts) and SKIPPED otherwise; CI's +// Database job covers the same functions through +// docs/audit/an-admin-digest-reaches-each-admin-once-check.sql. +// +// Synthetic data only. No provider is called: the engine tests use the fake +// provider that follows Resend's documented key semantics. +import { execSync } from 'node:child_process'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { + RESEND_KEY_RETENTION_MS, deliverDigestOccurrence, freezePlan, recipientKeyOf, resumeDigestOccurrence, + type BeginSendPolicy, type ClaimPolicy, type DigestDeliveryStore, type EngineConfig, type ProviderSendResult, +} from '@/lib/admin/digest-delivery'; +import { createPostgresDigestDeliveryStore, parseDeliveryRow, supabaseRpc, type RpcCall } from '@/lib/admin/digest-delivery-store'; +import { FakeClock, FakeResendProvider, HOUR, MINUTE, MemoryDigestDeliveryStore, deferred, never } from './helpers/digest-delivery-fakes'; +import { contractPlan, describeDigestDeliveryStoreContract } from './helpers/digest-delivery-store-contract'; +import { createPgFixture, pgFixtureEnabled, psql, type PgFixture } from './helpers/digest-delivery-postgres'; + +const T0 = '2026-09-30T12:31:00.000Z'; +const OCC = 'admin-digest:2026-09-30T12:30:00.000Z'; +const ONE = 'admin-one@example.test'; +const TWO = 'admin-two@example.test'; +const K1 = recipientKeyOf(ONE); +const CLAIM: ClaimPolicy = { leaseMs: 5 * MINUTE, maxAttempts: 4, providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, retentionSafetyMarginMs: HOUR }; +const BEGIN: BeginSendPolicy = { minLeaseRemainingMs: 150, providerKeyRetentionMs: RESEND_KEY_RETENTION_MS, retentionSafetyMarginMs: HOUR }; +const CONFIG: EngineConfig = { ...CLAIM, maxAttempts: 5, sendTimeoutMs: 2_000 }; + +// ── the adapter, strict about what it reads (always runs) ────────────────── + +describe('PostgreSQL adapter: strict mapping (fake transport)', () => { + const row = (over: Record = {}) => { + const d = freezePlan(contractPlan({ recipients: [ONE] }), new Date(T0)).deliveries[0]; + return { + occurrenceId: d.occurrenceId, recipientKey: d.recipientKey, idempotencyKey: d.idempotencyKey, payloadJson: d.payloadJson, + payloadHash: d.payloadHash, status: 'in_flight', attempts: 1, fence: 1, leaseOwner: 'w', leaseExpiresAt: '2026-09-30T12:36:00.123456+00:00', + sendStartedAt: null, firstSendAt: null, ambiguous: false, providerMessageId: null, lastError: null, updatedAt: '2026-09-30T12:31:00+00:00', ...over, + }; + }; + const storeAnswering = (answer: unknown) => { + const calls: [string, Record][] = []; + const rpc: RpcCall = async (fn, args) => { calls.push([fn, args]); return answer; }; + return { store: createPostgresDigestDeliveryStore(rpc), calls }; + }; + + it('reads database timestamps as the engine writes them', () => { + expect(parseDeliveryRow(row())).toMatchObject({ leaseExpiresAt: '2026-09-30T12:36:00.123Z', updatedAt: '2026-09-30T12:31:00.000Z' }); + }); + + it.each<[string, Record]>([ + ['an unknown status', { status: 'sent' }], + ['a fence that is not a count', { fence: -1 }], + ['a non-boolean ambiguity', { ambiguous: 'no' }], + ['bytes that are not the canonical serialization', { payloadJson: JSON.stringify({ to: ONE, from: 'x', subject: 's', html: 'h' }) }], + ['a timestamp that is not one', { leaseExpiresAt: 'soon' }], + ])('refuses a row with %s', (_n, over) => { + expect(() => parseDeliveryRow(row(over))).toThrow(/malformed/); + }); + + it('a claim is believed only if the row came back in flight under OUR name', async () => { + await expect(storeAnswering({ claimed: true, row: row({ leaseOwner: 'someone-else' }) }).store.claim(OCC, K1, 'w', CLAIM)).rejects.toThrow(/malformed/); + await expect(storeAnswering({ claimed: true, row: row({ status: 'pending', leaseOwner: null, leaseExpiresAt: null }) }).store.claim(OCC, K1, 'w', CLAIM)).rejects.toThrow(/malformed/); + await expect(storeAnswering({ claimed: false, reason: 'maybe' }).store.claim(OCC, K1, 'w', CLAIM)).rejects.toThrow(/malformed/); + }); + + it('beginSend and complete accept only their documented answers', async () => { + await expect(storeAnswering('yes').store.beginSend(OCC, K1, 1, BEGIN)).rejects.toThrow(/malformed/); + await expect(storeAnswering(null).store.complete(OCC, K1, 1, { kind: 'accepted', messageId: 'm' }, 3)).rejects.toThrow(/malformed/); + for (const a of ['ok', 'fenced_out', 'lease_expired', 'retention_passed']) expect(await storeAnswering(a).store.beginSend(OCC, K1, 1, BEGIN)).toBe(a); + }); + + it('freeze sends only identity and bytes, never a state', async () => { + const f = freezePlan(contractPlan({ recipients: [ONE] }), new Date(T0)); + const { store, calls } = storeAnswering({ created: true, occurrence: { ...f.occurrence }, deliveries: [row({ status: 'pending', attempts: 0, fence: 0, leaseOwner: null, leaseExpiresAt: null })] }); + await store.freeze(f.occurrence, f.deliveries.map((d) => ({ ...d, status: 'accepted' as const, providerMessageId: 'forged' }))); + expect(calls[0][0]).toBe('admin_digest_freeze'); + expect(Object.keys((calls[0][1].p_deliveries as object[])[0]).sort()).toEqual(['idempotencyKey', 'occurrenceId', 'payloadHash', 'payloadJson', 'recipientKey']); + }); + + it('the Supabase transport throws on an error instead of returning it', async () => { + const rpc = supabaseRpc({ rpc: async () => ({ data: null, error: { message: 'permission denied for function admin_digest_claim' } }) }); + await expect(rpc('admin_digest_claim', {})).rejects.toThrow('digest-delivery store: admin_digest_claim failed'); + const ok = supabaseRpc({ rpc: async () => ({ data: 'ok', error: null }) }); + expect(await ok('admin_digest_complete', {})).toBe('ok'); + }); +}); + +// ── against a real, disposable PostgreSQL database ───────────────────────── + +describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', () => { + let fx: PgFixture; + beforeAll(async () => { fx = await createPgFixture(); }, 60_000); + afterAll(async () => { await fx?.drop(); }, 60_000); + + /** A PostgreSQL store on a freshly emptied database, whose clock follows `clock`. */ + const pgStore = (clock?: FakeClock): DigestDeliveryStore => { + const ready = fx.reset(); + const rpc = fx.rpc(clock?.now); + return createPostgresDigestDeliveryStore(async (fn, args) => { await ready; return rpc(fn, args); }); + }; + /** A second store on the SAME data: a new process, new connections. */ + const reconnect = (clock?: FakeClock): DigestDeliveryStore => createPostgresDigestDeliveryStore(fx.rpc(clock?.now)); + + /** Wait until the backend named `app` is inside `select pg_sleep(...)`, i.e. past every statement before it. */ + const sleepingIn = (app: string) => vi.waitFor(async () => expect(await fx.sql( + `select count(*) from pg_stat_activity where application_name = '${app}' and state = 'active' and query like 'select pg_sleep%';`)).toBe('1'), + { timeout: 5_000, interval: 20 }); + + describeDigestDeliveryStoreContract('postgres store (0471)', (clock) => pgStore(clock)); + + describe('the SQL rules are the TypeScript rules (differential)', () => { + // Every step is applied to the in-memory store (the pure TypeScript rules) and to + // PostgreSQL (the four functions); answers and full state must agree after each one. + const mulberry = (seed: number) => () => { seed = (seed + 0x6D2B79F5) | 0; let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; + const RESULTS: ProviderSendResult[] = [ + { kind: 'accepted', messageId: 'm-1' }, { kind: 'accepted', messageId: '' }, { kind: 'payload_conflict' }, { kind: 'in_progress' }, + { kind: 'rejected', httpStatus: 429, code: 'rate_limit_exceeded', retryable: true }, { kind: 'rejected', httpStatus: 422, code: 'validation_error', retryable: false }, + { kind: 'unknown', reason: 'timeout' }, { kind: 'unknown', reason: 'server_error' }, + ]; + const STEPS = [0, 1_000, 4 * MINUTE, 5 * MINUTE, 6 * MINUTE, 2 * HOUR, 11 * HOUR, 23 * HOUR]; + + it.each([1, 2, 3])('seed %i: 160 random steps agree exactly', async (seed) => { + const rand = mulberry(seed); + const pick = (xs: readonly T[]) => xs[Math.floor(rand() * xs.length)]; + const clock = new FakeClock(T0); + const mem = new MemoryDigestDeliveryStore(clock.now); + const pg = pgStore(clock); + const frozen = freezePlan(contractPlan(), clock.now()); + await mem.freeze(frozen.occurrence, frozen.deliveries); + await pg.freeze(frozen.occurrence, frozen.deliveries); + const keys = frozen.occurrence.recipientKeys; + for (let i = 0; i < 160; i += 1) { + const key = pick(keys); + const current = mem.row(OCC, key)!.fence; + const fence = rand() < 0.8 ? current : Math.max(0, current - 1); + const op = pick(['claim', 'claim', 'begin', 'begin', 'complete', 'complete', 'advance'] as const); + let a: unknown; let b: unknown; + if (op === 'claim') { const owner = pick(['A', 'B']); a = await mem.claim(OCC, key, owner, CLAIM); b = await pg.claim(OCC, key, owner, CLAIM); } + else if (op === 'begin') { a = await mem.beginSend(OCC, key, fence, BEGIN); b = await pg.beginSend(OCC, key, fence, BEGIN); } + else if (op === 'complete') { const r = pick(RESULTS); a = await mem.complete(OCC, key, fence, r, CLAIM.maxAttempts); b = await pg.complete(OCC, key, fence, r, CLAIM.maxAttempts); } + else clock.advance(pick(STEPS)); + expect(b, `step ${i} ${op} ${key.slice(0, 6)} fence ${fence}`).toEqual(a); + expect(await pg.load(OCC), `state after step ${i}`).toEqual(await mem.load(OCC)); + } + }, 240_000); + }); + + describe('concurrency, crashes and restarts', () => { + it('forty concurrent claims of one delivery from forty connections: exactly one wins', async () => { + const clock = new FakeClock(T0); + const store = pgStore(clock); + const f = freezePlan(contractPlan(), clock.now()); + await store.freeze(f.occurrence, f.deliveries); + const results = await Promise.all(Array.from({ length: 40 }, (_, i) => reconnect(clock).claim(OCC, K1, `w${i}`, CLAIM))); + expect(results.filter((r) => r.claimed)).toHaveLength(1); + expect(results.filter((r) => !r.claimed && r.reason === 'leased')).toHaveLength(39); + expect((await store.load(OCC))!.deliveries.find((d) => d.recipientKey === K1)).toMatchObject({ fence: 1, attempts: 1, status: 'in_flight' }); + }, 60_000); + + it('database time is read AFTER the row lock: a claim that waited past a lease sees it expired (now() would not)', async () => { + const store = pgStore(); // the real clock + const f = freezePlan(contractPlan({ recipients: [ONE] }), new Date()); + await store.freeze(f.occurrence, f.deliveries); + const where = `occurrence_id = '${OCC}' and recipient_key = '${K1}'`; + const holdThenClaim = async () => { + await fx.sql(`update public.admin_digest_deliveries set lease_expires_at = clock_timestamp() + interval '800 milliseconds' where ${where};`); + const holder = fx.sql(`set application_name = 'lock_holder'; begin; select 1 from public.admin_digest_deliveries where ${where} for update; select pg_sleep(1.6); commit;`); + await sleepingIn('lock_holder'); // it holds the row lock + const claimed = await reconnect().claim(OCC, K1, 'waiter', CLAIM); // blocks until the holder commits + await holder; + return claimed; + }; + expect((await reconnect().claim(OCC, K1, 'first', CLAIM)).claimed).toBe(true); + const afterLock = await holdThenClaim(); + expect(afterLock).toMatchObject({ claimed: true, row: { fence: 2, leaseOwner: 'waiter' } }); + // The counterfactual, on this throwaway database only: with transaction-start time the same wait says "leased". + const original = await fx.sql(`select pg_get_functiondef('public.admin_digest_now()'::regprocedure);`); + await fx.sql(`create or replace function public.admin_digest_now() returns timestamptz language sql volatile set search_path = public, pg_temp as $$ select now() $$;`); + try { + expect(await holdThenClaim()).toEqual({ claimed: false, reason: 'leased' }); + } finally { + await fx.sql(`${original};`); + } + }, 30_000); + + it('a claim whose backend dies before commit leaves nothing behind', async () => { + const clock = new FakeClock(T0); + const store = pgStore(clock); + const f = freezePlan(contractPlan({ recipients: [ONE] }), clock.now()); + await store.freeze(f.occurrence, f.deliveries); + const dying = psql(fx.db, `set application_name = 'doomed_claim'; set admin_digest.test_now = '${T0}'; set role service_role; begin; + select public.admin_digest_claim('${OCC}', '${K1}', 'doomed', 300000, 4, ${RESEND_KEY_RETENTION_MS}, ${HOUR}); select pg_sleep(30); commit;`).catch((e: Error) => e); + await sleepingIn('doomed_claim'); // the claim has run inside the open transaction + await fx.sql(`select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'doomed_claim';`); + expect(await dying).toBeInstanceOf(Error); + expect((await reconnect(clock).load(OCC))!.deliveries[0]).toMatchObject({ status: 'pending', fence: 0, attempts: 0 }); + }, 30_000); + + it('a receipt whose backend dies before commit is not durable; the next claim treats the send as possibly made', async () => { + const clock = new FakeClock(T0); + const store = pgStore(clock); + const f = freezePlan(contractPlan({ recipients: [ONE] }), clock.now()); + await store.freeze(f.occurrence, f.deliveries); + const c = await store.claim(OCC, K1, 'w', CLAIM); + if (!c.claimed) throw new Error('setup'); + expect(await store.beginSend(OCC, K1, c.row.fence, BEGIN)).toBe('ok'); + const dying = psql(fx.db, `set application_name = 'doomed_receipt'; set admin_digest.test_now = '${T0}'; set role service_role; begin; + select public.admin_digest_complete('${OCC}', '${K1}', 1, '{"kind":"accepted","messageId":"m-lost"}'::jsonb, 4); select pg_sleep(30); commit;`).catch((e: Error) => e); + await sleepingIn('doomed_receipt'); // the receipt has been written inside the open transaction + await fx.sql(`select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'doomed_receipt';`); + await dying; + expect((await reconnect(clock).load(OCC))!.deliveries[0]).toMatchObject({ status: 'in_flight', providerMessageId: null }); + clock.advance(5 * MINUTE); + const again = await reconnect(clock).claim(OCC, K1, 'next', CLAIM); + expect(again).toMatchObject({ claimed: true, row: { ambiguous: true, fence: 2, firstSendAt: T0 } }); + }, 30_000); + + it.skipIf(!process.env.DIGEST_DELIVERY_PG_RESTART_CMD)('a committed receipt survives an immediate (crash) restart of the server', async () => { + const clock = new FakeClock(T0); + const store = pgStore(clock); + const f = freezePlan(contractPlan({ recipients: [ONE] }), clock.now()); + await store.freeze(f.occurrence, f.deliveries); + const c = await store.claim(OCC, K1, 'w', CLAIM); + if (!c.claimed) throw new Error('setup'); + await store.beginSend(OCC, K1, c.row.fence, BEGIN); + expect(await store.complete(OCC, K1, c.row.fence, { kind: 'accepted', messageId: 'm-durable' }, 4)).toBe('ok'); + execSync(process.env.DIGEST_DELIVERY_PG_RESTART_CMD!, { stdio: 'ignore' }); + await vi.waitFor(async () => expect(await fx.sql('select 1;')).toBe('1'), { timeout: 30_000, interval: 250 }); + expect((await reconnect(clock).load(OCC))!.deliveries[0]).toMatchObject({ status: 'accepted', providerMessageId: 'm-durable' }); + }, 60_000); + + it('the engine on PostgreSQL: two instances at once, each recipient once', async () => { + const clock = new FakeClock(T0); + const provider = new FakeResendProvider(clock.now); + const gate = deferred(); + provider.script = () => ({ do: 'delay_before_arrival', until: gate.promise, then: { do: 'accept' } }); + pgStore(clock); + const engine = (owner: string) => ({ store: reconnect(clock), provider, owner, config: CONFIG, now: clock.now }); + await fx.reset(); + const a = deliverDigestOccurrence(contractPlan(), engine('instance-a')); + const b = deliverDigestOccurrence(contractPlan(), engine('instance-b')); + await vi.waitFor(() => expect(provider.requests).toHaveLength(2), { timeout: 10_000 }); + gate.resolve(); + const [ra, rb] = await Promise.all([a, b]); + expect(provider.inbox.map((d) => d.to).sort()).toEqual([ONE, TWO]); + expect(ra.attempts.length + rb.attempts.length).toBe(2); + expect(ra.complete || rb.complete).toBe(true); + }, 30_000); + + it('the engine on PostgreSQL: crash after the provider accepted, then a new process resumes from the database alone', async () => { + const clock = new FakeClock(T0); + const provider = new FakeResendProvider(clock.now); + const base = pgStore(clock); + const crashing: DigestDeliveryStore = { ...base, complete: () => never() }; // the process dies before any receipt + void deliverDigestOccurrence(contractPlan({ recipients: [ONE] }), { store: crashing, provider, owner: 'dies', config: CONFIG, now: clock.now }); + await vi.waitFor(() => expect(provider.deliveredTo(ONE)).toBe(1), { timeout: 10_000 }); + clock.advance(6 * MINUTE); + const r = await resumeDigestOccurrence(OCC, { store: reconnect(clock), provider, owner: 'restarted', config: CONFIG, now: clock.now }); + expect(r.complete).toBe(true); + expect(provider.deliveredTo(ONE)).toBe(1); + const toOne = provider.requests.filter((q) => q.to === ONE); + expect(toOne).toHaveLength(2); + expect(new Set(toOne.map((q) => `${q.key}|${q.payloadJson}`)).size).toBe(1); + expect((await reconnect(clock).load(OCC))!.deliveries[0]).toMatchObject({ status: 'accepted', ambiguous: true, providerMessageId: provider.inbox[0].messageId }); + }, 30_000); + }); + + describe('access: the service role through the functions, nobody else, nothing direct', () => { + const denied = (role: string, stmt: string) => expect(fx.sql(`set role ${role}; ${stmt}`)).rejects.toThrow(/permission denied/); + const CALLS = [ + `select public.admin_digest_load('x');`, + `select public.admin_digest_freeze('{}'::jsonb, '[]'::jsonb);`, + `select public.admin_digest_claim('x', 'y', 'w', 1000, 1, 86400000, 3600000);`, + `select public.admin_digest_begin_send('x', 'y', 1, 0, 86400000, 3600000);`, + `select public.admin_digest_complete('x', 'y', 1, '{"kind":"unknown"}'::jsonb, 1);`, + ]; + const TABLE_STATEMENTS = [ + 'select * from public.admin_digest_deliveries;', + 'select * from public.admin_digest_occurrences;', + `insert into public.admin_digest_occurrences (occurrence_id, window_start, window_end, recipient_keys, payload_hash, engine_version, frozen_at) values ('x', now(), now() + interval '1 day', array['${'a'.repeat(64)}'], '${'b'.repeat(64)}', 1, now());`, + `update public.admin_digest_deliveries set status = 'accepted';`, + 'delete from public.admin_digest_deliveries;', + ]; + it.each(['anon', 'authenticated'])('%s can neither read, write nor call anything', async (role) => { + for (const s of [...TABLE_STATEMENTS, ...CALLS]) await denied(role, s); + }, 30_000); + it('service_role cannot touch the tables directly, nor call the internal clock', async () => { + for (const s of TABLE_STATEMENTS) await denied('service_role', s); + await denied('service_role', 'select public.admin_digest_now();'); + }, 30_000); + it('service_role can call the five functions', async () => { + await fx.reset(); + expect(await fx.sql(`set role service_role; ${CALLS[0]}`)).toBe(''); + expect(await fx.sql(`set role service_role; select public.admin_digest_claim('x', 'y', 'w', 1000, 1, 86400000, 3600000)::text;`)).toBe('{"reason": "not_found", "claimed": false}'); + }); + }); + + describe('frozen means frozen, in the database itself', () => { + const setup = async () => { + const clock = new FakeClock(T0); + const store = pgStore(clock); + const f = freezePlan(contractPlan({ recipients: [ONE] }), clock.now()); + await store.freeze(f.occurrence, f.deliveries); + return { clock, store, f, where: `occurrence_id = '${OCC}' and recipient_key = '${K1}'` }; + }; + it('bytes, key and recipient never change; rows are never deleted', async () => { + const { where } = await setup(); + await expect(fx.sql(`update public.admin_digest_deliveries set payload_json = replace(payload_json, '2 new', '9 new') where ${where};`)).rejects.toThrow(/keeps its recipient, key and bytes/); + await expect(fx.sql(`update public.admin_digest_deliveries set idempotency_key = 'bubaly/rotated' where ${where};`)).rejects.toThrow(/keeps its recipient, key and bytes/); + await expect(fx.sql(`delete from public.admin_digest_deliveries where ${where};`)).rejects.toThrow(/never deleted/); + await expect(fx.sql(`update public.admin_digest_occurrences set window_end = window_end + interval '1 day';`)).rejects.toThrow(/never changed or deleted/); + await expect(fx.sql(`delete from public.admin_digest_occurrences;`)).rejects.toThrow(/never changed or deleted/); + }); + it('an insert whose hash, recipient key or idempotency key disagrees with its bytes is refused', async () => { + const { f } = await setup(); + const d = f.deliveries[0]; + const other = 'admin-digest:2026-10-01T12:30:00.000Z'; + await fx.sql(`insert into public.admin_digest_occurrences (occurrence_id, window_start, window_end, recipient_keys, payload_hash, engine_version, frozen_at) values ('${other}', now(), now() + interval '1 day', array['${d.recipientKey}'], '${d.payloadHash}', 1, now());`); + const ins = (hash: string, key: string, rk = d.recipientKey) => fx.sql(`insert into public.admin_digest_deliveries (occurrence_id, recipient_key, idempotency_key, payload_json, payload_hash, status, updated_at) values ('${other}', '${rk}', '${key}', $j$${d.payloadJson}$j$, '${hash}', 'pending', now());`); + await expect(ins('0'.repeat(64), d.idempotencyKey)).rejects.toThrow(/payload_hash is not the sha256/); + await expect(ins(d.payloadHash, d.idempotencyKey, 'f'.repeat(64))).rejects.toThrow(/recipient_key is not the sha256/); + await expect(ins(d.payloadHash, d.idempotencyKey)).rejects.toThrow(/idempotency_key is not the key/); // the key names the OTHER occurrence + }); + it('a settled row stays settled; attempts, fence, ambiguity and the anchor only move forward', async () => { + const { store, where } = await setup(); + const c = await store.claim(OCC, K1, 'w', CLAIM); + if (!c.claimed) throw new Error('setup'); + await store.beginSend(OCC, K1, c.row.fence, BEGIN); + await store.complete(OCC, K1, c.row.fence, { kind: 'unknown', reason: 'timeout' }, 4); + await expect(fx.sql(`update public.admin_digest_deliveries set ambiguous = false where ${where};`)).rejects.toThrow(/only move forward/); + await expect(fx.sql(`update public.admin_digest_deliveries set fence = 0 where ${where};`)).rejects.toThrow(/only move forward/); + await expect(fx.sql(`update public.admin_digest_deliveries set first_send_at = first_send_at + interval '1 hour' where ${where};`)).rejects.toThrow(/only move forward/); + const d = await store.claim(OCC, K1, 'w2', CLAIM); + if (!d.claimed) throw new Error('setup'); + await store.beginSend(OCC, K1, d.row.fence, BEGIN); + await store.complete(OCC, K1, d.row.fence, { kind: 'accepted', messageId: 'm-1' }, 4); + await expect(fx.sql(`update public.admin_digest_deliveries set status = 'unknown', provider_message_id = null where ${where};`)).rejects.toThrow(/stays settled/); + }); + }); +}); diff --git a/tests/helpers/digest-delivery-postgres.ts b/tests/helpers/digest-delivery-postgres.ts new file mode 100644 index 000000000..428932b5e --- /dev/null +++ b/tests/helpers/digest-delivery-postgres.ts @@ -0,0 +1,119 @@ +// A disposable PostgreSQL fixture for migration 0471 and the PostgreSQL +// DigestDeliveryStore. Opt-in: set DIGEST_DELIVERY_PG=1 with a local cluster in +// PGHOST/PGPORT/PGUSER (default: the docs/audit/verify-pg.sh harness). Without +// it these suites are SKIPPED, and say so; CI's Database job proves the same +// functions through docs/audit/an-admin-digest-reaches-each-admin-once-check.sql. +// +// Each fixture is its own database, created for the run and dropped after it: +// 1. the Supabase roles and default privileges (grant ALL on new tables and +// functions to anon, authenticated and service_role), so 0471's revokes are +// tested against what Supabase would actually grant; +// 2. migration 0471, exactly as committed; +// 3. admin_digest_now() replaced by a version that honours a per-session +// `admin_digest.test_now`, so tests can move time. This replacement exists +// only in the throwaway database. +// Calls run as `service_role`, through psql, one process per call, so +// concurrent calls are concurrent transactions on separate connections. +import { spawn } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import type { RpcCall } from '@/lib/admin/digest-delivery-store'; + +export const pgFixtureEnabled = process.env.DIGEST_DELIVERY_PG === '1'; + +const MIGRATION = fileURLToPath(new URL('../../supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql', import.meta.url)); + +const env = () => ({ + ...process.env, + PGHOST: process.env.PGHOST ?? '/tmp/pgaudit_db', + PGPORT: process.env.PGPORT ?? '54399', + PGUSER: process.env.PGUSER ?? 'postgres', + PGOPTIONS: '', +}); + +/** Run SQL on stdin in one psql process; resolves to stdout (unaligned, tuples only). */ +export function psql(db: string, sql: string, opts: { signal?: AbortSignal } = {}): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn('psql', ['-X', '-q', '-At', '-v', 'ON_ERROR_STOP=1', '-d', db], { env: env(), signal: opts.signal }); + let out = ''; + let err = ''; + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { err += d; }); + child.on('error', reject); + child.on('close', (code) => (code === 0 ? resolvePromise(out.replace(/\n$/, '')) : reject(new Error(`psql exited ${code}: ${err.trim()}`)))); + child.stdin.end(sql); + }); +} + +/** A dollar-quoted literal whose tag cannot occur in the value. */ +function lit(value: string): string { + let tag: string; + do { tag = `q${randomBytes(6).toString('hex')}`; } while (value.includes(tag)); + return `$${tag}$${value}$${tag}$`; +} + +type ArgType = 'text' | 'jsonb' | 'integer' | 'bigint'; +const SIGNATURES: Record }> = { + admin_digest_freeze: { returns: 'jsonb', args: { p_occurrence: 'jsonb', p_deliveries: 'jsonb' } }, + admin_digest_load: { returns: 'jsonb', args: { p_occurrence_id: 'text' } }, + admin_digest_claim: { returns: 'jsonb', args: { p_occurrence_id: 'text', p_recipient_key: 'text', p_owner: 'text', p_lease_ms: 'integer', p_max_attempts: 'integer', p_retention_ms: 'bigint', p_margin_ms: 'bigint' } }, + admin_digest_begin_send: { returns: 'text', args: { p_occurrence_id: 'text', p_recipient_key: 'text', p_fence: 'bigint', p_min_lease_ms: 'integer', p_retention_ms: 'bigint', p_margin_ms: 'bigint' } }, + admin_digest_complete: { returns: 'text', args: { p_occurrence_id: 'text', p_recipient_key: 'text', p_fence: 'bigint', p_result: 'jsonb', p_max_attempts: 'integer' } }, +}; + +function arg(name: string, type: ArgType, value: unknown): string { + if (type === 'jsonb') return `${lit(JSON.stringify(value))}::jsonb`; + if (type === 'text') { + if (typeof value !== 'string') throw new TypeError(`${name} must be a string`); + return `${lit(value)}::text`; + } + if (typeof value !== 'number' || !Number.isSafeInteger(value)) throw new TypeError(`${name} must be an integer`); + return `${value}::${type}`; +} + +export type PgFixture = { + db: string; + /** SQL as the superuser that owns the fixture (setup, inspection, fault injection). */ + sql(text: string): Promise; + /** The adapter's transport: named-argument calls as `service_role`, at `now` when given. */ + rpc(now?: () => Date): RpcCall; + /** Empty both tables (TRUNCATE fires no row trigger; only the fixture owner can do it). */ + reset(): Promise; + drop(): Promise; +}; + +export async function createPgFixture(): Promise { + const db = `digest_delivery_${process.pid}_${randomBytes(4).toString('hex')}`; + await psql('postgres', `create database ${db};`); + const sql = (text: string) => psql(db, text); + await sql(` + do $$ begin create role anon nologin; exception when duplicate_object then null; end $$; + do $$ begin create role authenticated nologin; exception when duplicate_object then null; end $$; + do $$ begin create role service_role nologin bypassrls; exception when duplicate_object then null; end $$; + grant usage on schema public to anon, authenticated, service_role; + alter default privileges in schema public grant all on tables to anon, authenticated, service_role; + alter default privileges in schema public grant all on functions to anon, authenticated, service_role; + `); + await sql(readFileSync(MIGRATION, 'utf8')); + await sql(` + create or replace function public.admin_digest_now() + returns timestamptz language sql volatile set search_path = public, pg_temp + as $$ select coalesce(nullif(current_setting('admin_digest.test_now', true), '')::timestamptz, clock_timestamp()) $$; + `); + return { + db, + sql, + rpc: (now) => async (fn, args) => { + const sig = SIGNATURES[fn]; + if (!sig) throw new Error(`unknown function ${fn}`); + const named = Object.entries(sig.args).map(([k, t]) => `${k} => ${arg(k, t, args[k])}`).join(', '); + const pin = now ? `set admin_digest.test_now = ${lit(now().toISOString())};\n` : ''; + const out = await psql(db, `${pin}set role service_role;\nselect public.${fn}(${named})::text;`); + if (sig.returns === 'text') return out; + return out === '' ? null : JSON.parse(out); + }, + reset: async () => { await sql('truncate public.admin_digest_deliveries, public.admin_digest_occurrences;'); }, + drop: async () => { await psql('postgres', `drop database if exists ${db} with (force);`); }, + }; +} diff --git a/tests/helpers/digest-delivery-store-contract.ts b/tests/helpers/digest-delivery-store-contract.ts index e755e0a60..34eddd0e4 100644 --- a/tests/helpers/digest-delivery-store-contract.ts +++ b/tests/helpers/digest-delivery-store-contract.ts @@ -264,6 +264,14 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'unknown', ambiguous: true, providerMessageId: null }); }); + it('a completion without a mark is refused: a claim that never marked a send owns no provider answer', async () => { + const { store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + expect(await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3)).toBe('fenced_out'); + expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'in_flight', ambiguous: false, firstSendAt: null }); + }); + it('an unknown occurrence or recipient is not found, not created', async () => { const { store, id } = await setup(); expect(await store.claim(id, 'f'.repeat(64), 'a', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'not_found' }); From 6d035d9c0060aecdb8a7c7656ba9825ec8663c95 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 20:42:28 +0000 Subject: [PATCH 05/12] fix(admin-digest delivery): a mark whose answer arrives late cannot send late (dispatch deadline) Jimmy's P1 on #699 (head 5c58ae4f): storage could commit a valid send mark and return that answer after the lease or the key's safe retry deadline had passed. The engine then dispatched anyway. With the fake provider this gave a second copy under the same key once the key had expired, and a stalled worker that sent after a recovery worker had parked the row. - A granted mark is now an admission receipt: { ok: true, dispatchBy }. - dispatchBy is lease expiry minus the send deadline. For an ambiguous row it is also bounded by the first mark plus retention minus margin; a row only ever definitively refused has nothing to duplicate, so only the lease bounds it. - The engine re-reads its clock AFTER the store answers, immediately before the provider call. At or past dispatchBy it sends nothing and reports `dispatch_deadline_passed`. The row keeps its mark and is treated as possibly sent, which is the safe side. - The deadline is computed in decideBeginSend, the in-memory store, admin_digest_begin_send (0471 now returns jsonb) and the adapter, which refuses a grant without a deadline. - Late accepted completions under the same fence are still recorded, as asked. The conservative first-mark anchor, frozen bytes and keys, the fences and the delayed-claim correction are unchanged. Regressions, all with the fake clock and store hooks: - the mark commits 2 s before the cut-off and its answer is 2.5 s late: nothing is sent (inbox stays 1); - the answer is held past the cut-off while a recovery worker parks the row: the stalled worker sends nothing; - controls: a slow answer inside the deadline sends once; past the LEASE part, nothing is sent; - store contract: the deadline's value, for a plain row and an ambiguous one. Results: - engine suite 96/96; - PostgreSQL suite 47 + restart 1; - CI probe and plpgsql_check pass on the harness; - 0471 re-applies twice. The only repo test failing is migration-version-safety's shared next-version pin (0465 -> 0472). That is the register the coordinator reconciles; its hunk is proposed on #699, not committed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- ...n-digest-reaches-each-admin-once-check.sql | 8 +-- lib/admin/digest-delivery-store.ts | 7 ++- lib/admin/digest-delivery.ts | 38 ++++++++++--- ...n_admin_digest_reaches_each_admin_once.sql | 18 +++++-- tests/admin-digest-delivery-engine.test.ts | 54 +++++++++++++++++++ tests/admin-digest-delivery-postgres.test.ts | 9 ++-- tests/helpers/digest-delivery-fakes.ts | 2 +- tests/helpers/digest-delivery-postgres.ts | 2 +- .../helpers/digest-delivery-store-contract.ts | 27 +++++++--- 9 files changed, 135 insertions(+), 30 deletions(-) diff --git a/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql index 90599e693..4e4833d4a 100644 --- a/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql +++ b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql @@ -114,8 +114,8 @@ begin if public.admin_digest_complete(occ, k1, 1, '{"kind":"unknown","reason":"timeout"}', 3) <> 'fenced_out' then raise warning 'a completion without a mark was accepted'; failures := failures + 1; end if; - if public.admin_digest_begin_send(occ, k1, 0, 150, ret, mar) <> 'fenced_out' then raise warning 'a stale fence marked'; failures := failures + 1; end if; - if public.admin_digest_begin_send(occ, k1, 1, 150, ret, mar) <> 'ok' then raise warning 'the current fence could not mark'; failures := failures + 1; end if; + if public.admin_digest_begin_send(occ, k1, 0, 150, ret, mar) ->> 'answer' <> 'fenced_out' then raise warning 'a stale fence marked'; failures := failures + 1; end if; + if public.admin_digest_begin_send(occ, k1, 1, 150, ret, mar) ->> 'answer' <> 'ok' then raise warning 'the current fence could not mark'; failures := failures + 1; end if; if public.admin_digest_complete(occ, k1, 1, '{"kind":"accepted","messageId":""}', 3) <> 'ok' then raise warning 'complete refused'; failures := failures + 1; end if; res := public.admin_digest_load(occ); if (res -> 'deliveries') @> jsonb_build_array(jsonb_build_object('recipientKey', k1, 'status', 'accepted')) then @@ -129,12 +129,12 @@ begin -- A lease that lapses after a mark makes the row ambiguous; too little lease refuses the mark. res := public.admin_digest_claim(occ, k2, 'a', 300000, 3, ret, mar); - if public.admin_digest_begin_send(occ, k2, 1, 150, ret, mar) <> 'ok' then raise warning 'k2 mark'; failures := failures + 1; end if; + if public.admin_digest_begin_send(occ, k2, 1, 150, ret, mar) ->> 'answer' <> 'ok' then raise warning 'k2 mark'; failures := failures + 1; end if; perform set_config('admin_digest.test_now', (pin + interval '23 hours 5 minutes 1 second')::text, true); res := public.admin_digest_claim(occ, k2, 'b', 300000, 3, ret, mar); if not (res #>> '{row,ambiguous}')::boolean or (res #>> '{row,fence}')::int <> 2 then raise warning 'a lapsed marked lease was not ambiguous: %', res; failures := failures + 1; end if; perform set_config('admin_digest.test_now', (pin + interval '23 hours 10 minutes 0.9 seconds')::text, true); - if public.admin_digest_begin_send(occ, k2, 2, 150, ret, mar) <> 'lease_expired' then raise warning 'a nearly lapsed lease still marked'; failures := failures + 1; end if; + if public.admin_digest_begin_send(occ, k2, 2, 150, ret, mar) ->> 'answer' <> 'lease_expired' then raise warning 'a nearly lapsed lease still marked'; failures := failures + 1; end if; reset role; -- ── frozen means frozen, for everyone (these are the owner's own writes) ──── diff --git a/lib/admin/digest-delivery-store.ts b/lib/admin/digest-delivery-store.ts index 31ca0145d..66f6443e8 100644 --- a/lib/admin/digest-delivery-store.ts +++ b/lib/admin/digest-delivery-store.ts @@ -30,7 +30,7 @@ export function supabaseRpc(client: RpcClient): RpcCall { const STATUSES: readonly DeliveryStatus[] = ['pending', 'in_flight', 'failed', 'unknown', 'accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation']; const REFUSALS: readonly string[] = ['leased', 'not_found', 'accepted', 'rejected', 'conflict', 'exhausted', 'needs_reconciliation']; -const BEGIN_ANSWERS = ['ok', 'fenced_out', 'lease_expired', 'retention_passed'] as const; +const BEGIN_REFUSALS = ['fenced_out', 'lease_expired', 'retention_passed'] as const; function bad(what: string): never { throw new Error(`digest-delivery store: malformed ${what} from the database`); } const obj = (v: unknown, what: string) => (v && typeof v === 'object' && !Array.isArray(v) ? v as Record : bad(what)); @@ -141,7 +141,10 @@ export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryS p_min_lease_ms: ms(policy.minLeaseRemainingMs, 'minLease'), p_retention_ms: ms(policy.providerKeyRetentionMs, 'retention'), p_margin_ms: ms(policy.retentionSafetyMarginMs, 'margin'), }); - return (BEGIN_ANSWERS as readonly unknown[]).includes(out) ? out as typeof BEGIN_ANSWERS[number] : bad('begin_send'); + const a = obj(out, 'begin_send'); + if (a.answer === 'ok') return { ok: true, dispatchBy: iso(a.dispatchBy, 'begin_send.dispatchBy') }; + return (BEGIN_REFUSALS as readonly unknown[]).includes(a.answer) && a.dispatchBy === undefined + ? { ok: false, reason: a.answer as typeof BEGIN_REFUSALS[number] } : bad('begin_send'); }, async complete(occurrenceId, recipientKey, fence, result: ProviderSendResult, maxAttempts) { diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts index ca71bca90..f77b628f1 100644 --- a/lib/admin/digest-delivery.ts +++ b/lib/admin/digest-delivery.ts @@ -23,7 +23,10 @@ // be told apart: no mark means nothing reached the provider. It is // refused once the lease has too little time left for the send, or // once an ambiguous row is past the retention cut-off, so a worker -// that stalled after its claim cannot send late. +// that stalled after its claim cannot send late. A granted mark +// carries a DISPATCH DEADLINE, and the engine re-reads the time +// after the store answers and does not send once it has passed: a +// mark whose answer arrived late cannot send late either. // - complete the provider's answer, fenced: a worker whose lease was taken over // writes nothing. // @@ -207,7 +210,8 @@ export type BeginSendPolicy = Pick; /** Apply `decideClaim` atomically. */ claim(occurrenceId: string, recipientKey: string, owner: string, policy: ClaimPolicy): Promise<{ claimed: true; row: DeliveryRow } | { claimed: false; reason: ClaimRefusal }>; - /** Apply `decideBeginSend` atomically (persisting `next` when set). */ - beginSend(occurrenceId: string, recipientKey: string, fence: number, policy: BeginSendPolicy): Promise<'ok' | 'fenced_out' | 'lease_expired' | 'retention_passed'>; + /** Apply `decideBeginSend` atomically (persisting `next` when set); a granted mark returns its dispatch deadline. */ + beginSend(occurrenceId: string, recipientKey: string, fence: number, policy: BeginSendPolicy): Promise; /** Apply `decideCompletion` atomically. */ complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number): Promise<'ok' | 'fenced_out'>; } @@ -544,7 +559,7 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'not_sent', detail: 'stored_payload_integrity_failed' }); continue; } - let marked: Awaited>; + let marked: BeginSendAnswer; try { marked = await store.beginSend(id, key, row.fence, { minLeaseRemainingMs: config.sendTimeoutMs, @@ -555,8 +570,15 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc report.storageErrors.push({ stage: 'beginSend', recipientKey: key }); continue; } - if (marked !== 'ok') { - report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: marked === 'fenced_out' ? 'fenced_out' : 'not_sent', detail: marked }); + if (!marked.ok) { + report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: marked.reason === 'fenced_out' ? 'fenced_out' : 'not_sent', detail: marked.reason }); + continue; + } + // The store's answer may have arrived late. Re-read the time now, after the await and immediately + // before the call: past the deadline, the lease may be gone or the provider may forget the key, so + // the send does not happen. The row keeps its mark and is treated as possibly sent (the safe side). + if (!(deps.now().getTime() < Date.parse(marked.dispatchBy))) { + report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'not_sent', detail: 'dispatch_deadline_passed' }); continue; } const result = await sendWithDeadline(deps.provider, row, config.sendTimeoutMs); diff --git a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql index 28a026901..4a768a775 100644 --- a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql +++ b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql @@ -305,10 +305,14 @@ begin end $$; -- ── beginSend: decideBeginSend, the last check before the provider call ───── +-- A granted mark answers {"answer":"ok","dispatchBy":…}: the provider call must START before +-- then (lease minus the send deadline; for an ambiguous row also the first mark plus retention +-- minus margin), and the engine re-reads its clock after this answer arrives, so a late answer +-- cannot send late. create or replace function public.admin_digest_begin_send( p_occurrence_id text, p_recipient_key text, p_fence bigint, p_min_lease_ms integer, p_retention_ms bigint, p_margin_ms bigint -) returns text +) returns jsonb language plpgsql volatile security definer @@ -326,23 +330,27 @@ begin where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key for update; if not found or r.status <> 'in_flight' or r.fence <> p_fence then - return 'fenced_out'; + return jsonb_build_object('answer', 'fenced_out'); end if; v_now := public.admin_digest_now(); if r.lease_expires_at is null or r.lease_expires_at - v_now <= make_interval(secs => p_min_lease_ms / 1000.0) then - return 'lease_expired'; -- left for the next claim; nothing is written + return jsonb_build_object('answer', 'lease_expired'); -- left for the next claim; nothing is written end if; if r.ambiguous and (r.first_send_at is null or v_now >= r.first_send_at + make_interval(secs => (p_retention_ms - p_margin_ms) / 1000.0)) then update public.admin_digest_deliveries set status = 'needs_reconciliation', lease_owner = null, lease_expires_at = null, send_started_at = null, updated_at = v_now where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; - return 'retention_passed'; + return jsonb_build_object('answer', 'retention_passed'); end if; update public.admin_digest_deliveries set send_started_at = v_now, first_send_at = coalesce(r.first_send_at, v_now), updated_at = v_now where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key; - return 'ok'; + -- The retention part binds only an ambiguous row: one only ever refused has nothing to duplicate. + return jsonb_build_object('answer', 'ok', 'dispatchBy', case when r.ambiguous then least( + r.lease_expires_at - make_interval(secs => p_min_lease_ms / 1000.0), + coalesce(r.first_send_at, v_now) + make_interval(secs => (p_retention_ms - p_margin_ms) / 1000.0)) + else r.lease_expires_at - make_interval(secs => p_min_lease_ms / 1000.0) end); end $$; -- ── complete: decideCompletion, fenced, after a mark; only a message id is a receipt ── diff --git a/tests/admin-digest-delivery-engine.test.ts b/tests/admin-digest-delivery-engine.test.ts index 3bb47a898..c7d66035a 100644 --- a/tests/admin-digest-delivery-engine.test.ts +++ b/tests/admin-digest-delivery-engine.test.ts @@ -315,6 +315,60 @@ describe('a worker that stalls between its claim and its mark', () => { }); }); +describe('a mark whose answer arrives late cannot send late (dispatch deadline)', () => { + // 24 h retention, 1.5 s margin, 1 s send deadline, 5 min lease: a valid, deliberately tight config. + const TIGHT: EngineConfig = { ...CONFIG, retentionSafetyMarginMs: 1_500, sendTimeoutMs: 1_000 }; + + it('the mark commits 2 s before the cut-off but its answer arrives 2.5 s later: nothing is sent', async () => { + const { clock, store, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'accept_then_lose_answer', reason: 'timeout' } : { do: 'accept' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a', { config: TIGHT })); // first mark at T0; accepted, answer lost + clock.set(new Date(Date.parse(T0) + 24 * HOUR - 2_000).toISOString()); + store.hooks.push(hookOn('beginSend', 'after', K1, () => { clock.advance(2_500); })); // committed; the answer is slow + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b', { config: TIGHT })); + expect(r.attempts).toEqual([{ recipientKey: K1, fence: 2, result: 'not_sent', recorded: 'not_sent', detail: 'dispatch_deadline_passed' }]); + expect(provider.requests).toHaveLength(1); + expect(provider.inbox).toHaveLength(1); + }); + + it('a mark answer held past the cut-off while a recovery worker parks the row: the stalled worker still sends nothing', async () => { + const { clock, store, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'accept_then_lose_answer', reason: 'timeout' } : { do: 'accept' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); // first mark at T0 + clock.set(new Date(Date.parse(T0) + 22 * HOUR).toISOString()); + const entered = deferred(); + const release = deferred(); + store.hooks.push(hookOn('beginSend', 'after', K1, async () => { entered.resolve(); await release.promise; })); + const stalled = deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('stalled')); + await entered.promise; // the mark is committed; its answer is held + store.hooks = []; + clock.set(new Date(Date.parse(T0) + 25 * HOUR).toISOString()); + const recovery = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('recovery')); + expect(recovery.needsAttention).toEqual([{ recipientKey: K1, status: 'needs_reconciliation' }]); + release.resolve(); + const s = await stalled; + expect(s.attempts).toEqual([{ recipientKey: K1, fence: 2, result: 'not_sent', recorded: 'not_sent', detail: 'dispatch_deadline_passed' }]); + expect(provider.requests).toHaveLength(1); + expect(provider.inbox).toHaveLength(1); + }); + + it('control: a mark answer that is slow but inside the deadline still sends, once', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('beginSend', 'after', K1, () => { clock.advance(4 * MINUTE); })); // lease 5 min, send deadline 150 ms + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(r.complete).toBe(true); + expect(provider.inbox).toHaveLength(1); + }); + + it('control: past the LEASE part of the deadline (not retention) nothing is sent either', async () => { + const { clock, store, provider, engine } = world(); + store.hooks.push(hookOn('beginSend', 'after', K1, () => { clock.advance(5 * MINUTE - 100); })); // 100 ms of lease left < 150 ms send deadline + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a')); + expect(r.attempts[0]).toMatchObject({ result: 'not_sent', detail: 'dispatch_deadline_passed' }); + expect(provider.requests).toEqual([]); + }); +}); + describe('attempts are bounded even when no attempt ever completes', () => { it('five claims that each die before the mark: the sixth parks the recipient as exhausted, and nothing was sent', async () => { const { clock, store, provider, engine } = world(); diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts index 46fa3e1fb..db11a635d 100644 --- a/tests/admin-digest-delivery-postgres.test.ts +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -16,7 +16,7 @@ import { } from '@/lib/admin/digest-delivery'; import { createPostgresDigestDeliveryStore, parseDeliveryRow, supabaseRpc, type RpcCall } from '@/lib/admin/digest-delivery-store'; import { FakeClock, FakeResendProvider, HOUR, MINUTE, MemoryDigestDeliveryStore, deferred, never } from './helpers/digest-delivery-fakes'; -import { contractPlan, describeDigestDeliveryStoreContract } from './helpers/digest-delivery-store-contract'; +import { answerOf, contractPlan, describeDigestDeliveryStoreContract } from './helpers/digest-delivery-store-contract'; import { createPgFixture, pgFixtureEnabled, psql, type PgFixture } from './helpers/digest-delivery-postgres'; const T0 = '2026-09-30T12:31:00.000Z'; @@ -68,7 +68,10 @@ describe('PostgreSQL adapter: strict mapping (fake transport)', () => { it('beginSend and complete accept only their documented answers', async () => { await expect(storeAnswering('yes').store.beginSend(OCC, K1, 1, BEGIN)).rejects.toThrow(/malformed/); await expect(storeAnswering(null).store.complete(OCC, K1, 1, { kind: 'accepted', messageId: 'm' }, 3)).rejects.toThrow(/malformed/); - for (const a of ['ok', 'fenced_out', 'lease_expired', 'retention_passed']) expect(await storeAnswering(a).store.beginSend(OCC, K1, 1, BEGIN)).toBe(a); + expect(await storeAnswering({ answer: 'ok', dispatchBy: '2026-09-30T12:35:59.85+00:00' }).store.beginSend(OCC, K1, 1, BEGIN)).toEqual({ ok: true, dispatchBy: '2026-09-30T12:35:59.850Z' }); + for (const r of ['fenced_out', 'lease_expired', 'retention_passed']) expect(await storeAnswering({ answer: r }).store.beginSend(OCC, K1, 1, BEGIN)).toEqual({ ok: false, reason: r }); + await expect(storeAnswering({ answer: 'ok' }).store.beginSend(OCC, K1, 1, BEGIN)).rejects.toThrow(/malformed/); // a grant without a deadline + await expect(storeAnswering('ok').store.beginSend(OCC, K1, 1, BEGIN)).rejects.toThrow(/malformed/); }); it('freeze sends only identity and bytes, never a state', async () => { @@ -205,7 +208,7 @@ describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', ( await store.freeze(f.occurrence, f.deliveries); const c = await store.claim(OCC, K1, 'w', CLAIM); if (!c.claimed) throw new Error('setup'); - expect(await store.beginSend(OCC, K1, c.row.fence, BEGIN)).toBe('ok'); + expect(answerOf(await store.beginSend(OCC, K1, c.row.fence, BEGIN))).toBe('ok'); const dying = psql(fx.db, `set application_name = 'doomed_receipt'; set admin_digest.test_now = '${T0}'; set role service_role; begin; select public.admin_digest_complete('${OCC}', '${K1}', 1, '{"kind":"accepted","messageId":"m-lost"}'::jsonb, 4); select pg_sleep(30); commit;`).catch((e: Error) => e); await sleepingIn('doomed_receipt'); // the receipt has been written inside the open transaction diff --git a/tests/helpers/digest-delivery-fakes.ts b/tests/helpers/digest-delivery-fakes.ts index 7dab80da2..7b60561a5 100644 --- a/tests/helpers/digest-delivery-fakes.ts +++ b/tests/helpers/digest-delivery-fakes.ts @@ -147,7 +147,7 @@ export class MemoryDigestDeliveryStore implements DigestDeliveryStore { if (d.next) this.rows.set(k, copy(d.next)); // ── end ── await this.hook('beginSend', 'after', recipientKey); - return d.ok ? 'ok' as const : d.reason; + return d.ok ? { ok: true as const, dispatchBy: d.dispatchBy } : { ok: false as const, reason: d.reason }; } async complete(occurrenceId: string, recipientKey: string, fence: number, result: ProviderSendResult, maxAttempts: number) { diff --git a/tests/helpers/digest-delivery-postgres.ts b/tests/helpers/digest-delivery-postgres.ts index 428932b5e..53aebad62 100644 --- a/tests/helpers/digest-delivery-postgres.ts +++ b/tests/helpers/digest-delivery-postgres.ts @@ -58,7 +58,7 @@ const SIGNATURES: Record (a.ok ? 'ok' : a.reason); + export const contractPlan = (over: Partial = {}): OccurrencePlan => ({ occurrenceId: 'admin-digest:2026-09-30T12:30:00.000Z', window: { start: '2026-09-29T12:30:00.000Z', end: '2026-09-30T12:30:00.000Z' }, @@ -111,9 +114,9 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: clock.advance(5 * MINUTE); const b = await store.claim(id, key, 'b', CONTRACT_POLICY); if (!a.claimed || !b.claimed) throw new Error('setup'); - expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('fenced_out'); + expect(answerOf(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN))).toBe('fenced_out'); expect(await store.complete(id, key, a.row.fence, { kind: 'accepted', messageId: 'stale' }, 3)).toBe('fenced_out'); - expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toBe('ok'); + expect(answerOf(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN))).toBe('ok'); expect(await store.complete(id, key, b.row.fence, { kind: 'accepted', messageId: 'm-b' }, 3)).toBe('ok'); expect(await store.complete(id, key, b.row.fence, { kind: 'rejected', httpStatus: 422, code: 'x', retryable: false }, 3)).toBe('fenced_out'); const row = (await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!; @@ -158,7 +161,7 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: const a = await store.claim(id, key, 'a', CONTRACT_POLICY); const b = await store.claim(id, other, 'a', CONTRACT_POLICY); if (!a.claimed || !b.claimed) throw new Error('setup'); - expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('ok'); // marked; `other` is not + expect(answerOf(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN))).toBe('ok'); // marked; `other` is not clock.advance(5 * MINUTE); const again = await store.claim(id, key, 'c', CONTRACT_POLICY); const againOther = await store.claim(id, other, 'c', CONTRACT_POLICY); @@ -192,20 +195,20 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: const a = await store.claim(id, key, 'a', CONTRACT_POLICY); if (!a.claimed) throw new Error('setup'); clock.advance(5 * MINUTE - 1_000); - expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toBe('lease_expired'); + expect(answerOf(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN))).toBe('lease_expired'); expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'in_flight', sendStartedAt: null }); // An ambiguous row claimed just inside the window, marked just after it: clock.advance(1_000); const b = await store.claim(id, key, 'b', CONTRACT_POLICY); if (!b.claimed) throw new Error('setup'); - expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toBe('ok'); + expect(answerOf(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN))).toBe('ok'); await store.complete(id, key, b.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); const firstMark = Date.parse((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!.firstSendAt!); clock.set(new Date(firstMark + 23 * HOUR - 1).toISOString()); const c = await store.claim(id, key, 'c', { ...CONTRACT_POLICY, leaseMs: 2 * HOUR }); if (!c.claimed) throw new Error('setup'); clock.advance(1); - expect(await store.beginSend(id, key, c.row.fence, CONTRACT_BEGIN)).toBe('retention_passed'); + expect(answerOf(await store.beginSend(id, key, c.row.fence, CONTRACT_BEGIN))).toBe('retention_passed'); expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!.status).toBe('needs_reconciliation'); }); @@ -272,6 +275,18 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: expect((await store.load(id))!.deliveries.find((d) => d.recipientKey === key)!).toMatchObject({ status: 'in_flight', ambiguous: false, firstSendAt: null }); }); + it('a granted mark carries its dispatch deadline: lease minus the send deadline, and for an ambiguous row also first mark plus retention minus margin', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + expect(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN)).toEqual({ ok: true, dispatchBy: new Date(Date.parse(start) + 5 * MINUTE - 1_000).toISOString() }); + await store.complete(id, key, a.row.fence, { kind: 'unknown', reason: 'timeout' }, 3); + clock.set(new Date(Date.parse(start) + 23 * HOUR - 2 * MINUTE).toISOString()); // retention now binds before the lease + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!b.claimed) throw new Error('setup'); + expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toEqual({ ok: true, dispatchBy: new Date(Date.parse(start) + 23 * HOUR).toISOString() }); + }); + it('an unknown occurrence or recipient is not found, not created', async () => { const { store, id } = await setup(); expect(await store.claim(id, 'f'.repeat(64), 'a', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'not_found' }); From 6ae785b6aeeba8042fafea6b8f5e6be8a079dc34 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 20:43:56 +0000 Subject: [PATCH 06/12] docs(admin-digest delivery): the admission receipt, its dispatch deadline, and the clock-skew requirement Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index 378868f59..ef5cc0f96 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -46,6 +46,14 @@ The engine does not schedule. The caller (the cron route, later) decides and pas Together these stop a worker that stalled after its claim from sending late. +**A granted mark is an admission receipt, `{ ok: true, dispatchBy }`.** The provider call must **start** before `dispatchBy`: +- lease expiry minus the send deadline, so the call ends while the lease holds; +- for an ambiguous row, also no later than the first mark plus retention minus margin, so the request reaches the provider before it can forget the key. A row only ever definitively refused has nothing to duplicate. + +The engine re-reads its own clock **after** the store answers, immediately before the call. At or past `dispatchBy` it sends nothing (`dispatch_deadline_passed`), so a mark whose answer arrived late cannot send late. The row keeps its mark and is treated as possibly sent. + +**Clock skew.** `dispatchBy` is database time and the engine compares it with the application's clock. Skew between the two must stay well inside both `leaseMs − sendTimeoutMs` and `retentionSafetyMarginMs − sendTimeoutMs`. NTP-level skew does; the defaults are 5 min, 1 h and 15 s. + **Keys.** A key is `bubaly/admin-digest/v1//`: - It is a function of occurrence and recipient only. - It contains no address. @@ -71,7 +79,8 @@ A PostgreSQL `DigestDeliveryStore` must meet all of these. It must also pass `te 5. **`beginSend` commits before the provider call.** In one step it: - evaluates `decideBeginSend`: fence, remaining lease against the send deadline, and the retention cut-off for ambiguous rows; - sets `send_started_at`; - - sets `first_send_at` only if it is null. + - sets `first_send_at` only if it is null; + - returns the `dispatchBy` deadline, computed from the locked row. A refusal that parks a row (`retention_passed`) is written in that same step. The adapter returns only after the commit (`synchronous_commit` on, the default). 6. **Frozen means immutable.** `recipient_key`, `idempotency_key`, `payload_json` and `payload_hash` never change after insert. Enforce this with a trigger, not only in code. `idempotency_key` is unique across the table. `payload_hash = sha256(payload_json)` is checked on insert. From 4d2f82efc6ab5ce6d1874357e9cc13a9cb515616 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 21:40:08 +0000 Subject: [PATCH 07/12] fix(admin-digest delivery): the admission check and the provider call are one step; bound the lease Review P2 on 6ae785b6: the dispatch-deadline check ran one microtask before provider.send, so queued JavaScript could spend the admission budget between them. The check now runs inside sendWithDeadline, in the same synchronous step as the call, and answers an explicit no-send (dispatch_deadline_passed). Regression: 2.5 s of queued work at every microtask depth 0-12 never yields a second copy; both sides of the boundary are reached. It is red at depth 3 on the previous engine, and removing the new check fails 4 tests. From the adversarial review, triaged against head: - validateConfig and 0471's claim refuse a lease of retention minus margin or more (a worker that dies after its mark could never be retried), and 0471's claim and begin_send refuse a retention past the verified 24 h. - The key/bytes test calls #692's real sendEmail (fetch stubbed) instead of a copied regex and a hand-built body, with a control for a key it must refuse. - Store contract: a row that was only ever refused gets a lease-only dispatch deadline at any age (the SQL mutation S23 survived without it). - Contract: the duties the engine cannot enforce (recipient normalisation, removed recipients, no API-key rotation during open ambiguity, manual reconciliation, sendEmail's skipped answer, the prototype's incompatible key). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 27 ++++++++- lib/admin/digest-delivery.ts | 34 ++++++++--- ...n_admin_digest_reaches_each_admin_once.sql | 5 +- tests/admin-digest-delivery-engine.test.ts | 59 +++++++++++++++---- tests/admin-digest-delivery-postgres.test.ts | 9 +++ .../helpers/digest-delivery-store-contract.ts | 14 +++++ 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index ef5cc0f96..47f7d315a 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -21,6 +21,27 @@ The engine does not schedule. The caller (the cron route, later) decides and pas `deliverDigestOccurrence(plan)` stores the plan the first time it sees the occurrence. On every later call the **stored** plan is used. A recomputed plan with a different body or different recipients is reported in `planMismatch` and is never merged or sent. `resumeDigestOccurrence(id)` continues from stored state alone. +### Duties the engine cannot enforce + +These came out of an adversarial review of the engine. Each one belongs to the route wiring or to an operator. None is solved here. + +- **Normalise recipients before the plan.** + - `readSuperAdminRecipients` lowercases table addresses but does not trim them. + - `validatePlan` refuses the **whole occurrence** with a `TypeError` for any of these: a duplicate after trimming, an address that is not one plain address, or more than 200 recipients. Nothing is stored or sent. + - The route must trim, lowercase and dedupe first. It must also decide whether one bad address refuses the occurrence or is dropped and reported; that is a policy choice. Either way it catches the `TypeError` and answers non-200. +- **A recipient removed after the freeze is still sent on a retry of that occurrence.** + - The stored plan wins, and `planMismatch.recipientsRemoved` reports only a count. A row that was never ambiguous is retried at any age. + - Two options: the caller never resumes an old occurrence after an admin is removed, or a later migration adds a fenced `withdraw` step. That step is not in 0471. +- **Do not rotate `RESEND_API_KEY` while any row is `unknown` inside its window.** + - Resend documents 24-hour key retention, but not whether a key is scoped per API key or per team. + - If it is per API key, a retry after rotation is a new key, and the provider sends a second copy. + - 401/403 are retryable because an operator can fix them. That fix must not be a rotation while ambiguous rows are open. +- **Reconciliation is manual, and there is no store step for it yet.** + - Resend documents no lookup by idempotency key, and an ambiguous row holds no message id. A person checks Resend's logs by recipient, time and subject. + - `needs_reconciliation` is terminal, and 0471's triggers keep it that way. Recording the answer needs a later, fenced `reconcile` step. +- **`sendEmail` answers `{ ok: true, skipped: true }` when no API key is set.** A provider adapter must map that to a retryable refusal, never to `accepted` or `unknown`. +- **The scheduler prototype's route change must not ship alongside this.** Its key is `admin-digest//`; the engine's is `bubaly/admin-digest/v1//`. The two never deduplicate each other, so running both inside 24 hours sends twice. + ## 2. Per-recipient states | Status | Meaning | Next | @@ -50,10 +71,14 @@ Together these stop a worker that stalled after its claim from sending late. - lease expiry minus the send deadline, so the call ends while the lease holds; - for an ambiguous row, also no later than the first mark plus retention minus margin, so the request reaches the provider before it can forget the key. A row only ever definitively refused has nothing to duplicate. -The engine re-reads its own clock **after** the store answers, immediately before the call. At or past `dispatchBy` it sends nothing (`dispatch_deadline_passed`), so a mark whose answer arrived late cannot send late. The row keeps its mark and is treated as possibly sent. +The engine re-reads its own clock **after** the store answers, in the same synchronous step as the provider call: there is no await between the check and the call, so other queued work cannot spend the budget between them. At or past `dispatchBy` it sends nothing (`dispatch_deadline_passed`), so a mark whose answer arrived late cannot send late. The row keeps its mark and is treated as possibly sent. **Clock skew.** `dispatchBy` is database time and the engine compares it with the application's clock. Skew between the two must stay well inside both `leaseMs − sendTimeoutMs` and `retentionSafetyMarginMs − sendTimeoutMs`. NTP-level skew does; the defaults are 5 min, 1 h and 15 s. +**Policy bounds.** `validateConfig` refuses, and 0471's functions refuse again, two policies: +- a retention longer than the verified 24 hours; +- a lease of `retention − margin` or more. A worker that dies after its mark holds the row until the lease lapses, so a longer lease would park every such crash instead of retrying it. + **Keys.** A key is `bubaly/admin-digest/v1//`: - It is a function of occurrence and recipient only. - It contains no address. diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts index f77b628f1..6e30fcb51 100644 --- a/lib/admin/digest-delivery.ts +++ b/lib/admin/digest-delivery.ts @@ -416,6 +416,9 @@ export function validateConfig(c: EngineConfig): void { // The last safe retry must reach the provider before it forgets the key. if (c.retentionSafetyMarginMs <= c.sendTimeoutMs) planError('retentionSafetyMarginMs must exceed sendTimeoutMs'); if (c.retentionSafetyMarginMs >= c.providerKeyRetentionMs) planError('retentionSafetyMarginMs must be shorter than the retention window'); + // A worker that dies after its mark leaves the row locked until its lease lapses. The + // lease must lapse inside the retry window, or every such crash parks the row. + if (c.leaseMs >= c.providerKeyRetentionMs - c.retentionSafetyMarginMs) planError('leaseMs must be shorter than the retry window (retention minus margin)'); // Never assume the provider remembers a key longer than its documentation says. if (c.providerKeyRetentionMs > RESEND_KEY_RETENTION_MS) planError('providerKeyRetentionMs cannot exceed the verified 24 h'); } @@ -574,14 +577,16 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: marked.reason === 'fenced_out' ? 'fenced_out' : 'not_sent', detail: marked.reason }); continue; } - // The store's answer may have arrived late. Re-read the time now, after the await and immediately - // before the call: past the deadline, the lease may be gone or the provider may forget the key, so - // the send does not happen. The row keeps its mark and is treated as possibly sent (the safe side). - if (!(deps.now().getTime() < Date.parse(marked.dispatchBy))) { + // The store's answer may have arrived late, and other queued work may run before the call. The time + // is re-read inside sendWithDeadline, in the same synchronous step as the provider call: past the + // deadline, the lease may be gone or the provider may forget the key, so the send does not happen. + // The row keeps its mark and is treated as possibly sent (the safe side). + const dispatchBy = Date.parse(marked.dispatchBy); + const result = await sendWithDeadline(deps.provider, row, config.sendTimeoutMs, () => deps.now().getTime() < dispatchBy); + if (result === NOT_DISPATCHED) { report.attempts.push({ recipientKey: key, fence: row.fence, result: 'not_sent', recorded: 'not_sent', detail: 'dispatch_deadline_passed' }); continue; } - const result = await sendWithDeadline(deps.provider, row, config.sendTimeoutMs); let recorded: 'ok' | 'fenced_out'; try { recorded = await store.complete(id, key, row.fence, result, config.maxAttempts); @@ -607,16 +612,27 @@ async function run(stored: StoredOccurrence, deps: EngineDeps, report: Occurrenc return report; } -async function sendWithDeadline(provider: DigestEmailProvider, row: DeliveryRow, ms: number): Promise { +const NOT_DISPATCHED = Symbol('not_dispatched'); + +/** + * `admitted` and `provider.send` run in one synchronous step, with no await between them, so no + * other queued work can spend the admission budget after the check. The deadline starts with the call. + */ +async function sendWithDeadline(provider: DigestEmailProvider, row: DeliveryRow, ms: number, admitted: () => boolean): Promise { + if (!admitted()) return NOT_DISPATCHED; const controller = new AbortController(); let timer: ReturnType | undefined; const deadline = new Promise((resolve) => { timer = setTimeout(() => { controller.abort(); resolve({ kind: 'unknown', reason: 'timeout' }); }, ms); }); // A provider that throws, even synchronously, settles nothing: the request may have left. - const attempt = Promise.resolve() - .then(() => provider.send({ idempotencyKey: row.idempotencyKey, payload: { ...row.payload }, payloadJson: row.payloadJson, signal: controller.signal })) - .then(checkResult, (): ProviderSendResult => ({ kind: 'unknown', reason: 'network' })); + let sent: Promise; + try { + sent = Promise.resolve(provider.send({ idempotencyKey: row.idempotencyKey, payload: { ...row.payload }, payloadJson: row.payloadJson, signal: controller.signal })); + } catch { + sent = Promise.resolve({ kind: 'unknown', reason: 'network' }); + } + const attempt = sent.then(checkResult, (): ProviderSendResult => ({ kind: 'unknown', reason: 'network' })); try { return await Promise.race([attempt, deadline]); } finally { diff --git a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql index 4a768a775..940a3ba5a 100644 --- a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql +++ b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql @@ -254,7 +254,8 @@ declare begin if p_owner is null or length(p_owner) not between 1 and 200 or p_lease_ms is null or p_lease_ms < 1 or p_max_attempts is null or p_max_attempts < 1 or p_retention_ms is null or p_margin_ms is null - or p_margin_ms < 1 or p_margin_ms >= p_retention_ms then + or p_margin_ms < 1 or p_margin_ms >= p_retention_ms or p_retention_ms > 86400000 + or p_lease_ms >= p_retention_ms - p_margin_ms then raise exception 'admin_digest_claim: bad owner, lease, attempts or retention' using errcode = '22023'; end if; select * into r from public.admin_digest_deliveries @@ -323,7 +324,7 @@ declare v_now timestamptz; begin if p_min_lease_ms is null or p_min_lease_ms < 0 or p_retention_ms is null or p_margin_ms is null - or p_margin_ms < 1 or p_margin_ms >= p_retention_ms then + or p_margin_ms < 1 or p_margin_ms >= p_retention_ms or p_retention_ms > 86400000 then raise exception 'admin_digest_begin_send: bad lease or retention' using errcode = '22023'; end if; select * into r from public.admin_digest_deliveries diff --git a/tests/admin-digest-delivery-engine.test.ts b/tests/admin-digest-delivery-engine.test.ts index c7d66035a..ea3500a06 100644 --- a/tests/admin-digest-delivery-engine.test.ts +++ b/tests/admin-digest-delivery-engine.test.ts @@ -17,6 +17,7 @@ import { payloadJsonOf, recipientKeyOf, resumeDigestOccurrence, type EngineConfig, type EngineDeps, type OccurrencePlan, } from '@/lib/admin/digest-delivery'; +import { sendEmail } from '@/lib/server/email'; import { FakeClock, FakeResendProvider, HOUR, MINUTE, MemoryDigestDeliveryStore, deferred, never, type StoreHook, } from './helpers/digest-delivery-fakes'; @@ -41,9 +42,6 @@ const CONFIG: EngineConfig = { retentionSafetyMarginMs: HOUR, }; -/** Resend's own rule for a key, as #692's sendEmail enforces it. */ -const KEY_RULE = /^[\x21-\x7e](?:[\x20-\x7e]{0,254}[\x21-\x7e])?$/; - function world(start = T0) { const clock = new FakeClock(start); const store = new MemoryDigestDeliveryStore(clock.now); @@ -59,7 +57,7 @@ const hookOn = (method: string, phase: 'before' | 'after', key: string | null, a (m, p, k) => (m === method && p === phase && (key === null || k === key) ? act() : undefined); const fail = () => { throw new Error('synthetic storage failure'); }; -afterEach(() => { vi.useRealTimers(); }); +afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); // ── Inputs ───────────────────────────────────────────────────────────────── @@ -87,6 +85,8 @@ describe('inputs are checked before anything is stored or sent', () => { ['a margin as long as the retention window', { retentionSafetyMarginMs: RESEND_KEY_RETENTION_MS }], ['zero attempts', { maxAttempts: 0 }], ['a retention longer than the provider\'s verified 24 h', { providerKeyRetentionMs: 48 * HOUR }], + ['a lease that outlasts the retry window, so a worker that dies after its mark could never be retried', { leaseMs: 23 * HOUR }], + ['a lease too long to be a date at all', { leaseMs: 1e16 }], ])('a config with %s is refused', async (_name, over) => { const { store, provider, engine } = world(); await expect(deliverDigestOccurrence(plan(), engine('a', { config: { ...CONFIG, ...over } }))).rejects.toThrow(TypeError); @@ -96,12 +96,11 @@ describe('inputs are checked before anything is stored or sent', () => { }); describe('keys and bytes', () => { - it('each recipient has one stable key: a function of occurrence and recipient only, free of the address, within Resend\'s rule', () => { + it('each recipient has one stable key: a function of occurrence and recipient only, free of the address', () => { const a = freezePlan(plan(), new Date(T0)); const b = freezePlan(plan({ payload: { ...plan().payload, html: '

other

' }, recipients: [TWO.toUpperCase(), ONE] }), new Date('2027-01-01T00:00:00Z')); expect(a.deliveries.map((d) => d.idempotencyKey)).toEqual(b.deliveries.map((d) => d.idempotencyKey)); for (const d of a.deliveries) { - expect(d.idempotencyKey).toMatch(KEY_RULE); expect(d.idempotencyKey).not.toMatch(/admin-one|admin-two|example/); expect(d.idempotencyKey).toBe(idempotencyKeyFor(OCC, d.recipientKey)); } @@ -109,11 +108,24 @@ describe('keys and bytes', () => { expect(idempotencyKeyFor('admin-digest:2026-10-01T12:30:00.000Z', K1)).not.toBe(idempotencyKeyFor(OCC, K1)); }); - it('the stored bytes are the body sendEmail would build for the same message (from, to, subject, html; no reply_to)', () => { - const d = freezePlan(plan(), new Date(T0)).deliveries[0]; - const { from, to, subject, html } = d.payload; - expect(d.payloadJson).toBe(JSON.stringify({ from, to, subject, html, reply_to: undefined })); - expect(d.payloadJson).toBe(payloadJsonOf(d.payload)); + it('#692\'s real sendEmail takes each key unchanged and posts exactly the stored bytes (fetch stubbed; nothing leaves)', async () => { + const posted: { url: string; key: string | null; body: string }[] = []; + vi.stubEnv('RESEND_API_KEY', 're_synthetic_not_a_real_key'); + vi.stubGlobal('fetch', vi.fn(async (url: RequestInfo | URL, init: RequestInit = {}) => { + posted.push({ url: String(url), key: new Headers(init.headers).get('idempotency-key'), body: String(init.body) }); + return new Response(JSON.stringify({ id: 'msg_synthetic' }), { status: 200, headers: { 'content-type': 'application/json' } }); + })); + const { deliveries } = freezePlan(plan(), new Date(T0)); + for (const d of deliveries) { + const { from, to, subject, html } = d.payload; + await expect(sendEmail({ from, to, subject, html, idempotencyKey: d.idempotencyKey })).resolves.toEqual({ ok: true }); + expect(d.payloadJson).toBe(payloadJsonOf(d.payload)); + } + expect(posted).toEqual(deliveries.map((d) => ({ url: 'https://api.resend.com/emails', key: d.idempotencyKey, body: d.payloadJson }))); + // Control: the same helper refuses a key it would have to alter, so the acceptance above means something. + const { from, to, subject, html } = deliveries[0].payload; + await expect(sendEmail({ from, to, subject, html, idempotencyKey: `${deliveries[0].idempotencyKey} ` })).rejects.toThrow(TypeError); + expect(posted).toHaveLength(deliveries.length); }); }); @@ -352,6 +364,31 @@ describe('a mark whose answer arrives late cannot send late (dispatch deadline)' expect(provider.inbox).toHaveLength(1); }); + it('queued work that runs after the mark answer cannot spend the admission budget between the check and the call', async () => { + // Unrelated JavaScript taking 2.5 s is queued `depth` microtasks behind the mark's answer, at every + // depth from 0 to 12. Wherever it lands, it runs before the check (nothing sent) or after the + // call (sent inside the window). It never runs between them, so a second copy never arrives. + const outcomes = new Set(); + for (let depth = 0; depth <= 12; depth++) { + const { clock, store, provider, engine } = world(); + provider.script = ({ n }) => (n === 1 ? { do: 'accept_then_lose_answer', reason: 'timeout' } : { do: 'accept' }); + await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a', { config: TIGHT })); // first mark at T0 + clock.set(new Date(Date.parse(T0) + 24 * HOUR - 2_000).toISOString()); // 0.5 s of budget left at the check + store.hooks.push(hookOn('beginSend', 'after', K1, () => { + let queued = Promise.resolve(); + for (let i = 0; i < depth; i++) queued = queued.then(() => undefined); + void queued.then(() => { clock.advance(2_500); }); + })); + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b', { config: TIGHT })); + expect(provider.inbox, `depth ${depth}`).toHaveLength(1); + if (r.attempts[0]?.detail === 'dispatch_deadline_passed') outcomes.add('held before the call'); + else if (provider.requests.length === 2 && r.attempts[0]?.result === 'accepted') outcomes.add('sent inside the window'); + else outcomes.add(`unexpected at depth ${depth}: ${JSON.stringify(r.attempts)}`); + } + // Both sides of the boundary were reached, so the sweep covered the gap it guards. + expect([...outcomes].sort()).toEqual(['held before the call', 'sent inside the window']); + }); + it('control: a mark answer that is slow but inside the deadline still sends, once', async () => { const { clock, store, provider, engine } = world(); store.hooks.push(hookOn('beginSend', 'after', K1, () => { clock.advance(4 * MINUTE); })); // lease 5 min, send deadline 150 ms diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts index db11a635d..a5d0963f5 100644 --- a/tests/admin-digest-delivery-postgres.test.ts +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -298,6 +298,15 @@ describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', ( expect(await fx.sql(`set role service_role; ${CALLS[0]}`)).toBe(''); expect(await fx.sql(`set role service_role; select public.admin_digest_claim('x', 'y', 'w', 1000, 1, 86400000, 3600000)::text;`)).toBe('{"reason": "not_found", "claimed": false}'); }); + it('the functions refuse a policy the engine refuses: retention past the verified 24 h, a lease that outlasts the retry window', async () => { + const bad = (stmt: string) => expect(fx.sql(`set role service_role; ${stmt}`)).rejects.toThrow(/bad owner, lease, attempts or retention|bad lease or retention/); + await bad(`select public.admin_digest_claim('x', 'y', 'w', 1000, 1, 86400001, 3600000);`); + await bad(`select public.admin_digest_begin_send('x', 'y', 1, 0, 86400001, 3600000);`); + await bad(`select public.admin_digest_claim('x', 'y', 'w', ${23 * HOUR}, 1, 86400000, 3600000);`); + await bad(`select public.admin_digest_claim('x', 'y', 'w', 2147483647, 1, 86400000, 3600000);`); // the largest lease the integer parameter holds + // Control: the longest lease that still lapses inside the retry window is accepted. + expect(await fx.sql(`set role service_role; select public.admin_digest_claim('x', 'y', 'w', ${23 * HOUR - 1}, 1, 86400000, 3600000)::text;`)).toBe('{"reason": "not_found", "claimed": false}'); + }); }); describe('frozen means frozen, in the database itself', () => { diff --git a/tests/helpers/digest-delivery-store-contract.ts b/tests/helpers/digest-delivery-store-contract.ts index 1bbbb98a3..e4af49492 100644 --- a/tests/helpers/digest-delivery-store-contract.ts +++ b/tests/helpers/digest-delivery-store-contract.ts @@ -287,6 +287,20 @@ export function describeDigestDeliveryStoreContract(label: string, make: (clock: expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toEqual({ ok: true, dispatchBy: new Date(Date.parse(start) + 23 * HOUR).toISOString() }); }); + it('a row that was only ever refused is not bound by retention: its dispatch deadline is the lease alone, at any age', async () => { + const { clock, store, id, key } = await setup(); + const a = await store.claim(id, key, 'a', CONTRACT_POLICY); + if (!a.claimed) throw new Error('setup'); + expect(answerOf(await store.beginSend(id, key, a.row.fence, CONTRACT_BEGIN))).toBe('ok'); + await store.complete(id, key, a.row.fence, { kind: 'rejected', httpStatus: 429, code: 'rate_limit_exceeded', retryable: true }, 3); + const later = new Date(Date.parse(start) + 30 * HOUR).toISOString(); // long past first mark + retention − margin + clock.set(later); + const b = await store.claim(id, key, 'b', CONTRACT_POLICY); + if (!b.claimed) throw new Error('setup'); + expect(b.row).toMatchObject({ ambiguous: false, firstSendAt: start }); + expect(await store.beginSend(id, key, b.row.fence, CONTRACT_BEGIN)).toEqual({ ok: true, dispatchBy: new Date(Date.parse(later) + 5 * MINUTE - 1_000).toISOString() }); + }); + it('an unknown occurrence or recipient is not found, not created', async () => { const { store, id } = await setup(); expect(await store.claim(id, 'f'.repeat(64), 'a', CONTRACT_POLICY)).toEqual({ claimed: false, reason: 'not_found' }); From 053fb9b83a41babd8cf8035ec2ef0fd2a6703895 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 22:12:06 +0000 Subject: [PATCH 08/12] fix(admin-digest delivery): a NULL fence never matches (review P3) In admin_digest_begin_send and admin_digest_complete, `r.fence <> p_fence` is NULL for a NULL fence, so the IF skipped its refusal and the mark or the completion went ahead. Both functions now refuse a NULL fence as bad input (22023) and compare with `is distinct from`, so NULL cannot read as a match even without the guard. Stale integers keep their answer (fenced_out). The adapter sends only a non-negative safe integer fence; null, undefined, negatives, fractions, NaN, Infinity and strings throw before any RPC. Evidence: a PostgreSQL test (a NULL fence refused by both functions, with the row unchanged and no message id written) and two CI-probe checks. Reverting either function to `<>` without the guard fails the probe and the test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 3 +- ...n-digest-reaches-each-admin-once-check.sql | 11 ++++++- lib/admin/digest-delivery-store.ts | 9 ++++-- ...n_admin_digest_reaches_each_admin_once.sql | 11 +++++-- tests/admin-digest-delivery-postgres.test.ts | 30 +++++++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index 47f7d315a..c1f6d5890 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -100,6 +100,7 @@ A PostgreSQL `DigestDeliveryStore` must meet all of these. It must also pass `te 4. **The fence is a counter.** `fence` increments on every successful claim. `complete` also requires the claim's mark: a claim that never marked a send owns no provider answer. - `beginSend` and `complete` are each one `UPDATE … WHERE status = 'in_flight' AND fence = $fence RETURNING`. - Zero rows means `fenced_out`, and nothing is written. + - **A NULL fence never matches.** In SQL, `fence <> NULL` is NULL, and an `IF` on NULL skips its refusal. So 0471 refuses a NULL fence as bad input (`22023`) and compares with `is distinct from`. The adapter sends only a non-negative safe integer. A stale integer such as `0` still reaches the database, which answers `fenced_out`. - Owner equality alone is not enough: the same owner can re-claim after its own lease lapsed. 5. **`beginSend` commits before the provider call.** In one step it: - evaluates `decideBeginSend`: fence, remaining lease against the send deadline, and the retention cut-off for ambiguous rows; @@ -145,7 +146,7 @@ The engine takes a `DigestEmailProvider` whose `send` returns one classified out **It is not applied to production.** PROD-DB-0177 blocks every migration there. The shared `docs/PENDING_PROD_MIGRATIONS.md` entry is proposed to the coordinator for reconciliation, not written here. **Evidence:** -- **CI.** `docs/audit/an-admin-digest-reaches-each-admin-once-check.sql` runs on the full migration replay: access refusals, freeze-once, the fence, a completion needing a mark, retention parking and frozen identity. +- **CI.** `docs/audit/an-admin-digest-reaches-each-admin-once-check.sql` runs on the full migration replay: access refusals, freeze-once, the fence (a NULL fence included), a completion needing a mark, retention parking and frozen identity. - **Disposable database.** `tests/admin-digest-delivery-postgres.test.ts` needs `DIGEST_DELIVERY_PG=1` and a local cluster; the `docs/audit/verify-pg.sh` harness is the default. It runs: - the store contract suite against PostgreSQL; - a differential check in which random steps, applied to the TypeScript rules and to the SQL, must agree after every step; diff --git a/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql index 4e4833d4a..af5e3a27a 100644 --- a/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql +++ b/docs/audit/an-admin-digest-reaches-each-admin-once-check.sql @@ -7,7 +7,7 @@ -- anyone: changing a delivery's bytes or key, deleting a delivery, -- changing or deleting an occurrence; -- rules freeze stores a plan once and never replaces it; one claim wins; --- a superseded claim cannot mark or complete; a completion needs a +-- a superseded claim or a NULL fence cannot mark or complete; a completion needs a -- mark; a lease that lapses after the mark makes the row ambiguous; -- an ambiguous row is parked at retention minus margin, at claim and -- at mark; too little lease left refuses the mark; a success without @@ -56,6 +56,7 @@ declare res jsonb; txt text; refused boolean; + rejected boolean; -- bad input (invalid_parameter_value), not a privilege refusal failures int := 0; pin timestamptz := '2026-09-30T12:31:00Z'; begin @@ -111,11 +112,19 @@ begin if not (res ->> 'claimed')::boolean or (res #>> '{row,fence}')::int <> 1 then raise warning 'first claim: %', res; failures := failures + 1; end if; res := public.admin_digest_claim(occ, k1, 'b', 300000, 3, ret, mar); if res ->> 'reason' is distinct from 'leased' then raise warning 'a live lease did not refuse: %', res; failures := failures + 1; end if; + -- A NULL fence is bad input, never a match; the unmarked-completion check below also proves it wrote no mark. + rejected := false; + begin perform public.admin_digest_begin_send(occ, k1, null, 150, ret, mar); exception when invalid_parameter_value then rejected := true; end; + if not rejected then raise warning 'begin_send accepted a NULL fence'; failures := failures + 1; end if; if public.admin_digest_complete(occ, k1, 1, '{"kind":"unknown","reason":"timeout"}', 3) <> 'fenced_out' then raise warning 'a completion without a mark was accepted'; failures := failures + 1; end if; if public.admin_digest_begin_send(occ, k1, 0, 150, ret, mar) ->> 'answer' <> 'fenced_out' then raise warning 'a stale fence marked'; failures := failures + 1; end if; if public.admin_digest_begin_send(occ, k1, 1, 150, ret, mar) ->> 'answer' <> 'ok' then raise warning 'the current fence could not mark'; failures := failures + 1; end if; + -- The completion below succeeding proves this one wrote nothing. + rejected := false; + begin perform public.admin_digest_complete(occ, k1, null, '{"kind":"accepted","messageId":"msg-forged"}', 3); exception when invalid_parameter_value then rejected := true; end; + if not rejected then raise warning 'complete accepted a NULL fence'; failures := failures + 1; end if; if public.admin_digest_complete(occ, k1, 1, '{"kind":"accepted","messageId":""}', 3) <> 'ok' then raise warning 'complete refused'; failures := failures + 1; end if; res := public.admin_digest_load(occ); if (res -> 'deliveries') @> jsonb_build_array(jsonb_build_object('recipientKey', k1, 'status', 'accepted')) then diff --git a/lib/admin/digest-delivery-store.ts b/lib/admin/digest-delivery-store.ts index 66f6443e8..5e8ac5747 100644 --- a/lib/admin/digest-delivery-store.ts +++ b/lib/admin/digest-delivery-store.ts @@ -98,6 +98,11 @@ function parseStored(v: unknown): StoredOccurrence { } const ms = (n: number, what: string) => (Number.isSafeInteger(Math.round(n)) && n >= 0 ? Math.round(n) : bad(what)); +/** A fence is a count: a stale one is answered fenced_out by the database; a non-integer (null included) is never sent. */ +const fenceArg = (f: unknown) => { + if (typeof f === 'number' && Number.isSafeInteger(f) && f >= 0) return f; + throw new TypeError('digest-delivery store: a fence must be a non-negative integer'); +}; export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryStore { return { @@ -137,7 +142,7 @@ export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryS async beginSend(occurrenceId, recipientKey, fence, policy: BeginSendPolicy) { const out = await rpc('admin_digest_begin_send', { - p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fence, + p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fenceArg(fence), p_min_lease_ms: ms(policy.minLeaseRemainingMs, 'minLease'), p_retention_ms: ms(policy.providerKeyRetentionMs, 'retention'), p_margin_ms: ms(policy.retentionSafetyMarginMs, 'margin'), }); @@ -149,7 +154,7 @@ export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryS async complete(occurrenceId, recipientKey, fence, result: ProviderSendResult, maxAttempts) { const out = await rpc('admin_digest_complete', { - p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fence, p_result: result, p_max_attempts: maxAttempts, + p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fenceArg(fence), p_result: result, p_max_attempts: maxAttempts, }); return out === 'ok' || out === 'fenced_out' ? out : bad('complete'); }, diff --git a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql index 940a3ba5a..883855769 100644 --- a/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql +++ b/supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql @@ -327,10 +327,14 @@ begin or p_margin_ms < 1 or p_margin_ms >= p_retention_ms or p_retention_ms > 86400000 then raise exception 'admin_digest_begin_send: bad lease or retention' using errcode = '22023'; end if; + if p_fence is null then + raise exception 'admin_digest_begin_send: a fence is required' using errcode = '22023'; + end if; select * into r from public.admin_digest_deliveries where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key for update; - if not found or r.status <> 'in_flight' or r.fence <> p_fence then + -- is distinct from: a NULL can never read as a match, even without the guard above. + if not found or r.status <> 'in_flight' or r.fence is distinct from p_fence then return jsonb_build_object('answer', 'fenced_out'); end if; v_now := public.admin_digest_now(); @@ -373,11 +377,14 @@ begin or v_kind is null or v_kind not in ('accepted', 'rejected', 'payload_conflict', 'in_progress', 'unknown') then raise exception 'admin_digest_complete: bad result or attempts' using errcode = '22023'; end if; + if p_fence is null then + raise exception 'admin_digest_complete: a fence is required' using errcode = '22023'; + end if; select * into r from public.admin_digest_deliveries where occurrence_id = p_occurrence_id and recipient_key = p_recipient_key for update; -- Fenced, and only after a mark: a claim that never marked a send owns no provider answer. - if not found or r.status <> 'in_flight' or r.fence <> p_fence or r.send_started_at is null then + if not found or r.status <> 'in_flight' or r.fence is distinct from p_fence or r.send_started_at is null then return 'fenced_out'; end if; v_now := public.admin_digest_now(); diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts index a5d0963f5..41674e026 100644 --- a/tests/admin-digest-delivery-postgres.test.ts +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -74,6 +74,20 @@ describe('PostgreSQL adapter: strict mapping (fake transport)', () => { await expect(storeAnswering('ok').store.beginSend(OCC, K1, 1, BEGIN)).rejects.toThrow(/malformed/); }); + it('a fence that is not a non-negative integer never reaches the database', async () => { + for (const fence of [null, undefined, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, '1']) { + const { store, calls } = storeAnswering({ answer: 'ok', dispatchBy: '2026-09-30T12:35:59.85+00:00' }); + await expect(store.beginSend(OCC, K1, fence as never, BEGIN)).rejects.toThrow(TypeError); + await expect(store.complete(OCC, K1, fence as never, { kind: 'accepted', messageId: 'm' }, 3)).rejects.toThrow(TypeError); + expect(calls, String(fence)).toEqual([]); + } + // Controls: a claim's fence, and a stale 0, go through unchanged for the database to judge. + const { store, calls } = storeAnswering('ok'); + expect(await store.complete(OCC, K1, 7, { kind: 'accepted', messageId: 'm' }, 3)).toBe('ok'); + expect(await store.complete(OCC, K1, 0, { kind: 'accepted', messageId: 'm' }, 3)).toBe('ok'); + expect(calls.map((c) => c[1].p_fence)).toEqual([7, 0]); + }); + it('freeze sends only identity and bytes, never a state', async () => { const f = freezePlan(contractPlan({ recipients: [ONE] }), new Date(T0)); const { store, calls } = storeAnswering({ created: true, occurrence: { ...f.occurrence }, deliveries: [row({ status: 'pending', attempts: 0, fence: 0, leaseOwner: null, leaseExpiresAt: null })] }); @@ -325,6 +339,22 @@ describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', ( await expect(fx.sql(`update public.admin_digest_occurrences set window_end = window_end + interval '1 day';`)).rejects.toThrow(/never changed or deleted/); await expect(fx.sql(`delete from public.admin_digest_occurrences;`)).rejects.toThrow(/never changed or deleted/); }); + it('a NULL fence is refused by begin_send and complete, and writes nothing', async () => { + const { store, where } = await setup(); + const a = await store.claim(OCC, K1, 'a', CLAIM); + if (!a.claimed) throw new Error('setup'); + const rowNow = async () => (await store.load(OCC))!.deliveries[0]; + await expect(fx.sql(`set role service_role; select public.admin_digest_begin_send('${OCC}', '${K1}', null, 150, 86400000, 3600000);`)).rejects.toThrow(/a fence is required/); + expect(await rowNow()).toMatchObject({ status: 'in_flight', fence: 1, sendStartedAt: null, firstSendAt: null }); + expect(answerOf(await store.beginSend(OCC, K1, a.row.fence, BEGIN))).toBe('ok'); + const marked = await rowNow(); + await expect(fx.sql(`set role service_role; select public.admin_digest_complete('${OCC}', '${K1}', null, '{"kind":"accepted","messageId":"msg-forged"}'::jsonb, 4);`)).rejects.toThrow(/a fence is required/); + expect(await rowNow()).toEqual(marked); + expect(await fx.sql(`select count(*) from public.admin_digest_deliveries where ${where} and provider_message_id is not null;`)).toBe('0'); + // Control: the claim's own fence still completes. + expect(await store.complete(OCC, K1, a.row.fence, { kind: 'accepted', messageId: 'msg-1' }, 4)).toBe('ok'); + }); + it('an insert whose hash, recipient key or idempotency key disagrees with its bytes is refused', async () => { const { f } = await setup(); const d = f.deliveries[0]; From d965d690f6a51e9a6668c8d6b8583e4e4d5e4d32 Mon Sep 17 00:00:00 2001 From: NewWorldVenture Date: Wed, 30 Sep 2026 18:40:37 -0400 Subject: [PATCH 09/12] test: align digest migration high-water pin --- tests/migration-version-safety.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/migration-version-safety.test.ts b/tests/migration-version-safety.test.ts index 4e3c1f9a0..47d1cda93 100644 --- a/tests/migration-version-safety.test.ts +++ b/tests/migration-version-safety.test.ts @@ -500,7 +500,10 @@ describe('Supabase migration filename safety', () => { // makes a chat read receipt or reaction its reader's own (DB-RPC-M02). // 0464 makes an invited guest read-only on the eight household resources // /family/permissions shows them as read-only on (ROLE-M03). - expect(audit.nextVersion).toBe('0465'); + // 0471 adds the per-recipient admin digest delivery store. This literal + // tracks the checked-in high-water mark, not migration allocation: + // 0465-0470 remain NWV's, 0472 Support's, and 0473 the coordinator's. + expect(audit.nextVersion).toBe('0472'); }); it('flags a newly introduced collision instead of silently accepting it', () => { From a9b15c9e38602f38edf2a940d24f4fe8f8f4b304 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 22:42:47 +0000 Subject: [PATCH 10/12] test(admin-digest delivery): pin the clock for the direct NULL-fence calls; compare raw rows Review note on 053fb9b8: the direct SQL calls in the NULL-fence test ran on the real clock, so under the old NULL-blind comparison the lease could already have lapsed and the negative control would stop exercising the bypass. They now carry the test's pinned clock, and the test compares the raw row (row_to_json) before and after each refused call, not only the adapter's view. Counterfactual on 0471 as of 4d2f82ef: with the pin, begin_send(NULL) answers ok and marks the row, and this test fails. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- tests/admin-digest-delivery-postgres.test.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts index 41674e026..58037b6fe 100644 --- a/tests/admin-digest-delivery-postgres.test.ts +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -340,16 +340,21 @@ describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', ( await expect(fx.sql(`delete from public.admin_digest_occurrences;`)).rejects.toThrow(/never changed or deleted/); }); it('a NULL fence is refused by begin_send and complete, and writes nothing', async () => { - const { store, where } = await setup(); + const { clock, store, where } = await setup(); const a = await store.claim(OCC, K1, 'a', CLAIM); if (!a.claimed) throw new Error('setup'); - const rowNow = async () => (await store.load(OCC))!.deliveries[0]; - await expect(fx.sql(`set role service_role; select public.admin_digest_begin_send('${OCC}', '${K1}', null, 150, 86400000, 3600000);`)).rejects.toThrow(/a fence is required/); - expect(await rowNow()).toMatchObject({ status: 'in_flight', fence: 1, sendStartedAt: null, firstSendAt: null }); + // Direct calls carry the same pinned clock as the adapter, so the lease is live and the old, + // NULL-blind comparison would really have marked or completed: the refusal is what stops it. + const asService = (call: string) => fx.sql(`set admin_digest.test_now = '${clock.now().toISOString()}';\nset role service_role;\n${call}`); + const rawRow = () => fx.sql(`select row_to_json(d)::text from public.admin_digest_deliveries d where ${where};`); + const before = await rawRow(); + await expect(asService(`select public.admin_digest_begin_send('${OCC}', '${K1}', null, 150, 86400000, 3600000);`)).rejects.toThrow(/a fence is required/); + expect(await rawRow()).toBe(before); + expect((await store.load(OCC))!.deliveries[0]).toMatchObject({ status: 'in_flight', fence: 1, sendStartedAt: null, firstSendAt: null }); expect(answerOf(await store.beginSend(OCC, K1, a.row.fence, BEGIN))).toBe('ok'); - const marked = await rowNow(); - await expect(fx.sql(`set role service_role; select public.admin_digest_complete('${OCC}', '${K1}', null, '{"kind":"accepted","messageId":"msg-forged"}'::jsonb, 4);`)).rejects.toThrow(/a fence is required/); - expect(await rowNow()).toEqual(marked); + const marked = await rawRow(); + await expect(asService(`select public.admin_digest_complete('${OCC}', '${K1}', null, '{"kind":"accepted","messageId":"msg-forged"}'::jsonb, 4);`)).rejects.toThrow(/a fence is required/); + expect(await rawRow()).toBe(marked); expect(await fx.sql(`select count(*) from public.admin_digest_deliveries where ${where} and provider_message_id is not null;`)).toBe('0'); // Control: the claim's own fence still completes. expect(await store.complete(OCC, K1, a.row.fence, { kind: 'accepted', messageId: 'msg-1' }, 4)).toBe('ok'); From 6927f820b3d0689e4091a97719d6c63ee670bacd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 23:44:05 +0000 Subject: [PATCH 11/12] fix(admin-digest delivery): one admitted bound for stored bytes and message ids (0471 parity) Review 5373043643. The engine admitted what 0471 refuses, and 0471 altered what the engine accepted: - Stored bytes: the engine checked the HTML at 512 KiB of bytes, while 0471 limits payload_json to 600,000 characters. 300,000 quote characters pass the first and serialize to 600,125 characters. Reproduced with the real adapter: the engine and the memory store accept it, and PostgreSQL rejects it with admin_digest_deliveries_payload_json_check. freezePlan now refuses any stored bytes over MAX_PAYLOAD_JSON_CHARS, counted in PostgreSQL characters (code points), before anything is stored. - Message ids: the engine accepted any non-empty id, while 0471 truncates at 200. checkResult and decideCompletion now treat a longer id as unknown (never a receipt, never truncated). The adapter refuses to send one. Tests: - The review's counterexample is refused before storage (engine and PG). - At the exact bound, both the engine and PostgreSQL accept, including an emoji-bearing payload that is longer in UTF-16. One character more is refused first. - The declared SQL bounds equal the engine's. - A 201-character id is not a receipt; 200 is. - 5/5 new safeguards are caught when removed one at a time. - PG suite with the immediate-restart command: 160/160. A direct SQL call can still truncate an id; that needs a coordinated schema change (not 0471). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KP9rt5rVQ9jDMpZp2xBy3K --- docs/admin-digest-delivery-contract.md | 8 +++ lib/admin/digest-delivery-store.ts | 7 ++- lib/admin/digest-delivery.ts | 24 ++++++- tests/admin-digest-delivery-engine.test.ts | 55 +++++++++++++++- tests/admin-digest-delivery-postgres.test.ts | 66 +++++++++++++++++++- 5 files changed, 153 insertions(+), 7 deletions(-) diff --git a/docs/admin-digest-delivery-contract.md b/docs/admin-digest-delivery-contract.md index c1f6d5890..5f14099aa 100644 --- a/docs/admin-digest-delivery-contract.md +++ b/docs/admin-digest-delivery-contract.md @@ -75,6 +75,14 @@ The engine re-reads its own clock **after** the store answers, in the same synch **Clock skew.** `dispatchBy` is database time and the engine compares it with the application's clock. Skew between the two must stay well inside both `leaseMs − sendTimeoutMs` and `retentionSafetyMarginMs − sendTimeoutMs`. NTP-level skew does; the defaults are 5 min, 1 h and 15 s. +**Storage bounds: one admitted bound, not two.** 0471 limits `payload_json` to 600,000 and `provider_message_id` to 200 characters, counted as PostgreSQL's `length()` counts them (in code points). The engine applies the same numbers, in the same unit, before anything is stored: +- `freezePlan` refuses a plan whose stored bytes for any recipient exceed `MAX_PAYLOAD_JSON_CHARS`. The 512 KiB HTML byte bound alone is not enough: 300,000 quote characters are 300,000 bytes but serialize to more than 600,000 characters. +- A provider id longer than `MAX_PROVIDER_MESSAGE_ID_CHARS` is **not a receipt**, in both `checkResult` and `decideCompletion`. The row stays `unknown` and is retried under the same key, and the id is never truncated. +- The adapter refuses to send such an id to 0471, whose `left(v_mid, 200)` would truncate it. +- A test asserts that 0471 declares the same two numbers. + +**Residual, needs coordination:** a direct privileged call to `admin_digest_complete` with a longer id would still be truncated by the SQL. Making the SQL refuse it too needs a schema change, which I've put to the coordinator. + **Policy bounds.** `validateConfig` refuses, and 0471's functions refuse again, two policies: - a retention longer than the verified 24 hours; - a lease of `retention − margin` or more. A worker that dies after its mark holds the row until the lease lapses, so a longer lease would park every such crash instead of retrying it. diff --git a/lib/admin/digest-delivery-store.ts b/lib/admin/digest-delivery-store.ts index 5e8ac5747..b6701b4f8 100644 --- a/lib/admin/digest-delivery-store.ts +++ b/lib/admin/digest-delivery-store.ts @@ -9,7 +9,7 @@ // for a service-role Supabase client (later), or the psql transport the // disposable-database tests use (tests/helpers/digest-delivery-postgres.ts). import { - payloadJsonOf, + MAX_PROVIDER_MESSAGE_ID_CHARS, payloadJsonOf, pgLength, type BeginSendPolicy, type ClaimPolicy, type ClaimRefusal, type DeliveryPayload, type DeliveryRow, type DeliveryStatus, type DigestDeliveryStore, type FrozenOccurrence, type ProviderSendResult, type StoredOccurrence, } from '@/lib/admin/digest-delivery'; @@ -153,6 +153,11 @@ export function createPostgresDigestDeliveryStore(rpc: RpcCall): DigestDeliveryS }, async complete(occurrenceId, recipientKey, fence, result: ProviderSendResult, maxAttempts) { + // 0471 would truncate a longer id; a receipt is never altered, so it is never sent. (An empty id + // is sent: 0471, like decideCompletion, records it as unknown, not as a receipt.) + if (result.kind === 'accepted' && typeof result.messageId === 'string' && pgLength(result.messageId) > MAX_PROVIDER_MESSAGE_ID_CHARS) { + throw new TypeError(`digest-delivery store: an accepted message id must be 1-${MAX_PROVIDER_MESSAGE_ID_CHARS} characters`); + } const out = await rpc('admin_digest_complete', { p_occurrence_id: occurrenceId, p_recipient_key: recipientKey, p_fence: fenceArg(fence), p_result: result, p_max_attempts: maxAttempts, }); diff --git a/lib/admin/digest-delivery.ts b/lib/admin/digest-delivery.ts index 6e30fcb51..c8b0fe6d3 100644 --- a/lib/admin/digest-delivery.ts +++ b/lib/admin/digest-delivery.ts @@ -303,8 +303,8 @@ export function decideCompletion(row: DeliveryRow, fence: number, result: Provid const next: DeliveryRow = { ...row, leaseOwner: null, leaseExpiresAt: null, sendStartedAt: null, updatedAt: nowIso }; switch (result.kind) { case 'accepted': - // A receipt needs a message id; without one nothing is settled. - if (typeof result.messageId !== 'string' || result.messageId.length === 0) { + // A receipt needs a message id the store can hold whole; without one nothing is settled. + if (!isStorableMessageId(result.messageId)) { next.ambiguous = true; next.status = 'unknown'; next.lastError = 'outcome_unknown:unreadable_response'; break; } @@ -372,6 +372,21 @@ export function idempotencyKeyFor(occurrenceId: string, recipientKey: string): s /** Fixed key order, so the same payload is always the same bytes. */ export const payloadJsonOf = (p: DeliveryPayload) => JSON.stringify({ from: p.from, to: p.to, subject: p.subject, html: p.html }); + +/** + * 0471's bounds, in PostgreSQL characters (code points, what `length(text)` counts). The engine + * admits nothing the store would refuse: the stored bytes are checked before anything is stored, + * and a provider id the store could not hold whole is not a receipt (it is never truncated). + */ +export const MAX_PAYLOAD_JSON_CHARS = 600_000; +export const MAX_PROVIDER_MESSAGE_ID_CHARS = 200; +export function pgLength(s: string): number { + let n = 0; + for (const _ of s) n += 1; + return n; +} +export const isStorableMessageId = (id: unknown): id is string => + typeof id === 'string' && id.length > 0 && pgLength(id) <= MAX_PROVIDER_MESSAGE_ID_CHARS; const occurrencePayloadHash = (p: DigestPayload) => sha256(JSON.stringify({ from: p.from, subject: p.subject, html: p.html })); // ── Validation (before any storage call or send) ──────────────────────────── @@ -438,6 +453,9 @@ export function freezePlan(plan: OccurrencePlan, now: Date): { occurrence: Froze providerMessageId: null, lastError: null, updatedAt: nowIso, }; }).sort((a, b) => (a.recipientKey < b.recipientKey ? -1 : 1)); + for (const d of deliveries) { + if (pgLength(d.payloadJson) > MAX_PAYLOAD_JSON_CHARS) planError(`the stored bytes must be at most ${MAX_PAYLOAD_JSON_CHARS} characters (0471's bound)`); + } return { occurrence: { occurrenceId: plan.occurrenceId, window: { start: new Date(plan.window.start).toISOString(), end: new Date(plan.window.end).toISOString() }, @@ -646,7 +664,7 @@ const messageIdOf = (r: ProviderSendResult) => (r.kind === 'accepted' ? { messag function checkResult(r: ProviderSendResult): ProviderSendResult { if (!r || typeof r !== 'object') return { kind: 'unknown', reason: 'unreadable_response' }; switch (r.kind) { - case 'accepted': return typeof r.messageId === 'string' && r.messageId.length > 0 ? r : { kind: 'unknown', reason: 'unreadable_response' }; + case 'accepted': return isStorableMessageId(r.messageId) ? r : { kind: 'unknown', reason: 'unreadable_response' }; // A refusal is a 4xx: an adapter that calls a 5xx or a 2xx "rejected" has settled nothing. case 'rejected': return Number.isInteger(r.httpStatus) && r.httpStatus >= 400 && r.httpStatus < 500 && typeof r.retryable === 'boolean' ? { ...r, code: String(r.code ?? '') } : { kind: 'unknown', reason: 'unreadable_response' }; diff --git a/tests/admin-digest-delivery-engine.test.ts b/tests/admin-digest-delivery-engine.test.ts index ea3500a06..042306fb8 100644 --- a/tests/admin-digest-delivery-engine.test.ts +++ b/tests/admin-digest-delivery-engine.test.ts @@ -13,8 +13,8 @@ import { createHash } from 'node:crypto'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { - RESEND_KEY_RETENTION_MS, classifyResendResponse, deliverDigestOccurrence, freezePlan, idempotencyKeyFor, - payloadJsonOf, recipientKeyOf, resumeDigestOccurrence, + MAX_PAYLOAD_JSON_CHARS, MAX_PROVIDER_MESSAGE_ID_CHARS, RESEND_KEY_RETENTION_MS, classifyResendResponse, decideCompletion, + deliverDigestOccurrence, freezePlan, idempotencyKeyFor, payloadJsonOf, recipientKeyOf, resumeDigestOccurrence, type EngineConfig, type EngineDeps, type OccurrencePlan, } from '@/lib/admin/digest-delivery'; import { sendEmail } from '@/lib/server/email'; @@ -95,6 +95,57 @@ describe('inputs are checked before anything is stored or sent', () => { }); }); +describe('what the engine admits, the store can hold (0471\'s bounds)', () => { + const base = plan({ recipients: [ONE] }).payload; + const withHtml = (html: string) => plan({ recipients: [ONE], payload: { ...base, html } }); + const overhead = payloadJsonOf({ from: base.from, to: ONE, subject: base.subject, html: '' }).length; + + it('review counterexample: 300,000 quotes are valid HTML by bytes but not storable; refused before anything is stored or sent', async () => { + const { store, provider, engine } = world(); + await expect(deliverDigestOccurrence(withHtml('"'.repeat(300_000)), engine('a'))).rejects.toThrow(/at most 600000 characters/); + expect(store.calls).toEqual([]); + expect(provider.requests).toEqual([]); + }); + + it('the bound is exact, and counted in characters as PostgreSQL counts them', () => { + const at = '"'.repeat(100_000) + 'a'.repeat(MAX_PAYLOAD_JSON_CHARS - overhead - 200_000); + expect(freezePlan(withHtml(at), new Date(T0)).deliveries[0].payloadJson.length).toBe(MAX_PAYLOAD_JSON_CHARS); + expect(() => freezePlan(withHtml(`${at}a`), new Date(T0))).toThrow(/at most 600000 characters/); + // An emoji is two UTF-16 units but one PostgreSQL character: the bound follows PostgreSQL. (A control + // character is one byte that serializes to six characters, which keeps the HTML under 512 KiB.) + const emojis = 50_000; + const controls = Math.floor((MAX_PAYLOAD_JSON_CHARS - overhead - emojis) / 6); + const fill = MAX_PAYLOAD_JSON_CHARS - overhead - emojis - 6 * controls; + const mixed = '\u{1F600}'.repeat(emojis) + '\u0001'.repeat(controls) + 'a'.repeat(fill); + const f = freezePlan(withHtml(mixed), new Date(T0)); + expect(f.deliveries[0].payloadJson.length).toBe(MAX_PAYLOAD_JSON_CHARS + emojis); // UTF-16 units + expect(() => freezePlan(withHtml(`${mixed}a`), new Date(T0))).toThrow(/at most 600000 characters/); + }); + + it('the completion rule itself never records a receipt the store would have to truncate', () => { + const row = { ...freezePlan(plan({ recipients: [ONE] }), new Date(T0)).deliveries[0], status: 'in_flight' as const, fence: 1, attempts: 1, + leaseOwner: 'a', leaseExpiresAt: '2026-09-30T12:36:00.000Z', sendStartedAt: T0, firstSendAt: T0 }; + expect(decideCompletion(row, 1, { kind: 'accepted', messageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS + 1) }, new Date(T0), 5)) + .toMatchObject({ status: 'unknown', ambiguous: true, providerMessageId: null }); + expect(decideCompletion(row, 1, { kind: 'accepted', messageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS) }, new Date(T0), 5)) + .toMatchObject({ status: 'accepted', providerMessageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS) }); + }); + + it('a provider id the store could not hold whole is not a receipt: the row stays unknown and the retry is answered from the key', async () => { + const { store, engine } = world(); + let calls = 0; + const longId = { send: async () => { calls += 1; return { kind: 'accepted' as const, messageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS + 1) }; } }; + const r = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('a', { provider: longId })); + expect(r.attempts).toEqual([{ recipientKey: K1, fence: 1, result: 'unknown', recorded: 'ok' }]); + expect(store.row(OCC, K1)).toMatchObject({ status: 'unknown', ambiguous: true, providerMessageId: null }); + const atBound = { send: async () => ({ kind: 'accepted' as const, messageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS) }) }; + const again = await deliverDigestOccurrence(plan({ recipients: [ONE] }), engine('b', { provider: atBound })); + expect(again.complete).toBe(true); + expect(store.row(OCC, K1)!.providerMessageId).toHaveLength(MAX_PROVIDER_MESSAGE_ID_CHARS); + expect(calls).toBe(1); + }); +}); + describe('keys and bytes', () => { it('each recipient has one stable key: a function of occurrence and recipient only, free of the address', () => { const a = freezePlan(plan(), new Date(T0)); diff --git a/tests/admin-digest-delivery-postgres.test.ts b/tests/admin-digest-delivery-postgres.test.ts index 58037b6fe..fec0e4bec 100644 --- a/tests/admin-digest-delivery-postgres.test.ts +++ b/tests/admin-digest-delivery-postgres.test.ts @@ -9,9 +9,11 @@ // Synthetic data only. No provider is called: the engine tests use the fake // provider that follows Resend's documented key semantics. import { execSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { - RESEND_KEY_RETENTION_MS, deliverDigestOccurrence, freezePlan, recipientKeyOf, resumeDigestOccurrence, + MAX_PAYLOAD_JSON_CHARS, MAX_PROVIDER_MESSAGE_ID_CHARS, RESEND_KEY_RETENTION_MS, deliverDigestOccurrence, freezePlan, + payloadJsonOf, recipientKeyOf, resumeDigestOccurrence, type BeginSendPolicy, type ClaimPolicy, type DigestDeliveryStore, type EngineConfig, type ProviderSendResult, } from '@/lib/admin/digest-delivery'; import { createPostgresDigestDeliveryStore, parseDeliveryRow, supabaseRpc, type RpcCall } from '@/lib/admin/digest-delivery-store'; @@ -20,6 +22,7 @@ import { answerOf, contractPlan, describeDigestDeliveryStoreContract } from './h import { createPgFixture, pgFixtureEnabled, psql, type PgFixture } from './helpers/digest-delivery-postgres'; const T0 = '2026-09-30T12:31:00.000Z'; +const MIGRATION_0471 = 'supabase/migrations/0471_an_admin_digest_reaches_each_admin_once.sql'; const OCC = 'admin-digest:2026-09-30T12:30:00.000Z'; const ONE = 'admin-one@example.test'; const TWO = 'admin-two@example.test'; @@ -88,6 +91,27 @@ describe('PostgreSQL adapter: strict mapping (fake transport)', () => { expect(calls.map((c) => c[1].p_fence)).toEqual([7, 0]); }); + it('the bounds the engine enforces are the ones 0471 declares', () => { + const sql = readFileSync(MIGRATION_0471, 'utf8'); + expect(sql).toContain(`check (length(payload_json) between 2 and ${MAX_PAYLOAD_JSON_CHARS})`); + expect(sql).toContain(`length(provider_message_id) between 1 and ${MAX_PROVIDER_MESSAGE_ID_CHARS})`); + }); + + it('an accepted message id the store could not hold whole is never sent to it (0471 would truncate it)', async () => { + for (const messageId of ['x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS + 1), '\u{1F600}'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS + 1)]) { + const { store, calls } = storeAnswering('ok'); + await expect(store.complete(OCC, K1, 1, { kind: 'accepted', messageId }, 3)).rejects.toThrow(TypeError); + expect(calls).toEqual([]); + } + // Controls: exactly at the bound, counted in characters as PostgreSQL does (an emoji is one); and an + // empty id, which 0471 itself records as unknown, not as a receipt. + const { store, calls } = storeAnswering('ok'); + expect(await store.complete(OCC, K1, 1, { kind: 'accepted', messageId: 'x'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS) }, 3)).toBe('ok'); + expect(await store.complete(OCC, K1, 1, { kind: 'accepted', messageId: '\u{1F600}'.repeat(MAX_PROVIDER_MESSAGE_ID_CHARS) }, 3)).toBe('ok'); + expect(await store.complete(OCC, K1, 1, { kind: 'accepted', messageId: '' }, 3)).toBe('ok'); + expect(calls).toHaveLength(3); + }); + it('freeze sends only identity and bytes, never a state', async () => { const f = freezePlan(contractPlan({ recipients: [ONE] }), new Date(T0)); const { store, calls } = storeAnswering({ created: true, occurrence: { ...f.occurrence }, deliveries: [row({ status: 'pending', attempts: 0, fence: 0, leaseOwner: null, leaseExpiresAt: null })] }); @@ -323,6 +347,46 @@ describe.skipIf(!pgFixtureEnabled)('0471 on PostgreSQL (disposable database)', ( }); }); + describe('parity with 0471\'s bounds: what the engine admits, PostgreSQL stores', () => { + const base = contractPlan({ recipients: [ONE] }).payload; + const withHtml = (html: string) => contractPlan({ recipients: [ONE], payload: { ...base, html } }); + const overhead = payloadJsonOf({ from: base.from, to: ONE, subject: base.subject, html: '' }).length; + const occurrences = () => fx.sql('select count(*) from public.admin_digest_occurrences;'); + + it('review counterexample: 300,000 quotes are 300,000 bytes of HTML but 600,104 stored characters; refused before anything is stored', async () => { + await fx.reset(); + const plan = withHtml('"'.repeat(300_000)); + expect(payloadJsonOf({ ...plan.payload, to: ONE }).length).toBeGreaterThan(600_000); + expect(() => freezePlan(plan, new Date(T0))).toThrow(TypeError); + const store = pgStore(new FakeClock(T0)); + await expect(deliverDigestOccurrence(plan, { store, provider: new FakeResendProvider(() => new Date(T0)), owner: 'a', config: CONFIG, now: () => new Date(T0) })).rejects.toThrow(TypeError); + expect(await occurrences()).toBe('0'); + }); + + it('exactly at the bound, PostgreSQL stores it; one character over, the engine refuses it first', async () => { + await fx.reset(); + const quotes = 100_000; // each serializes to two characters, keeping the HTML under its 512 KiB byte bound + const atBound = withHtml('"'.repeat(quotes) + 'a'.repeat(MAX_PAYLOAD_JSON_CHARS - overhead - 2 * quotes)); + const f = freezePlan(atBound, new Date(T0)); + expect(f.deliveries[0].payloadJson.length).toBe(MAX_PAYLOAD_JSON_CHARS); + const stored = await pgStore(new FakeClock(T0)).freeze(f.occurrence, f.deliveries); + expect(stored.created).toBe(true); + expect(stored.deliveries[0].payloadJson).toBe(f.deliveries[0].payloadJson); + await fx.reset(); + const over = withHtml('"'.repeat(quotes) + 'a'.repeat(MAX_PAYLOAD_JSON_CHARS - overhead - 2 * quotes + 1)); + expect(() => freezePlan(over, new Date(T0))).toThrow(/at most 600000 characters/); + expect(await occurrences()).toBe('0'); + // Counted as PostgreSQL counts: an emoji-bearing payload exactly at the bound in characters (but + // longer in UTF-16 units) is admitted by the engine AND stored by the database. + const emojis = 50_000; + const controls = Math.floor((MAX_PAYLOAD_JSON_CHARS - overhead - emojis) / 6); + const mixed = withHtml('\u{1F600}'.repeat(emojis) + '\u0001'.repeat(controls) + 'a'.repeat(MAX_PAYLOAD_JSON_CHARS - overhead - emojis - 6 * controls)); + const g = freezePlan(mixed, new Date(T0)); + expect((await pgStore(new FakeClock(T0)).freeze(g.occurrence, g.deliveries)).created).toBe(true); + expect(await fx.sql(`select length(payload_json) from public.admin_digest_deliveries;`)).toBe(String(MAX_PAYLOAD_JSON_CHARS)); + }, 60_000); + }); + describe('frozen means frozen, in the database itself', () => { const setup = async () => { const clock = new FakeClock(T0); From 046bc33b8ac8e1ea70a09495eee1815374de9a81 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 13:15:37 +0000 Subject: [PATCH 12/12] E2E: schedule the social post a week ahead, not on a fixed date social-publish-consumers.spec.ts:269 filled the schedule field with '2026-10-01T12:00'. From 12:00 UTC on 2026-10-01 that is a past time, which the studio rightly refuses, so the test fails on every branch, main included (01b2c380 passed at 10:35 UTC; the same commit fails after 12:00, 1 failed / 11 passed locally, pinned Chromium 149). The time is now computed in the page as a week ahead, in the browser's own local time, which is what a datetime-local field takes. No assertion changed. Pinned Chromium, retries 0: 36/36 (x3). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TRY21ZKsFrfB3qtoP972A4 --- tests/e2e/social-publish-consumers.spec.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/e2e/social-publish-consumers.spec.ts b/tests/e2e/social-publish-consumers.spec.ts index 4be412212..dbdabc205 100644 --- a/tests/e2e/social-publish-consumers.spec.ts +++ b/tests/e2e/social-publish-consumers.spec.ts @@ -268,7 +268,14 @@ test('a lost action response does not offer a duplicate create after the provide test('a saved schedule links to its existing record and does not schedule or publish a second copy', async ({ page }) => { const { state } = await fixture(page); await compose(page); - await page.locator('input[type="datetime-local"]').fill('2026-10-01T12:00'); + // A week ahead, in the browser's own local time: a fixed date becomes a + // past schedule the day it arrives, and a past time is not schedulable. + const nextWeek = await page.evaluate(() => { + const d = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000); + const two = (n: number) => String(n).padStart(2, '0'); + return `${d.getFullYear()}-${two(d.getMonth() + 1)}-${two(d.getDate())}T${two(d.getHours())}:${two(d.getMinutes())}`; + }); + await page.locator('input[type="datetime-local"]').fill(nextWeek); await page.getByRole('button', { name: messages['studio.schedule'], exact: true }).click(); await resultCount(page, 1); expect(state.rows.social_posts).toHaveLength(1); expect(state.rows.social_schedules).toHaveLength(1); expect(state.rows.social_calendar_items).toHaveLength(1); expect(state.providerCalls).toBe(0);