diff --git a/docs/contributing/disaster-recovery.md b/docs/contributing/disaster-recovery.md index 435292e888..fb46fffaf8 100644 --- a/docs/contributing/disaster-recovery.md +++ b/docs/contributing/disaster-recovery.md @@ -370,8 +370,15 @@ config, and expect users to reauthorize OAuth and remote connectors. | Hourly `:45` | Control plane | D1 freshness (identity, size ceiling, manifest age ≤26h, R2 size/ETag) + seal recent complete days | Hourly freshness does not SHA-256 the SQL bytes; drills do. Page yourself on -`freshness-stale`, size-ceiling hits, missing manifests, seal failures, -`backup-unrestorable-statements`, or unexpected disablement of the enable gates. +`freshness-unrestorable` (the SQL contains statements D1 cannot import), +`freshness-stale`, size-ceiling hits, missing manifests or required SQL stats, +seal failures, `backup-unrestorable-statements`, or unexpected disablement of +the enable gates. Post-cutover unrestorable exports never receive a canonical +day manifest; catch-up retries can re-export the day after the oversized row or +write path is bounded. Historical bad days may already have canonical and full +manifests; immutable media is intentionally left unchanged, so freshness, +dashboard, drill, and production-restore gates remain essential until it ages +out. ## Offline CLI fallback diff --git a/packages/backup-control-plane/backup-control-plane-test-support.ts b/packages/backup-control-plane/backup-control-plane-test-support.ts index 0710ee633c..de46b34af4 100644 --- a/packages/backup-control-plane/backup-control-plane-test-support.ts +++ b/packages/backup-control-plane/backup-control-plane-test-support.ts @@ -6,6 +6,11 @@ import { canonicalBackupManifestPayload, type BackupManifestPayload, } from '@kody-internal/shared/backup-manifest.ts' +import { + backupSqlStatsKey, + backupSqlStatsSchemaVersion, + type BackupSqlStats, +} from '@kody-internal/shared/backup-sql-stats.ts' import { type BackupRuntimeStep } from './backup-runtime.ts' import { type DurableExportStep } from './durable-export.ts' import { BackupError, objectKeyForBookmark } from './backup-policy.ts' @@ -81,6 +86,7 @@ export class MemoryBucket { readonly puts: Array<{ key: string; options: R2PutOptions }> = [] private readonly objects = new Map() private readonly reportedSizes = new Map() + private readonly failedGets = new Set() private nextPutRace: { key: string; bytes: Uint8Array } | undefined private lockPolicyEnabled = false @@ -126,7 +132,12 @@ export class MemoryBucket { return this.objects.has(key) ? this.metadata(key) : null } + async delete(key: string): Promise { + this.objects.delete(key) + } + async get(key: string): Promise { + if (this.failedGets.has(key)) throw new Error('simulated R2 get failure') const bytes = this.objects.get(key) if (!bytes) return null const metadata = this.metadata(key) @@ -151,6 +162,10 @@ export class MemoryBucket { this.reportedSizes.set(key, size) } + failGetFor(key: string): void { + this.failedGets.add(key) + } + raceOnNextPut(key: string, value: string): void { this.nextPutRace = { key, bytes: new TextEncoder().encode(value) } } @@ -179,6 +194,36 @@ export const DRILL_ACCOUNT_ID = '33333333-3333-4333-8333-333333333333' export const BASELINE_SHA256 = 'b'.repeat(64) const manifestSigningKeys = generateKeyPairSync('ed25519') +export function badSqlStatsFixture( + day: string, + objectKey: string, +): BackupSqlStats { + return { + schemaVersion: backupSqlStatsSchemaVersion, + day, + objectKey, + maxStatementBytes: 1_495_663, + oversizedStatementCount: 1, + importStatementLimitBytes: 100_000, + } +} + +export async function putSqlStatsFixture( + bucket: MemoryBucket, + day: string, + objectKey: string, + options: { oversized?: boolean } = {}, +): Promise { + const stats = options.oversized + ? badSqlStatsFixture(day, objectKey) + : { + ...badSqlStatsFixture(day, objectKey), + maxStatementBytes: 50_000, + oversizedStatementCount: 0, + } + await bucket.put(backupSqlStatsKey(objectKey), JSON.stringify(stats)) +} + export function environment(bucket = new MemoryBucket()): BackupEnvironment { return { BACKUP_BUCKET: bucket as unknown as R2Bucket, @@ -307,11 +352,11 @@ export class RetryAfterCommitStep implements BackupRuntimeStep { export class CachedUploadStep implements BackupRuntimeStep { private readonly cache = new Map() - private readonly afterUpload: () => void + private readonly afterUpload: () => void | Promise private readonly afterFinalization?: () => Promise constructor( - afterUpload: () => void, + afterUpload: () => void | Promise, afterFinalization?: () => Promise, ) { this.afterUpload = afterUpload @@ -336,7 +381,7 @@ export class CachedUploadStep implements BackupRuntimeStep { : callback! const value = await execute() this.cache.set(name, value) - if (name === 'stream-export-to-immutable-r2') this.afterUpload() + if (name === 'stream-export-to-immutable-r2') await this.afterUpload() if ( name === 'verify-stored-object-and-write-immutable-manifest' && this.afterFinalization @@ -349,6 +394,43 @@ export class CachedUploadStep implements BackupRuntimeStep { async sleep(): Promise {} } +export class PreStatsUploadStep implements BackupRuntimeStep { + private readonly cache = new Map() + + async do( + name: string, + config: unknown, + callback: () => Promise, + ): Promise + async do(name: string, callback: () => Promise): Promise + async do( + name: string, + configOrCallback: unknown, + callback?: () => Promise, + ): Promise { + if (this.cache.has(name)) return this.cache.get(name) as T + const execute = + typeof configOrCallback === 'function' + ? (configOrCallback as () => Promise) + : callback! + const value = await execute() + const persisted = + name === 'stream-export-to-immutable-r2' && + value !== null && + typeof value === 'object' + ? Object.fromEntries( + Object.entries(value).filter( + ([key]) => key !== 'sqlStatementStats', + ), + ) + : value + this.cache.set(name, persisted) + return persisted as T + } + + async sleep(): Promise {} +} + export class RetryUploadStep implements BackupRuntimeStep { readonly uploadAttempts: number[] = [] private readonly cache = new Map() diff --git a/packages/backup-control-plane/backup-runtime.node.test.ts b/packages/backup-control-plane/backup-runtime.node.test.ts index 5ba0006731..99bf9cec38 100644 --- a/packages/backup-control-plane/backup-runtime.node.test.ts +++ b/packages/backup-control-plane/backup-runtime.node.test.ts @@ -15,8 +15,10 @@ import { CachedUploadStep, DATABASE_ID, MemoryBucket, + PreStatsUploadStep, RetryAfterCommitStep, RetryUploadStep, + badSqlStatsFixture, environment, exportEnvelope, identityEnvelope, @@ -287,6 +289,167 @@ test('workflow retry reuses an upload committed before step persistence and writ assert.ok(stats.maxStatementBytes > 0) }) +test('oversized SQL writes stats then fails retryably without a day manifest', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const env = environment(bucket) + const payload = backupPayload(env, new Date('2026-07-31T02:15:00Z')) + const objectKey = objectKeyForBookmark(payload.objectPrefix, 'bookmark-1') + const sql = `INSERT INTO t VALUES ('${'x'.repeat(100_001)}');` + + await assert.rejects( + runBackupRuntime( + env, + { + instanceId: workflowInstanceId(DATABASE_ID, payload.day), + payload, + timestamp: new Date('2026-07-31T02:15:01Z'), + }, + new CachedUploadStep(() => undefined), + { + api: { + fetcher: async (input) => + String(input).endsWith('/export') + ? exportEnvelope('complete') + : identityEnvelope(1_000), + sleep: async () => undefined, + }, + downloadFetcher: async () => + new Response(sql, { + headers: { 'content-length': String(sql.length) }, + }), + }, + ), + (error: unknown) => + error instanceof BackupError && + error.code === 'backup-unrestorable-statements' && + error.retryable, + ) + + const statsObject = await bucket.get(`${objectKey}.stats.json`) + assert.notEqual(statsObject, null) + const stats = (await statsObject!.json()) as { + oversizedStatementCount: number + } + assert.equal(stats.oversizedStatementCount, 1) + assert.equal(await bucket.head(payload.manifestKey), null) + const events = consoleError.mock.calls.map(([record]) => + JSON.parse(String(record)), + ) as Array<{ event: string }> + assert.ok( + events.some(({ event }) => event === 'backup-unrestorable-statements'), + ) + assert.ok(events.some(({ event }) => event === 'backup-failure')) +}) + +test('cached pre-stats uploads are allowed only for legacy backup days', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const consoleLog = vi.spyOn(console, 'log') + consoleLog.mockImplementation(() => undefined) + const options = { + api: { + fetcher: async (input: RequestInfo | URL) => + String(input).endsWith('/export') + ? exportEnvelope('complete') + : identityEnvelope(1_000), + sleep: async () => undefined, + }, + downloadFetcher: async () => + new Response('valid', { headers: { 'content-length': '5' } }), + } + + const legacyBucket = new MemoryBucket() + const legacyEnv = environment(legacyBucket) + const legacyPayload = backupPayload( + legacyEnv, + new Date('2026-07-27T02:15:00Z'), + ) + await runBackupRuntime( + legacyEnv, + { + instanceId: workflowInstanceId(DATABASE_ID, legacyPayload.day), + payload: legacyPayload, + timestamp: new Date('2026-07-27T02:15:01Z'), + }, + new PreStatsUploadStep(), + options, + ) + assert.notEqual(await legacyBucket.head(legacyPayload.manifestKey), null) + const legacyEvents = consoleLog.mock.calls.map(([record]) => + JSON.parse(String(record)), + ) as Array<{ event: string }> + assert.ok( + legacyEvents.some(({ event }) => event === 'backup-stats-legacy-missing'), + ) + + const requiredBucket = new MemoryBucket() + const requiredEnv = environment(requiredBucket) + const requiredPayload = backupPayload( + requiredEnv, + new Date('2026-07-28T02:15:00Z'), + ) + await assert.rejects( + runBackupRuntime( + requiredEnv, + { + instanceId: workflowInstanceId(DATABASE_ID, requiredPayload.day), + payload: requiredPayload, + timestamp: new Date('2026-07-28T02:15:01Z'), + }, + new PreStatsUploadStep(), + options, + ), + (error: unknown) => + error instanceof BackupError && + error.code === 'backup-sql-stats-missing' && + error.retryable, + ) + assert.equal(await requiredBucket.head(requiredPayload.manifestKey), null) +}) + +test('conflicting immutable SQL stats prevent manifest publication', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const env = environment(bucket) + const payload = backupPayload(env, new Date('2026-07-31T02:15:00Z')) + const objectKey = objectKeyForBookmark(payload.objectPrefix, 'bookmark-1') + + await assert.rejects( + runBackupRuntime( + env, + { + instanceId: workflowInstanceId(DATABASE_ID, payload.day), + payload, + timestamp: new Date('2026-07-31T02:15:01Z'), + }, + new CachedUploadStep(async () => { + await bucket.put( + `${objectKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(payload.day, objectKey)), + ) + }), + { + api: { + fetcher: async (input) => + String(input).endsWith('/export') + ? exportEnvelope('complete') + : identityEnvelope(1_000), + sleep: async () => undefined, + }, + downloadFetcher: async () => + new Response('valid', { headers: { 'content-length': '5' } }), + }, + ), + (error: unknown) => + error instanceof BackupError && + error.code === 'backup-sql-stats-conflict', + ) + assert.equal(await bucket.head(payload.manifestKey), null) +}) + test('initial upload ignores a stale cached signed URL and refreshes it in the callback', async () => { const bucket = new MemoryBucket() const env = environment(bucket) diff --git a/packages/backup-control-plane/backup-runtime.ts b/packages/backup-control-plane/backup-runtime.ts index 68d553731f..1fab7b2b2a 100644 --- a/packages/backup-control-plane/backup-runtime.ts +++ b/packages/backup-control-plane/backup-runtime.ts @@ -1,3 +1,10 @@ +import { + backupSqlStatsKey, + backupSqlStatsRequired, + backupSqlStatsSchemaVersion, + parseBackupSqlStats, +} from '@kody-internal/shared/backup-sql-stats.ts' + import { refreshCompletedD1Export, verifySourceDatabaseIdentity, @@ -30,9 +37,8 @@ import { signBackupManifest } from './manifest-signing.ts' /** * Persist per-object statement-length statistics next to the SQL object and * log them. An oversized statement means the object cannot be re-imported - * through the D1 import API (SQLITE_TOOBIG); the backup completes, but - * the condition is logged with failure status so observability and health - * checks can alert on an un-restorable day. + * through the D1 import API (SQLITE_TOOBIG), so the Workflow fails before it + * can publish a canonical day manifest. */ async function recordSqlStatementStats(input: { env: BackupEnvironment @@ -43,7 +49,71 @@ async function recordSqlStatementStats(input: { }): Promise { const { stats } = input // Step replays from pre-stats deployments have no measurements to record. - if (!stats) return + if (!stats) { + if (backupSqlStatsRequired(input.day)) { + throw new BackupError( + 'backup-sql-stats-missing', + `D1 SQL statement measurements are missing for ${input.day}`, + true, + ) + } + safeLog({ + event: 'backup-stats-legacy-missing', + status: 'stale-success', + day: input.day, + instanceId: input.instanceId, + objectKey: input.objectKey, + }) + return + } + const persistedStats = { + schemaVersion: backupSqlStatsSchemaVersion, + day: input.day, + objectKey: input.objectKey, + maxStatementBytes: stats.maxStatementBytes, + oversizedStatementCount: stats.oversizedStatementCount, + importStatementLimitBytes: stats.limit, + } + const body = JSON.stringify(persistedStats) + // An existing object from a replayed step wins and is left alone (the prefix + // is under the bucket's immutable lock, which rejects puts on existing keys + // with error 10069 before the conditional is evaluated). + const statsKey = backupSqlStatsKey(input.objectKey) + const putResult = await input.env.BACKUP_BUCKET.put(statsKey, body, { + onlyIf: { etagDoesNotMatch: '*' }, + httpMetadata: { contentType: 'application/json' }, + }).catch((error: unknown) => { + if (isBucketLockPolicyPutError(error)) return null + throw error + }) + if (putResult === null) { + const existing = await input.env.BACKUP_BUCKET.get(statsKey) + let existingStats + try { + existingStats = + existing === null + ? null + : parseBackupSqlStats(await existing.json()) + } catch { + existingStats = null + } + if ( + existingStats === null || + existingStats.schemaVersion !== persistedStats.schemaVersion || + existingStats.day !== persistedStats.day || + existingStats.objectKey !== persistedStats.objectKey || + existingStats.maxStatementBytes !== persistedStats.maxStatementBytes || + existingStats.oversizedStatementCount !== + persistedStats.oversizedStatementCount || + existingStats.importStatementLimitBytes !== + persistedStats.importStatementLimitBytes + ) { + throw new BackupError( + 'backup-sql-stats-conflict', + 'Existing immutable SQL stats differ from the streamed export', + ) + } + } safeLog({ event: 'backup-sql-stats', status: stats.oversizedStatementCount > 0 ? 'failure' : 'success', @@ -63,26 +133,12 @@ async function recordSqlStatementStats(input: { maxStatementBytes: stats.maxStatementBytes, oversizedStatementCount: stats.oversizedStatementCount, }) + throw new BackupError( + 'backup-unrestorable-statements', + `D1 export contains ${String(stats.oversizedStatementCount)} statement(s) above the import limit`, + true, + ) } - const body = JSON.stringify({ - schemaVersion: 1, - day: input.day, - objectKey: input.objectKey, - maxStatementBytes: stats.maxStatementBytes, - oversizedStatementCount: stats.oversizedStatementCount, - importStatementLimitBytes: stats.limit, - }) - // The stats object is advisory; an existing object from a replayed step - // wins and is left alone (the prefix is under the bucket's immutable - // lock, which rejects puts on existing keys with error 10069 before the - // conditional is evaluated). - await input.env.BACKUP_BUCKET.put(`${input.objectKey}.stats.json`, body, { - onlyIf: { etagDoesNotMatch: '*' }, - httpMetadata: { contentType: 'application/json' }, - }).catch((error: unknown) => { - if (isBucketLockPolicyPutError(error)) return null - throw error - }) } interface BackupRuntimeEvent { @@ -290,6 +346,18 @@ export async function runBackupRuntime( const completedAt = await step.do('record-completion-time', async () => new Date().toISOString(), ) + await step.do( + 'record-statement-stats', + { retries: { limit: 2, delay: '10 seconds' }, timeout: '2 minutes' }, + async () => + recordSqlStatementStats({ + env, + day: checkedPayload.day, + instanceId: event.instanceId, + objectKey: stored.objectKey, + stats: stored.sqlStatementStats, + }), + ) const unsignedManifest = { source: { accountId: env.SOURCE_ACCOUNT_ID, @@ -339,18 +407,6 @@ export async function runBackupRuntime( return signedManifest }, ) - await step.do( - 'record-statement-stats', - { retries: { limit: 2, delay: '10 seconds' }, timeout: '2 minutes' }, - async () => - recordSqlStatementStats({ - env, - day: checkedPayload.day, - instanceId: event.instanceId, - objectKey: stored.objectKey, - stats: stored.sqlStatementStats, - }), - ) safeLog({ event: 'backup-success', status: 'success', diff --git a/packages/backup-control-plane/backup-types.ts b/packages/backup-control-plane/backup-types.ts index dce35fcd55..55ab0d72cc 100644 --- a/packages/backup-control-plane/backup-types.ts +++ b/packages/backup-control-plane/backup-types.ts @@ -73,9 +73,8 @@ export type ExportState = ExportReady | ExportPending | ExportLost * Statement-length statistics for one exported SQL object, measured while * streaming during upload. `oversizedStatementCount > 0` means the object is * not restorable through the D1 import API (statement too long: - * SQLITE_TOOBIG); the backup completes, but the condition is logged as a - * failure-status event and persisted next to the object as - * `.stats.json` so dashboards and health checks can alert on it. + * SQLITE_TOOBIG); the condition is persisted next to the object before the + * Workflow fails without writing a day manifest. */ export interface SqlStatementStats { maxStatementBytes: number @@ -104,8 +103,10 @@ export interface LogRecord { | 'backup-failure' | 'backup-sql-stats' | 'backup-unrestorable-statements' + | 'backup-stats-legacy-missing' | 'freshness-success' | 'freshness-stale' + | 'freshness-unrestorable' | 'source-size-success' | 'source-size-failure' | 'full-backup-sealed' diff --git a/packages/backup-control-plane/control-plane-ui.node.test.ts b/packages/backup-control-plane/control-plane-ui.node.test.ts new file mode 100644 index 0000000000..1d1fe4a6a8 --- /dev/null +++ b/packages/backup-control-plane/control-plane-ui.node.test.ts @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' + +import { test, vi } from 'vitest' + +import { + MemoryBucket, + badSqlStatsFixture, + environment, + identityEnvelope, + manifest, + putSqlStatsFixture, + signedManifest, +} from './backup-control-plane-test-support.ts' +import { backupPayload, objectKeyForBookmark } from './backup-policy.ts' +import { collectDayStatuses, renderDashboard } from './control-plane-ui.ts' +import { putImmutableManifest } from './immutable-storage.ts' + +test('dashboard renders oversized D1 SQL as not restorable with a warning', async () => { + const bucket = new MemoryBucket() + const env = environment(bucket) + const day = '2026-07-31' + const payload = backupPayload(env, new Date(`${day}T12:00:00.000Z`)) + const sql = 'CREATE TABLE t(id INTEGER);\n' + const sqlObjectKey = objectKeyForBookmark(payload.objectPrefix, 'bookmark-1') + const template = manifest({ + bytes: sql.length, + sha256: createHash('sha256').update(sql).digest('hex'), + r2Etag: createHash('md5').update(sql).digest('hex'), + }) + await bucket.put(sqlObjectKey, sql) + await bucket.put( + `${sqlObjectKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, sqlObjectKey)), + ) + await putImmutableManifest( + bucket as unknown as R2Bucket, + payload.manifestKey, + signedManifest({ + ...template.payload, + export: { + ...template.payload.export, + scheduledAt: `${day}T02:15:00.000Z`, + startedAt: `${day}T02:15:01.000Z`, + completedAt: `${day}T02:16:00.000Z`, + }, + sql: { ...template.payload.sql, objectKey: sqlObjectKey }, + }), + ) + + const now = new Date(`${day}T12:00:00.000Z`) + const [status] = await collectDayStatuses(env, now) + assert.equal(status?.d1Restorable, false) + assert.ok( + status?.warnings.some((warning) => warning.includes('cannot be restored')), + ) + + const fetcher = vi.spyOn(globalThis, 'fetch') + fetcher.mockResolvedValue(identityEnvelope(1_000)) + const html = await renderDashboard(env, { now }) + assert.match(html, /D1 restorable<\/th>/) + assert.match(html, /no<\/td>/) + assert.match( + html, + /D1 SQL contains oversized statements and cannot be restored/, + ) + + await bucket.delete(`${sqlObjectKey}.stats.json`) + const [missingStatsStatus] = await collectDayStatuses(env, now) + assert.equal(missingStatsStatus?.d1Restorable, false) + const missingStatsHtml = await renderDashboard(env, { now }) + assert.match(missingStatsHtml, /no<\/td>/) + assert.match( + missingStatsHtml, + /D1 is not restorable: required SQL stats are missing/, + ) + + await putSqlStatsFixture(bucket, day, sqlObjectKey) + bucket.failGetFor(`${sqlObjectKey}.stats.json`) + const [statsReadFailure] = await collectDayStatuses(env, now) + assert.equal(statsReadFailure?.d1Verified, true) + assert.equal(statsReadFailure?.d1Restorable, false) + assert.ok( + statsReadFailure?.warnings.includes( + 'D1 is not restorable: SQL stats lookup failed', + ), + ) + assert.equal( + statsReadFailure?.warnings.includes('D1 manifest unreadable'), + false, + ) +}) diff --git a/packages/backup-control-plane/control-plane-ui.ts b/packages/backup-control-plane/control-plane-ui.ts index 1ed151fad8..68bc6723f5 100644 --- a/packages/backup-control-plane/control-plane-ui.ts +++ b/packages/backup-control-plane/control-plane-ui.ts @@ -6,7 +6,7 @@ import { } from '@kody-internal/shared/backup-staging.ts' import { isBackupEnabled, utcDay, backupPayload } from './backup-policy.ts' -import { type BackupEnvironment } from './backup-types.ts' +import { type BackupEnvironment, type BackupManifest } from './backup-types.ts' import { getD1Database } from './d1-import-api.ts' import { verifyBackupFullManifestSignature } from './full-manifest-signing.ts' import { readManifest } from './immutable-storage.ts' @@ -15,6 +15,7 @@ import { type DrillReport } from './restore-drill.ts' import { type ProductionRestoreProgress } from './production-restore.ts' import { type RestoreConfirmToken } from './restore-confirm-token.ts' import { readFullManifest } from './seal-full-backup.ts' +import { readSqlRestorability } from './sql-statement-stats.ts' import { type EnqueueResult } from './workflow-trigger.ts' const DASHBOARD_DAY_COUNT = 14 @@ -138,8 +139,9 @@ function statusLabel( ok: boolean | null, presentLabel = 'yes', absentLabel = 'no', + unknownLabel = '—', ): string { - if (ok === null) return '—' + if (ok === null) return unknownLabel return ok ? presentLabel : absentLabel } @@ -147,6 +149,7 @@ export type DayStatus = { day: string d1Present: boolean d1Verified: boolean | null + d1Restorable: boolean | null stagingPresent: boolean sealedPresent: boolean sealedVerified: boolean | null @@ -169,17 +172,61 @@ export async function collectDayStatuses( ).manifestKey let d1Present = false let d1Verified: boolean | null = null + let d1Restorable: boolean | null = null + let d1Manifest: BackupManifest | null = null try { - const manifest = await readManifest(env.BACKUP_BUCKET, d1Key) - d1Present = manifest !== null - if (manifest !== null) { - d1Verified = await verifyBackupManifestSignature(env, manifest) + d1Manifest = await readManifest(env.BACKUP_BUCKET, d1Key) + d1Present = d1Manifest !== null + if (d1Manifest !== null) { + d1Verified = await verifyBackupManifestSignature(env, d1Manifest) if (!d1Verified) warnings.push('D1 manifest signature invalid') } } catch { d1Verified = null warnings.push('D1 manifest unreadable') } + if (d1Manifest !== null && d1Verified === true) { + try { + const restorability = await readSqlRestorability( + env.BACKUP_BUCKET, + day, + d1Manifest.payload.sql.objectKey, + ) + switch (restorability.kind) { + case 'restorable': + d1Restorable = true + break + case 'unrestorable': + d1Restorable = false + warnings.push( + 'D1 SQL contains oversized statements and cannot be restored', + ) + break + case 'legacy-unknown': + warnings.push( + 'D1 restorability unknown: legacy backup has no SQL stats', + ) + break + case 'missing': + d1Restorable = false + warnings.push( + 'D1 is not restorable: required SQL stats are missing', + ) + break + case 'corrupt': + d1Restorable = false + warnings.push('D1 is not restorable: SQL stats are unreadable') + break + default: { + const exhaustive: never = restorability + throw exhaustive + } + } + } catch { + d1Restorable = false + warnings.push('D1 is not restorable: SQL stats lookup failed') + } + } const stagingPresent = (await env.BACKUP_BUCKET.head(stagingSummaryKey(day))) !== null @@ -205,6 +252,7 @@ export async function collectDayStatuses( day, d1Present, d1Verified, + d1Restorable, stagingPresent, sealedPresent, sealedVerified, @@ -241,6 +289,7 @@ export async function renderDashboard( ${escapeHtml(day.day)} ${escapeHtml(statusLabel(day.d1Present))} ${escapeHtml(statusLabel(day.d1Verified, 'verified', 'unverified'))} +${escapeHtml(statusLabel(day.d1Restorable, 'yes', 'no', 'unknown'))} ${escapeHtml(statusLabel(day.stagingPresent))} ${escapeHtml(statusLabel(day.sealedPresent))} ${escapeHtml(statusLabel(day.sealedVerified, 'verified', 'unverified'))} @@ -282,6 +331,7 @@ ${options.flashHtml ?? ''} Day D1 manifest D1 verified + D1 restorable Staging Sealed Seal verified diff --git a/packages/backup-control-plane/freshness-check.node.test.ts b/packages/backup-control-plane/freshness-check.node.test.ts index 837e9463c8..80466c2086 100644 --- a/packages/backup-control-plane/freshness-check.node.test.ts +++ b/packages/backup-control-plane/freshness-check.node.test.ts @@ -14,6 +14,7 @@ import { BackupError, objectKeyForBookmark } from './backup-policy.ts' import { DATABASE_ID, MemoryBucket, + badSqlStatsFixture, environment, identityApi, identityEnvelope, @@ -22,6 +23,8 @@ import { } from './backup-control-plane-test-support.ts' test('freshness accepts matching metadata and flags size/ETag drift or missing objects', async () => { + const consoleLog = vi.spyOn(console, 'log') + consoleLog.mockImplementation(() => undefined) const bucket = new MemoryBucket() const env = environment(bucket) const key = objectKeyForBookmark( @@ -43,6 +46,12 @@ test('freshness accepts matching metadata and flags size/ETag drift or missing o await checkFreshness(env, new Date('2026-07-22T03:45:00Z'), identityApi()), true, ) + const initialEvents = consoleLog.mock.calls.map(([record]) => + JSON.parse(String(record)), + ) as Array<{ event: string }> + assert.ok( + initialEvents.some(({ event }) => event === 'backup-stats-legacy-missing'), + ) bucket.setReportedSize(key, MAXIMUM_SINGLE_BACKUP_OBJECT_BYTES) assert.equal( await checkFreshness(env, new Date('2026-07-22T03:45:00Z'), identityApi()), @@ -122,6 +131,53 @@ test('freshness reports malformed manifest schema as stale', async () => { ) }) +test('freshness reports oversized SQL with a distinct unrestorable event', async () => { + const consoleLog = vi.spyOn(console, 'log') + consoleLog.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const env = environment(bucket) + const day = '2026-07-31' + const key = objectKeyForBookmark( + `daily/d1/${DATABASE_ID}/${day}`, + 'bookmark-1', + ) + const stored = await storeSignedDownload( + bucket as unknown as R2Bucket, + key, + 'https://download.example', + async () => new Response('valid', { headers: { 'content-length': '5' } }), + ) + const template = manifest(stored) + await putImmutableManifest( + bucket as unknown as R2Bucket, + `daily/d1/${DATABASE_ID}/${day}/manifest.json`, + signedManifest({ + ...template.payload, + export: { + ...template.payload.export, + scheduledAt: `${day}T02:15:00.000Z`, + startedAt: `${day}T02:15:01.000Z`, + completedAt: `${day}T02:16:00.000Z`, + }, + sql: { ...template.payload.sql, objectKey: key }, + }), + ) + await bucket.put( + `${key}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, key)), + ) + + assert.equal( + await checkFreshness(env, new Date(`${day}T03:45:00Z`), identityApi()), + false, + ) + const record = JSON.parse( + String(consoleLog.mock.calls.at(-1)?.[0]), + ) as Record + assert.equal(record.event, 'freshness-unrestorable') + assert.equal(record.oversizedStatementCount, 1) +}) + test('freshness requires a valid Ed25519 signature from the configured key', async () => { const bucket = new MemoryBucket() const env = environment(bucket) diff --git a/packages/backup-control-plane/freshness-check.ts b/packages/backup-control-plane/freshness-check.ts index 4be6f87950..0c570c2ad5 100644 --- a/packages/backup-control-plane/freshness-check.ts +++ b/packages/backup-control-plane/freshness-check.ts @@ -15,6 +15,7 @@ import { } from './backup-policy.ts' import { type BackupEnvironment } from './backup-types.ts' import { verifyBackupManifestSignature } from './manifest-signing.ts' +import { readSqlRestorability } from './sql-statement-stats.ts' function latestExpectedDate(scheduledAt: Date): Date { const expected = new Date(scheduledAt) @@ -59,6 +60,10 @@ export async function checkFreshness( throw error } let ageHours: number | undefined + let statsErrorCode: string | undefined + let unrestorableStats: + | { maxStatementBytes: number; oversizedStatementCount: number } + | undefined const signatureValid = manifest !== null && (await verifyBackupManifestSignature(env, manifest)) let stale = manifest === null || !signatureValid @@ -70,6 +75,38 @@ export async function checkFreshness( const object = validObjectKey ? await env.BACKUP_BUCKET.head(manifest.payload.sql.objectKey) : null + if (validObjectKey) { + const restorability = await readSqlRestorability( + env.BACKUP_BUCKET, + payload.day, + manifest.payload.sql.objectKey, + ) + switch (restorability.kind) { + case 'restorable': + break + case 'legacy-unknown': + safeLog({ + event: 'backup-stats-legacy-missing', + status: 'stale-success', + day: payload.day, + objectKey: manifest.payload.sql.objectKey, + }) + break + case 'unrestorable': + unrestorableStats = restorability.stats + break + case 'missing': + statsErrorCode = 'backup-sql-stats-missing' + break + case 'corrupt': + statsErrorCode = 'backup-sql-stats-corrupt' + break + default: { + const exhaustive: never = restorability + throw exhaustive + } + } + } ageHours = (scheduledAt.valueOf() - new Date(manifest.payload.export.completedAt).valueOf()) / @@ -84,6 +121,8 @@ export async function checkFreshness( env.SOURCE_DATABASE_ID.toLowerCase() || manifest.payload.source.databaseName !== env.SOURCE_DATABASE_NAME || !validObjectKey || + unrestorableStats !== undefined || + statsErrorCode !== undefined || manifest.payload.sql.bytes <= 0 || !/^[0-9a-f]{64}$/.test(manifest.payload.sql.sha256) || !manifest.payload.sql.r2Etag || @@ -94,12 +133,20 @@ export async function checkFreshness( object.etag !== manifest.payload.sql.r2Etag)) } safeLog({ - event: stale ? 'freshness-stale' : 'freshness-success', + event: + unrestorableStats !== undefined + ? 'freshness-unrestorable' + : stale + ? 'freshness-stale' + : 'freshness-success', status: stale ? 'stale-success' : 'success', day: payload.day, objectKey: manifest?.payload.sql.objectKey, manifestKey: payload.manifestKey, ageHours, + errorCode: statsErrorCode, + maxStatementBytes: unrestorableStats?.maxStatementBytes, + oversizedStatementCount: unrestorableStats?.oversizedStatementCount, }) return !stale } diff --git a/packages/backup-control-plane/immutable-storage.node.test.ts b/packages/backup-control-plane/immutable-storage.node.test.ts index fa08e152f2..bb7ef750b2 100644 --- a/packages/backup-control-plane/immutable-storage.node.test.ts +++ b/packages/backup-control-plane/immutable-storage.node.test.ts @@ -162,6 +162,14 @@ test('sql statement stats measure quote-aware statement lengths during upload', oversizedStatementCount: 1, limit: 10, }) + + const exactLimitScanner = createSqlStatementScanner(10) + exactLimitScanner.update(new TextEncoder().encode('SELECT 12;')) + assert.deepEqual(exactLimitScanner.finish(), { + maxStatementBytes: 10, + oversizedStatementCount: 0, + limit: 10, + }) }) test('truncated and interrupted downloads fail retryably, then a retry succeeds', async () => { diff --git a/packages/backup-control-plane/production-restore.node.test.ts b/packages/backup-control-plane/production-restore.node.test.ts index 439aa469fb..c347a1d96e 100644 --- a/packages/backup-control-plane/production-restore.node.test.ts +++ b/packages/backup-control-plane/production-restore.node.test.ts @@ -7,12 +7,13 @@ import { test, vi } from 'vitest' import { MemoryBucket, + badSqlStatsFixture, environment, exportEnvelope, manifest, signedManifest, } from './backup-control-plane-test-support.ts' -import { backupPayload } from './backup-policy.ts' +import { BackupError, backupPayload } from './backup-policy.ts' import { d1ImportForeignKeysOffPrefix } from './d1-import-api.ts' import { signBackupFullManifest } from './full-manifest-signing.ts' import { putImmutableManifest } from './immutable-storage.ts' @@ -36,10 +37,11 @@ async function seedSealedRestoreDay(bucket: MemoryBucket, day = '2026-07-22') { sha256: sha256Text(sqlBody), r2Etag: sqlMd5, }) - await bucket.put(template.payload.sql.objectKey, sqlBody) + const sqlObjectKey = template.payload.sql.objectKey.replace('2026-07-22', day) + await bucket.put(sqlObjectKey, sqlBody) const dayManifest = signedManifest({ ...template.payload, - sql: template.payload.sql, + sql: { ...template.payload.sql, objectKey: sqlObjectKey }, }) await putImmutableManifest( bucket as unknown as R2Bucket, @@ -70,7 +72,7 @@ async function seedSealedRestoreDay(bucket: MemoryBucket, day = '2026-07-22') { sealedFullManifestKey(day), serializeBackupFullManifest(full), ) - return { env, day, preparedImportMd5 } + return { env, day, preparedImportMd5, sqlObjectKey } } test('runProductionRestore returns failed progress when dr-restore emits warnings', async () => { @@ -154,3 +156,35 @@ test('runProductionRestore returns failed progress when dr-restore emits warning assert.ok(safetyExportKey) assert.ok(await bucket.head(safetyExportKey)) }) + +test('production restore refuses SQL with oversized statement stats', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const { env, day, sqlObjectKey } = await seedSealedRestoreDay( + bucket, + '2026-07-31', + ) + await bucket.put( + `${sqlObjectKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, sqlObjectKey)), + ) + + let fetchCalls = 0 + await assert.rejects( + runProductionRestore( + env, + { day, requestedAt: `${day}T12:00:00.000Z` }, + { + fetcher: async () => { + fetchCalls += 1 + throw new Error('restore should not start') + }, + }, + ), + (error: unknown) => + error instanceof BackupError && + error.code === 'backup-unrestorable-statements', + ) + assert.equal(fetchCalls, 0) +}) diff --git a/packages/backup-control-plane/production-restore.ts b/packages/backup-control-plane/production-restore.ts index 6d7a00c6c0..f829a7b306 100644 --- a/packages/backup-control-plane/production-restore.ts +++ b/packages/backup-control-plane/production-restore.ts @@ -12,6 +12,7 @@ import { verifyBackupFullManifestSignature } from './full-manifest-signing.ts' import { readManifest } from './immutable-storage.ts' import { verifyBackupManifestSignature } from './manifest-signing.ts' import { readFullManifest } from './seal-full-backup.ts' +import { assertSqlRestorable } from './sql-statement-stats.ts' export type DrRestoreChunkResponse = { done: boolean @@ -181,6 +182,11 @@ export async function validateSealedDayForRestore( `D1 manifest signature invalid for ${day}`, ) } + await assertSqlRestorable( + env.BACKUP_BUCKET, + day, + d1Manifest.payload.sql.objectKey, + ) return { fullManifestKey, d1ManifestKey: fullManifest.payload.d1ManifestKey, diff --git a/packages/backup-control-plane/readme.md b/packages/backup-control-plane/readme.md index 191f493358..94e56bcb0d 100644 --- a/packages/backup-control-plane/readme.md +++ b/packages/backup-control-plane/readme.md @@ -38,11 +38,18 @@ While streaming the upload, the runtime also measures SQL statement lengths ~100 KB statement limit with `statement too long: SQLITE_TOOBIG`, so an oversized statement means the object is not restorable through the D1 import API. The measurements are persisted as `.stats.json` next to the SQL -object and logged as `backup-sql-stats`; an oversized count above zero -additionally logs `backup-unrestorable-statements` with failure status. The -backup itself completes — application write paths bound row sizes (see -`packages/shared/src/backup-restore-safety.ts`), so a nonzero count indicates a -new unbounded write path that must be fixed. +object before manifest finalization and logged as `backup-sql-stats`. An +oversized count above zero additionally logs `backup-unrestorable-statements`, +fails the Workflow retryably, and leaves the day without a canonical manifest. +The hourly catch-up window can then re-export the same day after the offending +write path or row is bounded. + +Sealing, freshness, restore drills, production restore, and the dashboard all +read the SQL object's stats sibling. Oversized statements prevent sealing and +import, freshness emits `freshness-unrestorable`, and the dashboard marks the +day's D1 media as not restorable. Missing stats are tolerated with a structured +legacy warning only for retained objects before 2026-07-28; newer objects fail +closed. Every canonical manifest uses schema v2 and is an Ed25519-signed envelope. Its signature covers deterministic canonical JSON for the SQL key, bytes, SHA-256, @@ -143,7 +150,8 @@ Cloudflare dashboard. ## Integrity checks The hourly freshness check compares the immutable object's R2 size and ETag to -the canonical manifest. Run a separate periodic restore drill that downloads the -object, verifies its SHA-256 against the manifest, and restores it into a -non-production D1 database. The metadata freshness check is not a replacement -for that deep checksum and restore drill. +the canonical manifest and requires restorable SQL statement stats for new +backup days. Run a separate periodic restore drill that downloads the object, +verifies its SHA-256 against the manifest, and restores it into a non-production +D1 database. The metadata freshness check is not a replacement for that deep +checksum and restore drill. diff --git a/packages/backup-control-plane/restore-drill.node.test.ts b/packages/backup-control-plane/restore-drill.node.test.ts index 665074b469..be32442e69 100644 --- a/packages/backup-control-plane/restore-drill.node.test.ts +++ b/packages/backup-control-plane/restore-drill.node.test.ts @@ -1,10 +1,19 @@ import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' import { test } from 'vitest' -import { BackupError } from './backup-policy.ts' -import { ACCOUNT_ID, environment } from './backup-control-plane-test-support.ts' -import { assertDrillAccountIsolated } from './restore-drill.ts' +import { + ACCOUNT_ID, + MemoryBucket, + badSqlStatsFixture, + environment, + manifest, + signedManifest, +} from './backup-control-plane-test-support.ts' +import { BackupError, backupPayload } from './backup-policy.ts' +import { putImmutableManifest } from './immutable-storage.ts' +import { assertDrillAccountIsolated, runRestoreDrill } from './restore-drill.ts' test('restore drill refuses same account and accepts an isolated account', () => { const env = environment() @@ -17,3 +26,50 @@ test('restore drill refuses same account and accepts an isolated account', () => ) assert.doesNotThrow(() => assertDrillAccountIsolated(environment())) }) + +test('restore drill refuses SQL with oversized statement stats before import', async () => { + const bucket = new MemoryBucket() + const env = environment(bucket) + const day = '2026-07-31' + const payload = backupPayload(env, new Date(`${day}T12:00:00.000Z`)) + const sql = 'CREATE TABLE t(id INTEGER);\n' + const template = manifest({ + bytes: sql.length, + sha256: createHash('sha256').update(sql).digest('hex'), + r2Etag: createHash('md5').update(sql).digest('hex'), + }) + const sqlObjectKey = template.payload.sql.objectKey.replace('2026-07-22', day) + await bucket.put(sqlObjectKey, sql) + await bucket.put( + `${sqlObjectKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, sqlObjectKey)), + ) + await putImmutableManifest( + bucket as unknown as R2Bucket, + payload.manifestKey, + signedManifest({ + ...template.payload, + export: { + ...template.payload.export, + scheduledAt: `${day}T02:15:00.000Z`, + startedAt: `${day}T02:15:01.000Z`, + completedAt: `${day}T02:16:00.000Z`, + }, + sql: { ...template.payload.sql, objectKey: sqlObjectKey }, + }), + ) + + let fetchCalls = 0 + await assert.rejects( + runRestoreDrill(env, day, { + fetcher: async () => { + fetchCalls += 1 + throw new Error('import should not start') + }, + }), + (error: unknown) => + error instanceof BackupError && + error.code === 'backup-unrestorable-statements', + ) + assert.equal(fetchCalls, 0) +}) diff --git a/packages/backup-control-plane/restore-drill.ts b/packages/backup-control-plane/restore-drill.ts index eea9b2229a..dd96d464c9 100644 --- a/packages/backup-control-plane/restore-drill.ts +++ b/packages/backup-control-plane/restore-drill.ts @@ -9,6 +9,7 @@ import { import { type ApiOptions } from './d1-export-api.ts' import { readManifest } from './immutable-storage.ts' import { verifyBackupManifestSignature } from './manifest-signing.ts' +import { assertSqlRestorable } from './sql-statement-stats.ts' export type DrillReport = { day: string @@ -92,6 +93,7 @@ export async function runRestoreDrill( ) } const sqlObjectKey = manifest.payload.sql.objectKey + await assertSqlRestorable(env.BACKUP_BUCKET, day, sqlObjectKey) const sqlHead = await env.BACKUP_BUCKET.head(sqlObjectKey) if (sqlHead === null) { throw new BackupError('drill-sql-missing', `SQL object missing for ${day}`) diff --git a/packages/backup-control-plane/seal-full-backup.node.test.ts b/packages/backup-control-plane/seal-full-backup.node.test.ts index 5f36bd527c..dddc0d18e7 100644 --- a/packages/backup-control-plane/seal-full-backup.node.test.ts +++ b/packages/backup-control-plane/seal-full-backup.node.test.ts @@ -14,8 +14,10 @@ import { test, vi } from 'vitest' import { MemoryBucket, + badSqlStatsFixture, environment, manifest, + putSqlStatsFixture, signedManifest, } from './backup-control-plane-test-support.ts' import { BackupError, backupPayload } from './backup-policy.ts' @@ -119,7 +121,7 @@ async function seedCompleteDay( warnings: [], } await bucket.put(stagingSummaryKey(day), JSON.stringify(summary)) - return { env, day } + return { env, day, sqlKey } } test('sealFullBackupDay seals a complete day and is idempotent', async () => { @@ -195,6 +197,76 @@ test('sealFullBackupDay fails closed on staging sha mismatch', async () => { assert.equal(await bucket.head(sealedFullManifestKey(day)), null) }) +test('sealFullBackupDay refuses oversized SQL stats before sealing', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const { env, day, sqlKey } = await seedCompleteDay(bucket, '2026-07-31') + await bucket.put( + `${sqlKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, sqlKey)), + ) + + assert.deepEqual( + await sealFullBackupDay(env, day, new Date(`${day}T04:00:00.000Z`)), + { + kind: 'incomplete', + day, + reason: 'backup-unrestorable-statements', + }, + ) + assert.equal(await bucket.head(sealedFullManifestKey(day)), null) + const record = JSON.parse( + String(consoleError.mock.calls.at(-1)?.[0]), + ) as Record + assert.equal(record.event, 'full-backup-seal-skipped') + assert.equal(record.errorCode, 'backup-unrestorable-statements') +}) + +test('sealFullBackupDay reports an already-sealed unrestorable day as incomplete', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const { env, day, sqlKey } = await seedCompleteDay(bucket, '2026-07-31') + await putSqlStatsFixture(bucket, day, sqlKey) + const sealed = await sealFullBackupDay( + env, + day, + new Date(`${day}T04:00:00.000Z`), + ) + assert.equal(sealed.kind, 'sealed') + await bucket.put( + `${sqlKey}.stats.json`, + JSON.stringify(badSqlStatsFixture(day, sqlKey)), + ) + + assert.deepEqual( + await sealFullBackupDay(env, day, new Date(`${day}T04:01:00.000Z`)), + { + kind: 'incomplete', + day, + reason: 'backup-unrestorable-statements', + }, + ) +}) + +test('sealFullBackupDay fails closed when required SQL stats are missing', async () => { + const consoleError = vi.spyOn(console, 'error') + consoleError.mockImplementation(() => undefined) + const bucket = new MemoryBucket() + const { env, day } = await seedCompleteDay(bucket, '2026-07-31') + + assert.deepEqual( + await sealFullBackupDay(env, day, new Date(`${day}T04:00:00.000Z`)), + { + kind: 'incomplete', + day, + reason: 'backup-sql-stats-missing', + }, + ) + assert.equal(await bucket.head(sealedFullManifestKey(day)), null) +}) + test('sealFullBackupDay is incomplete when the 501st referenced blob is missing', async () => { const consoleError = vi.spyOn(console, 'error') consoleError.mockImplementation(() => undefined) diff --git a/packages/backup-control-plane/seal-full-backup.ts b/packages/backup-control-plane/seal-full-backup.ts index 0c37ba05a0..96445f2005 100644 --- a/packages/backup-control-plane/seal-full-backup.ts +++ b/packages/backup-control-plane/seal-full-backup.ts @@ -35,6 +35,7 @@ import { readManifest, } from './immutable-storage.ts' import { verifyBackupManifestSignature } from './manifest-signing.ts' +import { readSqlRestorability } from './sql-statement-stats.ts' const sha256Pattern = /^[0-9a-f]{64}$/ @@ -342,6 +343,16 @@ export type SealDayResult = | { kind: 'sealed'; day: string; manifestKey: string; alreadySealed: boolean } | { kind: 'incomplete'; day: string; reason: string } +function incompleteForSqlStats(day: string, reason: string): SealDayResult { + safeLog({ + event: 'full-backup-seal-skipped', + status: 'failure', + day, + errorCode: reason, + }) + return { kind: 'incomplete', day, reason } +} + export async function sealFullBackupDay( env: BackupEnvironment, day: string, @@ -350,6 +361,48 @@ export async function sealFullBackupDay( assertBackupDay(day) const manifestKey = sealedFullManifestKey(day) const existing = await readFullManifest(env.BACKUP_BUCKET, manifestKey) + + const d1Payload = backupPayload(env, new Date(`${day}T12:00:00.000Z`)) + const d1Manifest = await readManifest( + env.BACKUP_BUCKET, + d1Payload.manifestKey, + ) + if (d1Manifest === null) { + return { kind: 'incomplete', day, reason: 'd1-manifest-missing' } + } + if (!(await verifyBackupManifestSignature(env, d1Manifest))) { + throw new BackupError( + 'd1-manifest-signature-invalid', + `D1 manifest signature invalid for ${day}`, + ) + } + const restorability = await readSqlRestorability( + env.BACKUP_BUCKET, + day, + d1Manifest.payload.sql.objectKey, + ) + switch (restorability.kind) { + case 'restorable': + break + case 'legacy-unknown': + safeLog({ + event: 'backup-stats-legacy-missing', + status: 'stale-success', + day, + objectKey: d1Manifest.payload.sql.objectKey, + }) + break + case 'unrestorable': + return incompleteForSqlStats(day, 'backup-unrestorable-statements') + case 'missing': + return incompleteForSqlStats(day, 'backup-sql-stats-missing') + case 'corrupt': + return incompleteForSqlStats(day, 'backup-sql-stats-corrupt') + default: { + const exhaustive: never = restorability + throw exhaustive + } + } if (existing !== null) { if (!(await verifyBackupFullManifestSignature(env, existing))) { throw new BackupError( @@ -371,21 +424,6 @@ export async function sealFullBackupDay( }) return { kind: 'sealed', day, manifestKey, alreadySealed: true } } - - const d1Payload = backupPayload(env, new Date(`${day}T12:00:00.000Z`)) - const d1Manifest = await readManifest( - env.BACKUP_BUCKET, - d1Payload.manifestKey, - ) - if (d1Manifest === null) { - return { kind: 'incomplete', day, reason: 'd1-manifest-missing' } - } - if (!(await verifyBackupManifestSignature(env, d1Manifest))) { - throw new BackupError( - 'd1-manifest-signature-invalid', - `D1 manifest signature invalid for ${day}`, - ) - } const d1ManifestObject = await env.BACKUP_BUCKET.get(d1Payload.manifestKey) if (d1ManifestObject === null) { return { kind: 'incomplete', day, reason: 'd1-manifest-missing' } diff --git a/packages/backup-control-plane/sql-statement-stats.ts b/packages/backup-control-plane/sql-statement-stats.ts new file mode 100644 index 0000000000..edf99357a3 --- /dev/null +++ b/packages/backup-control-plane/sql-statement-stats.ts @@ -0,0 +1,81 @@ +import { + backupSqlStatsKey, + backupSqlStatsRequired, + parseBackupSqlStats, + type BackupSqlStats, +} from '@kody-internal/shared/backup-sql-stats.ts' +import { d1ImportMaxStatementBytes } from '@kody-internal/shared/backup-restore-safety.ts' + +import { BackupError, safeLog } from './backup-policy.ts' + +export type SqlRestorability = + | { kind: 'restorable'; stats: BackupSqlStats } + | { kind: 'unrestorable'; stats: BackupSqlStats } + | { kind: 'legacy-unknown' } + | { kind: 'missing' } + | { kind: 'corrupt' } + +export async function readSqlRestorability( + bucket: R2Bucket, + day: string, + objectKey: string, +): Promise { + const object = await bucket.get(backupSqlStatsKey(objectKey)) + if (object === null) { + return backupSqlStatsRequired(day) + ? { kind: 'missing' } + : { kind: 'legacy-unknown' } + } + let stats: BackupSqlStats + try { + stats = parseBackupSqlStats(await object.json()) + } catch { + return { kind: 'corrupt' } + } + if (stats.day !== day || stats.objectKey !== objectKey) { + return { kind: 'corrupt' } + } + return stats.oversizedStatementCount > 0 || + stats.maxStatementBytes > d1ImportMaxStatementBytes + ? { kind: 'unrestorable', stats } + : { kind: 'restorable', stats } +} + +export async function assertSqlRestorable( + bucket: R2Bucket, + day: string, + objectKey: string, +): Promise { + const result = await readSqlRestorability(bucket, day, objectKey) + switch (result.kind) { + case 'restorable': + return + case 'legacy-unknown': + safeLog({ + event: 'backup-stats-legacy-missing', + status: 'stale-success', + day, + objectKey, + }) + return + case 'unrestorable': + throw new BackupError( + 'backup-unrestorable-statements', + `D1 SQL for ${day} contains ${String(result.stats.oversizedStatementCount)} oversized statement(s)`, + ) + case 'missing': + throw new BackupError( + 'backup-sql-stats-missing', + `D1 SQL stats are missing for ${day}`, + ) + case 'corrupt': + throw new BackupError( + 'backup-sql-stats-corrupt', + `D1 SQL stats are corrupt for ${day}`, + ) + default: { + const exhaustive: never = result + throw exhaustive + } + } +} diff --git a/packages/shared/src/backup-sql-stats.node.test.ts b/packages/shared/src/backup-sql-stats.node.test.ts new file mode 100644 index 0000000000..0aff8ba1fc --- /dev/null +++ b/packages/shared/src/backup-sql-stats.node.test.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { d1ImportMaxStatementBytes } from './backup-restore-safety.ts' +import { + backupSqlStatsRequired, + backupSqlStatsSchemaVersion, + parseBackupSqlStats, +} from './backup-sql-stats.ts' + +test('backup SQL stats enforce the rollout cutoff and statement-limit consistency', () => { + assert.equal(backupSqlStatsRequired('2026-07-27'), false) + assert.equal(backupSqlStatsRequired('2026-07-28'), true) + + const valid = { + schemaVersion: backupSqlStatsSchemaVersion, + day: '2026-07-31', + objectKey: 'daily/d1/database/2026-07-31/backup-bookmark.sql', + maxStatementBytes: 50_000, + oversizedStatementCount: 0, + importStatementLimitBytes: d1ImportMaxStatementBytes, + } + assert.deepEqual(parseBackupSqlStats(valid), valid) + + const oversized = { + ...valid, + maxStatementBytes: d1ImportMaxStatementBytes + 1, + oversizedStatementCount: 1, + } + assert.deepEqual(parseBackupSqlStats(oversized), oversized) + const differentRecordedLimit = { + ...valid, + importStatementLimitBytes: d1ImportMaxStatementBytes + 1, + } + assert.deepEqual( + parseBackupSqlStats(differentRecordedLimit), + differentRecordedLimit, + ) + + for (const corrupt of [ + null, + { ...valid, schemaVersion: backupSqlStatsSchemaVersion + 1 }, + { ...valid, importStatementLimitBytes: 0 }, + { ...valid, maxStatementBytes: d1ImportMaxStatementBytes + 1 }, + { ...valid, maxStatementBytes: 50_000, oversizedStatementCount: 1 }, + ]) { + assert.throws(() => parseBackupSqlStats(corrupt)) + } +}) diff --git a/packages/shared/src/backup-sql-stats.ts b/packages/shared/src/backup-sql-stats.ts new file mode 100644 index 0000000000..280be81c53 --- /dev/null +++ b/packages/shared/src/backup-sql-stats.ts @@ -0,0 +1,51 @@ +export const backupSqlStatsSchemaVersion = 1 + +/** + * SQL statement statistics became mandatory for D1 backup objects after the + * stats-aware control plane was deployed. Older retained objects remain + * usable, but a missing stats sibling on this day or later is unsafe. + */ +export const backupSqlStatsRequiredFromDay = '2026-07-28' + +export type BackupSqlStats = { + schemaVersion: typeof backupSqlStatsSchemaVersion + day: string + objectKey: string + maxStatementBytes: number + oversizedStatementCount: number + importStatementLimitBytes: number +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function isNonNegativeSafeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && Number(value) >= 0 +} + +export function backupSqlStatsKey(objectKey: string): string { + return `${objectKey}.stats.json` +} + +export function backupSqlStatsRequired(day: string): boolean { + return day >= backupSqlStatsRequiredFromDay +} + +export function parseBackupSqlStats(value: unknown): BackupSqlStats { + if ( + !isRecord(value) || + value.schemaVersion !== backupSqlStatsSchemaVersion || + typeof value.day !== 'string' || + typeof value.objectKey !== 'string' || + !isNonNegativeSafeInteger(value.maxStatementBytes) || + !isNonNegativeSafeInteger(value.oversizedStatementCount) || + !Number.isSafeInteger(value.importStatementLimitBytes) || + Number(value.importStatementLimitBytes) <= 0 || + (value.oversizedStatementCount === 0) !== + Number(value.maxStatementBytes) <= Number(value.importStatementLimitBytes) + ) { + throw new Error('backup SQL stats are corrupt') + } + return value as BackupSqlStats +}