From b77934b593da0edb0072a8e95819e6926ca4f388 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:08:22 -0400 Subject: [PATCH 01/26] docs(plan): strengthen encryption-at-rest plan from doc review Fold multi-persona review findings into the plan: stage-then-promote restore (KTD10) so R9 holds on all paths, BACKUP_FORMAT_VERSION gate (KTD4), zip-slip blob path validation (KTD11), dedicated operator_audit table (KTD6), crash-safe durable maintenance flag + pool-safe advisory lock (KTD5), and differentiated U7 UI states. Signed-off-by: UncleSp1d3r --- ...01-feat-encryption-at-rest-backups-plan.md | 221 +++++++++++++++++- 1 file changed, 220 insertions(+), 1 deletion(-) diff --git a/docs/plans/2026-07-12-001-feat-encryption-at-rest-backups-plan.md b/docs/plans/2026-07-12-001-feat-encryption-at-rest-backups-plan.md index f86dd0e0..38b33b97 100644 --- a/docs/plans/2026-07-12-001-feat-encryption-at-rest-backups-plan.md +++ b/docs/plans/2026-07-12-001-feat-encryption-at-rest-backups-plan.md @@ -4,7 +4,7 @@ type: feat date: 2026-07-12 topic: encryption-at-rest-backups artifact_contract: ce-unified-plan/v1 -artifact_readiness: requirements-only +artifact_readiness: implementation-ready product_contract_source: ce-brainstorm execution: code --- @@ -141,3 +141,222 @@ Deferred to planning: - `STRATEGY.md` — the self-hosted, privacy-first identity; the backup doubles as a "take your data and leave" escape hatch. - Repo state (verified): Postgres + Drizzle, local-volume blob storage under `UPLOAD_DIR`, Better Auth admin plugin, `docker-compose.yml` + `Dockerfile`; no existing backup, export, or encryption code. - Prior-art scan (web, 2026): native encrypted whole-instance backup is the minority pattern among self-hosted apps (Home Assistant, Standard Notes); DB+blob apps (Immich, Paperless-ngx, Gitea, PhotoPrism, Firefly III) mostly document external tooling (`pg_dump`, `restic`, filesystem copy). Best-practice crypto is Argon2id + libsodium `secretstream` (Home Assistant's Trail-of-Bits-audited SecureTar v3). Restore-safety patterns worth copying: snapshot-before-restore + auto-rollback (Immich) and version-gated bundles that refuse cross-version restore (Paperless-ngx). References: OWASP Password Storage Cheat Sheet; libsodium `secretstream` docs; `age` (C2SP) spec; Home Assistant backup-encryption blog (2026); Immich and Paperless-ngx backup/restore docs; Docker secrets guidance and CIS container-hardening benchmark. + +--- + +## Planning Contract + +**Product Contract preservation:** unchanged. R1–R20, F1–F3, and AE1–AE5 are carried forward verbatim; planning adds the how below and resolves the three deferred Outstanding Questions (bundle format, version policy, in-app vs CLI restore) as decisions. R7 is enriched (not changed) with a maintenance lock + both-stores rollback per a code-review note; still refuse-unless-empty by default with force-replace behind type-to-confirm. + +**Planning enrichments from document review (2026-07-12):** the following HOW-level decisions were strengthened after a multi-persona review; none alter the Product Contract (WHAT). R9 is mechanized by stage-then-promote (KTD10) so live data is untouched until the whole bundle authenticates — this also closes the empty-instance (F2) rollback gap. R8's compatibility key changed from a strict migration-tag match to an explicit `BACKUP_FORMAT_VERSION` (KTD4) so same-instance DR survives routine migrations while still refusing genuinely incompatible bundles. R15's operator-event store is resolved to a dedicated `operator_audit` table (KTD6); `inventory_log` reuse is foreclosed by its CHECK constraints. Added: bundle path-traversal/zip-slip defense (KTD11), crash-safe durable maintenance flag + pool-safe advisory lock and temp-schema snapshot (KTD5). + +### Key Technical Decisions + +- KTD1. **libsodium for all crypto — the contract's stated pairing.** Argon2id via `crypto_pwhash` (OWASP `MODERATE`/`SENSITIVE` params, random per-bundle salt) derives the key; `crypto_secretstream_xchacha20poly1305` provides chunked, authenticated, tamper-evident streaming encryption (Home Assistant's audited SecureTar pairing). Binding: prefer `sodium-native` (native, streaming) with `libsodium-wrappers-sumo` (wasm) as the portable fallback — finalize at execution against the Bun/Docker build. No hand-rolled crypto, no `openssl enc`. This resolves the Product Contract's open "may substitute an equivalent vetted format (e.g., an `age` binding)" assumption: `age` is **not** adopted (its scrypt KDF and external-binary dependency fit this stack less cleanly than an in-process libsodium binding) — the Product Contract's fixed properties (password-based, authenticated, streaming, vetted-library) are all satisfied by the libsodium pairing. +- KTD2. **App-level Drizzle NDJSON database export, not `pg_dump`.** Enumerate every *persistent* table in FK-safe (dependency) order and stream rows as NDJSON; restore inserts in the same order. App-native (no postgres-client binary in the runtime image, no Postgres-major-version coupling), and its correctness is guarded by the schema-identity stamp (KTD4). Ephemeral tables — `session`, and any rate-limit/idempotency counters — are excluded so a restore is clean (R2). +- KTD3. **One streaming tar bundle piped through secretstream — never buffered whole (R13).** The bundle is a tar stream of `manifest.json` (versions + metadata), the DB NDJSON, and every blob under `UPLOAD_DIR`, encrypted chunk-by-chunk as it is produced (`tar-stream`/`node:stream` or web streams). Export streams straight to the browser download. Restore streams from the upload through decrypt → tar-extract → **stage** (see KTD10) — it never writes live data mid-stream. Nothing lands whole in memory on the happy path; staged data lands on disk in an isolated staging location, not the live stores. +- KTD10. **Restore stages, then promotes — this is how R9 "before changing any data" is actually met (both F2 and F3).** secretstream authenticates chunk-by-chunk, so a tamper late in a multi-GB bundle is only detected at that chunk — meaning a naive decrypt→apply pipeline would have already mutated live data before catching it. Instead restore imports DB rows into a **staging schema** and writes blobs into a **staging directory** (a sibling of `UPLOAD_DIR`) as it streams; only after the **final** secretstream chunk authenticates (whole-bundle integrity confirmed) does restore **promote** staging → live: DB rows move staging → live tables inside one transaction, and the staging blob directory is atomically swapped into place. A wrong password, tampered byte, or truncation anywhere in the stream fails **before** the promote step, so live data is never touched (R9/AE3) — this holds for the empty-instance path (F2) as well as force-replace (F3). This subsumes the earlier "F2 has no rollback" gap: F2 promote is a plain transaction+swap; F3 adds the wipe-and-snapshot envelope (KTD5) around the same promote. +- KTD4. **Compatibility key = an explicit `BACKUP_FORMAT_VERSION` integer, bumped only on restore-breaking schema changes (Paperless-ngx's version-gated-bundle pattern).** A strict latest-migration-tag match was rejected: the journal advances on nearly every feature PR (18 tags across ~11 PRs already), so tag-equality would refuse an operator restoring their *own* backup onto the *same* instance after any routine migration — breaking R10 and the "take your data and leave" escape-hatch. Instead: a single `BACKUP_FORMAT_VERSION` constant in code is bumped by hand **only** when a schema change would make an older bundle's NDJSON import fail or silently drop data against the new schema (a dropped/renamed column, a new NOT-NULL without default, a type change). The manifest stamps this integer plus the app version and latest migration tag (the latter two informational). Restore refuses when `bundle.backupFormatVersion !== instance.BACKUP_FORMAT_VERSION` (R8) — no migrate-on-restore, no range-guessing — but routine additive migrations don't bump it, so same-instance DR and forward restores across ordinary releases keep working. A doc comment on the constant must instruct: bump this whenever a migration would break restore of a prior bundle. +- KTD5. **Force-restore is atomic across both stores (R7, enriched).** Order: (1) enter maintenance mode — block all writes for the duration; (2) **stage** the incoming bundle and fully authenticate it (KTD10) — if authentication fails, abort here having touched nothing; (3) snapshot the current DB into a **temp schema inside Postgres** (not a filesystem dump — stays in the `db` container's own `pgdata` volume, inherits Postgres's at-rest posture, and avoids the cross-container dump + extra-headroom problem a same-app-volume dump would create) and move the current `UPLOAD_DIR` contents aside on the uploads volume; (4) wipe; (5) promote the already-staged bundle (KTD10); (6) on ANY failure, roll back BOTH the DB snapshot and the blob directory together, then exit maintenance; (7) on success, drop the snapshot, deterministically delete the moved-aside blobs, and exit maintenance. A DB-only rollback or a concurrent write must never leave rows and blobs divergent. + - **Write-blocking mechanism (maintenance flag + lock).** The maintenance flag must be **durable** (a DB-backed flag row, honored by the write path), not in-memory only — an in-memory flag silently evaporates if the process crashes mid-restore, leaving a half-wiped instance with no record. The concurrency guard uses `pg_advisory_xact_lock` wrapping the promote transaction (or a single explicitly-held `pool.connect()` client released only after unlock): a session-scoped `pg_advisory_lock` issued through the shared `pg` Pool (`src/db/client.ts`) is not safe — the lock belongs to whichever pooled connection acquired it and is dropped when that connection returns to the pool. + - **Crash safety.** Because the flag is durable and the DB snapshot lives in a temp schema, a process restart mid-restore can detect an interrupted force-restore (flag set, snapshot schema present) and either resume rollback or refuse to serve traffic until an operator resolves it — rather than serving a half-wiped instance. + - **Transient plaintext.** The temp-schema snapshot and moved-aside `UPLOAD_DIR` are plaintext for the restore window; they live on the same volumes the operator is directed to place on an encrypted host disk (R19), so operator docs (U9) must state the encrypted-volume guidance covers this transient data too. Step (7)'s cleanup must be deterministic even on the failure path. + - **Blocked-write UX.** While the maintenance flag is set, the write path returns a clear "instance under maintenance, try again shortly" response (a defined status code + message) rather than a generic 500, so another user's blocked write reads as intentional, not broken. +- KTD6. **Admin-gated, and every export/restore is an operator event.** Both actions gate on the repo's actual admin convention — an inline `user.role !== "admin"` check via a shared `requireAdmin()` helper, matching `app/(admin)/users/actions.ts` (the unused `isAdmin()` in `src/auth/session.ts` has zero callers today; the backup routes should not be its sole unverified first caller) — enforced at the route boundary (R14) and re-asserted inside the restore/export services as defense-in-depth. Events are recorded (actor, timestamp, outcome — R15) in a **new dedicated `operator_audit` table**, not `inventory_log`: `inventory_log` carries CHECK constraints pinning `parent_type` to `('firearm','magazine')` and `event_type` to firearm/magazine enums, so an instance-level export/restore event cannot be inserted there without a migration widening both constraints. A small purpose-built table is cleaner and is added to `table-order.ts` (KTD2) so it round-trips in backups. +- KTD7. **In-app streaming for v1; CLI-assisted restore deferred.** Export and restore both run through admin API routes with streaming bodies. A container entrypoint/CLI restore path for instances too large for an in-app request is out of v1 scope (see Deferred) — the risk is recorded, not solved. +- KTD8. **No server-side backup storage.** Export produces the bundle on demand and streams it to the operator's download; the app persists no backup, so there is no backup store to secure, expire, or leak. +- KTD11. **Bundle blob keys are untrusted input — validate every path (zip-slip defense).** A bundle is attacker-influenceable: anyone can craft one from scratch under a password of their choosing, so authenticated decryption proves the bundle wasn't tampered *after creation*, never that its *contents* are safe. Before writing any `blobs/` entry, resolve `path.join(stagingDir, storageKey)`, resolve symlinks, and assert the real path is still strictly under the staging directory; reject any entry that escapes (`../`, absolute paths), any symlink entry, and any non-regular-file entry — refusing the whole restore rather than partially extracting. Also bound per-entry size and total entry count against the manifest's declared counts, and precheck free disk before the force-restore wipe, so a decompression-bomb or entry-flood bundle is refused before it can exhaust the volume. This single choke point lives in the bundle reader (U2). +- KTD9. **Deploy hardening is shipped config + docs; the app never encrypts the host disk.** The compose supplies DB password/key material via Docker secrets (R16), structures the Postgres and `UPLOAD_DIR` volumes for a low-friction encrypted-host-disk mount (R17), and applies read-only rootfs + dropped caps + no-new-privileges where compatible (R18). Operator docs cover the LUKS/encrypted-cloud-volume how-to (R19) and the threat-coverage matrix (R20). + +### High-Level Technical Design + +Export / restore data flow (both stream; force-restore adds the maintenance-lock + rollback envelope): + +```mermaid +flowchart TB + subgraph Export [F1 Export] + XUI["admin backup UI (password + no-recovery warning)"] --> XR["/api/admin/backup/export (admin-gated)"] + XR --> KDF["deriveKey: Argon2id(password, salt)"] + XR --> DBX["db export: Drizzle → NDJSON (FK order)"] + XR --> BLOB["blob stream: UPLOAD_DIR files"] + DBX --> TAR["tar-stream: manifest + db.ndjson + blobs/"] + BLOB --> TAR + KDF --> ENC["secretstream encrypt (chunked)"] + TAR --> ENC --> DL["browser download (one file)"] + end + subgraph Restore [F2/F3 Restore] + RUP["upload bundle + password"] --> DEC["secretstream decrypt + auth (R9)"] + DEC --> STG["stage: rows→staging schema, blobs→staging dir; path-validate (KTD10/KTD11)"] + STG --> VER["full-bundle authenticated? · version match? (R8) · instance empty? (R6)"] + VER -->|empty| PROMO["promote staging→live (txn + dir swap)"] + VER -->|force-confirmed| MAINT["maintenance lock + snapshot DB & blobs (R7)"] + MAINT --> WIPE["wipe"] --> PROMO + PROMO -->|fail| RB["roll back → exit maintenance"] + PROMO -->|ok| DONE["drop snapshot → exit maintenance"] + end +``` + +Bundle layout (inside the encrypted stream): + +```text +bundle (secretstream-encrypted tar) +├── manifest.json # appVersion, migrationTag, createdAt, counts +├── db.ndjson # one line per row, table-tagged, FK-safe order +└── blobs/ # every file from UPLOAD_DIR, verbatim +``` + +Restore decision matrix: + +| Instance state | Password/integrity | Version | Action | +|---|---|---|---| +| Empty | valid (full-bundle auth) | match | Stage → promote (txn + dir swap), F2 (R10) | +| Non-empty, plain restore | — | — | Refuse: "instance not empty" (R6/AE1) | +| Non-empty, force + type-to-confirm | valid (full-bundle auth) | match | Stage → maintenance → snapshot → wipe → promote → rollback-on-fail (R7/AE2) | +| Any | wrong password / tampered | — | Refuse before promote — live data untouched (R9/AE3) | +| Any | valid | mismatch | Refuse before staging (R8/AE4) | + +### Assumptions & Dependencies + +- Adds `sodium-native` (or `libsodium-wrappers-sumo`) and a streaming tar dependency (`tar-stream` or equivalent). No `pg_dump`/`age` binaries in the runtime image. If `sodium-native` (native addon) is chosen, it **must** be added to `package.json`'s `trustedDependencies` alongside `sharp`/`unrs-resolver`, or Bun silently skips its postinstall native build and it fails to load in the Docker image. +- Depends on: the `UPLOAD_DIR` blob store and storage adapter (from #12), the `pg` Pool + Drizzle client (`src/db/`), the Drizzle migration journal (`src/db/migrations/meta/_journal.json`), and the admin-gating convention in `app/(admin)/users/actions.ts` (inline role check). +- **No true streaming precedent exists in the repo yet.** The cited `app/api/export/route.ts` buffers its CSV fully in memory and gates on any authenticated user (not admin); `app/api/documents/[id]/route.ts` returns a fully-materialized buffer; no `ReadableStream` is used anywhere today. U4/U5/U6 build the first genuinely streamed Route Handlers (both request and response) in this codebase — treat this as net-new engineering, not pattern-reuse, when estimating effort. A small streaming spike ahead of U4 is advisable. +- Assumption — the operator stores the bundle off-box; its at-rest safety is its own encryption, independent of where it lands. +- Assumption — a backup can be large (GB-scale with many document blobs); every path streams (R13). Very-large-instance restore beyond an in-app request budget is deferred (KTD7). + +### Sequencing + +U1 (crypto) and U3 (db export/import) are independent and can proceed in parallel. U2 (bundle archive) depends on U1. U4 (export service) depends on U1–U3. U5 (restore service) depends on U1–U3. U6 (routes + audit) depends on U4, U5; note U6 introduces the `operator_audit` table that U3's `table-order.ts` must include, so land U6's schema before U3 finalizes its list (or rely on U3's guard test to catch the omission). U7 (UI) depends on U6. U8 (compose/Dockerfile) and U9 (docs) are independent of the app code and can proceed anytime. + +**Recommended: ship U8+U9 as an early, standalone release ahead of the backup engine.** They need zero app code, carry none of the crypto/streaming/rollback risk, and already cover one of the two named threats (seized/stolen disk) plus the self-host trust signal STRATEGY cares about. Delivering them first banks a low-risk win instead of gating it behind U1–U7 (the riskiest engineering in the plan). + +--- + +## Implementation Units + +### U1. Crypto module — Argon2id KDF + secretstream + +- **Goal:** A vetted-primitive wrapper: derive a key from a password (Argon2id) and encrypt/decrypt a byte stream with authenticated, chunked `secretstream`, plus the bundle crypto header. +- **Requirements:** R3, R9, R11. +- **Dependencies:** none. +- **Files:** `src/backup/crypto.ts`, `src/backup/__tests__/crypto.test.ts`. +- **Approach:** `deriveKey(password, salt, params)` via libsodium `crypto_pwhash` (Argon2id, OWASP params, 16-byte random salt). `createEncryptStream(key)` / `createDecryptStream(key)` over `crypto_secretstream_xchacha20poly1305` push/pull, chunking the input. A small crypto header (magic bytes, format version, salt, KDF params, secretstream header) is written first on encrypt and parsed first on decrypt; a wrong password or tampered bytes fail authentication on the first chunk. No plaintext persisted. +- **Patterns to follow:** the storage adapter's stream handling in `src/storage/`; keep crypto in one module, no crypto elsewhere. +- **Test scenarios:** round-trip encrypt→decrypt returns identical bytes for small and multi-chunk inputs (Covers AE5 at the crypto layer); wrong password fails authenticated decryption and yields no plaintext (Covers AE3); a single flipped byte in the ciphertext fails authentication (Covers AE3); the derived key is deterministic for the same password+salt and differs across salts; header round-trips (salt/params recovered). + +### U2. Bundle format — streaming tar writer/reader + +- **Goal:** Serialize/deserialize the bundle (manifest + db.ndjson + blobs) as a single tar stream, composed with U1's encrypt/decrypt so nothing is buffered whole (R13). +- **Requirements:** R2, R4, R13. +- **Dependencies:** U1. +- **Files:** `src/backup/bundle.ts`, `src/backup/manifest.ts`, `src/backup/__tests__/bundle.test.ts`. +- **Approach:** `writeBundle({ manifest, dbStream, blobEntries }, encryptStream)` builds a tar stream (`manifest.json` first, then `db.ndjson`, then each `blobs/`) piped through U1's encrypt stream. `readBundle(decryptStream, { stagingDir })` yields the manifest, then the db stream, then blob entries in order. **This is the single choke point for KTD11 blob-key validation:** for every blob entry, resolve `path.join(stagingDir, storageKey)`, resolve symlinks, and assert the real path stays strictly under `stagingDir` — reject `../`, absolute paths, symlink entries, and non-regular-file entries by throwing (refuse the whole restore). Enforce per-entry size and total-entry-count bounds against the manifest counts. `manifest.ts` defines the manifest shape (`backupFormatVersion`, appVersion, migrationTag, createdAt, row/blob counts) and its stamping (R4). Streaming throughout — a large blob set never lands in memory. +- **Patterns to follow:** `src/storage/` stream read/write. +- **Test scenarios:** write→read round-trips a manifest, a multi-row db stream, and several blobs of varying sizes with identical bytes and order; manifest carries `backupFormatVersion` + appVersion + migrationTag (R4); reading a truncated/corrupt bundle surfaces an error rather than partial data (Covers AE3); an empty blob set round-trips (edge); **a bundle with a `blobs/../../etc/x` (path-traversal) entry is refused, not partially extracted (KTD11)**; a bundle with a symlink blob entry is refused (KTD11); a bundle whose entry count/size exceeds its manifest-declared bounds is refused before extraction (KTD11). + +### U3. Database export/import — Drizzle NDJSON, FK-safe + +- **Goal:** Stream every persistent table's rows out as NDJSON in dependency order, and import them back in the same order; exclude ephemeral tables. +- **Requirements:** R2, R10. +- **Dependencies:** none. +- **Files:** `src/backup/db-export.ts`, `src/backup/db-import.ts`, `src/backup/table-order.ts`, `src/backup/__tests__/db-roundtrip.test.ts`. +- **Approach:** `table-order.ts` lists the persistent tables in FK-safe insert order (users → firearms/magazines/ammo/accessories → children incl. `firearm_photo`/`firearm_document` → grants/joins/logs → `operator_audit` from U6), and the ephemeral exclusion set (`session`, rate-limit/idempotency). Because U6 introduces the new `operator_audit` table, U3's ordered list and its guard test must include it — the guard (below) catches it automatically if a future edit forgets, but land U6's schema before or alongside U3's list so the guard passes. `exportDatabase()` selects each table and emits NDJSON lines tagged by table; `importDatabase(stream)` reads lines and inserts per table in order, inside a transaction. Column types round-trip losslessly (timestamps, uuids, integers, text, booleans). New tables must be added to `table-order.ts` — call this out in its doc comment so a future schema addition isn't silently dropped from backups. +- **Execution note:** Start from a failing round-trip integration test (seed → export → wipe → import → assert equality) so the table list and ordering are proven, not assumed. +- **Patterns to follow:** `src/test-support/factories.ts` for seeding; Drizzle table exports in `src/db/schema.ts`. +- **Test scenarios:** (integration, `DATABASE_URL`) seed a full inventory (users, grants, firearms with photos+documents, magazines, ammo, logs) → export → wipe → import → every table's rows match by content (Covers AE5, R10); FK order lets a firearm_document import after its firearm with no constraint error; ephemeral `session` rows are NOT in the export (R2); a table absent from `table-order.ts` is detected by a guard test comparing the ordered list against the live schema's table set (regression guard against silent omission); timestamp/uuid/boolean columns round-trip exactly. + +### U4. Backup export service + +- **Goal:** Orchestrate a full export: admin-authorized, derive key, stream DB NDJSON + all `UPLOAD_DIR` blobs into one encrypted, version-stamped bundle. +- **Requirements:** R1, R2, R3, R4, R11, R13. +- **Dependencies:** U1, U2, U3. +- **Files:** `src/backup/export-service.ts`, `src/backup/__tests__/export-service.test.ts`. +- **Approach:** `createBackup(password): ReadableStream` — validate the caller is admin (defense-in-depth; the route also gates), build the manifest (appVersion + current migrationTag), derive the key, and compose U3's db export + a blob stream over `UPLOAD_DIR` into U2's bundle writer through U1's encrypt stream, returning a stream the route pipes to the download. No backup is written server-side (KTD8). +- **Patterns to follow:** `app/api/export/route.ts` (admin export + streaming response). +- **Test scenarios:** (integration) a seeded instance exports to a bundle that, decrypted with the password, contains the manifest, all rows, and all blobs (Covers AE5); the migrationTag in the manifest equals the instance's latest migration (R4); a large blob set streams without buffering the whole bundle (assert via a bounded-memory or chunked-consumption check, R13); the returned stream is consumable exactly once and the app retains no bundle file afterward (KTD8). + +### U5. Restore service — verify, guard, atomic force-replace + +- **Goal:** Restore from a bundle with full safety: authenticated decrypt, version guard, refuse-unless-empty by default, and an atomic force-replace (maintenance lock + both-store snapshot/rollback). +- **Requirements:** R5, R6, R7, R8, R9, R10. +- **Dependencies:** U1, U2, U3. +- **Files:** `src/backup/restore-service.ts`, `src/backup/maintenance.ts`, `src/backup/__tests__/restore-service.test.ts`. +- **Approach:** `restore(stream, password, { force })` — re-assert `requireAdmin()` internally (defense-in-depth; the route also gates — restore's blast radius warrants it). Refuse on `backupFormatVersion` mismatch (R8/AE4) after reading the manifest. Check instance emptiness — refuse a non-force restore on a non-empty instance (R6/AE1). Then **stage, don't apply** (KTD10): decrypt+authenticate the stream while importing DB rows into a **staging schema** and writing blobs into a **staging directory** (path-validated by U2's reader, KTD11); if authentication fails anywhere (wrong password / tampered / truncated), abort with live data untouched (R9/AE3). Only after the full bundle authenticates do we **promote**: for an empty instance (F2), promote staging→live in one transaction + atomic blob-dir swap; for `force` (F3, route enforces type-to-confirm), run the KTD5 envelope via `maintenance.ts` — set the durable maintenance flag, take a pool-safe advisory lock (`pg_advisory_xact_lock` / single held client), snapshot the DB into a temp schema and move `UPLOAD_DIR` aside, wipe, promote, and on any failure roll back BOTH stores and exit maintenance (AE2). A successful restore yields a functionally identical instance (R10/AE5). +- **Execution note:** Characterize the rollback path with a fault-injection test (make the promote step throw after wipe during force-restore) proving both stores return to their pre-restore state — this is the highest-risk behavior. Also assert the durable-flag crash-recovery contract at least at the unit level (a set flag + present snapshot schema signals an interrupted restore). +- **Patterns to follow:** transaction usage in `src/domain/*/service.ts`; `authorizeAndDeleteParent`'s transactional discipline; `requireAdmin()` in `app/(admin)/users/actions.ts`. +- **Test scenarios:** (integration) empty instance + valid bundle + matching version → promotes, instance equals source (Covers AE5, R10); non-empty instance + plain restore → refused, no data changed (Covers AE1, R6); non-empty + force + confirm → wipes and promotes (Covers AE2, R7); wrong password → refused, live data untouched (Covers AE3, R9); tampered bundle (byte flipped in the LAST chunk, after valid earlier chunks) → refused before promote, live data untouched (Covers AE3 — proves stage-then-promote, KTD10); version mismatch → refused before staging (Covers AE4, R8); **fault injected during promote of a force-restore → both DB and blobs roll back to the pre-restore snapshot** (Covers AE2/R7, the critical path); a concurrent write attempt during maintenance is blocked (KTD5); **a large-bundle restore (decrypt → stage → promote) streams without buffering the whole bundle in memory** (Covers R13); a path-traversal blob entry causes the restore to refuse with no file written outside the staging dir (Covers KTD11); non-admin caller is refused inside the service even if the route gate is bypassed (defense-in-depth, R14). + +### U6. Admin backup API routes + operator-event logging + +- **Goal:** Admin-gated export and restore endpoints, streaming, with each action recorded as an operator event. +- **Requirements:** R14, R15, R1, R5. +- **Dependencies:** U4, U5. +- **Files:** `app/api/admin/backup/export/route.ts`, `app/api/admin/backup/restore/route.ts`, `src/backup/audit.ts`, `src/db/operator-audit-schema.ts` (new `operator_audit` table) + the generated Drizzle migration, `src/backup/__tests__/routes.test.ts`. +- **Approach:** both routes use the repo's real admin convention — a shared `requireAdmin()` inline role check (mirroring `app/(admin)/users/actions.ts`), not the unused `isAdmin()`. `export` (POST): admin gate → read the password → return `createBackup()` as a streamed `Content-Disposition: attachment` download; a non-admin gets the app's existing not-authorized response (R14). `restore` (POST): admin gate → stream the uploaded bundle + password (+ `force`/confirmation) into `restore()`; return a **discriminated outcome** the UI can branch on (`ok` | `refused_not_empty` | `wrong_password_or_tampered` | `version_mismatch` | `rolled_back`). `audit.ts` writes `{ actor, action: export|restore, outcome, at }` to the new `operator_audit` table for both success and failure (R15). Restore uploads bypass the Server Action body cap (a route handler reading the request stream, not a buffered Server Action). +- **Patterns to follow:** `app/api/documents/[id]/route.ts` (session gating, streamed responses); `app/(admin)/users/actions.ts` (`requireAdmin()` inline check); a new small table schema modeled on the simplest existing table in `src/db/`. +- **Test scenarios:** (integration) non-admin → export and restore both refused (R14); admin export → 200 streamed attachment; admin restore of a valid bundle → success + an operator event recorded with actor/outcome (R15); a failed restore records a failure event (R15); the restore route accepts a large upload stream without the Server Action body-size limit applying. + +### U7. Admin backup UI + +- **Goal:** The admin backup screen: export (password set/confirm + no-recovery warning) and restore (upload + password + refuse-unless-empty message + force-replace type-to-confirm). +- **Requirements:** R12, R7, R6, R5, R1, R14. +- **Dependencies:** U6. +- **Files:** `app/(app)/admin/backup/page.tsx`, `app/(app)/admin/backup/backup-panel.tsx`, `app/(app)/admin/backup/__tests__/backup-panel.test.tsx` (or e2e in `e2e/backup.spec.ts`). +- **Approach:** Admin-only page (mirror how other admin surfaces gate). **Export:** set + confirm password, an explicit **no password recovery** warning shown before the operator commits (R12); the download is triggered as a **direct streamed response** (a form/navigation-triggered POST, or an anchor to the route) — **not** a client-side `fetch()`+blob, which would re-buffer the whole GB-scale bundle in the browser and undercut the R13 streaming guarantee. Show a pending/in-progress state while the download is being produced and a success confirmation when it starts. **Restore:** file picker + password; a plain restore on a non-empty instance surfaces the refuse message (R6). Force-replace is gated behind a **type-to-confirm phrase** — the literal phrase is the instance hostname (or a fixed sentinel like `REPLACE ALL DATA` if no instance identity is available), chosen so the operator must actively read what they are wiping; specify the exact phrase in code. The restore UI branches on U6's discriminated outcome to show **distinct, actionable messages** for each failure: wrong-password/tampered (AE3 → "check the password / bundle may be corrupt"), version mismatch (AE4 → "bundle is from an incompatible version"), and force-replace rollback (AE2 failure → "restore failed and your data was rolled back"). Show a **progress/pending state** across the multi-minute restore (verifying → applying → done/failed) so a static screen can't be mistaken for a hang and duplicate-submit. On **successful** restore, force session invalidation and redirect to login ("Instance restored — please sign in"), since the `users` table (including the acting admin's own row) was just replaced (R10). ARIA roles / accessible names / visible text only — no `data-testid`. Reuse shadcn/Radix primitives + `ConfirmDialog` for the type-to-confirm. +- **Patterns to follow:** the documents section's confirmation + panel patterns (`app/(app)/firearms/[id]/firearm-documents.tsx`); admin gating from the settings/users admin surface. +- **Test scenarios:** the no-recovery warning is present before export can be triggered (R12); a non-empty-instance plain restore shows the refuse message (Covers AE1); the force-replace action is disabled until the confirmation phrase is typed exactly (Covers AE2, R7); wrong-password, version-mismatch, and rollback outcomes each render their own distinct message (Covers AE3/AE4/AE2); a successful restore invalidates the session and redirects to login; a success state is shown for both export and restore; non-admins never see the page/actions (R14). UI behavior verified via e2e where practical. + +### U8. Deploy hardening — compose + Dockerfile + +- **Goal:** Ship a compose posture that uses Docker secrets, is encrypted-volume-ready, and applies baseline container hardening. +- **Requirements:** R16, R17, R18. +- **Dependencies:** none. +- **Files:** `docker-compose.yml`, `Dockerfile`, `.env.example` (document secret files). +- **Approach:** Move the DB password (and any key material) to Docker secrets (`secrets:` + `*_FILE` env convention) rather than plain env (R16). Structure the Postgres data volume and the `UPLOAD_DIR` volume as named mounts documented as the encrypted-host-disk attach points (R17). Apply `read_only: true` root filesystem with explicit writable `tmpfs`/volumes, `cap_drop: [ALL]` (+ minimal `cap_add`), and `security_opt: [no-new-privileges:true]` where compatible with Next.js + Postgres (R18). +- **Execution note:** Mostly config; verify by `docker compose config` validity and a boot smoke test (the stack starts, the app serves, uploads still write) rather than unit tests. +- **Patterns to follow:** the existing `docker-compose.yml`/`Dockerfile`. +- **Test scenarios:** `Test expectation: none — config/deploy. Verify: `docker compose config` is valid; the stack boots; the app serves and can read/write `UPLOAD_DIR`; DB connects via the secret; read-only rootfs doesn't break runtime writes.` + +### U9. Operator documentation + +- **Goal:** Document the encrypted-volume how-to and the threat-coverage matrix. +- **Requirements:** R19, R20. +- **Dependencies:** none. +- **Files:** `docs/operations/encryption-at-rest.md` (or the repo's docs site location), plus a link from `README`/deploy docs. +- **Approach:** A host-responsibility guide for running the Postgres and upload volumes on an encrypted host disk (LUKS or an encrypted cloud volume), stated plainly as something the app cannot do (R19). A matrix stating which layer covers which threat — encrypted volume ⇒ seized/stolen disk; encrypted backup ⇒ leaked bundle; neither ⇒ compromised running host (R20). Reference the backup screen and the no-recovery caveat. +- **Test scenarios:** `Test expectation: none — documentation. Verify: the threat matrix names all three threats and their coverage (R20); the LUKS/encrypted-volume steps are concrete and framed as a host responsibility (R19); links resolve.` + +--- + +## Verification Contract + +| Gate | Command | Applies to | +|---|---|---| +| Lint | `bun run lint` (Biome) | all | +| Type check | `bun run typecheck` | all | +| Unit + integration tests | `bun test` (needs `DATABASE_URL`; Testcontainers) | U1–U6 | +| End-to-end | `bun run test:e2e` (Docker) | U7 | +| Deploy smoke | `docker compose config` + boot/serve/upload check | U8 | +| Full pre-commit gate | `just ci-check` | all — must pass before every commit | + +--- + +## Definition of Done + +- All of R1–R20 are satisfied and traced to a unit above. +- Crypto is a thin wrapper over libsodium (Argon2id + secretstream); no hand-rolled crypto. Restore stages then promotes (KTD10): a wrong password or tampered bundle — including tampering in the final chunk — fails before any live data changes, on both the empty (F2) and force-replace (F3) paths (R9, R11, AE3). Untrusted blob keys are path-validated (KTD11); a path-traversal bundle is refused with nothing written outside staging. +- Export produces one streamed, password-encrypted, version-stamped bundle of the full DB + all blobs, retaining nothing server-side (R1–R4, R11, R13, KTD8). +- Restore refuses-unless-empty by default; enforces the `BACKUP_FORMAT_VERSION` compatibility gate (KTD4) so incompatible bundles are refused while same-instance DR survives routine migrations; the force-replace path is atomic across DB and blobs with proven both-store rollback on failure and a durable, crash-recoverable maintenance flag (R5–R10, KTD5). +- Both actions are admin-only (real `requireAdmin()` convention) and recorded as operator events in the dedicated `operator_audit` table (R14, R15). +- The shipped compose uses Docker secrets, is encrypted-volume-ready, and applies baseline hardening; operator docs cover the encrypted-volume how-to and the threat matrix (R16–R20). +- `bun run lint`, `bun run typecheck`, `bun test`, `bun run test:e2e` pass, and `just ci-check` is green. + +--- + +## Deferred / Open Questions + +- **CLI/entrypoint-assisted restore for very large instances** (KTD7) — v1 is in-app streaming only; an out-of-request restore path is deferred until an instance is genuinely too large. Risk recorded, not solved. +- **libsodium binding choice** — `sodium-native` (native, faster streaming) vs `libsodium-wrappers-sumo` (wasm, portable) to be finalized at execution against the Bun + Docker build; both satisfy the KTD1 primitives. If `sodium-native`, add it to `trustedDependencies` (see Assumptions). +- **In-app maintenance mode vs "stop the container first" (product consideration).** KTD5 builds an in-app write-blocking maintenance subsystem to guard force-replace against concurrent writes. For a single/small-operator self-hosted product an alternative is to document "stop the app container before a force-replace restore," dropping the in-app concurrency infra entirely while keeping the stage-then-promote + snapshot/rollback safety R7 actually requires. Kept in-app for v1 (works when the operator restores from a running instance), but worth revisiting if the maintenance-flag machinery proves costly to maintain. +- **App-level NDJSON export vs orchestrating `pg_dump`** — KTD2's hand-maintained `table-order.ts` is guarded against silent omission by a schema-diff test (U3), but an alternative is orchestrating `pg_dump`/`pg_restore` against the shipped Postgres container so the DB catalog handles FK ordering. Kept app-level for v1 (no postgres-client binary in the image, no PG-major coupling); the guard test mitigates the maintenance risk. +- **Snapshot mechanism resolved to temp-schema** (KTD5) — no longer open; on-volume dump was rejected for the cross-container/headroom cost. +- **Operator-event surface resolved to a dedicated `operator_audit` table** (KTD6) — no longer open; `inventory_log` reuse is foreclosed by its CHECK constraints. From c87748b481d16bf0c165be4e5fbc81e4cc0d7d7e Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:06:14 -0400 Subject: [PATCH 02/26] docs(operations): encrypted-volume how-to + threat matrix (U9) Signed-off-by: UncleSp1d3r --- README.md | 5 + docs/deployment.md | 8 ++ docs/operations/encryption-at-rest.md | 176 ++++++++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 docs/operations/encryption-at-rest.md diff --git a/README.md b/README.md index 9b1b1c7d..d80df0d4 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,11 @@ Everything lives in Postgres, so a normal `pg_dump` is your backup. Restoring it docker compose exec db pg_dump -U "$POSTGRES_USER" -Fc -d "$POSTGRES_DB" > magstacker.dump ``` +For running the Postgres and upload volumes on an encrypted host disk — and +a rundown of which threats disk encryption covers versus which an encrypted +in-app backup covers — see +[`docs/operations/encryption-at-rest.md`](docs/operations/encryption-at-rest.md). + ## Behind a reverse proxy Sign-in rides on cookies, so on any real network you run MagStacker behind a reverse proxy that terminates TLS rather than exposing port 3000 directly. Point the proxy at the app's published port and set `BETTER_AUTH_URL` in `.env` to the public `https://` address. It **must** match the origin you actually open, or Better Auth rejects the request. diff --git a/docs/deployment.md b/docs/deployment.md index d4200910..f9629a3a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -50,6 +50,14 @@ control — a homelab server, a NAS, a small VPS behind your own network. docker compose exec db pg_dump -U "$POSTGRES_USER" -Fc -d "$POSTGRES_DB" > magstacker.dump ``` +- The `magstacker-pgdata` and `magstacker-uploads` volumes above hold + everything sensitive this stack stores — putting them on an encrypted host + disk is on you, the operator; see + [`docs/operations/encryption-at-rest.md`](operations/encryption-at-rest.md) + for the LUKS and encrypted-cloud-volume how-to and a threat-coverage matrix + covering what disk encryption defends against versus an encrypted in-app + backup. + ## TLS / network exposure (important) Better Auth uses session **cookies**, and sign-in sends credentials. Do **not** diff --git a/docs/operations/encryption-at-rest.md b/docs/operations/encryption-at-rest.md new file mode 100644 index 00000000..7ca8e74c --- /dev/null +++ b/docs/operations/encryption-at-rest.md @@ -0,0 +1,176 @@ +# Encryption at rest + +MagStacker holds sensitive inventory — serial numbers, and (with document +attachments) receipts, warranties, and ATF forms. This page covers the two +layers MagStacker uses to keep that data safe when it's not sitting in a live, +authenticated session, and draws a clear line around what each layer does and +doesn't defend against. + +The two layers: + +- **Encrypted backups** — an admin-run, password-encrypted export of the whole + instance (database + document blobs), made from the **Admin → Backup** + screen. This is application-level: MagStacker does the encrypting. +- **Encrypted host volumes** — the disk(s) holding Postgres's data and the + document upload directory (`UPLOAD_DIR`) are encrypted at the storage layer. + This is **host-level, and it is your responsibility, not the app's.** + MagStacker cannot encrypt the disk it's running on; nothing in the app can + reach below the filesystem it's handed. + +Both matter, and they cover different threats — see the [threat-coverage +matrix](#threat-coverage-matrix) below. + +## Threat-coverage matrix + +| Threat | Scenario | Covered by | Not covered by | +| --- | --- | --- | --- | +| **Seized or stolen disk** | A drive is taken from a powered-off or decommissioned box — theft, hardware disposal, or state seizure. | Encrypted host volume (LUKS or an encrypted cloud volume) | Encrypted backups don't help here — they protect a bundle in transit or at rest *off*-box, not the live disk itself. | +| **Leaked or exfiltrated backup bundle** | A downloaded backup file ends up somewhere it shouldn't — emailed, left on a laptop, uploaded to the wrong place. | Encrypted backup (password-derived key, authenticated encryption; see the no-recovery caveat below) | An encrypted host volume doesn't help here — once the bundle leaves the box, disk encryption is irrelevant to it. | +| **Compromised running host** | An attacker gets code execution or root on the live, running server while it's up and the disk is unlocked. | **Neither layer.** Explicitly out of scope. | Both an unlocked encrypted volume and a decrypted-in-memory backup password are available to a live attacker with host access; disk encryption and backup encryption both assume an *offline* or *exfiltrated* artifact, not a compromised live process. | + +If you need to defend against a compromised running host specifically, that's +a different problem — hardening the host OS, minimizing attack surface, +network segmentation, and so on — and it's out of scope for this page. (For +what it's worth: this is also why MagStacker doesn't encrypt individual +database columns. An app-held key on a compromised host wouldn't meaningfully +help this threat, and it would break the server-side querying — serial-number +lookup and dedup — the product depends on.) + +## Host responsibility: encrypting the volumes + +This is a **host-level task you own** — MagStacker ships a deploy posture +that makes it a low-friction path, but it does not and cannot perform the +encryption itself. There are two volumes to place on encrypted storage, both +defined in `docker-compose.yml`: + +- `magstacker-pgdata` — the Postgres data directory (all inventory, users, + grants, and firearm records). +- `magstacker-uploads` — the `UPLOAD_DIR` mount (every document blob: + receipts, warranties, ATF forms). + +Docker's default `local` volume driver stores named volumes under +`/var/lib/docker/volumes/` on the host. That means the simplest way to +encrypt both at once is to encrypt the storage backing Docker's data root. +If you want to encrypt only these two volumes (leaving the rest of +`/var/lib/docker` alone), point them at a dedicated encrypted mount instead. +Both approaches are below. + +> **Transient restore data lives here too.** A force-replace restore (see +> [Backups and the no-recovery caveat](#backups-and-the-no-recovery-caveat)) +> briefly writes a plaintext snapshot of the outgoing database into a temp +> schema inside Postgres's data directory, and moves the outgoing +> `UPLOAD_DIR` contents aside on the same upload volume, before wiping and +> re-applying. Both live on the volumes covered by the guidance below, so +> encrypting `magstacker-pgdata` and `magstacker-uploads` also covers that +> transient window — there's nothing extra to configure for it. + +### Option A — encrypt the whole Docker data root (simplest) + +Do this at OS-install time, or by relocating Docker's data root onto an +already-encrypted disk. Because every named volume lives under +`/var/lib/docker/volumes/`, this transparently covers `magstacker-pgdata` and +`magstacker-uploads` (and any future volumes) with no compose changes. + +**Self-hosted, with LUKS (Linux):** + +1. Identify the disk or partition you'll dedicate to Docker (e.g. `/dev/sdb1`) + and encrypt it: + + ```bash + sudo cryptsetup luksFormat /dev/sdb1 + sudo cryptsetup open /dev/sdb1 docker_data + sudo mkfs.ext4 /dev/mapper/docker_data + ``` + +2. Mount it where Docker expects its data root: + + ```bash + sudo mkdir -p /var/lib/docker + sudo mount /dev/mapper/docker_data /var/lib/docker + ``` + + (If Docker was already initialized on this host, stop it first, move the + existing `/var/lib/docker` contents onto the new encrypted filesystem, + then remount.) + +3. Add the mapping to `/etc/crypttab` so the volume can be unlocked on boot, + and the mount to `/etc/fstab`. LUKS needs a passphrase or keyfile supplied + at unlock time — decide up front whether that's a manual passphrase entry + at boot (safest, but means the box doesn't come back up unattended after + a power cycle) or a keyfile (more automated, but the keyfile itself must + not live unencrypted next to the volume it unlocks — keep it on separate + protected storage, e.g. a TPM-backed unlock or a secrets manager your boot + process can reach). +4. `docker compose up -d` as normal — Docker now writes `magstacker-pgdata` + and `magstacker-uploads` onto the encrypted filesystem without any compose + changes. + +**Cloud, with an encrypted volume:** + +The exact steps vary by provider, but the shape is the same everywhere: + +1. Create (or attach) a block volume with encryption enabled at creation — + e.g. AWS EBS `--encrypted` (or turn on the account-level "always encrypt + new EBS volumes" default), a GCP persistent disk (encrypted by default, or + with a customer-managed key for stricter control), an Azure managed disk + (server-side encryption is on by default; add a customer-managed key if + you need to hold your own key), or your provider's equivalent. +2. Attach it to the instance and mount it at the path that will back Docker's + data root (or relocate Docker's `data-root` to it via + `/etc/docker/daemon.json`: `{ "data-root": "/mnt/encrypted-docker" }`, + then restart the Docker daemon). +3. `docker compose up -d` as normal. + +### Option B — encrypt just the MagStacker volumes (targeted) + +If you'd rather not touch Docker's whole data root, mount an encrypted +filesystem at a dedicated path and repoint only `magstacker-pgdata` and +`magstacker-uploads` at it using compose's bind-style `driver_opts`. Set up +the encrypted mount the same way as Option A (LUKS steps 1–3, or an attached +encrypted cloud volume), mounted at, say, `/mnt/magstacker-encrypted`, then +override the two volumes: + +```yaml +# docker-compose.override.yml +volumes: + magstacker-pgdata: + driver_opts: + type: none + o: bind + device: /mnt/magstacker-encrypted/pgdata + magstacker-uploads: + driver_opts: + type: none + o: bind + device: /mnt/magstacker-encrypted/uploads +``` + +Create the two subdirectories (`pgdata`, `uploads`) on the encrypted mount +before first `up` so Docker has somewhere to bind to. `docker compose up -d` +then reads the override automatically alongside `docker-compose.yml`. + +## Backups and the no-recovery caveat + +The other layer — encrypted backups — is covered from the **Admin → Backup** +screen in the app: an admin sets a password, and MagStacker exports the +entire instance (database plus every document blob) as a single +authenticated-encrypted file, downloaded on demand. Restoring reverses that: +upload the bundle, enter the password, and (on an empty instance) it applies +cleanly, or (on a non-empty instance) a guarded force-replace path snapshots, +wipes, and re-applies with automatic rollback on failure. + +**There is no password recovery.** The backup password derives the +encryption key directly — MagStacker never stores it, and there is no +"forgot password" path for a backup file. If you lose the password to a +backup, that backup is permanently unreadable; the export screen warns you +of this before you commit. Keep the password somewhere durable (a password +manager, not a sticky note on the server) and separate from the bundle +itself — storing them together defeats the point of encrypting the bundle. + +## See also + +- [`docs/deployment.md`](../deployment.md) — first-run setup, secrets, + TLS/reverse-proxy configuration, and the volumes defined in + `docker-compose.yml`. +- [`README.md`](../../README.md) — quick-start and the plain `pg_dump` + backup command for local tooling. From 9b0ffa04775330a3bb19fe446d3c3fa432a97077 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:13:55 -0400 Subject: [PATCH 03/26] feat(backup): add Argon2id + secretstream crypto module (U1) Adds src/backup/crypto.ts, a thin libsodium wrapper for the encrypted backup plan (KTD1): deriveKey() via Argon2id (crypto_pwhash, libsodium's OWASP-aligned MODERATE params) plus createEncryptStream/createDecryptStream over crypto_secretstream_xchacha20poly1305, chunked at 64 KiB. A fixed- length header (magic, version, salt, KDF params, secretstream header) precedes the ciphertext; writeHeader/readHeader (de)serialize it. Uses sodium-native (native, prebuilt via prebuildify - verified to load cleanly under Bun with no compile step) and adds it to package.json's trustedDependencies. Ships a small hand-written sodium-native.d.ts ambient declaration since the only published @types/sodium-native predates the installed v5.x line by several majors. 20 new tests cover round-trips (small/empty/multi-chunk/boundary-aligned/ unaligned writes), deriveKey determinism and salt/password sensitivity, header round-tripping and corruption handling, wrong-key decryption failure, and tamper detection (mid-stream and final-chunk byte flips, truncation) - all fail before any plaintext is produced. Signed-off-by: UncleSp1d3r --- bun.lock | 21 ++ package.json | 2 + src/backup/__tests__/crypto.test.ts | 279 ++++++++++++++++ src/backup/crypto.ts | 502 ++++++++++++++++++++++++++++ src/backup/sodium-native.d.ts | 62 ++++ 5 files changed, 866 insertions(+) create mode 100644 src/backup/__tests__/crypto.test.ts create mode 100644 src/backup/crypto.ts create mode 100644 src/backup/sodium-native.d.ts diff --git a/bun.lock b/bun.lock index 7bba62ce..7e08c816 100644 --- a/bun.lock +++ b/bun.lock @@ -21,6 +21,7 @@ "react": "^19.2.7", "react-dom": "^19.2.7", "sharp": "^0.35.3", + "sodium-native": "^5.1.0", "tailwind-merge": "^3.6.0", }, "devDependencies": { @@ -575,16 +576,30 @@ "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + "bare-addon-resolve": ["bare-addon-resolve@1.10.0", "", { "dependencies": { "bare-module-resolve": "^1.10.0", "bare-semver": "^1.0.0" }, "peerDependencies": { "bare-url": "*" }, "optionalPeers": ["bare-url"] }, "sha512-sSd0jieRJlDaODOzj0oe0RjFVC1QI0ZIjGIdPkbrTXsdVVtENg14c+lHHAhHwmWCZ2nQlMhy8jA3Y5LYPc/isA=="], + + "bare-ansi-escapes": ["bare-ansi-escapes@2.2.3", "", { "dependencies": { "bare-stream": "^2.6.5" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-02ES4/E2RbrtZSnHJ9LntBhYkLA6lPpSEeP8iqS3MccBIVhVBlEmruF1I7HZqx5Q8aiTeYfQVeqmrU9YO2yYoQ=="], + + "bare-assert": ["bare-assert@1.2.0", "", { "dependencies": { "bare-inspect": "^3.1.2" } }, "sha512-c6uvgvTJBspTDxtVnPgrBKmLgcpW3Fp72NVKDLg6oT4QjQbhGtvrkHMhGYMK1sh4vjBHOBmuUalyt9hSzV37fQ=="], + "bare-events": ["bare-events@2.9.1", "", { "peerDependencies": { "bare-abort-controller": "*" }, "optionalPeers": ["bare-abort-controller"] }, "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg=="], "bare-fs": ["bare-fs@4.7.2", "", { "dependencies": { "bare-events": "^2.5.4", "bare-path": "^3.0.0", "bare-stream": "^2.6.4", "bare-url": "^2.2.2", "fast-fifo": "^1.3.2" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg=="], + "bare-inspect": ["bare-inspect@3.1.4", "", { "dependencies": { "bare-ansi-escapes": "^2.1.0", "bare-type": "^1.0.0" } }, "sha512-jfW5KRA84o3REpI6Vr4nbvMn+hqVAw8GU1mMdRwUsY5yJovQamxYeKGVKGqdzs+8ZbG4jRzGUXP/3Ji/DnqfPg=="], + + "bare-module-resolve": ["bare-module-resolve@1.12.2", "", { "dependencies": { "bare-semver": "^1.0.0" }, "peerDependencies": { "bare-url": "*" }, "optionalPeers": ["bare-url"] }, "sha512-j+hiD5k99qec4KjJvYsI67q5AOBifmy9JG3oeMVxTmvrhn2sIdp8StrUvZu4YNgwTpO+NhniQG16N1ETDe1k5w=="], + "bare-os": ["bare-os@3.9.2", "", {}, "sha512-h530JsrkYi8518ZfR57GHaLoI5YzXkGGEV0Y+mf4KYPBn4OnNajiznwkDq7FgE+Vnmyss9Utnzi44y7sowiAXA=="], "bare-path": ["bare-path@3.0.1", "", { "dependencies": { "bare-os": "^3.0.1" } }, "sha512-ghj2DSK/2e99a1anTVPCV4m4YIYtrbXhfM7V3D7XZLOTsybnYyaJloymGqssQc8l/or0UoDyRtNQkmkEF/ysgQ=="], + "bare-semver": ["bare-semver@1.1.0", "", {}, "sha512-1Hw5qJ7hXdVt3uPUqjeFTuxyvBUJauvz5A1I2jk8gzjZMHp04n//6nV9MDbG9CMw78JHY2lGV0w6s//LrASm2w=="], + "bare-stream": ["bare-stream@2.13.3", "", { "dependencies": { "b4a": "^1.8.1", "streamx": "^2.25.0", "teex": "^1.0.1" }, "peerDependencies": { "bare-abort-controller": "*", "bare-buffer": "*", "bare-events": "*" }, "optionalPeers": ["bare-abort-controller", "bare-buffer", "bare-events"] }, "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ=="], + "bare-type": ["bare-type@1.1.0", "", {}, "sha512-LdtnnEEYldOc87Dr4GpsKnStStZk3zfgoEMXy8yvEZkXrcCv9RtYDrUYWFsBQHtaB0s1EUWmcvS6XmEZYIj3Bw=="], + "bare-url": ["bare-url@2.4.5", "", { "dependencies": { "bare-path": "^3.0.0" } }, "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ=="], "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], @@ -1163,6 +1178,8 @@ "recast": ["recast@0.23.12", "", { "dependencies": { "ast-types": "^0.16.1", "esprima": "~4.0.0", "source-map": "~0.6.1", "tiny-invariant": "^1.3.3", "tslib": "^2.0.1" } }, "sha512-dEWRjcINDu/F4l2dYx57ugBtD7HV9KXESyxhzw/MqWLeglJrsjJKqACPyUPg+6AF8mIgm+Zi0dZ3ACoIg+QtpA=="], + "require-addon": ["require-addon@1.2.0", "", { "dependencies": { "bare-addon-resolve": "^1.3.0" } }, "sha512-VNPDZlYgIYQwWp9jMTzljx+k0ZtatKlcvOhktZ/anNPI3dQ9NXk7cq2U4iJ1wd9IrytRnYhyEocFWbkdPb+MYA=="], + "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], @@ -1221,6 +1238,8 @@ "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], + "sodium-native": ["sodium-native@5.1.0", "", { "dependencies": { "bare-assert": "^1.2.0", "require-addon": "^1.1.0", "which-runtime": "^1.2.1" } }, "sha512-3RxgyWyJlhTsABPnJVpCI5CoTDANZTqqFrEPqr+kjfnRaBihpVtMUE3yTF40ukdoB1APXeoBNKF3MzZAIHg39g=="], + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], @@ -1331,6 +1350,8 @@ "which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], + "which-runtime": ["which-runtime@1.4.0", "", {}, "sha512-0ugbP4CJW4e2D20jvEcC4973dCgIaHI4Rw1PT+26U9zEve7FyYdWAIwUnoeOYvoCfn+wXHoHTKb1KhkYlb60Pw=="], + "wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], "wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], diff --git a/package.json b/package.json index 591d99cb..8f1b82a5 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "react": "^19.2.7", "react-dom": "^19.2.7", "sharp": "^0.35.3", + "sodium-native": "^5.1.0", "tailwind-merge": "^3.6.0" }, "devDependencies": { @@ -58,6 +59,7 @@ ], "trustedDependencies": [ "sharp", + "sodium-native", "unrs-resolver" ] } diff --git a/src/backup/__tests__/crypto.test.ts b/src/backup/__tests__/crypto.test.ts new file mode 100644 index 00000000..a1746563 --- /dev/null +++ b/src/backup/__tests__/crypto.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, test } from "bun:test"; +import { Readable } from "node:stream"; +import { expectRejects } from "@/src/test-support/assertions"; +import { + CHUNK_SIZE, + createDecryptStream, + createEncryptStream, + DEFAULT_KDF_PARAMS, + DecryptionAuthError, + deriveKey, + FORMAT_VERSION, + generateSalt, + InvalidHeaderError, + readHeader, + writeHeader, +} from "../crypto"; + +/** Drains a Readable/Transform stream into a single Buffer. */ +async function collect(stream: NodeJS.ReadableStream): Promise { + const parts: Buffer[] = []; + for await (const chunk of stream) { + parts.push(chunk as Buffer); + } + return Buffer.concat(parts); +} + +/** Encrypts `plaintext` with `key`/`salt` and returns the full encrypted output (header + ciphertext). */ +async function encrypt( + key: Buffer, + salt: Buffer, + plaintext: Buffer, +): Promise { + const source = Readable.from([plaintext]); + const encrypted = source.pipe(createEncryptStream(key, salt)); + return collect(encrypted); +} + +/** Decrypts a full encrypted buffer (header + ciphertext) with `key`. */ +async function decrypt(key: Buffer, encrypted: Buffer): Promise { + const source = Readable.from([encrypted]); + const decrypted = source.pipe(createDecryptStream(key)); + return collect(decrypted); +} + +describe("deriveKey", () => { + test("is deterministic for the same password and salt", () => { + const salt = generateSalt(); + + const key1 = deriveKey("correct horse battery staple", salt); + const key2 = deriveKey("correct horse battery staple", salt); + + expect(key1.equals(key2)).toBe(true); + }); + + test("differs across salts for the same password", () => { + const saltA = generateSalt(); + const saltB = generateSalt(); + + const keyA = deriveKey("same password", saltA); + const keyB = deriveKey("same password", saltB); + + expect(keyA.equals(keyB)).toBe(false); + }); + + test("differs across passwords for the same salt", () => { + const salt = generateSalt(); + + const keyA = deriveKey("password one", salt); + const keyB = deriveKey("password two", salt); + + expect(keyA.equals(keyB)).toBe(false); + }); + + test("generateSalt produces unique, correctly-sized salts", () => { + const salts = Array.from({ length: 20 }, () => generateSalt()); + const unique = new Set(salts.map((s) => s.toString("hex"))); + + expect(unique.size).toBe(20); + for (const salt of salts) { + expect(salt.byteLength).toBe(16); + } + }); +}); + +describe("header round-trip", () => { + test("writeHeader/readHeader recovers salt and KDF params", () => { + const salt = generateSalt(); + const secretstreamHeader = Buffer.alloc(24, 7); + + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: DEFAULT_KDF_PARAMS, + secretstreamHeader, + }); + const header = readHeader(bytes); + + expect(header.version).toBe(FORMAT_VERSION); + expect(header.salt.equals(salt)).toBe(true); + expect(header.kdfParams).toEqual(DEFAULT_KDF_PARAMS); + expect(header.secretstreamHeader.equals(secretstreamHeader)).toBe(true); + }); + + test("readHeader rejects bad magic bytes", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: DEFAULT_KDF_PARAMS, + secretstreamHeader: Buffer.alloc(24), + }); + bytes[0] = bytes[0] ^ 0xff; // corrupt the first magic byte + + expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); + }); + + test("readHeader rejects a buffer shorter than the header length", () => { + expect(() => readHeader(Buffer.alloc(4))).toThrow(InvalidHeaderError); + }); + + test("readHeader rejects an unsupported format version", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: DEFAULT_KDF_PARAMS, + secretstreamHeader: Buffer.alloc(24), + }); + bytes[4] = 99; // version byte follows the 4-byte magic + + expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); + }); +}); + +describe("encrypt/decrypt round-trip", () => { + test("round-trips a small single-chunk input", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.from("a small secret payload"); + + const encrypted = await encrypt(key, salt, plaintext); + const decrypted = await decrypt(key, encrypted); + + expect(decrypted.equals(plaintext)).toBe(true); + }); + + test("round-trips an empty input", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.alloc(0); + + const encrypted = await encrypt(key, salt, plaintext); + const decrypted = await decrypt(key, encrypted); + + expect(decrypted.length).toBe(0); + }); + + test("round-trips a multi-chunk input (several CHUNK_SIZE boundaries)", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + // 3.5 chunks worth of pseudo-random bytes, deterministic for the test. + const plaintext = Buffer.alloc(CHUNK_SIZE * 3.5); + for (let i = 0; i < plaintext.length; i++) { + plaintext[i] = i % 256; + } + + const encrypted = await encrypt(key, salt, plaintext); + const decrypted = await decrypt(key, encrypted); + + expect(decrypted.length).toBe(plaintext.length); + expect(decrypted.equals(plaintext)).toBe(true); + }); + + test("round-trips an input that lands exactly on a CHUNK_SIZE boundary", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.alloc(CHUNK_SIZE * 2, 0xab); + + const encrypted = await encrypt(key, salt, plaintext); + const decrypted = await decrypt(key, encrypted); + + expect(decrypted.equals(plaintext)).toBe(true); + }); + + test("input delivered across many small, unaligned writes still round-trips", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.alloc(CHUNK_SIZE + 777); + for (let i = 0; i < plaintext.length; i++) { + plaintext[i] = (i * 7) % 256; + } + + // Feed the encrypt stream in small, deliberately-unaligned pieces. + const source = Readable.from(chunkBuffer(plaintext, 333)); + const encryptedStream = source.pipe(createEncryptStream(key, salt)); + const encrypted = await collect(encryptedStream); + const decrypted = await decrypt(key, encrypted); + + expect(decrypted.equals(plaintext)).toBe(true); + }); + + test("wrong password fails authenticated decryption and yields no plaintext", async () => { + const salt = generateSalt(); + const rightKey = deriveKey("correct password", salt); + const wrongKey = deriveKey("wrong password", salt); + const plaintext = Buffer.from("top secret inventory data"); + + const encrypted = await encrypt(rightKey, salt, plaintext); + + await expectRejects(() => decrypt(wrongKey, encrypted)); + }); + + test("a single flipped byte in the ciphertext fails authentication", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.alloc(CHUNK_SIZE + 100, 0x42); + + const encrypted = await encrypt(key, salt, plaintext); + // Flip a byte well past the (unencrypted) header, inside the ciphertext. + const tampered = Buffer.from(encrypted); + const flipIndex = tampered.length - 5; + tampered[flipIndex] = tampered[flipIndex] ^ 0xff; + + await expectRejects(() => decrypt(key, tampered)); + }); + + test("a flipped byte in the final chunk fails authentication", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + // Two full chunks plus a partial final chunk so tampering the final + // chunk's ciphertext is distinguishable from tampering an earlier one. + const plaintext = Buffer.alloc(CHUNK_SIZE * 2 + 50, 0x11); + + const encrypted = await encrypt(key, salt, plaintext); + const lastByte = encrypted.length - 1; + const tampered = Buffer.from(encrypted); + tampered[lastByte] = tampered[lastByte] ^ 0xff; + + await expectRejects(() => decrypt(key, tampered)); + }); + + test("truncated ciphertext (missing final chunk) is refused, not silently accepted", async () => { + const salt = generateSalt(); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.alloc(CHUNK_SIZE + 500, 0x33); + + const encrypted = await encrypt(key, salt, plaintext); + const truncated = encrypted.subarray(0, encrypted.length - 10); + + await expectRejects(() => decrypt(key, truncated)); + }); + + test("throws when handed a decrypt key of the wrong length", () => { + expect(() => createDecryptStream(Buffer.alloc(10))).toThrow(); + }); + + test("throws when handed an encrypt key of the wrong length", () => { + expect(() => + createEncryptStream(Buffer.alloc(10), generateSalt()), + ).toThrow(); + }); +}); + +describe("DecryptionAuthError", () => { + test("is exported and is an Error subclass", () => { + const err = new DecryptionAuthError("boom"); + expect(err).toBeInstanceOf(Error); + expect(err.name).toBe("DecryptionAuthError"); + }); +}); + +/** Splits a buffer into pieces of at most `size` bytes, in order. */ +function chunkBuffer(buf: Buffer, size: number): Buffer[] { + const pieces: Buffer[] = []; + for (let offset = 0; offset < buf.length; offset += size) { + pieces.push(buf.subarray(offset, offset + size)); + } + return pieces; +} diff --git a/src/backup/crypto.ts b/src/backup/crypto.ts new file mode 100644 index 00000000..99ef81e9 --- /dev/null +++ b/src/backup/crypto.ts @@ -0,0 +1,502 @@ +/** + * Crypto module for MagStacker encrypted backups (plan Unit U1). + * + * A thin wrapper over libsodium — no hand-rolled crypto, no `openssl enc`. + * + * Binding choice (KTD1): `sodium-native` (native, prebuilt via prebuildify — + * no compile step, so Bun's install doesn't need to build anything) was + * verified to `import`/load cleanly under Bun 1.3 in this repo and is used + * here. `libsodium-wrappers-sumo` (wasm) remains the documented portable + * fallback if `sodium-native`'s native binding ever fails to load in a + * target environment (see the plan's Deferred/Open Questions). + * + * - Key derivation: Argon2id via libsodium `crypto_pwhash`, using + * libsodium's own OWASP-aligned "MODERATE" ops/mem-limit constants and a + * 16-byte random salt (R3, R11). + * - Bulk encryption: `crypto_secretstream_xchacha20poly1305`, chunked at + * {@link CHUNK_SIZE} (64 KiB), authenticated end-to-end — a wrong key or a + * single flipped ciphertext byte fails authentication before any + * plaintext is produced for that chunk (R9, R11, AE3). + * - A small, fixed-length, unencrypted header (magic bytes, format version, + * salt, KDF params, and the secretstream header) precedes the ciphertext + * so a bundle is self-describing. The header carries no secrets — it is + * exactly what a legitimate decryptor needs before it can even attempt to + * derive a key or authenticate the first chunk. + * + * No plaintext is ever persisted by this module; callers are responsible + * for piping streams straight through (KTD3). + */ + +import { Transform } from "node:stream"; +import * as sodium from "sodium-native"; + +/** Plaintext chunk size for the secretstream framing (KTD3). */ +export const CHUNK_SIZE = 64 * 1024; + +/** Crypto header format version. Bump when the header layout changes. */ +export const FORMAT_VERSION = 1; + +const MAGIC = Buffer.from("MSKB", "ascii"); // "MagStacker Key Backup" + +const SALT_BYTES = sodium.crypto_pwhash_SALTBYTES; +const SECRETSTREAM_HEADER_BYTES = + sodium.crypto_secretstream_xchacha20poly1305_HEADERBYTES; +const SECRETSTREAM_ABYTES = sodium.crypto_secretstream_xchacha20poly1305_ABYTES; +const SECRETSTREAM_KEY_BYTES = + sodium.crypto_secretstream_xchacha20poly1305_KEYBYTES; +const SECRETSTREAM_STATE_BYTES = + sodium.crypto_secretstream_xchacha20poly1305_STATEBYTES; +const TAG_MESSAGE = sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE; +const TAG_FINAL = sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL; + +/** Ciphertext size for one full {@link CHUNK_SIZE} plaintext chunk. */ +const CIPHERTEXT_CHUNK_SIZE = CHUNK_SIZE + SECRETSTREAM_ABYTES; + +/** Argon2id parameters for {@link deriveKey}. */ +export interface KdfParams { + readonly opslimit: number; + readonly memlimit: number; + readonly alg: number; +} + +/** + * OWASP-aligned Argon2id parameters — libsodium's own "MODERATE" preset + * (opslimit=3, memlimit=256 MiB), the pairing named in the plan (KTD1). + */ +export const DEFAULT_KDF_PARAMS: KdfParams = { + opslimit: sodium.crypto_pwhash_OPSLIMIT_MODERATE, + memlimit: sodium.crypto_pwhash_MEMLIMIT_MODERATE, + alg: sodium.crypto_pwhash_ALG_ARGON2ID13, +}; + +/** The bundle's unencrypted crypto header — see module doc comment. */ +export interface CryptoHeader { + readonly version: number; + readonly salt: Buffer; + readonly kdfParams: KdfParams; + readonly secretstreamHeader: Buffer; +} + +/** Fixed byte length of a serialized {@link CryptoHeader} (see {@link writeHeader}). */ +export const HEADER_BYTE_LENGTH = + MAGIC.byteLength + // magic + 1 + // version (uint8) + SALT_BYTES + // salt + 4 + // opslimit (uint32 LE) + 8 + // memlimit (uint64 LE) + 1 + // alg (uint8) + SECRETSTREAM_HEADER_BYTES; // secretstream header + +/** Thrown when a header buffer is malformed, truncated, or has an unsupported version. */ +export class InvalidHeaderError extends Error { + constructor(message: string) { + super(message); + this.name = "InvalidHeaderError"; + } +} + +/** + * Thrown when authenticated decryption fails — a wrong key, a tampered + * byte, or a truncated stream. Never carries recovered plaintext. + */ +export class DecryptionAuthError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "DecryptionAuthError"; + } +} + +function toError(err: unknown): Error { + return err instanceof Error ? err : new Error(String(err)); +} + +/** Generates a fresh 16-byte random salt for {@link deriveKey}. */ +export function generateSalt(): Buffer { + const salt = Buffer.alloc(SALT_BYTES); + sodium.randombytes_buf(salt); + return salt; +} + +/** + * Derives a 32-byte secretstream key from `password` and `salt` via + * Argon2id (`crypto_pwhash`). Deterministic for the same password + salt + + * params; differs whenever any of those inputs differ. + */ +export function deriveKey( + password: string, + salt: Buffer, + params: KdfParams = DEFAULT_KDF_PARAMS, +): Buffer { + if (salt.byteLength !== SALT_BYTES) { + throw new RangeError(`salt must be ${SALT_BYTES} bytes`); + } + const passwordBytes = Buffer.from(password, "utf8"); + const key = Buffer.alloc(SECRETSTREAM_KEY_BYTES); + sodium.crypto_pwhash( + key, + passwordBytes, + salt, + params.opslimit, + params.memlimit, + params.alg, + ); + return key; +} + +/** Serializes a {@link CryptoHeader} to its fixed-length wire format. */ +export function writeHeader(header: CryptoHeader): Buffer { + if (header.salt.byteLength !== SALT_BYTES) { + throw new RangeError(`header salt must be ${SALT_BYTES} bytes`); + } + if (header.secretstreamHeader.byteLength !== SECRETSTREAM_HEADER_BYTES) { + throw new RangeError( + `secretstreamHeader must be ${SECRETSTREAM_HEADER_BYTES} bytes`, + ); + } + + const buf = Buffer.alloc(HEADER_BYTE_LENGTH); + let offset = 0; + + MAGIC.copy(buf, offset); + offset += MAGIC.byteLength; + + buf.writeUInt8(header.version, offset); + offset += 1; + + header.salt.copy(buf, offset); + offset += SALT_BYTES; + + buf.writeUInt32LE(header.kdfParams.opslimit, offset); + offset += 4; + + buf.writeBigUInt64LE(BigInt(header.kdfParams.memlimit), offset); + offset += 8; + + buf.writeUInt8(header.kdfParams.alg, offset); + offset += 1; + + header.secretstreamHeader.copy(buf, offset); + offset += SECRETSTREAM_HEADER_BYTES; + + return buf; +} + +/** + * Parses a {@link CryptoHeader} from the first {@link HEADER_BYTE_LENGTH} + * bytes of `buf`. Throws {@link InvalidHeaderError} on bad magic, an + * unsupported version, or a buffer shorter than the header. + */ +export function readHeader(buf: Buffer): CryptoHeader { + if (buf.byteLength < HEADER_BYTE_LENGTH) { + throw new InvalidHeaderError( + `buffer is too short to contain a crypto header: got ${buf.byteLength} bytes, need ${HEADER_BYTE_LENGTH}`, + ); + } + + let offset = 0; + + const magic = buf.subarray(offset, offset + MAGIC.byteLength); + offset += MAGIC.byteLength; + if (!magic.equals(MAGIC)) { + throw new InvalidHeaderError("bad magic bytes: not a MagStacker backup"); + } + + const version = buf.readUInt8(offset); + offset += 1; + if (version !== FORMAT_VERSION) { + throw new InvalidHeaderError( + `unsupported crypto header version: ${version}`, + ); + } + + const salt = Buffer.from(buf.subarray(offset, offset + SALT_BYTES)); + offset += SALT_BYTES; + + const opslimit = buf.readUInt32LE(offset); + offset += 4; + + const memlimit = Number(buf.readBigUInt64LE(offset)); + offset += 8; + + const alg = buf.readUInt8(offset); + offset += 1; + + const secretstreamHeader = Buffer.from( + buf.subarray(offset, offset + SECRETSTREAM_HEADER_BYTES), + ); + offset += SECRETSTREAM_HEADER_BYTES; + + return { + version, + salt, + kdfParams: { opslimit, memlimit, alg }, + secretstreamHeader, + }; +} + +/** + * Accumulates buffers and lets a caller pull off exact byte counts as they + * become available — the framing primitive both stream transforms below use + * to align arbitrary-sized writes to {@link CHUNK_SIZE}/ + * {@link CIPHERTEXT_CHUNK_SIZE} boundaries. + */ +class ByteAccumulator { + private chunks: Buffer[] = []; + private length = 0; + + push(chunk: Buffer): void { + if (chunk.length === 0) return; + this.chunks.push(chunk); + this.length += chunk.length; + } + + get size(): number { + return this.length; + } + + /** Removes and returns exactly `n` buffered bytes. Caller must check `size >= n` first. */ + take(n: number): Buffer { + const combined = + this.chunks.length === 1 + ? this.chunks[0] + : Buffer.concat(this.chunks, this.length); + const result = Buffer.from(combined.subarray(0, n)); + const rest = combined.subarray(n); + this.chunks = rest.length > 0 ? [Buffer.from(rest)] : []; + this.length = rest.length; + return result; + } + + /** Removes and returns all buffered bytes. */ + takeAll(): Buffer { + return this.take(this.length); + } +} + +/** + * Builds a `Transform` that encrypts a plaintext byte stream into a + * MagStacker backup bundle stream: the fixed-length {@link CryptoHeader} + * first, then `crypto_secretstream_xchacha20poly1305` ciphertext chunked at + * {@link CHUNK_SIZE} plaintext bytes per chunk, with the last chunk tagged + * `TAG_FINAL` (even for zero-byte input, so every encrypted stream ends in + * exactly one authenticated final chunk). + * + * `salt` and `kdfParams` are only needed to make the header self-describing + * — `key` must already be `deriveKey(password, salt, kdfParams)`. + */ +export function createEncryptStream( + key: Buffer, + salt: Buffer, + kdfParams: KdfParams = DEFAULT_KDF_PARAMS, +): Transform { + if (key.byteLength !== SECRETSTREAM_KEY_BYTES) { + throw new RangeError(`key must be ${SECRETSTREAM_KEY_BYTES} bytes`); + } + if (salt.byteLength !== SALT_BYTES) { + throw new RangeError(`salt must be ${SALT_BYTES} bytes`); + } + + const state = Buffer.alloc(SECRETSTREAM_STATE_BYTES); + const secretstreamHeader = Buffer.alloc(SECRETSTREAM_HEADER_BYTES); + sodium.crypto_secretstream_xchacha20poly1305_init_push( + state, + secretstreamHeader, + key, + ); + + const pending = new ByteAccumulator(); + let headerWritten = false; + + function encryptChunk( + stream: Transform, + plaintext: Buffer, + tag: number, + ): void { + const ciphertext = Buffer.alloc(plaintext.length + SECRETSTREAM_ABYTES); + sodium.crypto_secretstream_xchacha20poly1305_push( + state, + ciphertext, + plaintext, + null, + tag, + ); + stream.push(ciphertext); + } + + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + if (!headerWritten) { + this.push( + writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams, + secretstreamHeader, + }), + ); + headerWritten = true; + } + + pending.push(chunk); + while (pending.size >= CHUNK_SIZE) { + encryptChunk(this, pending.take(CHUNK_SIZE), TAG_MESSAGE); + } + callback(); + } catch (err) { + callback(toError(err)); + } + }, + flush(callback) { + try { + if (!headerWritten) { + this.push( + writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams, + secretstreamHeader, + }), + ); + headerWritten = true; + } + // Final chunk carries TAG_FINAL even when empty (zero-byte input), + // so every stream this function produces ends in one authenticated + // final chunk a decryptor can rely on (KTD10's authenticate-before- + // trust contract starts here). + encryptChunk(this, pending.takeAll(), TAG_FINAL); + callback(); + } catch (err) { + callback(toError(err)); + } + }, + }); +} + +/** + * Builds a `Transform` that decrypts a MagStacker backup bundle stream + * produced by {@link createEncryptStream}: it parses the leading + * {@link CryptoHeader} itself (the header is unencrypted preamble, so no key + * is needed to read it), then authenticates and decrypts each ciphertext + * chunk with `key`. + * + * Throws {@link InvalidHeaderError} for a malformed/truncated header and + * {@link DecryptionAuthError} for a wrong key, a tampered ciphertext byte + * (anywhere, including the final chunk), or a stream that ends before an + * authenticated final chunk — in every failure case, no unauthenticated + * plaintext is ever pushed downstream. + */ +export function createDecryptStream(key: Buffer): Transform { + if (key.byteLength !== SECRETSTREAM_KEY_BYTES) { + throw new RangeError(`key must be ${SECRETSTREAM_KEY_BYTES} bytes`); + } + + const state = Buffer.alloc(SECRETSTREAM_STATE_BYTES); + const headerBuf = new ByteAccumulator(); + const ciphertext = new ByteAccumulator(); + let initialized = false; + let finalized = false; + + function decryptChunk(stream: Transform, ct: Buffer): void { + if (ct.byteLength < SECRETSTREAM_ABYTES) { + throw new DecryptionAuthError( + "ciphertext chunk shorter than the authentication tag: truncated or corrupt bundle", + ); + } + const plaintext = Buffer.alloc(ct.byteLength - SECRETSTREAM_ABYTES); + const tag = Buffer.alloc(1); + try { + sodium.crypto_secretstream_xchacha20poly1305_pull( + state, + plaintext, + tag, + ct, + null, + ); + } catch (err) { + throw new DecryptionAuthError( + "authenticated decryption failed: wrong password/key or a tampered/corrupt bundle", + { cause: err }, + ); + } + stream.push(plaintext); + if (tag[0] === TAG_FINAL) { + finalized = true; + } + } + + function drainFullChunks(stream: Transform): void { + while (!finalized && ciphertext.size >= CIPHERTEXT_CHUNK_SIZE) { + decryptChunk(stream, ciphertext.take(CIPHERTEXT_CHUNK_SIZE)); + } + } + + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + if (finalized) { + throw new DecryptionAuthError( + "received data after the final authenticated chunk: trailing/corrupt bytes", + ); + } + + if (!initialized) { + headerBuf.push(chunk); + if (headerBuf.size < HEADER_BYTE_LENGTH) { + callback(); + return; + } + const header = readHeader(headerBuf.take(HEADER_BYTE_LENGTH)); + try { + sodium.crypto_secretstream_xchacha20poly1305_init_pull( + state, + header.secretstreamHeader, + key, + ); + } catch (err) { + throw new DecryptionAuthError( + "failed to initialize decryption from the bundle header", + { cause: err }, + ); + } + initialized = true; + ciphertext.push(headerBuf.takeAll()); + } else { + ciphertext.push(chunk); + } + + drainFullChunks(this); + callback(); + } catch (err) { + callback(toError(err)); + } + }, + flush(callback) { + try { + if (!initialized) { + throw new InvalidHeaderError( + "bundle ended before a complete crypto header was read", + ); + } + if (finalized) { + callback(); + return; + } + if (ciphertext.size === 0) { + throw new DecryptionAuthError( + "bundle ended before its final authenticated chunk: truncated bundle", + ); + } + decryptChunk(this, ciphertext.takeAll()); + if (!finalized) { + throw new DecryptionAuthError( + "bundle ended without a final authenticated chunk: truncated bundle", + ); + } + callback(); + } catch (err) { + callback(toError(err)); + } + }, + }); +} diff --git a/src/backup/sodium-native.d.ts b/src/backup/sodium-native.d.ts new file mode 100644 index 00000000..8ae32009 --- /dev/null +++ b/src/backup/sodium-native.d.ts @@ -0,0 +1,62 @@ +/** + * Minimal ambient types for `sodium-native`, scoped to exactly the API + * surface `crypto.ts` (U1) uses. + * + * `sodium-native` ships no types of its own, and the only published + * `@types/sodium-native` (2.3.9, DefinitelyTyped) predates the installed + * v5.x line by several major versions — pinning to it risks a silent + * mismatch against the actual runtime API. A small hand-written surface for + * the handful of low-level libsodium primitives used here (which have been + * stable across sodium-native's major versions) is safer and easier to + * audit than trusting a stale third-party `.d.ts`. + */ +declare module "sodium-native" { + // Argon2id key derivation (crypto_pwhash). + export const crypto_pwhash_SALTBYTES: number; + export const crypto_pwhash_ALG_ARGON2ID13: number; + export const crypto_pwhash_OPSLIMIT_MODERATE: number; + export const crypto_pwhash_MEMLIMIT_MODERATE: number; + export function crypto_pwhash( + out: Buffer, + passwd: Buffer, + salt: Buffer, + opslimit: number, + memlimit: number, + alg: number, + ): void; + + // Randomness. + export function randombytes_buf(buffer: Buffer): void; + + // Authenticated streaming encryption (crypto_secretstream_xchacha20poly1305). + export const crypto_secretstream_xchacha20poly1305_HEADERBYTES: number; + export const crypto_secretstream_xchacha20poly1305_ABYTES: number; + export const crypto_secretstream_xchacha20poly1305_KEYBYTES: number; + export const crypto_secretstream_xchacha20poly1305_STATEBYTES: number; + export const crypto_secretstream_xchacha20poly1305_TAG_MESSAGE: number; + export const crypto_secretstream_xchacha20poly1305_TAG_FINAL: number; + export function crypto_secretstream_xchacha20poly1305_init_push( + state: Buffer, + header: Buffer, + key: Buffer, + ): void; + export function crypto_secretstream_xchacha20poly1305_init_pull( + state: Buffer, + header: Buffer, + key: Buffer, + ): void; + export function crypto_secretstream_xchacha20poly1305_push( + state: Buffer, + ciphertext: Buffer, + message: Buffer, + additionalData: Buffer | null, + tag: number, + ): number; + export function crypto_secretstream_xchacha20poly1305_pull( + state: Buffer, + message: Buffer, + tag: Buffer, + ciphertext: Buffer, + additionalData: Buffer | null, + ): number; +} From a86babdc7bea5580295f17fe665ddec63cae1872 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:22:47 -0400 Subject: [PATCH 04/26] feat(backup): NDJSON DB export/import + operator_audit table (U3) Add the DB export/import primitives for the encryption-at-rest backup plan: exportDatabase()/importDatabase() stream the full database as NDJSON in FK-safe order, wipeDatabase() clears it in reverse order for a future force-replace restore, and table-order.ts enumerates every persistent table (ephemeral session/rate_limit/idempotency excluded) so new tables can't silently drop out of a backup. Also adds the operator_audit table (id, actor, action, outcome, at) to record backup export/restore events, wired into the schema barrel with its migration. Verified with a Testcontainers-backed round-trip integration test: full seed -> export -> wipe -> import reproduces every table's rows exactly, FK order lets firearm_document import right after firearm, ephemeral tables are excluded, and a regression guard compares table-order.ts against the live information_schema. Signed-off-by: UncleSp1d3r --- src/backup/__tests__/db-roundtrip.test.ts | 378 ++++ src/backup/db-export.ts | 52 + src/backup/db-import.ts | 127 ++ src/backup/table-order.ts | 81 + .../migrations/0018_spotty_blonde_phantom.sql | 10 + src/db/migrations/meta/0018_snapshot.json | 1928 +++++++++++++++++ src/db/migrations/meta/_journal.json | 7 + src/db/operator-audit-schema.ts | 40 + src/db/schema.ts | 3 + 9 files changed, 2626 insertions(+) create mode 100644 src/backup/__tests__/db-roundtrip.test.ts create mode 100644 src/backup/db-export.ts create mode 100644 src/backup/db-import.ts create mode 100644 src/backup/table-order.ts create mode 100644 src/db/migrations/0018_spotty_blonde_phantom.sql create mode 100644 src/db/migrations/meta/0018_snapshot.json create mode 100644 src/db/operator-audit-schema.ts diff --git a/src/backup/__tests__/db-roundtrip.test.ts b/src/backup/__tests__/db-roundtrip.test.ts new file mode 100644 index 00000000..fc4f69e3 --- /dev/null +++ b/src/backup/__tests__/db-roundtrip.test.ts @@ -0,0 +1,378 @@ +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { Readable } from "node:stream"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { getTableName, sql } from "drizzle-orm"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import * as schema from "../../db/schema"; +import { + accessory, + account, + ammo, + firearm, + firearmDocument, + firearmPhoto, + grant, + inventoryLog, + magazine, + magazineFirearm, + magazineLabelPrefix, + operatorAudit, + rangeSession, + rangeSessionAccessory, + rateLimit, + session, + user, + verification, +} from "../../db/schema"; +import { type ExportedRow, exportDatabase } from "../db-export"; +import { importDatabase, wipeDatabase } from "../db-import"; +import { EPHEMERAL_TABLE_NAMES, EXPORT_TABLE_ORDER } from "../table-order"; + +/** + * Round-trip integration test for the NDJSON export/import pair (U3). Runs + * against an ephemeral Testcontainers Postgres — NOT the ambient dev + * database — because these tests wipe every table, which would be + * destructive against a shared/ambient DB. + * + * Same pinned image as `e2e/start-test-server.ts` (AWS ECR Public mirror — + * avoids Docker Hub's unauthenticated per-IP pull limit on shared runners). + */ +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +type Db = NodePgDatabase; + +interface SeededInventory { + ownerId: string; + granteeId: string; + firearmId: string; + documentId: string; +} + +/** + * Seed one of every non-ephemeral row type the export must cover, plus a + * `session` and `rate_limit` row (ephemeral — must NOT survive into the + * export). Mirrors `src/test-support/factories.ts`'s shapes but is + * parameterized on a locally-constructed `db` bound to the test container, + * rather than the process-wide singleton in `src/db/client.ts`. + */ +async function seedFullInventory(db: Db): Promise { + const ownerId = `owner-${randomUUID()}`; + const granteeId = `grantee-${randomUUID()}`; + + await db.insert(user).values([ + { id: ownerId, name: "Owner", email: `${ownerId}@example.test` }, + { id: granteeId, name: "Grantee", email: `${granteeId}@example.test` }, + ]); + + await db.insert(verification).values({ + id: randomUUID(), + identifier: `verify-${ownerId}`, + value: "token", + expiresAt: new Date(Date.now() + 3_600_000), + }); + + await db.insert(account).values({ + id: randomUUID(), + accountId: ownerId, + providerId: "credential", + userId: ownerId, + password: "hashed", + }); + + const [firearmRow] = await db + .insert(firearm) + .values({ ownerId, name: "Round Trip FA", caliber: "9mm", isNfa: true }) + .returning(); + + const [magazineRow] = await db + .insert(magazine) + .values({ + ownerId, + brandModel: "Round Trip MG", + caliber: "9mm", + baseCapacity: 17, + }) + .returning(); + + await db + .insert(ammo) + .values({ ownerId, caliber: "9mm", brand: "Round Trip Ammo Co" }); + + const [accessoryRow] = await db + .insert(accessory) + .values({ ownerId, category: "optic", currentFirearmId: firearmRow.id }) + .returning(); + + await db.insert(magazineLabelPrefix).values({ ownerId, prefix: "RT" }); + + await db.insert(magazineFirearm).values({ + magazineId: magazineRow.id, + firearmId: firearmRow.id, + ordinal: 0, + }); + + const [rangeSessionRow] = await db + .insert(rangeSession) + .values({ firearmId: firearmRow.id, date: "2026-01-01", roundsFired: 50 }) + .returning(); + + await db.insert(rangeSessionAccessory).values({ + rangeSessionId: rangeSessionRow.id, + accessoryId: accessoryRow.id, + }); + + await db.insert(firearmPhoto).values({ + firearmId: firearmRow.id, + storageKey: `${randomUUID()}.jpg`, + mimeType: "image/jpeg", + sizeBytes: 1024, + width: 800, + height: 600, + sortOrder: 0, + }); + + const [documentRow] = await db + .insert(firearmDocument) + .values({ + firearmId: firearmRow.id, + storageKey: `${randomUUID()}.pdf`, + filename: "receipt.pdf", + mimeType: "application/pdf", + sizeBytes: 2048, + docType: "receipt", + }) + .returning(); + + await db.insert(grant).values({ + ownerId, + granteeId, + parentType: "firearm", + parentId: firearmRow.id, + permission: "view", + }); + + await db.insert(inventoryLog).values({ + parentType: "firearm", + parentId: firearmRow.id, + eventType: "inventoried", + actorId: ownerId, + }); + + await db.insert(operatorAudit).values({ + actor: ownerId, + action: "export", + outcome: "success", + }); + + // Ephemeral rows — must never appear in an export. + await db.insert(session).values({ + id: randomUUID(), + expiresAt: new Date(Date.now() + 3_600_000), + token: randomUUID(), + userId: ownerId, + }); + await db.insert(rateLimit).values({ + id: randomUUID(), + key: `rl-${randomUUID()}`, + count: 1, + lastRequest: Date.now(), + }); + + return { + ownerId, + granteeId, + firearmId: firearmRow.id, + documentId: documentRow.id, + }; +} + +async function streamToString(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function parseNdjson(text: string): ExportedRow[] { + return text + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as ExportedRow); +} + +/** Snapshot every exported table's rows, order-independent (sorted by content). */ +async function snapshotTables( + db: Db, +): Promise[]>> { + const snapshot: Record[]> = {}; + for (const table of EXPORT_TABLE_ORDER) { + const rows = await selectAll(db, table); + snapshot[getTableName(table)] = rows.sort((a, b) => + JSON.stringify(a).localeCompare(JSON.stringify(b)), + ); + } + return snapshot; +} + +/** + * `EXPORT_TABLE_ORDER` is deliberately heterogeneous (every persistent + * table), so this is the one place `.from()` loses its concrete row type. + */ +async function selectAll( + db: Db, + table: (typeof EXPORT_TABLE_ORDER)[number], +): Promise[]> { + // biome-ignore lint/suspicious/noExplicitAny: see function doc comment. + const rows = await db.select().from(table as any); + return rows as Record[]; +} + +describe("DB export/import round trip (U3)", () => { + let container: StartedPostgreSqlContainer; + let pool: Pool; + let db: Db; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_backup_test") + .start(); + pool = new Pool({ connectionString: container.getConnectionUri() }); + db = drizzle(pool, { schema }); + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + }, 120_000); + + afterAll(async () => { + await pool?.end(); + await container?.stop(); + }); + + // Every test starts from a clean slate regardless of execution order — + // wipeDatabase is exercised here as much as it is by the tests themselves. + beforeEach(async () => { + await wipeDatabase(db); + }); + + test("full seed round-trips through export -> wipe -> import with every table matching exactly (AE5/R10)", async () => { + await seedFullInventory(db); + const before = await snapshotTables(db); + + const exportText = await streamToString(exportDatabase(db)); + await wipeDatabase(db); + + // Confirm the wipe actually cleared every exported table before import + // "fixes" it back up — otherwise a no-op import could pass by accident. + const afterWipe = await snapshotTables(db); + for (const rows of Object.values(afterWipe)) { + expect(rows).toHaveLength(0); + } + + await importDatabase(db, Readable.from([exportText])); + const after = await snapshotTables(db); + + expect(after).toEqual(before); + }); + + test("FK-safe order lets firearm_document import immediately after its firearm with no constraint violation (R2 ordering)", async () => { + const seeded = await seedFullInventory(db); + const exportText = await streamToString(exportDatabase(db)); + await wipeDatabase(db); + + // If table-order.ts ever put firearm_document before firearm, this + // insert would throw a foreign-key violation and the transaction (and + // this test) would fail. + await importDatabase(db, Readable.from([exportText])); + + const [document] = await db + .select() + .from(firearmDocument) + .where(sql`${firearmDocument.id} = ${seeded.documentId}`); + expect(document).toBeDefined(); + expect(document.firearmId).toBe(seeded.firearmId); + }); + + test("ephemeral session, rate_limit, and idempotency rows are excluded from the export", async () => { + await seedFullInventory(db); + const exportText = await streamToString(exportDatabase(db)); + const rows = parseNdjson(exportText); + const exportedTableNames = new Set(rows.map((r) => r.table)); + + for (const ephemeralTable of EPHEMERAL_TABLE_NAMES) { + expect(exportedTableNames.has(ephemeralTable)).toBe(false); + } + // Sanity check the export isn't just empty — it does carry real rows. + expect(rows.length).toBeGreaterThan(0); + }); + + test("table-order.ts's export order plus the ephemeral exclusion set covers exactly the live schema's tables (regression guard)", async () => { + const liveTables = await db.execute<{ table_name: string }>( + sql`select table_name from information_schema.tables where table_schema = 'public' and table_type = 'BASE TABLE'`, + ); + const liveTableNames = new Set(liveTables.rows.map((r) => r.table_name)); + + const declaredTableNames = new Set([ + ...EXPORT_TABLE_ORDER.map((t) => getTableName(t)), + ...EPHEMERAL_TABLE_NAMES, + ]); + + const missingFromTableOrder = [...liveTableNames].filter( + (name) => !declaredTableNames.has(name), + ); + const staleInTableOrder = [...declaredTableNames].filter( + (name) => !liveTableNames.has(name), + ); + + expect(missingFromTableOrder).toEqual([]); + expect(staleInTableOrder).toEqual([]); + }); + + test("timestamptz, uuid, boolean, and integer columns round-trip exactly, not just structurally", async () => { + const seeded = await seedFullInventory(db); + const [beforeFirearm] = await db + .select() + .from(firearm) + .where(sql`${firearm.id} = ${seeded.firearmId}`); + const [beforeMagazine] = await db.select().from(magazine); + + const exportText = await streamToString(exportDatabase(db)); + await wipeDatabase(db); + await importDatabase(db, Readable.from([exportText])); + + const [afterFirearm] = await db + .select() + .from(firearm) + .where(sql`${firearm.id} = ${seeded.firearmId}`); + const [afterMagazine] = await db.select().from(magazine); + + // uuid (primary key) — same string, not merely equal-looking. + expect(afterFirearm.id).toBe(beforeFirearm.id); + // boolean. + expect(afterFirearm.isNfa).toBe(true); + expect(beforeFirearm.isNfa).toBe(true); + // timestamptz — reconstructed as a real Date with the same instant. + expect(afterFirearm.createdAt).toBeInstanceOf(Date); + expect(afterFirearm.createdAt.getTime()).toBe( + beforeFirearm.createdAt.getTime(), + ); + expect(afterFirearm.updatedAt.getTime()).toBe( + beforeFirearm.updatedAt.getTime(), + ); + // integer. + expect(afterMagazine.baseCapacity).toBe(beforeMagazine.baseCapacity); + expect(afterMagazine.baseCapacity).toBe(17); + }); +}); diff --git a/src/backup/db-export.ts b/src/backup/db-export.ts new file mode 100644 index 00000000..fa82dffd --- /dev/null +++ b/src/backup/db-export.ts @@ -0,0 +1,52 @@ +import { Readable } from "node:stream"; +import { getTableName } from "drizzle-orm"; +import type { PgTable } from "drizzle-orm/pg-core"; +import type { DbOrTx } from "../db/client"; +import { EXPORT_TABLE_ORDER } from "./table-order"; + +/** One NDJSON line: a table name plus the raw row it names. */ +export interface ExportedRow { + table: string; + row: Record; +} + +/** + * Export the full database as NDJSON (U3, R2/R13) — one JSON line per row, + * each tagged with its table name, in FK-safe insert order + * (`EXPORT_TABLE_ORDER`). Ephemeral tables (session, rate-limit, idempotency) + * are never touched — they simply aren't in that list. + * + * Returned as a Node `Readable` so a caller can pipe it straight into an + * encryption/archive stage without buffering the whole export in memory: each + * table's rows are fetched and yielded table-by-table (not held alongside + * every other table's rows at once), and the generator only pulls the next + * table once the consumer has drained the current one. + */ +export function exportDatabase(db: DbOrTx): Readable { + async function* generate(): AsyncGenerator { + for (const table of EXPORT_TABLE_ORDER) { + const tableName = getTableName(table); + const rows = await selectAllRows(db, table); + for (const row of rows) { + yield `${JSON.stringify({ table: tableName, row } satisfies ExportedRow)}\n`; + } + } + } + + return Readable.from(generate()); +} + +/** + * `EXPORT_TABLE_ORDER` is deliberately heterogeneous (every persistent + * table), so this is the one place `.from()` loses its concrete row type — + * every row coming out of it is an opaque bag of columns anyway, which is all + * `exportDatabase` needs. + */ +async function selectAllRows( + db: DbOrTx, + table: PgTable, +): Promise[]> { + // biome-ignore lint/suspicious/noExplicitAny: see function doc comment. + const rows = await db.select().from(table as any); + return rows as Record[]; +} diff --git a/src/backup/db-import.ts b/src/backup/db-import.ts new file mode 100644 index 00000000..53d8ff33 --- /dev/null +++ b/src/backup/db-import.ts @@ -0,0 +1,127 @@ +import { createInterface } from "node:readline"; +import type { Readable } from "node:stream"; +import { getTableColumns, getTableName } from "drizzle-orm"; +import type { PgTable } from "drizzle-orm/pg-core"; +import type { Database, Transaction } from "../db/client"; +import type { ExportedRow } from "./db-export"; +import { EXPORT_TABLE_ORDER, WIPE_TABLE_ORDER } from "./table-order"; + +const TABLES_BY_NAME: ReadonlyMap = new Map( + EXPORT_TABLE_ORDER.map((table) => [getTableName(table), table]), +); + +/** + * JS property keys (not SQL column names) on `table` whose values are + * date/timestamp columns. Cached per table by `importDatabase` since the same + * table appears across many NDJSON lines. + */ +function dateColumnKeys(table: PgTable): readonly string[] { + const columns = getTableColumns(table); + return Object.entries(columns) + .filter(([, column]) => column.dataType === "date") + .map(([key]) => key); +} + +/** + * Revive a row parsed back out of NDJSON: `JSON.stringify` turns every `Date` + * into an ISO string, but Drizzle's date/timestamp columns call + * `value.toISOString()` on write (`mapToDriverValue`) — handing them a plain + * string throws. Returns a new object; never mutates `row` (immutability). + */ +function reviveRow( + row: Record, + dateKeys: readonly string[], +): Record { + if (dateKeys.length === 0) return row; + const revived: Record = { ...row }; + for (const key of dateKeys) { + const value = revived[key]; + if (typeof value === "string") { + revived[key] = new Date(value); + } + } + return revived; +} + +/** + * Import an NDJSON export produced by `exportDatabase` (U3, R5/R10). + * + * Reads the stream line by line (never buffering the whole file) and inserts + * each row inside ONE transaction, so a failure partway through — a parse + * error, an unknown table, a constraint violation — rolls back every row + * already inserted rather than leaving a half-restored database. Insert order + * is whatever order the file already carries its rows in: `exportDatabase` + * always writes tables in `EXPORT_TABLE_ORDER` (FK-safe), so import trusts + * that order instead of re-sorting or buffering the file to re-derive it. + * + * Refuse-unless-empty, force-replace, and version-compatibility checks are + * the caller's job (a later restore-flow unit) — this function is the raw + * insert primitive only. + */ +export async function importDatabase( + db: Database, + stream: Readable, +): Promise { + await db.transaction(async (tx) => { + const dateKeyCache = new Map(); + const lines = createInterface({ + input: stream, + crlfDelay: Number.POSITIVE_INFINITY, + }); + + for await (const line of lines) { + if (line.trim() === "") continue; + + const parsed = JSON.parse(line) as ExportedRow; + const table = TABLES_BY_NAME.get(parsed.table); + if (!table) { + throw new Error( + `Backup references unknown table "${parsed.table}" — it isn't in EXPORT_TABLE_ORDER (src/backup/table-order.ts). Refusing to import a row this build doesn't know how to place.`, + ); + } + + let dateKeys = dateKeyCache.get(parsed.table); + if (!dateKeys) { + dateKeys = dateColumnKeys(table); + dateKeyCache.set(parsed.table, dateKeys); + } + + await insertRow(tx, table, reviveRow(parsed.row, dateKeys)); + } + }); +} + +/** + * `TABLES_BY_NAME` is deliberately heterogeneous (every persistent table), + * and the row shape is only known at runtime — from the NDJSON line itself — + * so this is the one place `.insert()` loses its concrete row type. + */ +async function insertRow( + tx: Transaction, + table: PgTable, + row: Record, +): Promise { + // biome-ignore lint/suspicious/noExplicitAny: see function doc comment. + await tx.insert(table as any).values(row); +} + +/** + * Delete every row from every persistent table, in FK-safe reverse-insert + * order (`WIPE_TABLE_ORDER`), inside one transaction. This is the primitive a + * force-replace restore uses to clear existing data before applying a bundle + * (R7) — it does not itself implement the refuse-unless-empty guard or the + * snapshot/rollback safety net around it; those are the caller's job. + */ +export async function wipeDatabase(db: Database): Promise { + await db.transaction(async (tx) => { + for (const table of WIPE_TABLE_ORDER) { + await deleteAllRows(tx, table); + } + }); +} + +/** See `insertRow`'s doc comment — same reason, for `.delete()`. */ +async function deleteAllRows(tx: Transaction, table: PgTable): Promise { + // biome-ignore lint/suspicious/noExplicitAny: see insertRow's doc comment. + await tx.delete(table as any); +} diff --git a/src/backup/table-order.ts b/src/backup/table-order.ts new file mode 100644 index 00000000..9c3db63a --- /dev/null +++ b/src/backup/table-order.ts @@ -0,0 +1,81 @@ +import type { PgTable } from "drizzle-orm/pg-core"; +import { account, user, verification } from "../db/auth-schema"; +import { + accessory, + ammo, + firearm, + firearmDocument, + firearmPhoto, + grant, + inventoryLog, + magazine, + magazineFirearm, + magazineLabelPrefix, + rangeSession, + rangeSessionAccessory, +} from "../db/inventory-schema"; +import { operatorAudit } from "../db/operator-audit-schema"; + +/** + * FK-safe insert order for EVERY persistent table in the database (U3). + * + * New tables MUST be added here or they are silently dropped from backups. + * + * Order rationale: + * - `user` is the root parent everything else (directly or transitively) + * depends on; `verification` and `account` are the remaining auth tables + * that persist across restarts (`account` FKs to `user`). + * - The four owned inventory parents (`firearm`, `magazine`, `ammo`, + * `accessory`) FK to `user`. `accessory` additionally holds a nullable FK to + * `firearm` (its optional current mount), so it must come after `firearm`. + * - `magazine_label_prefix` FKs to `user` only. + * - The remaining tables are children of the parents above and are ordered so + * every FK target has already been inserted: `magazine_firearm` (magazine + + * firearm), `range_session` (firearm), `range_session_accessory` + * (range_session + accessory), `firearm_photo` (firearm), `firearm_document` + * (firearm). + * - `grant` and `inventory_log` FK to `user` only (their polymorphic + * `parent_id` carries no FK — see the schema doc comments) so they can sit + * anywhere after `user`; placed last among inventory tables since they + * logically depend on the items they reference existing first. + * - `operator_audit` has no FK to anything and is appended last. + */ +export const EXPORT_TABLE_ORDER: readonly PgTable[] = [ + user, + verification, + account, + firearm, + magazine, + ammo, + accessory, + magazineLabelPrefix, + magazineFirearm, + rangeSession, + rangeSessionAccessory, + firearmPhoto, + firearmDocument, + grant, + inventoryLog, + operatorAudit, +]; + +/** + * Insert order in reverse — the FK-safe order to WIPE tables before a + * force-replace restore (children before the parents they reference). + */ +export const WIPE_TABLE_ORDER: readonly PgTable[] = [ + ...EXPORT_TABLE_ORDER, +].reverse(); + +/** + * Ephemeral tables intentionally excluded from every backup: session state, + * the DB-stored rate-limit counters, and the short-lived idempotency dedup + * store. None of these belong in a restored instance — restoring them would + * either resurrect stale sessions/rate-limit windows or reintroduce + * already-expired dedup rows. + */ +export const EPHEMERAL_TABLE_NAMES: readonly string[] = [ + "session", + "rate_limit", + "idempotency", +]; diff --git a/src/db/migrations/0018_spotty_blonde_phantom.sql b/src/db/migrations/0018_spotty_blonde_phantom.sql new file mode 100644 index 00000000..f55fb01f --- /dev/null +++ b/src/db/migrations/0018_spotty_blonde_phantom.sql @@ -0,0 +1,10 @@ +CREATE TABLE "operator_audit" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "actor" text NOT NULL, + "action" text NOT NULL, + "outcome" text NOT NULL, + "at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "operator_audit_action_valid" CHECK ("operator_audit"."action" in ('export', 'restore')) +); +--> statement-breakpoint +CREATE INDEX "operator_audit_at_idx" ON "operator_audit" USING btree ("at"); diff --git a/src/db/migrations/meta/0018_snapshot.json b/src/db/migrations/meta/0018_snapshot.json new file mode 100644 index 00000000..70bdabd9 --- /dev/null +++ b/src/db/migrations/meta/0018_snapshot.json @@ -0,0 +1,1928 @@ +{ + "id": "2193bdc4-6bb6-4152-88b8-279f14d7875b", + "prevId": "ddce0ee2-3865-41e5-8121-f739ddde54ed", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit": { + "name": "rate_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "last_request": { + "name": "last_request", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "rate_limit_key_unique": { + "name": "rate_limit_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "magpul_mode": { + "name": "magpul_mode", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.accessory": { + "name": "accessory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_firearm_id": { + "name": "current_firearm_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "brand": { + "name": "brand", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "serial_number": { + "name": "serial_number", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "installed_date": { + "name": "installed_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "cost_cents": { + "name": "cost_cents", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "is_nfa": { + "name": "is_nfa", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accessory_owner_id_idx": { + "name": "accessory_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "accessory_current_firearm_id_idx": { + "name": "accessory_current_firearm_id_idx", + "columns": [ + { + "expression": "current_firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accessory_owner_id_user_id_fk": { + "name": "accessory_owner_id_user_id_fk", + "tableFrom": "accessory", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "accessory_current_firearm_id_firearm_id_fk": { + "name": "accessory_current_firearm_id_firearm_id_fk", + "tableFrom": "accessory", + "tableTo": "firearm", + "columnsFrom": ["current_firearm_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "accessory_cost_cents_min": { + "name": "accessory_cost_cents_min", + "value": "\"accessory\".\"cost_cents\" >= 0" + }, + "accessory_installed_date_requires_mount": { + "name": "accessory_installed_date_requires_mount", + "value": "\"accessory\".\"installed_date\" IS NULL OR \"accessory\".\"current_firearm_id\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.ammo": { + "name": "ammo", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "brand": { + "name": "brand", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "caliber": { + "name": "caliber", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "grain": { + "name": "grain", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "quantity_rounds": { + "name": "quantity_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "low_stock_threshold": { + "name": "low_stock_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "acquired_date": { + "name": "acquired_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ammo_owner_id_idx": { + "name": "ammo_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "ammo_owner_id_user_id_fk": { + "name": "ammo_owner_id_user_id_fk", + "tableFrom": "ammo", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ammo_grain_min": { + "name": "ammo_grain_min", + "value": "\"ammo\".\"grain\" >= 0" + }, + "ammo_quantity_min": { + "name": "ammo_quantity_min", + "value": "\"ammo\".\"quantity_rounds\" >= 0" + }, + "ammo_threshold_min": { + "name": "ammo_threshold_min", + "value": "\"ammo\".\"low_stock_threshold\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.firearm": { + "name": "firearm", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "nickname": { + "name": "nickname", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "manufacturer": { + "name": "manufacturer", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "caliber": { + "name": "caliber", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unspecified'" + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unspecified'" + }, + "subtype": { + "name": "subtype", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "serial_number": { + "name": "serial_number", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "is_nfa": { + "name": "is_nfa", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "firearm_owner_id_idx": { + "name": "firearm_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "firearm_owner_id_user_id_fk": { + "name": "firearm_owner_id_user_id_fk", + "tableFrom": "firearm", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "firearm_type_valid": { + "name": "firearm_type_valid", + "value": "\"firearm\".\"type\" in ('pistol', 'revolver', 'rifle', 'shotgun', 'pcc', 'other', 'unspecified')" + }, + "firearm_action_valid": { + "name": "firearm_action_valid", + "value": "\"firearm\".\"action\" in ('semi-auto', 'bolt', 'lever', 'pump', 'break', 'single-shot', 'unspecified')" + } + }, + "isRLSEnabled": false + }, + "public.firearm_document": { + "name": "firearm_document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "firearm_id": { + "name": "firearm_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "doc_type": { + "name": "doc_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'other'" + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "firearm_document_firearm_id_idx": { + "name": "firearm_document_firearm_id_idx", + "columns": [ + { + "expression": "firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "firearm_document_firearm_id_firearm_id_fk": { + "name": "firearm_document_firearm_id_firearm_id_fk", + "tableFrom": "firearm_document", + "tableTo": "firearm", + "columnsFrom": ["firearm_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "firearm_document_mime_type_valid": { + "name": "firearm_document_mime_type_valid", + "value": "\"firearm_document\".\"mime_type\" in ('application/pdf', 'image/jpeg', 'image/png', 'image/webp', 'image/avif')" + }, + "firearm_document_doc_type_valid": { + "name": "firearm_document_doc_type_valid", + "value": "\"firearm_document\".\"doc_type\" in ('receipt', 'warranty', 'atf-form-1', 'atf-form-4', 'manual', 'insurance', 'other')" + }, + "firearm_document_size_bytes_min": { + "name": "firearm_document_size_bytes_min", + "value": "\"firearm_document\".\"size_bytes\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.firearm_photo": { + "name": "firearm_photo", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "firearm_id": { + "name": "firearm_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_primary": { + "name": "is_primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "firearm_photo_firearm_id_idx": { + "name": "firearm_photo_firearm_id_idx", + "columns": [ + { + "expression": "firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "firearm_photo_one_primary_per_firearm": { + "name": "firearm_photo_one_primary_per_firearm", + "columns": [ + { + "expression": "firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"firearm_photo\".\"is_primary\"", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "firearm_photo_firearm_id_firearm_id_fk": { + "name": "firearm_photo_firearm_id_firearm_id_fk", + "tableFrom": "firearm_photo", + "tableTo": "firearm", + "columnsFrom": ["firearm_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "firearm_photo_sort_order_min": { + "name": "firearm_photo_sort_order_min", + "value": "\"firearm_photo\".\"sort_order\" >= 0" + }, + "firearm_photo_mime_type_valid": { + "name": "firearm_photo_mime_type_valid", + "value": "\"firearm_photo\".\"mime_type\" in ('image/jpeg', 'image/png', 'image/webp', 'image/avif')" + }, + "firearm_photo_size_bytes_min": { + "name": "firearm_photo_size_bytes_min", + "value": "\"firearm_photo\".\"size_bytes\" > 0" + }, + "firearm_photo_width_min": { + "name": "firearm_photo_width_min", + "value": "\"firearm_photo\".\"width\" > 0" + }, + "firearm_photo_height_min": { + "name": "firearm_photo_height_min", + "value": "\"firearm_photo\".\"height\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.grant": { + "name": "grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grantee_id": { + "name": "grantee_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_type": { + "name": "parent_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "allow_create_on_behalf": { + "name": "allow_create_on_behalf", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grant_grantee_parent_type_idx": { + "name": "grant_grantee_parent_type_idx", + "columns": [ + { + "expression": "grantee_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grant_owner_id_user_id_fk": { + "name": "grant_owner_id_user_id_fk", + "tableFrom": "grant", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_grantee_id_user_id_fk": { + "name": "grant_grantee_id_user_id_fk", + "tableFrom": "grant", + "tableTo": "user", + "columnsFrom": ["grantee_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "grant_grantee_parent_unique": { + "name": "grant_grantee_parent_unique", + "nullsNotDistinct": false, + "columns": ["grantee_id", "parent_type", "parent_id"] + } + }, + "policies": {}, + "checkConstraints": { + "grant_parent_type_valid": { + "name": "grant_parent_type_valid", + "value": "\"grant\".\"parent_type\" in ('firearm', 'magazine', 'ammo')" + }, + "grant_permission_valid": { + "name": "grant_permission_valid", + "value": "\"grant\".\"permission\" in ('view', 'edit')" + } + }, + "isRLSEnabled": false + }, + "public.idempotency": { + "name": "idempotency", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_expires_at_idx": { + "name": "idempotency_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "idempotency_user_id_user_id_fk": { + "name": "idempotency_user_id_user_id_fk", + "tableFrom": "idempotency", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "idempotency_user_id_idempotency_key_pk": { + "name": "idempotency_user_id_idempotency_key_pk", + "columns": ["user_id", "idempotency_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inventory_log": { + "name": "inventory_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "parent_type": { + "name": "parent_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inventory_log_parent_idx": { + "name": "inventory_log_parent_idx", + "columns": [ + { + "expression": "parent_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inventory_log_actor_id_user_id_fk": { + "name": "inventory_log_actor_id_user_id_fk", + "tableFrom": "inventory_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "inventory_log_parent_type_valid": { + "name": "inventory_log_parent_type_valid", + "value": "\"inventory_log\".\"parent_type\" in ('firearm', 'magazine')" + }, + "inventory_log_event_type_valid": { + "name": "inventory_log_event_type_valid", + "value": "(\"inventory_log\".\"parent_type\" = 'firearm' AND \"inventory_log\".\"event_type\" in ('inventoried', 'cleaned', 'lubed')) OR (\"inventory_log\".\"parent_type\" = 'magazine' AND \"inventory_log\".\"event_type\" in ('inventoried'))" + } + }, + "isRLSEnabled": false + }, + "public.magazine": { + "name": "magazine", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "brand_model": { + "name": "brand_model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "caliber": { + "name": "caliber", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_capacity": { + "name": "base_capacity", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "extension_rounds": { + "name": "extension_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "acquired_date": { + "name": "acquired_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "magazine_owner_id_idx": { + "name": "magazine_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "magazine_owner_id_user_id_fk": { + "name": "magazine_owner_id_user_id_fk", + "tableFrom": "magazine", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "magazine_base_capacity_min": { + "name": "magazine_base_capacity_min", + "value": "\"magazine\".\"base_capacity\" >= 1" + }, + "magazine_extension_rounds_min": { + "name": "magazine_extension_rounds_min", + "value": "\"magazine\".\"extension_rounds\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.magazine_firearm": { + "name": "magazine_firearm", + "schema": "", + "columns": { + "magazine_id": { + "name": "magazine_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "firearm_id": { + "name": "firearm_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "magazine_firearm_firearm_id_idx": { + "name": "magazine_firearm_firearm_id_idx", + "columns": [ + { + "expression": "firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "magazine_firearm_magazine_id_magazine_id_fk": { + "name": "magazine_firearm_magazine_id_magazine_id_fk", + "tableFrom": "magazine_firearm", + "tableTo": "magazine", + "columnsFrom": ["magazine_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "magazine_firearm_firearm_id_firearm_id_fk": { + "name": "magazine_firearm_firearm_id_firearm_id_fk", + "tableFrom": "magazine_firearm", + "tableTo": "firearm", + "columnsFrom": ["firearm_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "magazine_firearm_magazine_id_firearm_id_pk": { + "name": "magazine_firearm_magazine_id_firearm_id_pk", + "columns": ["magazine_id", "firearm_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.magazine_label_prefix": { + "name": "magazine_label_prefix", + "schema": "", + "columns": { + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "magazine_label_prefix_owner_id_user_id_fk": { + "name": "magazine_label_prefix_owner_id_user_id_fk", + "tableFrom": "magazine_label_prefix", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "magazine_label_prefix_owner_id_prefix_pk": { + "name": "magazine_label_prefix_owner_id_prefix_pk", + "columns": ["owner_id", "prefix"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.range_session": { + "name": "range_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "firearm_id": { + "name": "firearm_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "rounds_fired": { + "name": "rounds_fired", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "ammo_id": { + "name": "ammo_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "range_session_firearm_id_idx": { + "name": "range_session_firearm_id_idx", + "columns": [ + { + "expression": "firearm_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "range_session_firearm_id_firearm_id_fk": { + "name": "range_session_firearm_id_firearm_id_fk", + "tableFrom": "range_session", + "tableTo": "firearm", + "columnsFrom": ["firearm_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "range_session_rounds_fired_min": { + "name": "range_session_rounds_fired_min", + "value": "\"range_session\".\"rounds_fired\" >= 1" + } + }, + "isRLSEnabled": false + }, + "public.range_session_accessory": { + "name": "range_session_accessory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "range_session_id": { + "name": "range_session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "accessory_id": { + "name": "accessory_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "range_session_accessory_session_id_idx": { + "name": "range_session_accessory_session_id_idx", + "columns": [ + { + "expression": "range_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "range_session_accessory_accessory_id_idx": { + "name": "range_session_accessory_accessory_id_idx", + "columns": [ + { + "expression": "accessory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "range_session_accessory_range_session_id_range_session_id_fk": { + "name": "range_session_accessory_range_session_id_range_session_id_fk", + "tableFrom": "range_session_accessory", + "tableTo": "range_session", + "columnsFrom": ["range_session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "range_session_accessory_accessory_id_accessory_id_fk": { + "name": "range_session_accessory_accessory_id_accessory_id_fk", + "tableFrom": "range_session_accessory", + "tableTo": "accessory", + "columnsFrom": ["accessory_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "range_session_accessory_unique": { + "name": "range_session_accessory_unique", + "nullsNotDistinct": false, + "columns": ["range_session_id", "accessory_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.operator_audit": { + "name": "operator_audit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor": { + "name": "actor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "at": { + "name": "at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "operator_audit_at_idx": { + "name": "operator_audit_at_idx", + "columns": [ + { + "expression": "at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "operator_audit_action_valid": { + "name": "operator_audit_action_valid", + "value": "\"operator_audit\".\"action\" in ('export', 'restore')" + } + }, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index 3d3d94b7..a1ae28de 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1783858219596, "tag": "0017_certain_mentallo", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1783890417560, + "tag": "0018_spotty_blonde_phantom", + "breakpoints": true } ] } diff --git a/src/db/operator-audit-schema.ts b/src/db/operator-audit-schema.ts new file mode 100644 index 00000000..5377fd38 --- /dev/null +++ b/src/db/operator-audit-schema.ts @@ -0,0 +1,40 @@ +import { sql } from "drizzle-orm"; +import { + check, + index, + pgTable, + text, + timestamp, + uuid, +} from "drizzle-orm/pg-core"; + +/** + * Operator audit trail (U3, R15 — encryption-at-rest backups plan). + * + * Append-only record of admin-run backup export/restore actions: who ran it + * (`actor` — the acting user id or email), what they ran (`action`), and how + * it turned out (`outcome`, e.g. "success"/"failure"/"refused"). Deliberately + * carries NO FK to `user`: an audit row must outlive the account that + * produced it (account deletion must never delete or block on its own audit + * trail), and `actor` may record an email rather than a live user id. `at` + * defaults to the write time; the row is never updated after insert. + */ +export const operatorAudit = pgTable( + "operator_audit", + { + id: uuid("id").primaryKey().defaultRandom(), + actor: text("actor").notNull(), + action: text("action").notNull(), + outcome: text("outcome").notNull(), + at: timestamp("at", { withTimezone: true }).defaultNow().notNull(), + }, + (t) => [ + // Chronological audit lookup. + index("operator_audit_at_idx").on(t.at), + // R26-style backstop — domain validation is the primary surface. + check( + "operator_audit_action_valid", + sql`${t.action} in ('export', 'restore')`, + ), + ], +); diff --git a/src/db/schema.ts b/src/db/schema.ts index a05b0ed8..dfb6da90 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -7,6 +7,8 @@ * - Inventory tables (firearms, magazines, the compatibility join, the * polymorphic grant table, and the idempotency store) live in * `inventory-schema.ts` (U3). + * - The operator audit trail (backup export/restore events) lives in + * `operator-audit-schema.ts` (U3 — encryption-at-rest backups plan). * * Regenerate auth tables with: * bun x @better-auth/cli@latest generate --output src/db/auth-schema.ts -y @@ -14,3 +16,4 @@ export * from "./auth-schema"; export * from "./inventory-schema"; +export * from "./operator-audit-schema"; From aada2f31ae1ca7a15585e5ef0a579e906a82483b Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:30:35 -0400 Subject: [PATCH 05/26] feat(deploy): docker secrets + hardening + encrypted-volume-ready mounts (U8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the Postgres password and Better Auth signing secret out of plain compose environment variables and into Docker secrets (files under ./secrets/, mounted read-only at /run/secrets/*), resolved into DATABASE_URL/BETTER_AUTH_SECRET at container start by a new docker-entrypoint.sh (R16). Apply baseline container hardening — read-only rootfs, explicit writable tmpfs, dropped capabilities, no-new-privileges — to the db, migrate, and app services (R18), and document the Postgres data and upload volumes as the attach points for an encrypted host disk (R17). Update setup.sh, README, CONTRIBUTING, and docs/deployment.md so the documented setup/dev flows create and read the new secret files instead of plaintext .env values. Signed-off-by: UncleSp1d3r --- .dockerignore | 2 + .env.example | 28 ++++++++--- .gitignore | 5 ++ CONTRIBUTING.md | 10 +++- Dockerfile | 8 +++ README.md | 30 ++++++++--- docker-compose.yml | 115 ++++++++++++++++++++++++++++++++++++++----- docker-entrypoint.sh | 36 ++++++++++++++ docs/deployment.md | 36 +++++++++++--- secrets/README.md | 59 ++++++++++++++++++++++ setup.sh | 47 +++++++++++------- 11 files changed, 321 insertions(+), 55 deletions(-) create mode 100755 docker-entrypoint.sh create mode 100644 secrets/README.md diff --git a/.dockerignore b/.dockerignore index 995008bc..12f4d11c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,8 @@ .env .env.* !.env.example +secrets/* +!secrets/README.md node_modules .next diff --git a/.env.example b/.env.example index 2d5e7fe6..0fd25f86 100644 --- a/.env.example +++ b/.env.example @@ -1,21 +1,35 @@ # Copy to `.env` (gitignored) and fill in. Never commit real secrets. +# +# The database password and the Better Auth signing secret are NOT set here — +# docker-compose.yml reads them as Docker secrets (plain files under +# ./secrets/, mounted read-only into the containers, never a plain env var — +# R16). Before first `docker compose up`, create both files: +# mkdir -p secrets +# openssl rand -hex 24 > secrets/postgres_password.txt +# openssl rand -hex 32 > secrets/better_auth_secret.txt +# (hex, not base64 — the password is embedded unescaped in a connection URL, +# and base64's `/+=` characters are not valid there unescaped) +# See secrets/README.md for details, rotation notes, and how to read the +# password back out for local (non-Docker) tooling. # --- Database --------------------------------------------------------------- POSTGRES_USER=magstacker -POSTGRES_PASSWORD=change-me-in-production POSTGRES_DB=magstacker # Host port the db is published on (kept off 5432 to avoid clashing with a host # Postgres). The app container reaches the db by service name on 5432. POSTGRES_HOST_PORT=5544 -# DATABASE_URL is NOT set here: docker compose builds it per-service inline -# (host `db`). For local tooling (`bun test`, `bun run db:migrate`) export it -# yourself pointing at the published host port, e.g.: -# export DATABASE_URL=postgres://magstacker:change-me-in-production@localhost:5544/magstacker +# DATABASE_URL is NOT set here: the app/migrate containers build it at +# startup from secrets/postgres_password.txt (see docker-entrypoint.sh). For +# local tooling (`bun test`, `bun run db:migrate`) export it yourself, +# pointing at the published host port and reading the password from the +# secret file: +# export DATABASE_URL="postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker" # --- Auth (Better Auth, added in U2) ---------------------------------------- -# Generate a strong random secret, e.g. `openssl rand -base64 32`. -BETTER_AUTH_SECRET=change-me-generate-a-strong-random-secret +# The signing secret lives in secrets/better_auth_secret.txt (see above), not +# here. For local (non-Docker) tooling that needs BETTER_AUTH_SECRET directly: +# export BETTER_AUTH_SECRET="$(cat secrets/better_auth_secret.txt)" BETTER_AUTH_URL=http://localhost:3000 # First-admin bootstrap for `bun run seed:admin` (one-time, fresh deployment). diff --git a/.gitignore b/.gitignore index 77b1ac94..801c965f 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,11 @@ yarn-error.log* .env* !.env.example +# Docker secrets (R16) — plaintext secret files read by docker-compose.yml. +# Never commit real secret values; see secrets/README.md. +/secrets/* +!/secrets/README.md + # vercel .vercel diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05459a15..452589fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,15 @@ just env-setup # create .env.local from .env.example just install-hooks # install the pre-commit hooks (once) ``` -Then set `DATABASE_URL` in `.env.local` so `mise` loads it into your shell — for the local Postgres below that's `postgres://magstacker:@localhost:5544/magstacker`. Also fill in `BETTER_AUTH_SECRET` and, if you want a seeded admin, `ADMIN_EMAIL` / `ADMIN_PASSWORD`. Now bring up the database and start the app: +The database password and Better Auth signing secret are Docker secrets (R16), not `.env` values — create them once (see [`secrets/README.md`](secrets/README.md)): + +```bash +mkdir -p secrets +openssl rand -hex 24 > secrets/postgres_password.txt +openssl rand -hex 32 > secrets/better_auth_secret.txt +``` + +Then set `DATABASE_URL` and `BETTER_AUTH_SECRET` in `.env.local` so `mise` loads them into your shell — for the local Postgres below that's `postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker`. If you want a seeded admin, also fill in `ADMIN_EMAIL` / `ADMIN_PASSWORD`. Now bring up the database and start the app: ```bash docker compose up -d db # local Postgres on host port 5544 diff --git a/Dockerfile b/Dockerfile index 7c741efc..8e6a867a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,10 +46,18 @@ COPY --from=builder /app/scripts ./scripts # first upload fails EACCES. RUN mkdir -p /data/uploads && chown bun:bun /data/uploads +# Resolves Docker-secrets `*_FILE` env vars into the plain env vars the app +# expects (POSTGRES_PASSWORD, BETTER_AUTH_SECRET, DATABASE_URL) before exec'ing +# the real command — see docker-entrypoint.sh (R16). Shared by the `app` and +# `migrate` services in docker-compose.yml. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + # Run as the unprivileged user shipped in the bun image. USER bun EXPOSE 3000 ENV PORT=3000 HOSTNAME=0.0.0.0 +ENTRYPOINT ["docker-entrypoint.sh"] CMD ["bun", "run", "start"] diff --git a/README.md b/README.md index d80df0d4..f4502ca3 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,16 @@ You don't need to clone the repo for this. Grab the two files the stack needs an curl -O https://raw.githubusercontent.com/unclesp1d3r/mag_stacker/main/docker-compose.yml curl -o .env https://raw.githubusercontent.com/unclesp1d3r/mag_stacker/main/.env.example -# 2. Fill in .env: a database password, a long random BETTER_AUTH_SECRET -# (try `openssl rand -base64 32`), your first admin email and password, and -# BETTER_AUTH_URL set to the address you'll actually open it at. +# 2. Fill in .env: your first admin email and password, and BETTER_AUTH_URL +# set to the address you'll actually open it at. -# 3. Pull the published image and start the stack +# 3. Create the two Docker secret files (R16) — the database password and the +# Better Auth signing secret are NOT set in .env: +mkdir -p secrets +openssl rand -hex 24 > secrets/postgres_password.txt +openssl rand -hex 32 > secrets/better_auth_secret.txt + +# 4. Pull the published image and start the stack docker compose pull docker compose up -d # migrates, seeds your first admin, starts the app ``` @@ -68,9 +73,14 @@ To build the image yourself instead of pulling the published one, clone the repo ```bash cp .env.example .env -# Fill in .env: a database password, a long random BETTER_AUTH_SECRET -# (try `openssl rand -base64 32`), your first admin email and password, and -# BETTER_AUTH_URL set to the address you'll actually open it at. +# Fill in .env: your first admin email and password, and BETTER_AUTH_URL set +# to the address you'll actually open it at. + +# Create the two Docker secret files (R16) — the database password and the +# Better Auth signing secret are NOT set in .env: +mkdir -p secrets +openssl rand -hex 24 > secrets/postgres_password.txt +openssl rand -hex 32 > secrets/better_auth_secret.txt docker compose up --build -d # migrates, seeds your first admin, starts the app ``` @@ -115,8 +125,10 @@ MagStacker is the original Go/Wails (later Avalonia) desktop app rebuilt as a mu Stack: Next.js 16 (App Router), React 19, Bun, Drizzle ORM, Postgres, Better Auth, Tailwind v4, Biome. Use Bun and Biome, not ESLint/Prettier/pnpm (see `AGENTS.md`). ```bash +mkdir -p secrets # once, if not already created +openssl rand -hex 24 > secrets/postgres_password.txt # (see secrets/README.md) docker compose up -d db # local Postgres on host port 5544 -export DATABASE_URL=postgres://magstacker:@localhost:5544/magstacker +export DATABASE_URL="postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker" bun install bun run db:migrate bun run dev # http://localhost:3000 @@ -128,6 +140,8 @@ bun test # unit + integration ``` > `mise` (`mise.toml`) pins the toolchain and loads `.env` into your shell, then caches it. After you edit `.env`, run `mise cache clear`, or a stale value can shadow both your tooling and `docker compose`. +> +> The db service reads its password from `secrets/postgres_password.txt` (a Docker secret, R16), not from `.env` — see [`secrets/README.md`](secrets/README.md). The README's demo images and walkthrough gif are generated from the live UI. Regenerate them all before a release with `just demo-images` (needs Docker + ffmpeg). The generators are `e2e/demo-*.spec.ts`, gated behind `DEMO=1` so they stay out of the normal test run, and they share one sample dataset from `e2e/fixtures/demo-seed.ts`. diff --git a/docker-compose.yml b/docker-compose.yml index d51c6d30..4d66c7ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,14 +12,27 @@ # docker compose up --build -d # Builds the Dockerfile and tags it with the same image name. # -# Secrets (POSTGRES_PASSWORD, BETTER_AUTH_SECRET, ...) are supplied at runtime -# via a gitignored `.env` file or the host environment — never committed here -# or baked into the image. Copy `.env.example` to `.env` and fill it in. +# Secrets: the database password and the Better Auth signing secret are +# supplied as Docker secrets — plaintext files under ./secrets/, mounted +# read-only at /run/secrets/* inside the containers that need them, never as +# plain environment variables and never baked into the image (R16). Create +# them before first `up`: +# mkdir -p secrets +# openssl rand -hex 24 > secrets/postgres_password.txt +# openssl rand -hex 32 > secrets/better_auth_secret.txt +# (hex, not base64 — the password is embedded unescaped in DATABASE_URL by +# docker-entrypoint.sh, and base64's `/+=` characters break a connection URL.) +# `secrets/` is gitignored — see secrets/README.md. `docker-entrypoint.sh` +# resolves the `*_FILE` vars into the app/migrate process env at container +# start; the official postgres image resolves POSTGRES_PASSWORD_FILE itself. # -# Required variables — the stack refuses to start (a `${VAR:?...}` guard aborts +# Everything else (POSTGRES_USER/DB, ADMIN_EMAIL/PASSWORD, ports, ...) stays a +# plain env var via a gitignored `.env` file or the host environment. Copy +# `.env.example` to `.env` and fill it in. +# +# Required — the stack refuses to start (a `${VAR:?...}` guard aborts # `docker compose` before anything runs) if these are unset or empty: -# • POSTGRES_PASSWORD — database password -# • BETTER_AUTH_SECRET — auth signing secret (openssl rand -base64 32) +# • secrets/postgres_password.txt, secrets/better_auth_secret.txt (files, not env vars) # Optional (sensible defaults): POSTGRES_USER/POSTGRES_DB (magstacker), # BETTER_AUTH_URL (http://localhost:3000 — set to your real origin behind a # proxy), APP_HOST_PORT/POSTGRES_HOST_PORT, MAGSTACKER_VERSION (latest), and @@ -27,6 +40,23 @@ # # Network exposure on a home network should sit behind a TLS-terminating # reverse proxy so session cookies and credentials are never sent in cleartext. +# +# Encrypted-volume-ready (R17): `magstacker-pgdata` (Postgres data) and +# `magstacker-uploads` (document/photo blobs, UPLOAD_DIR) are the two named +# volumes holding data at rest. Both are plain named volumes with no +# host-path binding here, so the turnkey path to running them on an +# encrypted host disk is: point Docker's data-root (or these volumes' +# driver_opts, per-volume) at a directory that lives on a LUKS-encrypted (or +# equivalent encrypted cloud) block device, then `docker compose up -d` as +# normal — the app never has to know the underlying disk is encrypted. See +# docs/deployment.md for the exact steps; that's a host/operator +# responsibility the app cannot perform (R19, R20). + +secrets: + postgres_password: + file: ./secrets/postgres_password.txt + better_auth_secret: + file: ./secrets/better_auth_secret.txt services: db: @@ -34,9 +64,12 @@ services: restart: unless-stopped environment: POSTGRES_USER: ${POSTGRES_USER:-magstacker} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password POSTGRES_DB: ${POSTGRES_DB:-magstacker} + secrets: + - postgres_password volumes: + # Attach point for an encrypted host disk (R17) — see the note above. - magstacker-pgdata:/var/lib/postgresql/data ports: # Host port is configurable; defaults away from 5432 to avoid clashing @@ -47,6 +80,26 @@ services: interval: 5s timeout: 5s retries: 10 + # Hardening (R18): read-only rootfs, explicit writable tmpfs for the + # paths Postgres' entrypoint and runtime need (PGDATA itself is the + # writable named volume above, not part of the rootfs). Postgres' + # entrypoint runs as root first to chown PGDATA/socket dir, then drops to + # the `postgres` user — CHOWN/DAC_OVERRIDE/FOWNER/SETUID/SETGID cover + # that; every other capability is dropped. + read_only: true + tmpfs: + - /tmp + - /var/run/postgresql:mode=1777 + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - SETUID + - SETGID + security_opt: + - no-new-privileges:true # One-shot bootstrap: apply migrations, then seed the first admin. Both steps # are idempotent — already-applied migrations are skipped, and the seed no-ops @@ -72,17 +125,36 @@ services: echo "ADMIN_EMAIL/ADMIN_PASSWORD unset — skipping first-admin seed." fi environment: - DATABASE_URL: postgres://${POSTGRES_USER:-magstacker}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@db:5432/${POSTGRES_DB:-magstacker} + POSTGRES_USER: ${POSTGRES_USER:-magstacker} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: ${POSTGRES_DB:-magstacker} + DB_HOST: db # seed:admin imports `@/auth`, which needs the secret at module load. - BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET:?set BETTER_AUTH_SECRET in .env} + # docker-entrypoint.sh (the image's ENTRYPOINT) resolves this file into + # BETTER_AUTH_SECRET before `command` above runs. + BETTER_AUTH_SECRET_FILE: /run/secrets/better_auth_secret BETTER_AUTH_URL: ${BETTER_AUTH_URL:-http://localhost:3000} - # Optional: when set, the first admin is created on an empty DB. + # Optional: when set, the first admin is created on an empty DB. Not a + # Docker secret — it's a one-time bootstrap value the operator already + # chose and typed into .env; only the long-lived DB/auth secrets are. ADMIN_EMAIL: ${ADMIN_EMAIL:-} ADMIN_PASSWORD: ${ADMIN_PASSWORD:-} + secrets: + - postgres_password + - better_auth_secret depends_on: db: condition: service_healthy restart: "no" + # Hardening (R18): no local files are written by migrate/seed — only a + # /tmp tmpfs for anything Bun/Node wants as scratch space. + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true app: image: ghcr.io/unclesp1d3r/mag_stacker:${MAGSTACKER_VERSION:-latest} @@ -91,17 +163,36 @@ services: dockerfile: Dockerfile restart: unless-stopped environment: - DATABASE_URL: postgres://${POSTGRES_USER:-magstacker}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@db:5432/${POSTGRES_DB:-magstacker} - BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET:?set BETTER_AUTH_SECRET in .env} + POSTGRES_USER: ${POSTGRES_USER:-magstacker} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: ${POSTGRES_DB:-magstacker} + DB_HOST: db + # docker-entrypoint.sh resolves this file into BETTER_AUTH_SECRET + # before `bun run start` runs. + BETTER_AUTH_SECRET_FILE: /run/secrets/better_auth_secret BETTER_AUTH_URL: ${BETTER_AUTH_URL:-http://localhost:3000} UPLOAD_DIR: /data/uploads + secrets: + - postgres_password + - better_auth_secret volumes: + # Attach point for an encrypted host disk (R17) — see the note above. - magstacker-uploads:/data/uploads ports: - "${APP_HOST_PORT:-3000}:3000" depends_on: migrate: condition: service_completed_successfully + # Hardening (R18): read-only rootfs; /data/uploads is the writable named + # volume above (UPLOAD_DIR), /tmp covers Sharp's image-processing scratch + # files and any other transient writes `next start` wants. + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true volumes: magstacker-pgdata: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..1019999b --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Resolves Docker-secrets-style `*_FILE` env vars (POSTGRES_PASSWORD_FILE, +# BETTER_AUTH_SECRET_FILE) into the plain env vars the app/migrate steps read, +# then builds DATABASE_URL from the resolved password if the caller didn't +# already supply one. Runs inside the container only — secret material never +# touches the host process environment or `docker compose config` output, +# only the `/run/secrets/*` files Compose mounts (R16). +# +# Used as the Dockerfile ENTRYPOINT for both the `app` and `migrate` services; +# `docker-compose.yml`'s per-service `command`/default CMD become "$@" below. +set -eu + +# Reads a secret file's contents into the named var, trimming one trailing +# newline (some `openssl rand`/`echo`-created secret files carry one). +resolve_secret() { + var_name="$1" + file_path="$2" + if [ -n "${file_path}" ] && [ -f "${file_path}" ]; then + value="$(cat "${file_path}")" + export "${var_name}=${value}" + fi +} + +resolve_secret POSTGRES_PASSWORD "${POSTGRES_PASSWORD_FILE:-}" +resolve_secret BETTER_AUTH_SECRET "${BETTER_AUTH_SECRET_FILE:-}" + +# DATABASE_URL is not itself a Docker secret (it also carries the non-secret +# host/user/db name), so it's assembled here from the resolved password +# rather than being passed through compose-level `${...}` interpolation -- +# that would require the plaintext password in the host's `.env`/shell +# environment, defeating the point of the secret file. +if [ -z "${DATABASE_URL:-}" ] && [ -n "${POSTGRES_PASSWORD:-}" ]; then + export DATABASE_URL="postgres://${POSTGRES_USER:-magstacker}:${POSTGRES_PASSWORD}@${DB_HOST:-db}:${DB_PORT:-5432}/${POSTGRES_DB:-magstacker}" +fi + +exec "$@" diff --git a/docs/deployment.md b/docs/deployment.md index f9629a3a..7b246e98 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -5,15 +5,30 @@ control — a homelab server, a NAS, a small VPS behind your own network. ## First run -1. Copy the env template and fill in **real** secrets (never commit `.env`): +1. Copy the env template and fill in the non-secret values (never commit `.env`): ```bash cp .env.example .env - # set POSTGRES_PASSWORD, BETTER_AUTH_SECRET (openssl rand -base64 32), - # ADMIN_EMAIL, ADMIN_PASSWORD, and BETTER_AUTH_URL (your reverse-proxy URL) + # set ADMIN_EMAIL, ADMIN_PASSWORD, and BETTER_AUTH_URL (your reverse-proxy URL) ``` -2. Build and start the stack. The `migrate` service applies the database +2. Create the two Docker secret files. The database password and the Better + Auth signing secret are **not** set in `.env` — they're Docker secrets, + plain files under `secrets/` that Compose mounts read-only into the + containers rather than plaintext environment variables (R16): + + ```bash + mkdir -p secrets + openssl rand -hex 24 > secrets/postgres_password.txt + openssl rand -hex 32 > secrets/better_auth_secret.txt + ``` + + Use `-hex`, not `-base64` — the password is embedded unescaped in a + connection URL, and base64's `/+=` characters break it there. See + `secrets/README.md` for rotation notes. `docker compose up` refuses to + start until both files exist. + +3. Build and start the stack. The `migrate` service applies the database migrations and — when `ADMIN_EMAIL` / `ADMIN_PASSWORD` are set in `.env` — seeds the first operator account, both before the `app` service starts: @@ -39,10 +54,15 @@ control — a homelab server, a NAS, a small VPS behind your own network. ## Secrets -- `DATABASE_URL` and `BETTER_AUTH_SECRET` are supplied at runtime via `.env` / - the host environment — never baked into the image. `.dockerignore` excludes - `.env*` from the build context. The image build uses throwaway placeholder - values that never open a connection. +- The Postgres password and the Better Auth signing secret are Docker + secrets — files under `secrets/` (see `secrets/README.md`), mounted + read-only at `/run/secrets/*` inside the containers that need them, never + a plaintext environment variable (R16). `docker-entrypoint.sh` resolves + them into `DATABASE_URL`/`BETTER_AUTH_SECRET` at container start; the + official `postgres` image resolves `POSTGRES_PASSWORD_FILE` itself. + `.dockerignore`/`.gitignore` exclude both `.env*` and `secrets/*` (except + `secrets/README.md`) from the build context and git. The image build uses + throwaway placeholder values that never open a connection. - Back up Postgres with the standard tooling; a `pg_dump` / `pg_restore` round-trip reproduces inventory, ownership, and grant state exactly. diff --git a/secrets/README.md b/secrets/README.md new file mode 100644 index 00000000..eaf3c6f0 --- /dev/null +++ b/secrets/README.md @@ -0,0 +1,59 @@ +# Docker secrets + +`docker-compose.yml` reads the database password and the Better Auth signing +secret as [Docker secrets](https://docs.docker.com/compose/how-tos/use-secrets/) +— plain files in this directory, bind-mounted read-only into the containers +that need them at `/run/secrets/*`, rather than plaintext environment +variables (R16). This keeps them out of `docker compose config`, `docker +inspect`, and process-environment dumps on the host. + +This directory is gitignored except this file — never commit real secret +values. + +## Create the secret files + +Before the first `docker compose up`, create both files (run from the repo +root, next to `docker-compose.yml`): + +```bash +mkdir -p secrets +openssl rand -hex 24 > secrets/postgres_password.txt +openssl rand -hex 32 > secrets/better_auth_secret.txt +``` + +Use `-hex`, not `-base64`: `docker-entrypoint.sh` embeds the password +unescaped in `DATABASE_URL` (`postgres://user:PASSWORD@host/db`), and +base64's `/`, `+`, `=` characters are not valid there unescaped — a base64 +password with a `/` or `@` in it breaks the connection string. + +- `secrets/postgres_password.txt` — the Postgres database password. Consumed + natively by the official `postgres` image via `POSTGRES_PASSWORD_FILE` + (only applied the first time the data volume is created — changing this + file later does not rotate an existing database's password). +- `secrets/better_auth_secret.txt` — the Better Auth session/token signing + secret. Consumed by `docker-entrypoint.sh`, which resolves + `BETTER_AUTH_SECRET_FILE` into `BETTER_AUTH_SECRET` before the app or the + `migrate` bootstrap step starts. + +`docker compose up` refuses to start (`secret ... not found`) until both +files exist. + +## Local (non-Docker) tooling + +`bun test`, `bun run db:migrate`, `bun run dev`, etc. run outside the +containers and build their own `DATABASE_URL` — they don't read these files +automatically. Read the password out when you need it: + +```bash +export DATABASE_URL="postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:${POSTGRES_HOST_PORT:-5544}/magstacker" +``` + +## Rotating a secret + +- **Better Auth secret:** overwrite `secrets/better_auth_secret.txt` and + restart the `app`/`migrate` services. Rotating it invalidates existing + sessions (users are signed out). +- **Postgres password:** overwriting the file alone does *not* change the + running database's password (Postgres only reads it on first init). Change + the password inside Postgres too (`ALTER USER ... PASSWORD ...`) and keep + the file in sync, or recreate the data volume for a fresh instance. diff --git a/setup.sh b/setup.sh index 7c9feced..f5bec1b6 100755 --- a/setup.sh +++ b/setup.sh @@ -16,10 +16,9 @@ set -euo pipefail # Placeholder values shipped in .env.example — refuse to start with these so a -# deployment never comes up with a default database password, auth secret, or -# admin login. -PLACEHOLDER_POSTGRES_PASSWORD="change-me-in-production" -PLACEHOLDER_AUTH_SECRET="change-me-generate-a-strong-random-secret" +# deployment never comes up with a default admin login. The database password +# and auth secret aren't in .env at all — they're Docker secrets (R16), +# checked separately below. PLACEHOLDER_ADMIN_EMAIL="admin@example.com" PLACEHOLDER_ADMIN_PASSWORD="change-me-strong-admin-password" @@ -69,40 +68,50 @@ if [[ ! -f .env ]]; then echo "Created .env from .env.example." echo "" echo "Edit .env and fill in real values before continuing:" - echo " - POSTGRES_PASSWORD (a real database password)" - echo " - BETTER_AUTH_SECRET (generate with: openssl rand -base64 32)" echo " - ADMIN_EMAIL / ADMIN_PASSWORD (your first admin login)" echo " - BETTER_AUTH_URL (must match the origin you'll open the app at)" echo "" - echo "Postgres only applies POSTGRES_PASSWORD the first time its data volume" - echo "is created, so set a real password *before* the database starts." + echo "Then create the two Docker secret files (R16) — the database password" + echo "and the Better Auth signing secret are NOT set in .env:" + echo " mkdir -p secrets" + echo " openssl rand -hex 24 > secrets/postgres_password.txt" + echo " openssl rand -hex 32 > secrets/better_auth_secret.txt" + echo "(hex, not base64 — the password lands unescaped in a connection URL.)" echo "" - echo "Re-run ./setup.sh once .env is filled in." + echo "Postgres only applies the password the first time its data volume is" + echo "created, so create secrets/postgres_password.txt *before* first boot." + echo "" + echo "Re-run ./setup.sh once .env and secrets/ are filled in." exit 0 fi echo ".env found — leaving it untouched." echo "" -# --- Read .env for presence/placeholder checks (values are never printed) -- +# --- Docker secrets (R16) — files under secrets/, never printed ----------- -postgres_password="$(env_value POSTGRES_PASSWORD)" -auth_secret="$(env_value BETTER_AUTH_SECRET)" -admin_email="$(env_value ADMIN_EMAIL)" -admin_password="$(env_value ADMIN_PASSWORD)" +postgres_password_file="secrets/postgres_password.txt" +auth_secret_file="secrets/better_auth_secret.txt" fail=0 -if [[ -z "${postgres_password}" || "${postgres_password}" == "${PLACEHOLDER_POSTGRES_PASSWORD}" ]]; then - echo "Error: set a real POSTGRES_PASSWORD in .env (still the placeholder)." >&2 +if [[ ! -s "${postgres_password_file}" ]]; then + echo "Error: ${postgres_password_file} is missing or empty." >&2 + echo " Create it: openssl rand -hex 24 > ${postgres_password_file}" >&2 fail=1 fi -if [[ -z "${auth_secret}" || "${auth_secret}" == "${PLACEHOLDER_AUTH_SECRET}" ]]; then - echo "Error: set a real BETTER_AUTH_SECRET in .env (openssl rand -base64 32)." >&2 +if [[ ! -s "${auth_secret_file}" ]]; then + echo "Error: ${auth_secret_file} is missing or empty." >&2 + echo " Create it: openssl rand -hex 32 > ${auth_secret_file}" >&2 fail=1 fi +# --- Read .env for presence/placeholder checks (values are never printed) -- + +admin_email="$(env_value ADMIN_EMAIL)" +admin_password="$(env_value ADMIN_PASSWORD)" + # The admin seed is optional — leave both unset to skip it. But if both are set # (so compose will seed) and either is still the shipped placeholder, refuse: # otherwise the stack comes up with a known-password admin account. @@ -116,7 +125,7 @@ fi if [[ "${fail}" -ne 0 ]]; then echo "" >&2 - echo "Fix the values above in .env, then re-run ./setup.sh" >&2 + echo "Fix the issues above (in .env and/or secrets/), then re-run ./setup.sh" >&2 exit 1 fi From 94052bb2bc427eb421206a241e4787466ec37ff9 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 17:36:30 -0400 Subject: [PATCH 06/26] feat(backup): streaming tar bundle format + zip-slip guard (U2) Signed-off-by: UncleSp1d3r --- bun.lock | 5 + package.json | 4 +- src/backup/__tests__/bundle.test.ts | 431 ++++++++++++++++++++++++++++ src/backup/bundle.ts | 418 +++++++++++++++++++++++++++ src/backup/manifest.ts | 193 +++++++++++++ 5 files changed, 1050 insertions(+), 1 deletion(-) create mode 100644 src/backup/__tests__/bundle.test.ts create mode 100644 src/backup/bundle.ts create mode 100644 src/backup/manifest.ts diff --git a/bun.lock b/bun.lock index 7e08c816..1b37db94 100644 --- a/bun.lock +++ b/bun.lock @@ -23,6 +23,7 @@ "sharp": "^0.35.3", "sodium-native": "^5.1.0", "tailwind-merge": "^3.6.0", + "tar-stream": "^3.2.0", }, "devDependencies": { "@biomejs/biome": "2.5.2", @@ -34,6 +35,7 @@ "@types/pg": "^8.20.0", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", + "@types/tar-stream": "^3.1.4", "babel-plugin-react-compiler": "^1.0.0", "drizzle-kit": "^0.31.10", "shadcn": "^4.13.0", @@ -46,6 +48,7 @@ }, "trustedDependencies": [ "sharp", + "sodium-native", ], "packages": { "@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="], @@ -536,6 +539,8 @@ "@types/ssh2-streams": ["@types/ssh2-streams@0.1.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA=="], + "@types/tar-stream": ["@types/tar-stream@3.1.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg=="], + "@types/validate-npm-package-name": ["@types/validate-npm-package-name@4.0.2", "", {}, "sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw=="], "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], diff --git a/package.json b/package.json index 8f1b82a5..c695d7d9 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "react-dom": "^19.2.7", "sharp": "^0.35.3", "sodium-native": "^5.1.0", - "tailwind-merge": "^3.6.0" + "tailwind-merge": "^3.6.0", + "tar-stream": "^3.2.0" }, "devDependencies": { "@biomejs/biome": "2.5.2", @@ -46,6 +47,7 @@ "@types/pg": "^8.20.0", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", + "@types/tar-stream": "^3.1.4", "babel-plugin-react-compiler": "^1.0.0", "drizzle-kit": "^0.31.10", "shadcn": "^4.13.0", diff --git a/src/backup/__tests__/bundle.test.ts b/src/backup/__tests__/bundle.test.ts new file mode 100644 index 00000000..457f02b9 --- /dev/null +++ b/src/backup/__tests__/bundle.test.ts @@ -0,0 +1,431 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { Readable } from "node:stream"; +import * as tar from "tar-stream"; +import { PathTraversalError } from "@/src/storage/local-fs-adapter"; +import { expectRejects } from "@/src/test-support/assertions"; +import { + type BundleBlobEntry, + type BundleEvent, + BundleFormatError, + BundleIntegrityError, + readBundle, + UnsafeBundleEntryError, + type WriteBundleInput, + writeBundle, +} from "../bundle"; +import { + createDecryptStream, + createEncryptStream, + deriveKey, + generateSalt, +} from "../crypto"; +import { + BACKUP_FORMAT_VERSION, + type BackupManifest, + buildManifest, + currentAppVersion, + latestMigrationTag, +} from "../manifest"; + +const PASSWORD = "correct horse battery staple"; + +/** Drains a Readable into one Buffer. */ +async function collect(stream: NodeJS.ReadableStream): Promise { + const parts: Buffer[] = []; + for await (const chunk of stream) { + parts.push(chunk as Buffer); + } + return Buffer.concat(parts); +} + +/** Encrypts a raw plaintext buffer, returning both the ciphertext and the key needed to decrypt it. Used to craft raw/malicious bundles directly (bypassing `writeBundle`). */ +async function encryptWithKey( + plaintext: Buffer, +): Promise<{ encrypted: Buffer; key: Buffer }> { + const salt = generateSalt(); + const key = deriveKey(PASSWORD, salt); + const source = Readable.from([plaintext]); + const encrypted = await collect(source.pipe(createEncryptStream(key, salt))); + return { encrypted, key }; +} + +/** Runs `writeBundle` end to end and returns the encrypted output plus the key needed to decrypt it. */ +async function writeBundleEncrypted( + input: WriteBundleInput, +): Promise<{ encrypted: Buffer; key: Buffer }> { + const salt = generateSalt(); + const key = deriveKey(PASSWORD, salt); + const encrypted = await collect( + writeBundle(input, createEncryptStream(key, salt)), + ); + return { encrypted, key }; +} + +/** Builds a decrypt stream fed from `encrypted`, ready to hand to `readBundle`. */ +function decryptStreamFor(encrypted: Buffer, key: Buffer) { + const decryptStream = createDecryptStream(key); + Readable.from([encrypted]).pipe(decryptStream); + return decryptStream; +} + +function makeStagingDir(): string { + return mkdtempSync(join(tmpdir(), "magstacker-bundle-")); +} + +function ndjsonStream(rows: readonly string[]): Readable { + return Readable.from(rows.map((row) => `${row}\n`)); +} + +function blobEntry(storageKey: string, content: Buffer): BundleBlobEntry { + return { + storageKey, + size: content.byteLength, + stream: Readable.from([content]), + }; +} + +/** Builds a raw tar buffer directly (bypassing writeBundle), so tests can inject malformed/malicious entries. */ +function buildRawTar( + entries: ReadonlyArray<{ + name: string; + type?: "file" | "symlink" | "directory"; + data?: Buffer; + size?: number; + linkname?: string; + }>, +): Promise { + const pack = tar.pack(); + const chunks: Buffer[] = []; + pack.on("data", (chunk: Buffer) => chunks.push(chunk)); + + return new Promise((resolvePromise, rejectPromise) => { + pack.on("end", () => resolvePromise(Buffer.concat(chunks))); + pack.on("error", rejectPromise); + + (async () => { + for (const e of entries) { + await new Promise((res, rej) => { + if (e.type === "symlink") { + pack.entry( + { + name: e.name, + type: "symlink", + linkname: e.linkname ?? "elsewhere", + }, + (err) => (err ? rej(err) : res()), + ); + } else { + pack.entry( + { + name: e.name, + size: e.data ? e.data.byteLength : (e.size ?? 0), + type: e.type ?? "file", + }, + e.data, + (err) => (err ? rej(err) : res()), + ); + } + }); + } + pack.finalize(); + })().catch(rejectPromise); + }); +} + +function testManifest(counts: BackupManifest["counts"]): BackupManifest { + return buildManifest({ + counts, + createdAt: new Date("2026-07-12T00:00:00.000Z"), + }); +} + +interface CapturedEvents { + manifest: BackupManifest | undefined; + dbLines: string[]; + blobEvents: Array<{ storageKey: string; path: string; size: number }>; +} + +/** Fully drains a `readBundle` async generator (including the "db" stream event, which the reader is required to consume before advancing — see readBundle's doc comment) and captures everything for assertions. */ +async function collectEvents( + generator: AsyncGenerator, +): Promise { + const result: CapturedEvents = { + manifest: undefined, + dbLines: [], + blobEvents: [], + }; + for await (const event of generator) { + if (event.kind === "manifest") { + result.manifest = event.manifest; + } else if (event.kind === "db") { + const buf = await collect(event.stream); + result.dbLines = buf + .toString("utf8") + .split("\n") + .filter((line) => line.trim() !== ""); + } else { + result.blobEvents.push({ + storageKey: event.storageKey, + path: event.path, + size: event.size, + }); + } + } + return result; +} + +describe("manifest fields (R4)", () => { + test("buildManifest stamps backupFormatVersion, appVersion, and migrationTag", () => { + const manifest = testManifest({ rows: 3, blobs: 0, totalBlobBytes: 0 }); + + expect(manifest.backupFormatVersion).toBe(BACKUP_FORMAT_VERSION); + expect(manifest.appVersion).toBe(currentAppVersion()); + expect(manifest.migrationTag).toBe(latestMigrationTag()); + expect(manifest.createdAt).toBe("2026-07-12T00:00:00.000Z"); + }); +}); + +describe("writeBundle / readBundle round-trip", () => { + let stagingDir: string; + + beforeEach(() => { + stagingDir = makeStagingDir(); + }); + + afterEach(() => { + rmSync(stagingDir, { recursive: true, force: true }); + }); + + test("round-trips manifest + multi-row db stream + several blobs of varying sizes, in order", async () => { + const rows = [ + JSON.stringify({ table: "firearm", row: { id: 1 } }), + JSON.stringify({ table: "firearm", row: { id: 2 } }), + JSON.stringify({ table: "magazine", row: { id: 1 } }), + ]; + const blobs = [ + { key: "empty.bin", content: Buffer.alloc(0) }, + { key: "small.bin", content: Buffer.from("hello world") }, + { key: "large.bin", content: Buffer.alloc(200_000, 0xab) }, + ]; + const totalBlobBytes = blobs.reduce( + (sum, b) => sum + b.content.byteLength, + 0, + ); + const manifest = testManifest({ + rows: rows.length, + blobs: blobs.length, + totalBlobBytes, + }); + + const { encrypted, key } = await writeBundleEncrypted({ + manifest, + dbStream: ndjsonStream(rows), + blobEntries: blobs.map((b) => blobEntry(b.key, b.content)), + }); + + const events = await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + + expect(events.manifest).toEqual(manifest); + expect(events.dbLines).toEqual(rows); + expect(events.blobEvents.map((e) => e.storageKey)).toEqual( + blobs.map((b) => b.key), + ); + for (const [index, blobEvent] of events.blobEvents.entries()) { + const expected = blobs[index]; + expect(expected).toBeDefined(); + if (!expected) continue; + expect(blobEvent.size).toBe(expected.content.byteLength); + const written = readFileSync(blobEvent.path); + expect(written.equals(expected.content)).toBe(true); + } + }); + + test("an empty blob set round-trips", async () => { + const rows = [JSON.stringify({ table: "firearm", row: { id: 1 } })]; + const manifest = testManifest({ + rows: rows.length, + blobs: 0, + totalBlobBytes: 0, + }); + + const { encrypted, key } = await writeBundleEncrypted({ + manifest, + dbStream: ndjsonStream(rows), + blobEntries: [], + }); + + const events = await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + + expect(events.manifest).toEqual(manifest); + expect(events.dbLines).toEqual(rows); + expect(events.blobEvents).toEqual([]); + }); + + test("a truncated/corrupt bundle surfaces an error, not partial data", async () => { + const rows = [JSON.stringify({ table: "firearm", row: { id: 1 } })]; + const content = Buffer.alloc(5_000, 0x11); + const manifest = testManifest({ + rows: rows.length, + blobs: 1, + totalBlobBytes: content.byteLength, + }); + + const { encrypted, key } = await writeBundleEncrypted({ + manifest, + dbStream: ndjsonStream(rows), + blobEntries: [blobEntry("a.bin", content)], + }); + const truncated = encrypted.subarray(0, encrypted.length - 20); + + await expectRejects(() => + collectEvents( + readBundle(decryptStreamFor(truncated, key), { stagingDir }), + ), + ); + }); +}); + +describe("KTD11 — untrusted blob-entry validation", () => { + let stagingDir: string; + + beforeEach(() => { + stagingDir = makeStagingDir(); + }); + + afterEach(() => { + rmSync(stagingDir, { recursive: true, force: true }); + }); + + test("a path-traversal blob entry (blobs/../../etc/x) is refused, nothing written outside stagingDir", async () => { + const manifest = testManifest({ rows: 0, blobs: 1, totalBlobBytes: 10 }); + const raw = await buildRawTar([ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/../../etc/x", data: Buffer.from("pwned") }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(PathTraversalError); + expect(readdirSync(stagingDir)).toEqual([]); + const escapeTarget = resolve(stagingDir, "../../etc/x"); + expect(() => readFileSync(escapeTarget)).toThrow(); + }); + + test("a symlink blob entry is refused", async () => { + const manifest = testManifest({ rows: 0, blobs: 1, totalBlobBytes: 10 }); + const raw = await buildRawTar([ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/evil-link", type: "symlink", linkname: "/etc/passwd" }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(UnsafeBundleEntryError); + expect(readdirSync(stagingDir)).toEqual([]); + }); + + test("a bundle whose blob count exceeds its manifest-declared count is refused before extraction", async () => { + const manifest = testManifest({ rows: 0, blobs: 1, totalBlobBytes: 20 }); + const raw = await buildRawTar([ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/a.bin", data: Buffer.from("aaaaa") }, + { name: "blobs/b.bin", data: Buffer.from("bbbbb") }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(BundleIntegrityError); + }); + + test("a blob entry whose declared size exceeds the manifest's total-bytes bound is refused before extraction", async () => { + const manifest = testManifest({ rows: 0, blobs: 1, totalBlobBytes: 10 }); + const oversized = Buffer.alloc(1_000, 0x42); + const raw = await buildRawTar([ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/big.bin", data: oversized }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(BundleIntegrityError); + expect(readdirSync(stagingDir)).toEqual([]); + }); + + test("refuses a bundle whose first entry isn't manifest.json", async () => { + const raw = await buildRawTar([ + { name: "db.ndjson", data: Buffer.alloc(0) }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(BundleFormatError); + }); + + test("refuses a non-regular, non-symlink entry (directory type)", async () => { + const manifest = testManifest({ rows: 0, blobs: 1, totalBlobBytes: 10 }); + const raw = await buildRawTar([ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/a-directory", type: "directory" }, + ]); + const { encrypted, key } = await encryptWithKey(raw); + + let thrown: unknown; + try { + await collectEvents( + readBundle(decryptStreamFor(encrypted, key), { stagingDir }), + ); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(UnsafeBundleEntryError); + }); +}); diff --git a/src/backup/bundle.ts b/src/backup/bundle.ts new file mode 100644 index 00000000..ac06d50d --- /dev/null +++ b/src/backup/bundle.ts @@ -0,0 +1,418 @@ +/** + * Backup bundle tar format — writer and reader (plan Unit U2, R2/R13). + * + * A bundle is a tar stream (`manifest.json` first, then `db.ndjson`, then + * every `blobs/`) piped through U1's crypto streams + * (`createEncryptStream`/`createDecryptStream` in `./crypto`). This module + * never buffers a blob's content beyond stream backpressure — a large + * document blob store (R13, KTD3) is streamed straight from source to the + * encrypted output. + * + * `db.ndjson` is the one deliberate exception: the tar format requires an + * exact byte length in an entry's header *before* any of its content is + * written, so a truly unbounded-size streaming write isn't possible without + * either buffering the content or a two-pass write-to-temp-file dance. Given + * the DB export is JSON-per-row text (not the binary attachments R13/KTD3 are + * actually worried about), `writeBundle` buffers `dbStream` in memory to + * learn its length, then writes it as one tar entry. On read, `db.ndjson` is + * handed back to the caller as a live, unbuffered stream — so importing a + * very large export still never requires the whole NDJSON payload in memory. + * + * **KTD11 choke point:** `readBundle` is the single place that turns + * untrusted `blobs/` tar entries into filesystem writes. A bundle + * is attacker-influenceable (anyone can craft one under a password of their + * choosing), so authenticated decryption only proves the bytes weren't + * tampered with after creation — never that the bundle's *contents* are + * safe. Every blob entry's key is path-validated the same way + * `LocalFilesystemAdapter` validates storage keys (see + * `src/storage/local-fs-adapter.ts`), non-regular-file/symlink entries are + * refused outright, and per-entry/total size and entry-count are bounded + * against the manifest's declared counts. + */ + +import { constants as fsConstants } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import { dirname, resolve, sep } from "node:path"; +import type { Readable, Transform } from "node:stream"; +import * as tar from "tar-stream"; +import { PathTraversalError } from "@/src/storage/local-fs-adapter"; +import { + type BackupManifest, + parseManifest, + serializeManifest, +} from "./manifest"; + +const MANIFEST_ENTRY_NAME = "manifest.json"; +const DB_ENTRY_NAME = "db.ndjson"; +const BLOB_ENTRY_PREFIX = "blobs/"; + +/** Safety cap on the manifest entry itself — it should always be tiny. */ +const MAX_MANIFEST_BYTES = 1 * 1024 * 1024; // 1 MiB + +/** Thrown when a bundle's tar structure doesn't match the expected shape (entry order/names). */ +export class BundleFormatError extends Error { + constructor(message: string) { + super(message); + this.name = "BundleFormatError"; + } +} + +/** Thrown when a bundle's actual contents violate its own manifest-declared counts/bounds (KTD11). */ +export class BundleIntegrityError extends Error { + constructor(message: string) { + super(message); + this.name = "BundleIntegrityError"; + } +} + +/** Thrown when a blob entry is refused for safety reasons other than path traversal — a symlink or non-regular-file entry (KTD11). */ +export class UnsafeBundleEntryError extends Error { + constructor(message: string) { + super(message); + this.name = "UnsafeBundleEntryError"; + } +} + +function toError(err: unknown): Error { + return err instanceof Error ? err : new Error(String(err)); +} + +/** One blob to write into the bundle. `size` must be the exact byte length of `stream`'s content (tar requires it upfront). */ +export interface BundleBlobEntry { + readonly storageKey: string; + readonly size: number; + readonly stream: Readable; +} + +export interface WriteBundleInput { + readonly manifest: BackupManifest; + /** NDJSON database export (see `./db-export.ts`). Buffered once to learn its length — see the module doc comment. */ + readonly dbStream: Readable; + readonly blobEntries: + | AsyncIterable + | Iterable; +} + +/** Reads a Readable fully into one Buffer. Used only for the manifest and `db.ndjson` — see the module doc comment for why those two are the deliberate exceptions to "never buffer a whole stream". */ +async function bufferStream(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +/** Writes one whole-buffer tar entry and resolves once it's flushed. */ +function packBufferEntry( + pack: tar.Pack, + name: string, + data: Buffer, +): Promise { + return new Promise((resolvePromise, rejectPromise) => { + pack.entry({ name, size: data.byteLength, type: "file" }, data, (err) => { + if (err) rejectPromise(err); + else resolvePromise(); + }); + }); +} + +/** Streams `source` into one tar entry of declared `size` without buffering its content. */ +function packStreamEntry( + pack: tar.Pack, + name: string, + size: number, + source: Readable, +): Promise { + return new Promise((resolvePromise, rejectPromise) => { + const sink = pack.entry({ name, size, type: "file" }, (err) => { + if (err) rejectPromise(err); + else resolvePromise(); + }); + source.on("error", (err) => sink.destroy(err)); + source.pipe(sink); + }); +} + +/** + * Builds the bundle: `manifest.json`, then `db.ndjson`, then every + * `blobs/`, tarred and piped through `encryptStream` (R2, R13). + * Returns the encrypted output stream — nothing is written server-side + * (KTD8 is the caller's concern; this function only produces the stream). + */ +export function writeBundle( + input: WriteBundleInput, + encryptStream: Transform, +): Readable { + const pack = tar.pack(); + + // Errors on the tar-pack side don't automatically propagate through + // `.pipe()` to the encrypt stream (a well-known Node stream gotcha) — + // forward them explicitly so a write-side failure surfaces to whoever is + // consuming the encrypted output rather than hanging. + pack.on("error", (err) => encryptStream.destroy(toError(err))); + + (async () => { + try { + const manifestBytes = serializeManifest(input.manifest); + await packBufferEntry(pack, MANIFEST_ENTRY_NAME, manifestBytes); + + const dbBytes = await bufferStream(input.dbStream); + await packBufferEntry(pack, DB_ENTRY_NAME, dbBytes); + + for await (const blob of input.blobEntries) { + await packStreamEntry( + pack, + `${BLOB_ENTRY_PREFIX}${blob.storageKey}`, + blob.size, + blob.stream, + ); + } + + pack.finalize(); + } catch (err) { + pack.destroy(toError(err)); + } + })(); + + return pack.pipe(encryptStream); +} + +export interface ReadBundleOptions { + /** Directory blob entries are written under — a sibling staging directory, never the live upload store (KTD10). */ + readonly stagingDir: string; +} + +export type BundleEvent = + | { readonly kind: "manifest"; readonly manifest: BackupManifest } + | { readonly kind: "db"; readonly stream: Readable } + | { + readonly kind: "blob"; + readonly storageKey: string; + readonly path: string; + readonly size: number; + }; + +/** Resolves `storageKey` under `stagingRoot`, rejecting any path that would escape it — the same pattern `LocalFilesystemAdapter.resolvePath` uses (KTD11). */ +function resolveStagingBlobPath( + stagingRoot: string, + storageKey: string, +): string { + const resolved = resolve(stagingRoot, storageKey); + const rootPrefix = stagingRoot.endsWith(sep) + ? stagingRoot + : `${stagingRoot}${sep}`; + const isRootItself = resolved === stagingRoot; + const isInsideRoot = resolved.startsWith(rootPrefix); + if (!isRootItself && !isInsideRoot) { + throw new PathTraversalError(storageKey); + } + return resolved; +} + +/** + * Streams a tar entry's content to `destPath`, refusing to follow a symlink + * at the destination (`O_NOFOLLOW`, when the platform supports it) and + * aborting the write the moment more than `maxBytes` has been streamed — + * independent of whatever size the entry's tar header claims, since that + * header is attacker-controlled (KTD11). + */ +async function writeEntryToFile( + entry: Readable, + destPath: string, + maxBytes: number, +): Promise { + const flags = + fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_TRUNC | + (fsConstants.O_NOFOLLOW ?? 0); + // `O_NOFOLLOW` makes `open` itself throw (ELOOP) if `destPath` is a + // symlink, so a prior malicious entry can't have this write silently + // follow it elsewhere (KTD11 defense-in-depth beyond the entry-type check + // above). + const handle = await open(destPath, flags, 0o600); + const dest = handle.createWriteStream(); + + return new Promise((resolvePromise, rejectPromise) => { + let written = 0; + let settled = false; + + function fail(err: unknown): void { + if (settled) return; + settled = true; + entry.destroy(); + dest.destroy(); + rejectPromise(toError(err)); + } + + entry.on("data", (chunk: Buffer) => { + written += chunk.byteLength; + if (written > maxBytes) { + fail( + new BundleIntegrityError( + `blob entry streamed more than its ${maxBytes}-byte bound: "${destPath}"`, + ), + ); + } + }); + entry.on("error", fail); + dest.on("error", fail); + dest.on("finish", () => { + if (settled) return; + settled = true; + resolvePromise(written); + }); + + entry.pipe(dest); + }); +} + +/** + * Reads a bundle: decrypts `decryptStream` via U1's authenticated + * decryption, un-tars it, and yields the manifest, then the db NDJSON + * stream, then each blob entry in order. + * + * Blob content is written under `stagingDir` as it streams — never the live + * upload store (KTD10) — with every entry validated per the module doc + * comment's KTD11 choke point. + * + * The caller MUST fully drain the `"db"` event's stream before requesting + * the next event: tar's sequential framing means the reader can't advance to + * the next entry until the current one has been read to completion. + */ +export async function* readBundle( + decryptStream: Transform, + options: ReadBundleOptions, +): AsyncGenerator { + const extract = tar.extract(); + decryptStream.pipe(extract); + decryptStream.on("error", (err) => extract.destroy(toError(err))); + + const stagingRoot = resolve(options.stagingDir); + + let manifest: BackupManifest | undefined; + let entryCount = 0; + let blobsSeen = 0; + let blobBytesSeen = 0; + + for await (const entry of extract) { + entryCount++; + const name = entry.header.name; + + if (entryCount === 1) { + if (name !== MANIFEST_ENTRY_NAME) { + entry.resume(); + throw new BundleFormatError( + `expected first bundle entry "${MANIFEST_ENTRY_NAME}", got "${name}"`, + ); + } + const raw = await bufferBoundedEntry( + entry, + MAX_MANIFEST_BYTES, + MANIFEST_ENTRY_NAME, + ); + manifest = parseManifest(raw); + yield { kind: "manifest", manifest }; + continue; + } + + if (entryCount === 2) { + if (name !== DB_ENTRY_NAME) { + entry.resume(); + throw new BundleFormatError( + `expected second bundle entry "${DB_ENTRY_NAME}", got "${name}"`, + ); + } + yield { kind: "db", stream: entry }; + continue; + } + + // Every entry from here on must be a validated blob (KTD11). + if (!manifest) { + entry.resume(); + throw new BundleFormatError( + "internal: manifest missing before blob entries", + ); + } + if (!name.startsWith(BLOB_ENTRY_PREFIX)) { + entry.resume(); + throw new BundleFormatError( + `unexpected bundle entry outside "${BLOB_ENTRY_PREFIX}": "${name}"`, + ); + } + + const storageKey = name.slice(BLOB_ENTRY_PREFIX.length); + + blobsSeen++; + if (blobsSeen > manifest.counts.blobs) { + entry.resume(); + throw new BundleIntegrityError( + `bundle manifest declares ${manifest.counts.blobs} blob(s) but the bundle contains more`, + ); + } + + if (entry.header.type !== "file") { + entry.resume(); + throw new UnsafeBundleEntryError( + `refusing non-regular-file bundle entry (type "${entry.header.type}"): "${storageKey}"`, + ); + } + + const destPath = resolveStagingBlobPath(stagingRoot, storageKey); + + const declaredSize = entry.header.size ?? 0; + if (declaredSize > manifest.counts.totalBlobBytes) { + entry.resume(); + throw new BundleIntegrityError( + `blob entry "${storageKey}" declares ${declaredSize} bytes, exceeding the manifest's total blob-bytes bound (${manifest.counts.totalBlobBytes})`, + ); + } + + await mkdir(dirname(destPath), { recursive: true, mode: 0o700 }); + const remainingBudget = manifest.counts.totalBlobBytes - blobBytesSeen; + const writtenSize = await writeEntryToFile( + entry, + destPath, + remainingBudget, + ); + blobBytesSeen += writtenSize; + + yield { kind: "blob", storageKey, path: destPath, size: writtenSize }; + } + + if (entryCount === 0) { + throw new BundleFormatError("bundle is empty: no entries found"); + } + if (entryCount === 1) { + throw new BundleFormatError( + `bundle ended after "${MANIFEST_ENTRY_NAME}": missing "${DB_ENTRY_NAME}"`, + ); + } + if (manifest && blobsSeen !== manifest.counts.blobs) { + throw new BundleIntegrityError( + `bundle manifest declares ${manifest.counts.blobs} blob(s) but the bundle contained ${blobsSeen}`, + ); + } +} + +/** Buffers a small, bounded entry (the manifest) — refuses to buffer past `maxBytes`. */ +async function bufferBoundedEntry( + stream: Readable, + maxBytes: number, + label: string, +): Promise { + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of stream) { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buf.byteLength; + if (total > maxBytes) { + stream.destroy(); + throw new BundleIntegrityError( + `${label} exceeds the ${maxBytes}-byte safety cap`, + ); + } + chunks.push(buf); + } + return Buffer.concat(chunks); +} diff --git a/src/backup/manifest.ts b/src/backup/manifest.ts new file mode 100644 index 00000000..a48ff461 --- /dev/null +++ b/src/backup/manifest.ts @@ -0,0 +1,193 @@ +/** + * Backup bundle manifest (plan Unit U2, R2/R4). + * + * The manifest is the bundle's first tar entry (`manifest.json`) — it stamps + * the bundle with the compatibility key restore checks before touching any + * data (R8, KTD4), plus informational version/timestamp fields and the + * row/blob counts `bundle.ts` uses to bound untrusted bundle contents during + * restore (KTD11). + */ + +import packageJson from "../../package.json"; +import migrationsJournal from "../db/migrations/meta/_journal.json"; + +/** + * Compatibility key restore checks against the running instance's own + * `BACKUP_FORMAT_VERSION` (R8, KTD4). Bump this **by hand**, and only when a + * schema change would make an older bundle's NDJSON import fail or silently + * drop data against the new schema (a dropped/renamed column, a new + * NOT-NULL column without a default, a type change). Routine additive + * migrations do NOT bump this — the migration tag advances on nearly every + * feature PR, so gating on it directly would refuse an operator restoring + * their own backup onto the same instance after an ordinary release. Owned + * here (U2) since U3 (db export/import) and U5 (restore service) both + * consume it — import from here rather than redefining it. + */ +export const BACKUP_FORMAT_VERSION = 1; + +interface MigrationJournalEntry { + readonly tag: string; +} + +interface MigrationJournal { + readonly entries: readonly MigrationJournalEntry[]; +} + +const journal = migrationsJournal as MigrationJournal; + +/** Row/blob counts a bundle declares, used by `bundle.ts` to bound untrusted contents during restore (KTD11). */ +export interface BackupManifestCounts { + /** Total database rows across every exported table. */ + readonly rows: number; + /** Total document blob entries. */ + readonly blobs: number; + /** Sum of every blob's byte size. */ + readonly totalBlobBytes: number; +} + +/** The bundle manifest — always the first entry in the tar (`manifest.json`). */ +export interface BackupManifest { + readonly backupFormatVersion: number; + readonly appVersion: string; + readonly migrationTag: string; + /** ISO 8601 timestamp. */ + readonly createdAt: string; + readonly counts: BackupManifestCounts; +} + +/** Thrown when a manifest buffer/string is not well-formed JSON or fails shape validation. Manifests are untrusted input on restore (KTD11). */ +export class InvalidManifestError extends Error { + constructor(message: string) { + super(message); + this.name = "InvalidManifestError"; + } +} + +/** The running instance's app version, read from `package.json` (R4). */ +export function currentAppVersion(): string { + return packageJson.version; +} + +/** The latest applied migration tag, read from the Drizzle migration journal (R4, informational — not the compatibility key, see {@link BACKUP_FORMAT_VERSION}). */ +export function latestMigrationTag(): string { + const entries = journal.entries; + if (entries.length === 0) { + throw new Error( + "migration journal has no entries: cannot determine the latest migration tag", + ); + } + const last = entries[entries.length - 1]; + if (!last) { + throw new Error("migration journal entry is unexpectedly undefined"); + } + return last.tag; +} + +export interface BuildManifestInput { + readonly counts: BackupManifestCounts; + /** Defaults to now; override in tests for deterministic timestamps. */ + readonly createdAt?: Date; +} + +/** Builds a manifest stamped with the running instance's versions (R4). */ +export function buildManifest(input: BuildManifestInput): BackupManifest { + return { + backupFormatVersion: BACKUP_FORMAT_VERSION, + appVersion: currentAppVersion(), + migrationTag: latestMigrationTag(), + createdAt: (input.createdAt ?? new Date()).toISOString(), + counts: input.counts, + }; +} + +/** Serializes a manifest to its `manifest.json` tar-entry bytes. */ +export function serializeManifest(manifest: BackupManifest): Buffer { + return Buffer.from(JSON.stringify(manifest), "utf8"); +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +/** + * Parses and validates a manifest from raw `manifest.json` bytes. The + * manifest is untrusted input on restore (KTD11: "a bundle is + * attacker-influenceable") — every field is checked before being trusted by + * the rest of the restore pipeline. + */ +export function parseManifest(raw: Buffer | string): BackupManifest { + let parsed: unknown; + try { + parsed = JSON.parse(typeof raw === "string" ? raw : raw.toString("utf8")); + } catch (err) { + throw new InvalidManifestError( + `manifest.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + if (typeof parsed !== "object" || parsed === null) { + throw new InvalidManifestError("manifest.json must be a JSON object"); + } + const obj = parsed as Record; + + if (!isNonNegativeInteger(obj.backupFormatVersion)) { + throw new InvalidManifestError( + "manifest.backupFormatVersion must be a non-negative integer", + ); + } + if (!isNonEmptyString(obj.appVersion)) { + throw new InvalidManifestError( + "manifest.appVersion must be a non-empty string", + ); + } + if (!isNonEmptyString(obj.migrationTag)) { + throw new InvalidManifestError( + "manifest.migrationTag must be a non-empty string", + ); + } + if ( + !isNonEmptyString(obj.createdAt) || + Number.isNaN(Date.parse(obj.createdAt)) + ) { + throw new InvalidManifestError( + "manifest.createdAt must be a valid ISO 8601 timestamp string", + ); + } + + const counts = obj.counts; + if (typeof counts !== "object" || counts === null) { + throw new InvalidManifestError("manifest.counts must be an object"); + } + const countsObj = counts as Record; + if (!isNonNegativeInteger(countsObj.rows)) { + throw new InvalidManifestError( + "manifest.counts.rows must be a non-negative integer", + ); + } + if (!isNonNegativeInteger(countsObj.blobs)) { + throw new InvalidManifestError( + "manifest.counts.blobs must be a non-negative integer", + ); + } + if (!isNonNegativeInteger(countsObj.totalBlobBytes)) { + throw new InvalidManifestError( + "manifest.counts.totalBlobBytes must be a non-negative integer", + ); + } + + return { + backupFormatVersion: obj.backupFormatVersion, + appVersion: obj.appVersion, + migrationTag: obj.migrationTag, + createdAt: obj.createdAt, + counts: { + rows: countsObj.rows, + blobs: countsObj.blobs, + totalBlobBytes: countsObj.totalBlobBytes, + }, + }; +} From 9f593c2a3a8e873fd981fd48e3fb486768671257 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 18:05:36 -0400 Subject: [PATCH 07/26] feat(backup): backup export service (U4) Signed-off-by: UncleSp1d3r --- src/backup/__tests__/export-service.test.ts | 336 ++++++++++++++++++++ src/backup/export-service.ts | 169 ++++++++++ 2 files changed, 505 insertions(+) create mode 100644 src/backup/__tests__/export-service.test.ts create mode 100644 src/backup/export-service.ts diff --git a/src/backup/__tests__/export-service.test.ts b/src/backup/__tests__/export-service.test.ts new file mode 100644 index 00000000..5ba76d2e --- /dev/null +++ b/src/backup/__tests__/export-service.test.ts @@ -0,0 +1,336 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +// `storage` is a lazily-constructed singleton shared across the whole test +// process (module state, not per-file): whichever test file's `storage`/ +// `activeStorageRoot()` access happens first "wins" `UPLOAD_DIR` for every +// file in the run, so this only takes effect when this file is the first to +// touch storage (mirrors `src/domain/firearm-documents/__tests__/serving.test.ts`). +// Every test below therefore resolves the *actual* active root via +// `activeStorageRoot()` rather than trusting this constant, so the suite is +// correct regardless of file execution order. +process.env.UPLOAD_DIR = mkdtempSync(join(tmpdir(), "export-service-uploads-")); + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + mock, + test, +} from "bun:test"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import { NotAuthorizedError } from "@/src/auth/errors"; +import { activeStorageRoot } from "@/src/storage/index"; +import { expectRejects } from "@/src/test-support/assertions"; +import * as schema from "../../db/schema"; +import { firearm, user } from "../../db/schema"; +import type { BundleEvent } from "../bundle"; +import { readBundle } from "../bundle"; +import { + createDecryptStream, + deriveKey, + HEADER_BYTE_LENGTH, + readHeader, +} from "../crypto"; +import { wipeDatabase } from "../db-import"; +import { createBackup } from "../export-service"; +import { + BACKUP_FORMAT_VERSION, + type BackupManifest, + latestMigrationTag, +} from "../manifest"; + +/** + * Controls the mocked `isAdmin()` result for the "non-admin caller" test — + * a mutable holder lets each test drive the outcome without re-registering + * the module mock (mirrors `serving.test.ts`'s `currentUserId` pattern). + */ +let currentIsAdmin = true; +mock.module("@/src/auth/session", () => ({ + isAdmin: async () => currentIsAdmin, +})); + +const PASSWORD = "correct horse battery staple"; + +// Same pinned image as `db-roundtrip.test.ts` / `e2e/start-test-server.ts` +// (AWS ECR Public mirror — avoids Docker Hub's unauthenticated per-IP pull +// limit on shared runners). +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +type Db = NodePgDatabase; + +/** Drains a Readable into one Buffer, tracking chunking behavior along the way (R13 evidence). */ +async function collectWithStats(stream: Readable): Promise<{ + buffer: Buffer; + maxChunkBytes: number; + chunkCount: number; +}> { + const chunks: Buffer[] = []; + let maxChunkBytes = 0; + let chunkCount = 0; + for await (const chunk of stream) { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + chunks.push(buf); + maxChunkBytes = Math.max(maxChunkBytes, buf.byteLength); + chunkCount += 1; + } + return { buffer: Buffer.concat(chunks), maxChunkBytes, chunkCount }; +} + +/** Drains a Readable into one Buffer (no stats needed). */ +async function collect(stream: NodeJS.ReadableStream): Promise { + const parts: Buffer[] = []; + for await (const chunk of stream) { + parts.push(chunk as Buffer); + } + return Buffer.concat(parts); +} + +interface CapturedEvents { + manifest: BackupManifest | undefined; + dbLines: string[]; + blobEvents: Array<{ storageKey: string; path: string; size: number }>; +} + +/** + * The real restore-side read path: extract the salt from the bundle's own + * unencrypted header, derive the key from the operator's password, then + * decrypt and un-tar (U1/U2's read path) — exactly as a real restore would, + * since `createBackup` only takes a password, never handing the caller a + * salt/key directly. + */ +async function decryptAndCollect( + encrypted: Buffer, + password: string, + stagingDir: string, +): Promise { + const header = readHeader(encrypted.subarray(0, HEADER_BYTE_LENGTH)); + const key = deriveKey(password, header.salt); + const decryptStream = createDecryptStream(key); + Readable.from([encrypted]).pipe(decryptStream); + + const result: CapturedEvents = { + manifest: undefined, + dbLines: [], + blobEvents: [], + }; + const generator = readBundle(decryptStream, { stagingDir }); + for await (const event of generator as AsyncGenerator< + BundleEvent, + void, + void + >) { + if (event.kind === "manifest") { + result.manifest = event.manifest; + } else if (event.kind === "db") { + const buf = await collect(event.stream); + result.dbLines = buf + .toString("utf8") + .split("\n") + .filter((line) => line.trim() !== ""); + } else { + result.blobEvents.push({ + storageKey: event.storageKey, + path: event.path, + size: event.size, + }); + } + } + return result; +} + +/** + * Always resolves the singleton's *actual* root — see the top-of-file note on + * why this must not be assumed to equal the constant this file set + * `UPLOAD_DIR` to. Recreated on demand: whichever test file's directory won + * the singleton race may have already been torn down by that file's own + * `afterAll` by the time this file's tests run, so every access here is + * defensive about the directory having gone missing (mirrors + * `listUploadBlobs`'s own ENOENT tolerance in `export-service.ts`). + */ +function resolvedUploadDir(): string { + const dir = activeStorageRoot(); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function writeUploadBlob(name: string, content: Buffer): void { + writeFileSync(join(resolvedUploadDir(), name), content); +} + +function listUploadFiles(): string[] { + return readdirSync(resolvedUploadDir()); +} + +function clearUploadDir(): void { + const dir = resolvedUploadDir(); + for (const name of readdirSync(dir)) { + rmSync(join(dir, name), { force: true }); + } +} + +function makeStagingDir(): string { + return mkdtempSync(join(tmpdir(), "export-service-staging-")); +} + +describe("backup export service (U4)", () => { + let container: StartedPostgreSqlContainer; + let pool: Pool; + let db: Db; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_export_service_test") + .start(); + pool = new Pool({ connectionString: container.getConnectionUri() }); + db = drizzle(pool, { schema }); + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + }, 120_000); + + afterAll(async () => { + await pool?.end(); + await container?.stop(); + }); + + beforeEach(async () => { + await wipeDatabase(db); + clearUploadDir(); + }); + + afterEach(() => { + currentIsAdmin = true; + }); + + test("exports a bundle that decrypts to the manifest, all rows, and all blobs (AE5 round trip)", async () => { + const ownerId = `owner-${randomUUID()}`; + await db + .insert(user) + .values({ id: ownerId, name: "Owner", email: `${ownerId}@example.test` }); + const [firearmRow] = await db + .insert(firearm) + .values({ ownerId, name: "Export Test FA", caliber: "9mm" }) + .returning(); + + const blobs = [ + { key: `${randomUUID()}.bin`, content: randomBytes(1024) }, + { key: `${randomUUID()}.bin`, content: randomBytes(2048) }, + { key: `${randomUUID()}.bin`, content: Buffer.alloc(0) }, + ]; + for (const blob of blobs) { + writeUploadBlob(blob.key, blob.content); + } + + const stream = await createBackup(PASSWORD, { db }); + const { buffer } = await collectWithStats(stream); + + const stagingDir = makeStagingDir(); + try { + const events = await decryptAndCollect(buffer, PASSWORD, stagingDir); + + expect(events.manifest).toBeDefined(); + expect(events.manifest?.backupFormatVersion).toBe(BACKUP_FORMAT_VERSION); + expect(events.manifest?.migrationTag).toBe(latestMigrationTag()); + expect(events.manifest?.counts.rows).toBe(events.dbLines.length); + expect(events.manifest?.counts.blobs).toBe(blobs.length); + + const parsedRows = events.dbLines.map((line) => JSON.parse(line)); + const exportedFirearmRow = parsedRows.find( + (row) => row.table === "firearm" && row.row.id === firearmRow.id, + ); + expect(exportedFirearmRow).toBeDefined(); + + expect(events.blobEvents).toHaveLength(blobs.length); + const blobsByKey = new Map(blobs.map((b) => [b.key, b.content])); + for (const blobEvent of events.blobEvents) { + const expectedContent = blobsByKey.get(blobEvent.storageKey); + expect(expectedContent).toBeDefined(); + if (!expectedContent) continue; + expect(blobEvent.size).toBe(expectedContent.byteLength); + const written = await Bun.file(blobEvent.path).arrayBuffer(); + expect(Buffer.from(written).equals(expectedContent)).toBe(true); + } + } finally { + rmSync(stagingDir, { recursive: true, force: true }); + } + }); + + test("a large blob set streams without buffering the whole bundle in memory (R13)", async () => { + const blobCount = 4; + const blobSize = 1_500_000; // 1.5 MiB each, ~6 MiB total + for (let i = 0; i < blobCount; i++) { + writeUploadBlob(`${randomUUID()}.bin`, randomBytes(blobSize)); + } + const totalPlaintextBytes = blobCount * blobSize; + + const stream = await createBackup(PASSWORD, { db }); + const { buffer, maxChunkBytes, chunkCount } = + await collectWithStats(stream); + + // The encrypted output is at least as large as the plaintext blobs (plus + // headers/tar framing/auth tags) — sanity check we actually captured the + // whole bundle before asserting on how it arrived. + expect(buffer.byteLength).toBeGreaterThan(totalPlaintextBytes); + + // Chunked delivery, not one whole-bundle buffer: many chunks arrived, and + // no single chunk carried anywhere near the full blob set (which a + // whole-buffer implementation would produce as one multi-megabyte chunk). + expect(chunkCount).toBeGreaterThan(20); + expect(maxChunkBytes).toBeLessThan(totalPlaintextBytes / 4); + }); + + test("the returned stream is consumable once and no bundle file remains on disk afterward", async () => { + writeUploadBlob(`${randomUUID()}.bin`, randomBytes(512)); + const beforeFiles = new Set(listUploadFiles()); + + const stream = await createBackup(PASSWORD, { db }); + await collectWithStats(stream); + + expect(stream.readableEnded).toBe(true); + + // Re-draining an already-ended Readable yields nothing further — proves + // the stream isn't secretly re-derivable/replayable. + let extraChunks = 0; + for await (const _chunk of stream) { + extraChunks += 1; + } + expect(extraChunks).toBe(0); + + // No bundle (or any other) file was left behind under UPLOAD_DIR — the + // only files there are the ones the test itself wrote (KTD8: nothing is + // retained server-side). + const afterFiles = new Set(listUploadFiles()); + expect(afterFiles).toEqual(beforeFiles); + }); + + test("a non-admin caller is rejected", async () => { + currentIsAdmin = false; + + await expectRejects(() => createBackup(PASSWORD, { db })); + + try { + await createBackup(PASSWORD, { db }); + throw new Error("expected createBackup to reject for a non-admin caller"); + } catch (error) { + expect(error).toBeInstanceOf(NotAuthorizedError); + } + }); +}); diff --git a/src/backup/export-service.ts b/src/backup/export-service.ts new file mode 100644 index 00000000..49a32255 --- /dev/null +++ b/src/backup/export-service.ts @@ -0,0 +1,169 @@ +/** + * Backup export service (plan Unit U4, R1/R2/R3/R4/R11/R13). + * + * Orchestrates a full instance export: an admin-authorized caller supplies a + * password, and this module streams the entire database (U3's NDJSON export) + * plus every document blob under `UPLOAD_DIR` into one authenticated, + * password-encrypted, version-stamped bundle (U2's tar format, piped through + * U1's crypto stream). The route that eventually serves this pipes the + * returned stream straight to the operator's browser download — no bundle is + * ever written to disk here (KTD8): `createBackup` only builds and returns a + * `Readable`. + */ + +import { createReadStream } from "node:fs"; +import { readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { NotAuthorizedError } from "@/src/auth/errors"; +import { isAdmin } from "@/src/auth/session"; +import type { DbOrTx } from "@/src/db/client"; +import { activeStorageRoot } from "@/src/storage/index"; +import { type BundleBlobEntry, writeBundle } from "./bundle"; +import { createEncryptStream, deriveKey, generateSalt } from "./crypto"; +import { exportDatabase } from "./db-export"; +import { buildManifest } from "./manifest"; + +/** + * Re-asserts the admin role inside the service itself — the future admin + * route (U6) also gates, but a backup export's blast radius (the whole + * instance, in the clear once decrypted) warrants defense-in-depth here too. + * Delegates to `isAdmin()` (`src/auth/session.ts`) rather than duplicating + * the role check. + */ +async function requireAdmin(): Promise { + if (!(await isAdmin())) { + throw new NotAuthorizedError("Only admins can export a backup"); + } +} + +/** One blob file discovered under `UPLOAD_DIR`, before it is streamed. */ +interface BlobFileInfo { + readonly storageKey: string; + readonly size: number; +} + +/** + * Lists every blob file directly under the upload root, mirroring the + * non-recursive, `node:fs`-direct scan `orphanSweep` uses + * (`src/storage/orphan-sweep.ts`) rather than going through the + * `StorageService` interface, which intentionally has no `list()` method + * (YAGNI — see that module's doc comment). A missing `UPLOAD_DIR` (fresh + * install, nothing uploaded yet) is not an error: there is simply nothing to + * bundle. + */ +async function listUploadBlobs(): Promise { + const uploadDir = activeStorageRoot(); + const entries = await readdir(uploadDir, { withFileTypes: true }).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }, + ); + const fileNames = entries.filter((entry) => entry.isFile()); + + const infos: BlobFileInfo[] = []; + for (const entry of fileNames) { + const info = await stat(join(uploadDir, entry.name)); + infos.push({ storageKey: entry.name, size: info.size }); + } + return infos; +} + +/** + * Lazily streams each listed blob's content from disk as the bundle writer + * consumes it — one file open/read at a time, never all of them buffered + * together (R13, KTD3). Sizes are the `stat` results already gathered by + * `listUploadBlobs`, matching what `writeBundle` requires up front per tar + * entry. + */ +async function* blobEntriesFor( + infos: readonly BlobFileInfo[], +): AsyncGenerator { + const uploadDir = activeStorageRoot(); + for (const info of infos) { + yield { + storageKey: info.storageKey, + size: info.size, + stream: createReadStream(join(uploadDir, info.storageKey)), + }; + } +} + +/** + * Buffers `exportDatabase`'s NDJSON output once so the row count is known + * before the manifest is built (the manifest must exist before `writeBundle` + * starts streaming — it is the bundle's first tar entry). This mirrors + * `bundle.ts`'s own documented exception: `db.ndjson` is JSON-per-row text, + * not the binary attachments R13/KTD3 are actually concerned with, and + * `writeBundle` buffers it internally regardless (it needs an exact byte + * length up front for the tar header). Buffering it once here — instead of + * once here and again inside `writeBundle` — would require re-plumbing + * `writeBundle`'s API, which U4 does not own; the double-buffer cost is a + * small NDJSON payload, not the large blob set R13 is about. + */ +async function bufferDbExport( + db: DbOrTx, +): Promise<{ text: string; rowCount: number }> { + const chunks: Buffer[] = []; + for await (const chunk of exportDatabase(db)) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const text = Buffer.concat(chunks).toString("utf8"); + const rowCount = text.split("\n").filter((line) => line.trim() !== "").length; + return { text, rowCount }; +} + +export interface CreateBackupOptions { + /** Drizzle handle to export from. Production callers pass the shared `db` + * from `@/src/db/client`; tests pass a handle bound to their own instance + * (e.g. a Testcontainers Postgres). */ + readonly db: DbOrTx; +} + +/** + * Exports the full instance as one password-encrypted backup stream (F1). + * + * 1. Re-asserts the caller is an admin (defense-in-depth, R14). + * 2. Builds the manifest — row/blob counts, `BACKUP_FORMAT_VERSION`, app + * version, and the latest migration tag (R4). + * 3. Derives a fresh-salt key from `password` (R3, R11). + * 4. Composes U3's NDJSON export and a lazy blob stream over `UPLOAD_DIR` + * into U2's tar bundle writer, piped through U1's encrypt stream. + * + * Returns the encrypted bundle stream; the caller (an admin route, U6) pipes + * it straight to the operator's download. Nothing is written server-side — + * no bundle file, no plaintext (KTD8, R13). + */ +export async function createBackup( + password: string, + options: CreateBackupOptions, +): Promise { + await requireAdmin(); + + const [{ text: dbText, rowCount }, blobInfos] = await Promise.all([ + bufferDbExport(options.db), + listUploadBlobs(), + ]); + const totalBlobBytes = blobInfos.reduce((sum, blob) => sum + blob.size, 0); + + const manifest = buildManifest({ + counts: { + rows: rowCount, + blobs: blobInfos.length, + totalBlobBytes, + }, + }); + + const salt = generateSalt(); + const key = deriveKey(password, salt); + + return writeBundle( + { + manifest, + dbStream: Readable.from([dbText]), + blobEntries: blobEntriesFor(blobInfos), + }, + createEncryptStream(key, salt), + ); +} From 839b5a80d256ff399f58c1a2e64f27e4e9476bcd Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 18:09:22 -0400 Subject: [PATCH 08/26] feat(backup): stage-then-promote restore service + maintenance (U5) Signed-off-by: UncleSp1d3r --- src/backup/__tests__/restore-service.test.ts | 642 +++++++++++++++++++ src/backup/crypto.ts | 87 +++ src/backup/maintenance.ts | 137 ++++ src/backup/restore-service.ts | 501 +++++++++++++++ 4 files changed, 1367 insertions(+) create mode 100644 src/backup/__tests__/restore-service.test.ts create mode 100644 src/backup/maintenance.ts create mode 100644 src/backup/restore-service.ts diff --git a/src/backup/__tests__/restore-service.test.ts b/src/backup/__tests__/restore-service.test.ts new file mode 100644 index 00000000..8382a77d --- /dev/null +++ b/src/backup/__tests__/restore-service.test.ts @@ -0,0 +1,642 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { getTableName } from "drizzle-orm"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import * as tar from "tar-stream"; +import { NotAuthorizedError } from "../../auth/errors"; +import * as schema from "../../db/schema"; +import { firearm, firearmDocument, user } from "../../db/schema"; +import { type BundleBlobEntry, writeBundle } from "../bundle"; +import { createEncryptStream, deriveKey, generateSalt } from "../crypto"; +import { exportDatabase } from "../db-export"; +import { importDatabase, wipeDatabase } from "../db-import"; +import { + BACKUP_FORMAT_VERSION, + type BackupManifest, + buildManifest, +} from "../manifest"; +import { restore } from "../restore-service"; +import { EXPORT_TABLE_ORDER } from "../table-order"; + +/** + * Integration tests for the restore service (U5) — the most safety-critical + * unit in the backup feature. Every test runs against an ephemeral + * Testcontainers Postgres (never the ambient dev DB) plus a per-test + * temporary "UPLOAD_DIR" on the real filesystem, since `restore()` performs + * real directory renames as part of its stage-then-promote sequence. + * + * Same pinned image as `db-roundtrip.test.ts` / `e2e/start-test-server.ts`. + */ +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +const PASSWORD = "correct horse battery staple"; + +type Db = NodePgDatabase; + +async function selectAll( + db: Db, + table: (typeof EXPORT_TABLE_ORDER)[number], +): Promise[]> { + // biome-ignore lint/suspicious/noExplicitAny: EXPORT_TABLE_ORDER is deliberately heterogeneous. + const rows = await db.select().from(table as any); + return rows as Record[]; +} + +async function snapshotTables( + db: Db, +): Promise[]>> { + const snapshot: Record[]> = {}; + for (const table of EXPORT_TABLE_ORDER) { + const rows = await selectAll(db, table); + snapshot[getTableName(table)] = rows.sort((a, b) => + JSON.stringify(a).localeCompare(JSON.stringify(b)), + ); + } + return snapshot; +} + +interface SeededInventory { + ownerId: string; + firearmId: string; + documentStorageKey: string; +} + +/** Seeds one user + one firearm + one document (with a real blob file under `uploadDir`). */ +async function seedInventory( + db: Db, + uploadDir: string, +): Promise { + const ownerId = `owner-${randomUUID()}`; + await db + .insert(user) + .values({ id: ownerId, name: "Owner", email: `${ownerId}@example.test` }); + + const [firearmRow] = await db + .insert(firearm) + .values({ ownerId, name: "Pre-existing FA", caliber: "9mm" }) + .returning(); + if (!firearmRow) throw new Error("seed: firearm insert returned no row"); + + const documentStorageKey = `${randomUUID()}.pdf`; + await db.insert(firearmDocument).values({ + firearmId: firearmRow.id, + storageKey: documentStorageKey, + filename: "receipt.pdf", + mimeType: "application/pdf", + sizeBytes: 11, + docType: "receipt", + }); + await writeFile(join(uploadDir, documentStorageKey), "pre-existing"); + + return { ownerId, firearmId: firearmRow.id, documentStorageKey }; +} + +async function countAllRows(db: Db): Promise { + let total = 0; + for (const table of EXPORT_TABLE_ORDER) { + total += (await selectAll(db, table)).length; + } + return total; +} + +interface FakeBlob { + readonly key: string; + readonly content: Buffer; +} + +/** Builds a real encrypted+authenticated bundle from `db`'s current state plus `blobs`. */ +async function buildEncryptedBundle( + db: Db, + options: { + password?: string; + blobs?: readonly FakeBlob[]; + backupFormatVersion?: number; + } = {}, +): Promise { + const blobs = options.blobs ?? []; + const totalBlobBytes = blobs.reduce( + (sum, b) => sum + b.content.byteLength, + 0, + ); + const baseManifest = buildManifest({ + counts: { + rows: await countAllRows(db), + blobs: blobs.length, + totalBlobBytes, + }, + }); + const manifest: BackupManifest = + options.backupFormatVersion === undefined + ? baseManifest + : { ...baseManifest, backupFormatVersion: options.backupFormatVersion }; + + const salt = generateSalt(); + const key = deriveKey(options.password ?? PASSWORD, salt); + const blobEntries: BundleBlobEntry[] = blobs.map((b) => ({ + storageKey: b.key, + size: b.content.byteLength, + stream: Readable.from([b.content]), + })); + + const encrypted = writeBundle( + { manifest, dbStream: exportDatabase(db), blobEntries }, + createEncryptStream(key, salt), + ); + return collect(encrypted); +} + +async function collect(stream: NodeJS.ReadableStream): Promise { + const parts: Buffer[] = []; + for await (const chunk of stream) { + parts.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(parts); +} + +/** + * Builds an incoming bundle from data that's genuinely DIFFERENT from `db`'s + * current live rows, then restores `db` back to its original live state (an + * exact NDJSON export/import round-trip). Without this, a fault-injection + * test that builds its bundle from `db`'s own unchanged state can't tell "the + * promote was rolled back" apart from "the promote succeeded, but happened to + * write back the same rows it started with" — the bundle must actually carry + * different data for a rollback assertion to mean anything. + */ +async function buildDistinctIncomingBundle( + db: Db, + blobKey: string, + blobContent: Buffer, +): Promise { + const originalNdjson = (await collect(exportDatabase(db))).toString("utf8"); + + await wipeDatabase(db); + const distinctOwner = `owner-${randomUUID()}`; + await db.insert(user).values({ + id: distinctOwner, + name: "Distinct Incoming Owner", + email: `${distinctOwner}@example.test`, + }); + const [distinctFirearm] = await db + .insert(firearm) + .values({ + ownerId: distinctOwner, + name: "Distinct Incoming FA", + caliber: "7.62", + }) + .returning(); + if (!distinctFirearm) throw new Error("seed failed"); + await db.insert(firearmDocument).values({ + firearmId: distinctFirearm.id, + storageKey: blobKey, + filename: "incoming.pdf", + mimeType: "application/pdf", + sizeBytes: blobContent.byteLength, + docType: "receipt", + }); + + const bundle = await buildEncryptedBundle(db, { + blobs: [{ key: blobKey, content: blobContent }], + }); + + await wipeDatabase(db); + await importDatabase(db, Readable.from([originalNdjson])); + + return bundle; +} + +/** Builds a raw (unencrypted) tar buffer directly, bypassing writeBundle, so a test can inject a malicious entry — then encrypts it under a real key so it authenticates. */ +async function buildRawTarThenEncrypt( + entries: ReadonlyArray<{ + name: string; + type?: "file" | "symlink"; + data?: Buffer; + linkname?: string; + }>, + password: string, +): Promise { + const pack = tar.pack(); + const chunks: Buffer[] = []; + pack.on("data", (chunk: Buffer) => chunks.push(chunk)); + const raw = await new Promise((resolvePromise, rejectPromise) => { + pack.on("end", () => resolvePromise(Buffer.concat(chunks))); + pack.on("error", rejectPromise); + (async () => { + for (const e of entries) { + await new Promise((res, rej) => { + if (e.type === "symlink") { + pack.entry( + { + name: e.name, + type: "symlink", + linkname: e.linkname ?? "elsewhere", + }, + (err) => (err ? rej(err) : res()), + ); + } else { + pack.entry( + { + name: e.name, + size: e.data ? e.data.byteLength : 0, + type: "file", + }, + e.data, + (err) => (err ? rej(err) : res()), + ); + } + }); + } + pack.finalize(); + })().catch(rejectPromise); + }); + + const salt = generateSalt(); + const key = deriveKey(password, salt); + const encrypted = collect( + Readable.from([raw]).pipe(createEncryptStream(key, salt)), + ); + return encrypted; +} + +/** Flips the last byte of `buf` — used to tamper the final secretstream chunk. */ +function tamperLastByte(buf: Buffer): Buffer { + const tampered = Buffer.from(buf); + tampered[tampered.length - 1] = (tampered[tampered.length - 1] ?? 0) ^ 0xff; + return tampered; +} + +async function readUploadDirKeys(uploadDir: string): Promise { + try { + return (await readdir(uploadDir)).sort(); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return []; + throw err; + } +} + +describe("restore service (U5)", () => { + let container: StartedPostgreSqlContainer; + let pool: Pool; + let db: Db; + let uploadDir: string; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_restore_test") + .start(); + pool = new Pool({ connectionString: container.getConnectionUri() }); + db = drizzle(pool, { schema }); + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + }, 120_000); + + afterAll(async () => { + await pool?.end(); + await container?.stop(); + }); + + beforeEach(async () => { + await wipeDatabase(db); + uploadDir = await mkdtemp(join(tmpdir(), "magstacker-restore-upload-")); + }); + + afterEach(async () => { + await rm(uploadDir, { recursive: true, force: true }).catch(() => {}); + }); + + function restoreOptions(extra: Parameters[2] = {}) { + return { + db, + pool, + uploadDir, + checkIsAdmin: async () => true, + ...extra, + }; + } + + test("empty instance + valid bundle + matching version promotes; the instance ends up equal to the source (AE5/R10)", async () => { + const sourceOwner = `owner-${randomUUID()}`; + await db.insert(user).values({ + id: sourceOwner, + name: "Source", + email: `${sourceOwner}@example.test`, + }); + const [sourceFirearm] = await db + .insert(firearm) + .values({ ownerId: sourceOwner, name: "Source FA", caliber: ".223" }) + .returning(); + if (!sourceFirearm) throw new Error("seed failed"); + const docKey = `${randomUUID()}.pdf`; + await db.insert(firearmDocument).values({ + firearmId: sourceFirearm.id, + storageKey: docKey, + filename: "warranty.pdf", + mimeType: "application/pdf", + sizeBytes: 5, + docType: "warranty", + }); + + const blobContent = Buffer.from("hello"); + const bundle = await buildEncryptedBundle(db, { + blobs: [{ key: docKey, content: blobContent }], + }); + const beforeSnapshot = await snapshotTables(db); + + // Wipe live data — restore is expected to reproduce it exactly (F2 into + // an empty instance), not merely leave it unchanged. + await wipeDatabase(db); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions(), + ); + + expect(outcome.kind).toBe("ok"); + const afterSnapshot = await snapshotTables(db); + expect(afterSnapshot).toEqual(beforeSnapshot); + + const files = await readUploadDirKeys(uploadDir); + expect(files).toEqual([docKey]); + const restoredBlob = await readFile(join(uploadDir, docKey)); + expect(restoredBlob.equals(blobContent)).toBe(true); + }); + + test("non-empty instance + plain restore is refused; nothing changes (AE1/R6)", async () => { + const seeded = await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + + const bundle = await buildEncryptedBundle(db, { + blobs: [{ key: "unrelated.bin", content: Buffer.from("x") }], + }); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions(), + ); + + expect(outcome.kind).toBe("refused_not_empty"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + void seeded; + }); + + test("non-empty instance + force restore wipes then promotes (AE2/R7)", async () => { + await seedInventory(db, uploadDir); + + const newOwner = `owner-${randomUUID()}`; + const sourceDb = db; // build the incoming bundle from a distinct dataset + await wipeDatabase(sourceDb); + await sourceDb.insert(user).values({ + id: newOwner, + name: "New Owner", + email: `${newOwner}@example.test`, + }); + const [newFirearm] = await sourceDb + .insert(firearm) + .values({ ownerId: newOwner, name: "New FA", caliber: "5.56" }) + .returning(); + if (!newFirearm) throw new Error("seed failed"); + const newDocKey = `${randomUUID()}.pdf`; + await sourceDb.insert(firearmDocument).values({ + firearmId: newFirearm.id, + storageKey: newDocKey, + filename: "new-doc.pdf", + mimeType: "application/pdf", + sizeBytes: 3, + docType: "receipt", + }); + const newBlobContent = Buffer.from("new"); + const bundle = await buildEncryptedBundle(sourceDb, { + blobs: [{ key: newDocKey, content: newBlobContent }], + }); + const expectedSnapshot = await snapshotTables(sourceDb); + + // Re-seed the "current" (about-to-be-overwritten) live instance and its + // upload dir with DIFFERENT data than the bundle carries. + await wipeDatabase(db); + await rm(uploadDir, { recursive: true, force: true }); + await mkdir(uploadDir, { recursive: true }); + const preExisting = await seedInventory(db, uploadDir); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions({ force: true }), + ); + + expect(outcome.kind).toBe("ok"); + expect(await snapshotTables(db)).toEqual(expectedSnapshot); + const files = await readUploadDirKeys(uploadDir); + expect(files).toEqual([newDocKey]); + expect(files).not.toContain(preExisting.documentStorageKey); + }); + + test("wrong password is refused; live data untouched (AE3/R9)", async () => { + await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + + const bundle = await buildEncryptedBundle(db, { + password: "the-real-password", + }); + + const outcome = await restore( + Readable.from([bundle]), + "definitely-wrong-password", + restoreOptions(), + ); + + expect(outcome.kind).toBe("wrong_password_or_tampered"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + }); + + test("a tampered byte in the LAST chunk is refused before promote — proves stage-then-promote (KTD10)", async () => { + const beforeSnapshot = await snapshotTables(db); // empty instance + // A blob large enough to force multiple secretstream chunks, so the + // "last chunk" is meaningfully distinct from the header/first chunk. + const bigBlob = Buffer.alloc(150_000, 0xab); + const bundle = await buildEncryptedBundle(db, { + blobs: [{ key: "big.bin", content: bigBlob }], + }); + const tampered = tamperLastByte(bundle); + + const outcome = await restore( + Readable.from([tampered]), + PASSWORD, + restoreOptions(), + ); + + expect(outcome.kind).toBe("wrong_password_or_tampered"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual([]); + }); + + test("version mismatch is refused before staging (AE4/R8)", async () => { + await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + + const bundle = await buildEncryptedBundle(db, { + backupFormatVersion: BACKUP_FORMAT_VERSION + 1, + }); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions(), + ); + + expect(outcome.kind).toBe("version_mismatch"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + }); + + test("fault injected inside the wipe+promote transaction rolls back both DB and blobs (critical path, pre-commit)", async () => { + const preExisting = await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + const beforeBlobContent = await readFile( + join(uploadDir, preExisting.documentStorageKey), + ); + + const bundle = await buildDistinctIncomingBundle( + db, + "incoming.bin", + Buffer.from("incoming"), + ); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions({ + force: true, + _testFaultInjection: (point) => { + if (point === "pre-commit") { + throw new Error("injected fault: pre-commit"); + } + }, + }), + ); + + expect(outcome.kind).toBe("rolled_back"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + expect( + (await readFile(join(uploadDir, preExisting.documentStorageKey))).equals( + beforeBlobContent, + ), + ).toBe(true); + }); + + test("fault injected after the DB commits but before the blob swap rolls back both stores via the snapshot schema (critical path, post-commit)", async () => { + const preExisting = await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + const beforeBlobContent = await readFile( + join(uploadDir, preExisting.documentStorageKey), + ); + + const bundle = await buildDistinctIncomingBundle( + db, + "incoming.bin", + Buffer.from("incoming"), + ); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions({ + force: true, + _testFaultInjection: (point) => { + if (point === "post-commit-pre-blob-swap") { + throw new Error("injected fault: post-commit-pre-blob-swap"); + } + }, + }), + ); + + expect(outcome.kind).toBe("rolled_back"); + // The DB side had already committed the new data when the fault hit — + // proving this assertion requires the snapshot-schema restore path, not + // just Postgres's own transaction rollback. + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + expect( + (await readFile(join(uploadDir, preExisting.documentStorageKey))).equals( + beforeBlobContent, + ), + ).toBe(true); + }); + + test("a path-traversal blob entry is refused; nothing is written outside staging (KTD11)", async () => { + const beforeSnapshot = await snapshotTables(db); + const manifest = buildManifest({ + counts: { rows: 0, blobs: 1, totalBlobBytes: 10 }, + }); + const bundle = await buildRawTarThenEncrypt( + [ + { name: "manifest.json", data: Buffer.from(JSON.stringify(manifest)) }, + { name: "db.ndjson", data: Buffer.alloc(0) }, + { name: "blobs/../../etc/pwned", data: Buffer.from("pwned") }, + ], + PASSWORD, + ); + + const outcome = await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions(), + ); + + expect(outcome.kind).toBe("wrong_password_or_tampered"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual([]); + }); + + test("a non-admin caller is refused inside the service, independent of any route gate (R14)", async () => { + const beforeSnapshot = await snapshotTables(db); + const bundle = await buildEncryptedBundle(db, {}); + + let thrown: unknown; + try { + await restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions({ checkIsAdmin: async () => false }), + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(NotAuthorizedError); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + }); +}); diff --git a/src/backup/crypto.ts b/src/backup/crypto.ts index 99ef81e9..e0af09a9 100644 --- a/src/backup/crypto.ts +++ b/src/backup/crypto.ts @@ -386,6 +386,93 @@ export function createEncryptStream( * authenticated final chunk — in every failure case, no unauthenticated * plaintext is ever pushed downstream. */ +/** + * Builds a `Transform` that decrypts a MagStacker backup bundle stream from a + * `password` alone (U5's restore entry point). {@link createDecryptStream} + * needs the derived key up front, but the key can only be derived from the + * salt and KDF params that live inside the bundle's own unencrypted + * {@link CryptoHeader} preamble — a chicken-and-egg problem solved here by + * peeking the first {@link HEADER_BYTE_LENGTH} bytes off the stream first. + * + * Once enough bytes have arrived, this parses the header, derives the key via + * {@link deriveKey}, builds the real {@link createDecryptStream}, and + * re-prepends the buffered header bytes to it — so nothing is lost. From the + * caller's side this behaves exactly like {@link createDecryptStream}: pipe + * the raw encrypted stream in, read decrypted plaintext out. + * + * Throws {@link InvalidHeaderError} if the stream ends before a complete + * header is read, and {@link DecryptionAuthError} for a wrong password or a + * tampered/truncated bundle — including, for a tamper in the final chunk, + * only once the whole stream has been consumed (see + * {@link createDecryptStream}'s doc comment; this is what lets a + * stage-then-promote restore catch a late tamper before anything live is + * touched). + */ +export function createDecryptStreamFromPassword(password: string): Transform { + const headerAcc = new ByteAccumulator(); + let inner: Transform | undefined; + + const outer: Transform = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + if (inner) { + inner.write(chunk, (err) => callback(err ? toError(err) : undefined)); + return; + } + + headerAcc.push(chunk); + if (headerAcc.size < HEADER_BYTE_LENGTH) { + callback(); + return; + } + + // May include ciphertext bytes beyond the header if this write + // delivered more than HEADER_BYTE_LENGTH bytes at once — that's fine, + // `readHeader` only reads its fixed-length prefix, and the whole + // buffer (header + any trailing ciphertext) is exactly what a raw + // stream into `createDecryptStream` looks like, so it's forwarded + // whole below. + const combined = headerAcc.takeAll(); + const header = readHeader(combined); + const key = deriveKey(password, header.salt, header.kdfParams); + inner = createDecryptStream(key); + inner.on("data", (data: Buffer) => outer.push(data)); + // Errors already surface through the write/flush callback chain + // below (which destroys `outer` the normal way); this listener only + // exists so an unhandled 'error' event on `inner` doesn't crash the + // process. + inner.on("error", () => {}); + + inner.write(combined, (err) => + callback(err ? toError(err) : undefined), + ); + } catch (err) { + callback(toError(err)); + } + }, + flush(callback) { + try { + if (!inner) { + // Never reached a full header. This always throws + // InvalidHeaderError (buffer too short) — the explicit throw below + // is an unreachable safety net in case that ever changes. + readHeader(headerAcc.takeAll()); + throw new InvalidHeaderError( + "bundle stream ended before a complete crypto header was read", + ); + } + inner.end((err?: Error | null) => + callback(err ? toError(err) : undefined), + ); + } catch (err) { + callback(toError(err)); + } + }, + }); + + return outer; +} + export function createDecryptStream(key: Buffer): Transform { if (key.byteLength !== SECRETSTREAM_KEY_BYTES) { throw new RangeError(`key must be ${SECRETSTREAM_KEY_BYTES} bytes`); diff --git a/src/backup/maintenance.ts b/src/backup/maintenance.ts new file mode 100644 index 00000000..a033941b --- /dev/null +++ b/src/backup/maintenance.ts @@ -0,0 +1,137 @@ +/** + * Force-restore maintenance envelope (plan Unit U5, KTD5): a durable, + * crash-recoverable "restore in progress" flag plus a pool-safe advisory + * lock, both scoped OUTSIDE the `public` schema so they never show up as + * live application tables (and so U3's `db-roundtrip.test.ts` regression + * guard — which asserts `EXPORT_TABLE_ORDER` + `EPHEMERAL_TABLE_NAMES` cover + * every `public`-schema table exactly — stays green without needing to know + * about restore's own bookkeeping). + * + * **Durable flag.** A single-row table (`restore_ops.maintenance_flag`) + * rather than an in-memory flag: the flag must survive a process restart so + * a crash mid-force-restore is still visible afterward. Crash-recovery + * contract (consumed by future tooling, not built here — U5 only owns the + * flag primitive): `active = true` together with a still-present + * `restore_snapshot` schema (see `restore-service.ts`) signals an + * interrupted force-restore; `active = true` with no snapshot schema means + * the crash happened before the risky section began and nothing live was + * touched. + * + * **Pool-safe advisory lock.** `withRestoreAdvisoryLock` holds ONE + * `pool.connect()`-checked-out client for its entire duration and issues a + * *session*-scoped `pg_advisory_lock`/`pg_advisory_unlock` pair on that same + * connection, explicitly unlocking before releasing it back to the pool. + * A session-scoped lock acquired through the *shared* pool (i.e. letting the + * pool hand out whichever connection is free for the lock call, then a + * different one for later statements) would be unsafe — the lock would be + * held by a connection this code no longer has a handle on, and releasing + * that connection back to the pool without unlocking would leak the lock for + * the connection's lifetime. Dedicating and holding a single connection for + * the whole envelope avoids that. + */ + +import { sql } from "drizzle-orm"; +import type { Pool, PoolClient } from "pg"; +import type { DbOrTx } from "@/src/db/client"; + +const MAINTENANCE_SCHEMA = "restore_ops"; +const MAINTENANCE_TABLE = "maintenance_flag"; + +/** + * Fixed application-specific advisory-lock key for the force-restore + * envelope. Arbitrary but must stay stable and must not collide with any + * other advisory lock key introduced elsewhere in the app (none exist yet). + */ +const RESTORE_ADVISORY_LOCK_KEY = 847_362_910_123; + +function qualified(schemaName: string, name: string) { + return sql`${sql.identifier(schemaName)}.${sql.identifier(name)}`; +} + +/** + * Creates the maintenance schema/table/singleton-row if they don't already + * exist. Idempotent and cheap (`IF NOT EXISTS` / `ON CONFLICT DO NOTHING`) — + * safe to call before every read or write of the flag rather than requiring + * a separate migration step. + */ +async function ensureMaintenanceInfrastructure(db: DbOrTx): Promise { + await db.execute( + sql`CREATE SCHEMA IF NOT EXISTS ${sql.identifier(MAINTENANCE_SCHEMA)}`, + ); + await db.execute(sql` + CREATE TABLE IF NOT EXISTS ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} ( + id boolean PRIMARY KEY DEFAULT true, + active boolean NOT NULL DEFAULT false, + reason text, + started_at timestamptz, + CONSTRAINT maintenance_flag_singleton CHECK (id) + ) + `); + await db.execute(sql` + INSERT INTO ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} (id, active) + VALUES (true, false) + ON CONFLICT (id) DO NOTHING + `); +} + +/** True while a force-restore's write-blocking envelope is in progress. */ +export async function isMaintenanceActive(db: DbOrTx): Promise { + await ensureMaintenanceInfrastructure(db); + const result = await db.execute<{ active: boolean }>(sql` + SELECT active FROM ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + WHERE id = true + `); + return result.rows[0]?.active ?? false; +} + +/** Sets the durable flag active. Must be called before the risky section of a force-restore begins. */ +export async function enterMaintenance( + db: DbOrTx, + reason: string, +): Promise { + await ensureMaintenanceInfrastructure(db); + await db.execute(sql` + UPDATE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + SET active = true, reason = ${reason}, started_at = now() + WHERE id = true + `); +} + +/** Clears the durable flag. Always called from a `finally`, on both success and rollback. */ +export async function exitMaintenance(db: DbOrTx): Promise { + await ensureMaintenanceInfrastructure(db); + await db.execute(sql` + UPDATE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + SET active = false, reason = NULL, started_at = NULL + WHERE id = true + `); +} + +/** + * Runs `fn` with the force-restore advisory lock held on a single dedicated + * connection for `fn`'s entire duration (see module doc comment for why this + * must not be a session lock acquired through the shared pool at large). + * `fn` receives that same `PoolClient` so it can run its own + * transactions/queries on the identical connection without contending with + * itself for the lock. + */ +export async function withRestoreAdvisoryLock( + pool: Pool, + fn: (client: PoolClient) => Promise, +): Promise { + const client = await pool.connect(); + try { + await client.query("SELECT pg_advisory_lock($1)", [ + RESTORE_ADVISORY_LOCK_KEY, + ]); + try { + return await fn(client); + } finally { + await client.query("SELECT pg_advisory_unlock($1)", [ + RESTORE_ADVISORY_LOCK_KEY, + ]); + } + } finally { + client.release(); + } +} diff --git a/src/backup/restore-service.ts b/src/backup/restore-service.ts new file mode 100644 index 00000000..1df855a2 --- /dev/null +++ b/src/backup/restore-service.ts @@ -0,0 +1,501 @@ +/** + * Restore service (plan Unit U5, R5-R10, KTD4/KTD5/KTD10/KTD11). + * + * The most safety-critical unit in the backup feature: `restore()` NEVER + * touches live data until the entire uploaded bundle has authenticated + * end-to-end (stage-then-promote, KTD10). Concretely: + * + * 1. Re-assert admin (defense-in-depth — the route also gates this). + * 2. Decrypt the upload with `createDecryptStreamFromPassword` and drive + * `readBundle` over an isolated staging area: DB rows land in a Postgres + * `restore_staging` schema (via U3's `importDatabase`, redirected there + * with a `search_path` trick rather than a modified copy — U3 is + * consumed as-is); blobs land in a staging directory that `readBundle` + * itself path-validates (KTD11). A wrong password, a tampered byte + * ANYWHERE (including the last secretstream chunk), or a truncated + * stream throws before staging completes, or is only discovered once + * staging finishes (secretstream authenticates the final chunk at + * stream-end) — either way, nothing live has been touched yet. + * 3. The manifest's `backupFormatVersion` is checked the moment it's read + * (the manifest is always the bundle's first entry), before any DB rows + * or blobs are staged (R8/AE4). + * 4. Only once the whole bundle has authenticated does `restore()` check + * instance emptiness (R6/AE1) and, if empty (or `force`), promote staging + * to live. + * 5. A `force` restore additionally runs the KTD5 envelope (`maintenance.ts`): + * durable maintenance flag, pool-safe advisory lock, a committed + * `restore_snapshot` schema of the pre-restore DB, and the pre-restore + * blob directory moved aside — so a failure at ANY point in the + * wipe+promote step (including after the DB side has already committed, + * but before the blob directory has been swapped in) rolls both stores + * back together. + */ + +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rename, rm, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import type { Readable, Transform } from "node:stream"; +import { count, getTableName, sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { NotAuthorizedError } from "@/src/auth/errors"; +import { isAdmin } from "@/src/auth/session"; +import { + type Database, + db as defaultDb, + pool as defaultPool, + type Transaction, +} from "@/src/db/client"; +import * as schema from "@/src/db/schema"; +import { accessory, ammo, firearm, magazine } from "@/src/db/schema"; +import { activeStorageRoot } from "@/src/storage"; +import { readBundle } from "./bundle"; +import { createDecryptStreamFromPassword } from "./crypto"; +import { importDatabase } from "./db-import"; +import { + enterMaintenance, + exitMaintenance, + withRestoreAdvisoryLock, +} from "./maintenance"; +import { BACKUP_FORMAT_VERSION, type BackupManifest } from "./manifest"; +import { EXPORT_TABLE_ORDER, WIPE_TABLE_ORDER } from "./table-order"; + +/** Postgres schema DB rows are staged into before the whole bundle authenticates (KTD10). Recreated fresh on every restore attempt. */ +const STAGING_SCHEMA = "restore_staging"; + +/** Postgres schema a force-restore's pre-restore live data is copied into before the wipe (KTD5) — the DB-side rollback source if promote fails after committing. */ +const SNAPSHOT_SCHEMA = "restore_snapshot"; + +/** Discriminated outcome of a restore attempt. Every branch carries an operator-facing `message`; none of them throw for expected restore-flow refusals — only a genuine programming/authorization error (see `restore`'s admin check) throws. */ +export type RestoreOutcome = + | { readonly kind: "ok"; readonly message: string } + | { readonly kind: "refused_not_empty"; readonly message: string } + | { readonly kind: "wrong_password_or_tampered"; readonly message: string } + | { readonly kind: "version_mismatch"; readonly message: string } + | { readonly kind: "rolled_back"; readonly message: string }; + +/** Thrown internally when a force-restore's promote step fails after entering the risky section; `restore()` converts this into a `'rolled_back'` outcome. Not exported — callers observe it only via `RestoreOutcome`. */ +class RestoreRolledBackError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "RestoreRolledBackError"; + } +} + +export interface RestoreOptions { + /** Force-replace an already-populated instance (R7/F3). Defaults to false (refuse-unless-empty, R6/F2). */ + readonly force?: boolean; + /** DI seam for tests — defaults to the shared singleton (`src/db/client.ts`). */ + readonly db?: Database; + /** DI seam for tests — defaults to the shared singleton pool (`src/db/client.ts`). Must be the same pool `db` is bound to. */ + readonly pool?: Pool; + /** DI seam for tests — defaults to the shared storage root (`src/storage`). */ + readonly uploadDir?: string; + /** DI seam for tests — defaults to the real session-backed `isAdmin()`, which requires a Next.js request context `restore()` won't have outside a route/action. */ + readonly checkIsAdmin?: () => Promise; + /** + * Test-only fault-injection hook for the force-restore promote envelope. + * Never supplied in production. See `restore-service.test.ts`'s + * fault-injection tests for both trip points this can throw from: still + * inside the wipe+promote transaction (before it commits) and after it has + * committed but before the blob directory has been swapped in — the two + * distinct failure windows KTD5's snapshot exists to cover. + */ + readonly _testFaultInjection?: ( + point: "pre-commit" | "post-commit-pre-blob-swap", + ) => void | Promise; +} + +function qualified(schemaName: string, name: string) { + return sql`${sql.identifier(schemaName)}.${sql.identifier(name)}`; +} + +function toError(err: unknown): Error { + return err instanceof Error ? err : new Error(String(err)); +} + +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return false; + throw err; + } +} + +/** + * Restore the whole instance from an encrypted backup bundle (F2/F3). + * See the module doc comment for the full stage-then-promote sequence. + */ +export async function restore( + stream: Readable, + password: string, + options: RestoreOptions = {}, +): Promise { + const checkIsAdmin = options.checkIsAdmin ?? isAdmin; + if (!(await checkIsAdmin())) { + // Not a modeled RestoreOutcome: an unauthorized caller is a programming/ + // authorization error, not a legitimate restore-flow branch (R14, + // defense-in-depth — the route is expected to have already gated this). + throw new NotAuthorizedError("only an admin may restore a backup"); + } + + const db = options.db ?? defaultDb; + const pool = options.pool ?? defaultPool; + const uploadDir = options.uploadDir ?? activeStorageRoot(); + const force = options.force ?? false; + + await recreateStagingSchema(db); + const stagingBlobDir = await mkdtemp( + join(dirname(uploadDir), "restore-staging-"), + ); + + try { + const decryptStream = createDecryptStreamFromPassword(password); + stream.on("error", (err) => decryptStream.destroy(err)); + stream.pipe(decryptStream); + + try { + // Version and emptiness are both checked inside stageBundle, the + // moment the manifest (always the bundle's first entry) is read — + // before any row/blob is staged (R8/R6, AE4/AE1). Only once both pass + // does stageBundle continue on to actually stage the bundle's rows and + // blobs, so a full authentication check of the whole stream (including + // a tamper in the final chunk, KTD10) only happens for restores that + // could otherwise proceed. + await stageBundle(decryptStream, stagingBlobDir, pool, { + checkEmptiness: !force, + instanceHasInventoryData: () => instanceHasInventoryData(db), + }); + } catch (err) { + if (err instanceof VersionMismatchSignal) { + return { kind: "version_mismatch", message: err.message }; + } + if (err instanceof NotEmptySignal) { + return { kind: "refused_not_empty", message: err.message }; + } + return { + kind: "wrong_password_or_tampered", + message: `bundle failed to authenticate: ${toError(err).message}`, + }; + } + + try { + if (force) { + await forcePromote({ + db, + pool, + uploadDir, + stagingBlobDir, + testFaultInjection: options._testFaultInjection, + }); + } else { + await emptyInstancePromote({ db, uploadDir, stagingBlobDir }); + } + } catch (err) { + if (err instanceof RestoreRolledBackError) { + return { kind: "rolled_back", message: err.message }; + } + throw err; + } + + return { kind: "ok", message: "restore completed successfully" }; + } finally { + await db + .execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(STAGING_SCHEMA)} CASCADE`, + ) + .catch(() => {}); + await rm(stagingBlobDir, { recursive: true, force: true }).catch(() => {}); + } +} + +class VersionMismatchSignal extends Error {} +class NotEmptySignal extends Error {} + +/** (Re)creates an empty `restore_staging` schema with one table per `EXPORT_TABLE_ORDER` entry, structurally mirroring `public` (KTD10 staging area). */ +async function recreateStagingSchema(db: Database): Promise { + await db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(STAGING_SCHEMA)} CASCADE`, + ); + await db.execute(sql`CREATE SCHEMA ${sql.identifier(STAGING_SCHEMA)}`); + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await db.execute(sql` + CREATE TABLE ${qualified(STAGING_SCHEMA, name)} + (LIKE ${qualified("public", name)} INCLUDING ALL) + `); + } +} + +interface StageBundleOptions { + /** Whether to refuse a non-empty instance (skipped for `force`). */ + readonly checkEmptiness: boolean; + readonly instanceHasInventoryData: () => Promise; +} + +/** + * Drives `readBundle` to completion: checks the manifest's + * `backupFormatVersion` and (unless `force`) instance emptiness the moment + * the manifest is read — the bundle's always-first entry — refusing BEFORE + * any row/blob is staged (R8/R6, AE4/AE1). Only once both checks pass does it + * import the `db.ndjson` stream into the staging schema and let `readBundle` + * itself stage every blob (KTD11 path validation lives there). Only returns + * once the underlying decrypt stream has authenticated the WHOLE bundle, + * including its final secretstream chunk (KTD10) — a tamper anywhere, + * including the last chunk, surfaces as a thrown `DecryptionAuthError` here + * instead. + */ +async function stageBundle( + decryptStream: Transform, + stagingBlobDir: string, + pool: Pool, + options: StageBundleOptions, +): Promise<{ manifest: BackupManifest }> { + const generator = readBundle(decryptStream, { stagingDir: stagingBlobDir }); + + let manifest: BackupManifest | undefined; + for await (const event of generator) { + if (event.kind === "manifest") { + manifest = event.manifest; + if (manifest.backupFormatVersion !== BACKUP_FORMAT_VERSION) { + throw new VersionMismatchSignal( + `bundle backupFormatVersion ${manifest.backupFormatVersion} is incompatible with this instance's ${BACKUP_FORMAT_VERSION}`, + ); + } + if ( + options.checkEmptiness && + (await options.instanceHasInventoryData()) + ) { + throw new NotEmptySignal( + "the instance already holds inventory data; use force-replace to overwrite it", + ); + } + } else if (event.kind === "db") { + await importIntoStaging(pool, event.stream); + } + // "blob" events: readBundle has already written + path-validated the + // file under stagingBlobDir (KTD11) — nothing further to do here. + } + + if (!manifest) { + throw new Error("internal: readBundle completed without a manifest event"); + } + return { manifest }; +} + +/** + * Imports `dbStream` into the staging schema by reusing U3's + * `importDatabase` UNMODIFIED: a dedicated connection has its `search_path` + * redirected to `restore_staging` first, so `importDatabase`'s unqualified + * `INSERT INTO "tablename"` statements land there instead of `public`. + */ +async function importIntoStaging( + pool: Pool, + dbStream: Readable, +): Promise { + const client = await pool.connect(); + try { + await client.query(`SET search_path TO "${STAGING_SCHEMA}", public`); + const stagingDb = drizzle(client, { schema }); + await importDatabase(stagingDb, dbStream); + } finally { + await client.query("RESET search_path").catch(() => {}); + client.release(); + } +} + +/** True when the instance already holds owned inventory data (R6's "instance already holds inventory data"). Auth tables (user/verification/account) are deliberately excluded — an admin must already exist to reach restore at all, so a live user row alone doesn't mean "non-empty" for this purpose. */ +async function instanceHasInventoryData(db: Database): Promise { + const [firearmCount, magazineCount, ammoCount, accessoryCount] = + await Promise.all([ + db.select({ n: count() }).from(firearm), + db.select({ n: count() }).from(magazine), + db.select({ n: count() }).from(ammo), + db.select({ n: count() }).from(accessory), + ]); + return [firearmCount, magazineCount, ammoCount, accessoryCount].some( + (rows) => (rows[0]?.n ?? 0) > 0, + ); +} + +interface BlobSwapHandle { + readonly hadExistingDir: boolean; + readonly movedAsideDir: string; +} + +/** Moves `uploadDir` aside (if present) so the staged blob directory can take its place. Reversible via `undoBlobSwap`/`commitBlobSwap`. */ +async function beginBlobSwap(uploadDir: string): Promise { + const movedAsideDir = `${uploadDir}.pre-restore-${randomUUID()}`; + const hadExistingDir = await pathExists(uploadDir); + if (hadExistingDir) { + await rename(uploadDir, movedAsideDir); + } + return { hadExistingDir, movedAsideDir }; +} + +/** Swaps the staged blob directory into `uploadDir`'s now-vacated path. */ +async function finishBlobSwap( + stagingBlobDir: string, + uploadDir: string, +): Promise { + await rename(stagingBlobDir, uploadDir); +} + +/** Undoes `beginBlobSwap` (and any partial `finishBlobSwap`): removes whatever now sits at `uploadDir` and restores the original contents. */ +async function undoBlobSwap( + handle: BlobSwapHandle, + uploadDir: string, +): Promise { + await rm(uploadDir, { recursive: true, force: true }).catch(() => {}); + if (handle.hadExistingDir) { + await rename(handle.movedAsideDir, uploadDir).catch(() => {}); + } else { + await mkdir(uploadDir, { recursive: true, mode: 0o700 }).catch(() => {}); + } +} + +/** Discards the moved-aside pre-restore blobs after a successful promote. */ +async function commitBlobSwap(handle: BlobSwapHandle): Promise { + if (handle.hadExistingDir) { + await rm(handle.movedAsideDir, { recursive: true, force: true }).catch( + () => {}, + ); + } +} + +async function copySchemaToLive( + tx: Transaction, + fromSchema: string, +): Promise { + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await tx.execute(sql` + INSERT INTO ${qualified("public", name)} + SELECT * FROM ${qualified(fromSchema, name)} + `); + } +} + +async function wipeLive(tx: Transaction): Promise { + for (const table of WIPE_TABLE_ORDER) { + const name = getTableName(table); + await tx.execute(sql`DELETE FROM ${qualified("public", name)}`); + } +} + +/** + * F2 (empty instance) promote: swap the staged blob directory in, then + * promote staging rows into live in one transaction. Either step failing + * undoes the other — no live change survives a partial failure. + */ +async function emptyInstancePromote(ctx: { + db: Database; + uploadDir: string; + stagingBlobDir: string; +}): Promise { + const swap = await beginBlobSwap(ctx.uploadDir); + try { + await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); + } catch (err) { + await undoBlobSwap(swap, ctx.uploadDir); + throw new RestoreRolledBackError( + `blob promotion failed: ${toError(err).message}`, + { cause: err }, + ); + } + + try { + await ctx.db.transaction(async (tx) => { + await copySchemaToLive(tx, STAGING_SCHEMA); + }); + } catch (err) { + await undoBlobSwap(swap, ctx.uploadDir); + throw new RestoreRolledBackError( + `database promotion failed: ${toError(err).message}`, + { cause: err }, + ); + } + + await commitBlobSwap(swap); +} + +/** + * F3 (force-replace) promote — the KTD5 envelope: maintenance flag, pool-safe + * advisory lock, a committed pre-restore snapshot schema, wipe+promote in one + * transaction, and a blob-directory swap. A failure anywhere after the + * snapshot is committed rolls BOTH the DB (restored from the snapshot, if the + * wipe+promote transaction had already committed) and the blobs (restored + * from the moved-aside directory) back together, then always exits + * maintenance. + */ +async function forcePromote(ctx: { + db: Database; + pool: Pool; + uploadDir: string; + stagingBlobDir: string; + testFaultInjection?: RestoreOptions["_testFaultInjection"]; +}): Promise { + await enterMaintenance(ctx.db, "force-restore"); + try { + await withRestoreAdvisoryLock(ctx.pool, async (client) => { + const opsDb = drizzle(client, { schema }); + + await opsDb.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, + ); + await opsDb.execute( + sql`CREATE SCHEMA ${sql.identifier(SNAPSHOT_SCHEMA)}`, + ); + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await opsDb.execute(sql` + CREATE TABLE ${qualified(SNAPSHOT_SCHEMA, name)} + AS TABLE ${qualified("public", name)} + `); + } + + const swap = await beginBlobSwap(ctx.uploadDir); + + let dbCommitted = false; + try { + await opsDb.transaction(async (tx) => { + await wipeLive(tx); + await copySchemaToLive(tx, STAGING_SCHEMA); + await ctx.testFaultInjection?.("pre-commit"); + }); + dbCommitted = true; + + await ctx.testFaultInjection?.("post-commit-pre-blob-swap"); + await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); + } catch (err) { + if (dbCommitted) { + // The wipe+promote transaction already committed new data — the + // only way back is to explicitly restore from the snapshot. + await opsDb.transaction(async (tx) => { + await wipeLive(tx); + await copySchemaToLive(tx, SNAPSHOT_SCHEMA); + }); + } + await undoBlobSwap(swap, ctx.uploadDir); + await opsDb + .execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, + ) + .catch(() => {}); + throw new RestoreRolledBackError( + `force-restore promotion failed and was rolled back: ${toError(err).message}`, + { cause: err }, + ); + } + + await opsDb.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, + ); + await commitBlobSwap(swap); + }); + } finally { + await exitMaintenance(ctx.db); + } +} From 655e16fa3041316842a70423425033065acf8dff Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 18:55:42 -0400 Subject: [PATCH 09/26] feat(backup): admin backup API routes + operator-event logging (U6) Signed-off-by: UncleSp1d3r --- app/api/admin/backup/export/route.ts | 97 +++++++++ app/api/admin/backup/restore/route.ts | 102 +++++++++ src/backup/__tests__/routes.test.ts | 301 ++++++++++++++++++++++++++ src/backup/audit.ts | 53 +++++ 4 files changed, 553 insertions(+) create mode 100644 app/api/admin/backup/export/route.ts create mode 100644 app/api/admin/backup/restore/route.ts create mode 100644 src/backup/__tests__/routes.test.ts create mode 100644 src/backup/audit.ts diff --git a/app/api/admin/backup/export/route.ts b/app/api/admin/backup/export/route.ts new file mode 100644 index 00000000..b3117814 --- /dev/null +++ b/app/api/admin/backup/export/route.ts @@ -0,0 +1,97 @@ +import { Readable } from "node:stream"; +import { getCurrentUser } from "@/src/auth/session"; +import { recordOperatorEvent } from "@/src/backup/audit"; +import { createBackup } from "@/src/backup/export-service"; +import { db } from "@/src/db/client"; + +/** + * Admin backup export (plan Unit U6, R1/R14/R15). + * + * `POST` with a JSON body `{ "password": string }` — the request side is + * small (just a password), so it's read with `request.json()` rather than + * streamed; R13's streaming guarantee is about the response, which IS + * streamed straight through from U4's `createBackup()` (a Node `Readable`, + * bridged to the Web `ReadableStream` `Response` expects via + * `Readable.toWeb`) with no buffering and nothing written server-side + * (KTD8). + * + * Gating mirrors `app/(admin)/users/actions.ts`'s inline `requireAdmin()` + * convention (KTD6): an unauthenticated caller gets 401, an authenticated + * non-admin gets 403 — both with no body, since there's no per-resource + * existence to hide here (unlike `app/api/documents/[id]/route.ts`'s 404 + * collapse), only a whole admin feature to gate. + * + * Every attempt that reaches the admin gate is recorded to `operator_audit` + * (R15), success or failure. "Success" is recorded once the encrypted + * stream has been composed and handed to the platform for delivery — the + * finest-grained signal available; neither the Node `Readable`/Web + * `ReadableStream` APIs nor U4 itself expose "the browser received every + * byte". + */ +export async function POST(request: Request): Promise { + const user = await getCurrentUser(); + if (!user) return new Response(null, { status: 401 }); + if (user.role !== "admin") return new Response(null, { status: 403 }); + + let password: string; + try { + password = await readPassword(request); + } catch (error) { + await recordOperatorEvent({ + actor: user.email, + action: "export", + outcome: `failure: ${errorMessage(error)}`, + }).catch(() => {}); + return Response.json( + { error: "a non-empty password is required" }, + { status: 400 }, + ); + } + + let bundle: Readable; + try { + bundle = await createBackup(password, { db }); + } catch (error) { + await recordOperatorEvent({ + actor: user.email, + action: "export", + outcome: `failure: ${errorMessage(error)}`, + }).catch(() => {}); + return Response.json({ error: "backup export failed" }, { status: 500 }); + } + + await recordOperatorEvent({ + actor: user.email, + action: "export", + outcome: "success", + }); + + const filename = `magstacker-backup-${timestampForFilename()}.magstacker-backup`; + return new Response(Readable.toWeb(bundle) as unknown as ReadableStream, { + status: 200, + headers: { + "Content-Type": "application/octet-stream", + "Content-Disposition": `attachment; filename="${filename}"`, + // Bytes decrypt to the whole instance — never cache on disk (mirrors + // the documents route's same PII posture). + "Cache-Control": "private, no-store", + }, + }); +} + +async function readPassword(request: Request): Promise { + const body = (await request.json()) as { password?: unknown }; + if (typeof body.password !== "string" || body.password.length === 0) { + throw new Error("password is required"); + } + return body.password; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** `YYYY-MM-DDTHH-MM-SS-mmmZ`, filesystem-safe (no `:`). */ +function timestampForFilename(): string { + return new Date().toISOString().replace(/[:.]/g, "-"); +} diff --git a/app/api/admin/backup/restore/route.ts b/app/api/admin/backup/restore/route.ts new file mode 100644 index 00000000..c8073d45 --- /dev/null +++ b/app/api/admin/backup/restore/route.ts @@ -0,0 +1,102 @@ +import { Readable } from "node:stream"; +import type { ReadableStream as NodeWebReadableStream } from "node:stream/web"; +import { getCurrentUser } from "@/src/auth/session"; +import { recordOperatorEvent } from "@/src/backup/audit"; +import { type RestoreOutcome, restore } from "@/src/backup/restore-service"; + +/** + * Admin backup restore (plan Unit U6, R5/R14/R15). + * + * The upload can be GB-scale (R13), so this is a genuinely streamed Route + * Handler on BOTH sides: `request.body` (a Web `ReadableStream`) is bridged + * to a Node `Readable` via `Readable.fromWeb` and handed straight to U5's + * `restore()` — never buffered, never routed through a `"use server"` + * Server Action (whose body size cap this therefore bypasses, per plan). + * + * Request contract (documented, deliberately simple — not multipart, so no + * streaming multipart parser is needed to keep the body unbuffered): + * - `X-Backup-Password` header — required, the bundle's password. + * - `X-Backup-Force` header — optional, `"true"` to force-replace a + * non-empty instance (R7); anything else (including absent) means the + * safe refuse-unless-empty default (R6). + * - The raw encrypted bundle bytes as the request body + * (`Content-Type: application/octet-stream`). + * + * Metadata travels in headers rather than a query string so the password + * never lands in a URL (server access logs, browser history, proxies). + * + * Gating mirrors the export route: 401 unauthenticated, 403 non-admin, both + * with no body (KTD6). The response is a discriminated JSON outcome the UI + * can branch on (R6/R7/R8/R9/AE1-AE4): `{ outcome, message }`, `outcome` + * mirroring U5's `RestoreOutcome["kind"]` one-for-one, mapped to the HTTP + * status below. Every attempt that reaches the admin gate is recorded to + * `operator_audit` (R15), success or failure. + */ +const OUTCOME_STATUS: Record = { + ok: 200, + refused_not_empty: 409, + wrong_password_or_tampered: 400, + version_mismatch: 409, + rolled_back: 500, +}; + +export async function POST(request: Request): Promise { + const user = await getCurrentUser(); + if (!user) return new Response(null, { status: 401 }); + if (user.role !== "admin") return new Response(null, { status: 403 }); + + const password = request.headers.get("x-backup-password"); + if (!password) { + return Response.json( + { outcome: "bad_request", message: "a password is required" }, + { status: 400 }, + ); + } + if (!request.body) { + return Response.json( + { outcome: "bad_request", message: "a backup bundle body is required" }, + { status: 400 }, + ); + } + const force = request.headers.get("x-backup-force") === "true"; + const bundleStream = toNodeReadable(request.body); + + let outcome: RestoreOutcome; + try { + outcome = await restore(bundleStream, password, { force }); + } catch (error) { + await recordOperatorEvent({ + actor: user.email, + action: "restore", + outcome: `failure: ${errorMessage(error)}`, + }).catch(() => {}); + return Response.json( + { outcome: "error", message: "restore failed unexpectedly" }, + { status: 500 }, + ); + } + + await recordOperatorEvent({ + actor: user.email, + action: "restore", + outcome: outcome.kind, + }); + + return Response.json( + { outcome: outcome.kind, message: outcome.message }, + { status: OUTCOME_STATUS[outcome.kind] }, + ); +} + +/** Bridges the Web `ReadableStream` `request.body` is typed as (the DOM lib + * global) into the Node `Readable` `restore()` (U5) expects. The runtime + * shapes are compatible (Node's fetch implementation IS a Web stream); only + * the two ambient type declarations (DOM's `lib.dom.d.ts` vs. `node:stream/web`) + * disagree, hence the cast. */ +function toNodeReadable(body: ReadableStream): Readable { + return Readable.fromWeb(body as unknown as NodeWebReadableStream); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/backup/__tests__/routes.test.ts b/src/backup/__tests__/routes.test.ts new file mode 100644 index 00000000..92911415 --- /dev/null +++ b/src/backup/__tests__/routes.test.ts @@ -0,0 +1,301 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +// `storage` (src/storage/index.ts) is a lazily-constructed singleton, so +// `UPLOAD_DIR` must be set before ANY test body first touches it — set it +// here, ahead of the rest of this file's imports being evaluated (mirrors +// `export-service.test.ts` / `restore-service.test.ts`). +process.env.UPLOAD_DIR = mkdtempSync(join(tmpdir(), "backup-routes-uploads-")); + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + mock, + test, +} from "bun:test"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { eq } from "drizzle-orm"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { wipeDatabase } from "@/src/backup/db-import"; +import { createBackup } from "@/src/backup/export-service"; +import { closePool, db } from "@/src/db/client"; +import { operatorAudit } from "@/src/db/operator-audit-schema"; + +/** + * Uses the REAL `@/src/db/client` singleton (`db`/`closePool`) rather than + * mocking it. `mock.module()` overrides are process-global, and — unlike + * `mock()`/`spyOn()` fn-mocks — a later `mock.module()` "restore" call does + * NOT retroactively fix an already-linked static `import { db } from + * "@/src/db/client"` in some OTHER test file: verified empirically (a + * minimal 2-file repro) that once another file's static import has resolved + * against this file's mock, re-registering the real module in `afterAll` + * does not unstick it, so running the full `bun test src` suite left every + * other file importing the real singleton pointed at this file's + * already-torn-down pool ("Cannot use a pool after calling end on the + * pool"). + * + * Instead this leans on `client.ts`'s OWN test seam: `db`/`pool` are lazy — + * nothing connects until first property access — and `closePool()` resets + * that cached connection. `beforeAll` points `DATABASE_URL` at this file's + * own ephemeral Testcontainers Postgres (so first access connects there); + * `afterAll` closes that pool and restores `DATABASE_URL`, so whichever file + * runs next reconnects (lazily, on ITS first access) to the original target. + * `db`'s two consumers here are always the SAME shared object identity every + * other file also imports, so there is nothing to "leak" once restored. + */ +const ORIGINAL_DATABASE_URL = process.env.DATABASE_URL; + +interface MockUser { + readonly id: string; + readonly email: string; + readonly role: string | null; +} +let currentUser: MockUser | null = null; +mock.module("@/src/auth/session", () => ({ + getCurrentUser: async () => currentUser, + isAdmin: async () => currentUser?.role === "admin", +})); + +// Same pinned image as the rest of the backup suite (AWS ECR Public mirror — +// avoids Docker Hub's unauthenticated per-IP pull limit on shared runners). +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +const PASSWORD = "correct horse battery staple"; +const ADMIN: MockUser = { + id: "admin-1", + email: "admin@example.test", + role: "admin", +}; +const NON_ADMIN: MockUser = { + id: "user-1", + email: "user@example.test", + role: "user", +}; + +function exportRequest(password: unknown): Request { + return new Request("http://localhost/api/admin/backup/export", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); +} + +function restoreRequest(options: { + password?: string; + force?: boolean; + body?: Readable | null; +}): Request { + const headers: Record = {}; + if (options.password !== undefined) + headers["x-backup-password"] = options.password; + if (options.force) headers["x-backup-force"] = "true"; + + const init: RequestInit & { duplex?: "half" } = { + method: "POST", + headers, + }; + if (options.body !== undefined && options.body !== null) { + init.body = Readable.toWeb(options.body) as unknown as ReadableStream; + init.duplex = "half"; + } + return new Request("http://localhost/api/admin/backup/restore", init); +} + +/** Drains a Readable into one Buffer. */ +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +/** Builds a real encrypted bundle (via U4's `createBackup`) as one Buffer, + * so restore tests exercise the actual wire format rather than a fixture. */ +async function buildValidBundle(): Promise { + currentUser = ADMIN; + try { + const stream = await createBackup(PASSWORD, { db }); + return await collect(stream); + } finally { + currentUser = null; + } +} + +async function auditRowsFor(action: "export" | "restore") { + return db + .select() + .from(operatorAudit) + .where(eq(operatorAudit.action, action)); +} + +describe("admin backup API routes (U6)", () => { + let container: StartedPostgreSqlContainer; + let exportPost: (request: Request) => Promise; + let restorePost: (request: Request) => Promise; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_backup_routes_test") + .start(); + process.env.DATABASE_URL = container.getConnectionUri(); + // First real access to the shared `db`/`pool` singleton — `connect()` + // (src/db/client.ts) lazily binds it to `DATABASE_URL` right here. + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + + ({ POST: exportPost } = await import( + "@/app/api/admin/backup/export/route" + )); + ({ POST: restorePost } = await import( + "@/app/api/admin/backup/restore/route" + )); + }, 120_000); + + afterAll(async () => { + await closePool(); + await container?.stop(); + // Restore `DATABASE_URL` so whichever file runs next reconnects (lazily, + // on its own first access) to the original target instead of this file's + // now-stopped container. + if (ORIGINAL_DATABASE_URL === undefined) { + process.env.DATABASE_URL = undefined; + } else { + process.env.DATABASE_URL = ORIGINAL_DATABASE_URL; + } + }); + + beforeEach(async () => { + await wipeDatabase(db); + }); + + afterEach(() => { + currentUser = null; + }); + + test("non-admin caller is refused on both export and restore (R14)", async () => { + currentUser = NON_ADMIN; + const exportRes = await exportPost(exportRequest(PASSWORD)); + expect(exportRes.status).toBe(403); + + const restoreRes = await restorePost( + restoreRequest({ + password: PASSWORD, + body: Readable.from([Buffer.alloc(0)]), + }), + ); + expect(restoreRes.status).toBe(403); + + // Unauthenticated (no session at all) is refused too, distinctly (401). + currentUser = null; + const unauthRes = await exportPost(exportRequest(PASSWORD)); + expect(unauthRes.status).toBe(401); + + expect(await auditRowsFor("export")).toHaveLength(0); + expect(await auditRowsFor("restore")).toHaveLength(0); + }); + + test("admin export returns a 200 streamed attachment with the expected headers", async () => { + currentUser = ADMIN; + const res = await exportPost(exportRequest(PASSWORD)); + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("application/octet-stream"); + const disposition = res.headers.get("content-disposition") ?? ""; + expect(disposition).toContain("attachment"); + expect(disposition).toContain(".magstacker-backup"); + expect(res.body).not.toBeNull(); + + // The bundle itself must be non-empty (proves the response body is the + // real encrypted stream, not a stub). + const bytes = await res.arrayBuffer(); + expect(bytes.byteLength).toBeGreaterThan(0); + + const rows = await auditRowsFor("export"); + expect(rows).toHaveLength(1); + expect(rows[0]?.actor).toBe(ADMIN.email); + expect(rows[0]?.outcome).toBe("success"); + }); + + test("admin restore of a valid bundle succeeds and records an operator_audit row (R15)", async () => { + const bundle = await buildValidBundle(); + + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ password: PASSWORD, body: Readable.from([bundle]) }), + ); + const json = (await res.json()) as { outcome: string; message: string }; + + expect(res.status).toBe(200); + expect(json.outcome).toBe("ok"); + + const rows = await auditRowsFor("restore"); + expect(rows).toHaveLength(1); + expect(rows[0]?.actor).toBe(ADMIN.email); + expect(rows[0]?.outcome).toBe("ok"); + }); + + test("a wrong-password restore returns the discriminated failure outcome and records a failure event", async () => { + const bundle = await buildValidBundle(); + + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ + password: "not the password", + body: Readable.from([bundle]), + }), + ); + const json = (await res.json()) as { outcome: string; message: string }; + + expect(res.status).toBe(400); + expect(json.outcome).toBe("wrong_password_or_tampered"); + + const rows = await auditRowsFor("restore"); + expect(rows).toHaveLength(1); + expect(rows[0]?.actor).toBe(ADMIN.email); + expect(rows[0]?.outcome).toBe("wrong_password_or_tampered"); + }); + + test("the restore route accepts a multi-megabyte upload stream — no Server-Action-style body cap applies", async () => { + // A single ~3MB blob comfortably exceeds Next.js's default Server Action + // body-size limit (1MB) — if this route were, or went through, a + // buffered Server Action, this would be rejected before ever reaching + // `restore()`. It succeeding proves the route reads `request.body` as a + // genuine stream straight into U5's `restore()`. + currentUser = ADMIN; + const { writeFileSync } = await import("node:fs"); + const { randomBytes, randomUUID } = await import("node:crypto"); + const { activeStorageRoot } = await import("@/src/storage"); + const largeBlob = randomBytes(3 * 1024 * 1024); + writeFileSync(join(activeStorageRoot(), `${randomUUID()}.bin`), largeBlob); + + const bundle = await buildValidBundle(); + expect(bundle.byteLength).toBeGreaterThan(3 * 1024 * 1024); + + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ password: PASSWORD, body: Readable.from([bundle]) }), + ); + const json = (await res.json()) as { outcome: string; message: string }; + + expect(res.status).toBe(200); + expect(json.outcome).toBe("ok"); + }); + + test("restore refuses with 400 when no password header is supplied", async () => { + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ body: Readable.from([Buffer.alloc(0)]) }), + ); + expect(res.status).toBe(400); + }); +}); diff --git a/src/backup/audit.ts b/src/backup/audit.ts new file mode 100644 index 00000000..75ed4eef --- /dev/null +++ b/src/backup/audit.ts @@ -0,0 +1,53 @@ +/** + * Operator-event logging (plan Unit U6, R15, KTD6). + * + * Records every admin-run backup export/restore attempt to the + * `operator_audit` table (U3's schema — `src/db/operator-audit-schema.ts`): + * actor, action, outcome, and timestamp (defaulted by the column). Both + * routes (`app/api/admin/backup/export/route.ts`, + * `app/api/admin/backup/restore/route.ts`) call this for success AND failure + * outcomes, so a force-replace restore — the most destructive action in the + * app — always leaves a trail of who ran it and how it went (R15). + */ + +import { type DbOrTx, db as defaultDb } from "@/src/db/client"; +import { operatorAudit } from "@/src/db/operator-audit-schema"; + +/** Matches the `operator_audit_action_valid` CHECK constraint. */ +export type OperatorAction = "export" | "restore"; + +export interface RecordOperatorEventInput { + /** + * The acting user's identity. This repo's audit table deliberately carries + * no FK to `user` (the row must outlive the account that produced it), so + * callers pass the admin's email — a value that still identifies them after + * account deletion, mirroring `operator-audit-schema.ts`'s doc comment. + */ + readonly actor: string; + readonly action: OperatorAction; + /** + * Free-text outcome description — e.g. `"success"`, a `RestoreOutcome` + * `kind` (`"ok"`, `"refused_not_empty"`, ...), or `"failure: "`. + * Kept as free text rather than a second CHECK-constrained enum: the set of + * things that can go wrong (crypto errors, DB errors, bad requests) is + * open-ended, and the audit trail's job is to record what happened, not to + * validate it. + */ + readonly outcome: string; + /** DI seam for tests — defaults to the shared singleton (`src/db/client.ts`). */ + readonly db?: DbOrTx; +} + +/** Appends one row to `operator_audit`. Callers decide how to handle a write + * failure (e.g. swallow it so a real export/restore result isn't masked by a + * logging failure) — this function itself does not swallow errors. */ +export async function recordOperatorEvent( + input: RecordOperatorEventInput, +): Promise { + const db = input.db ?? defaultDb; + await db.insert(operatorAudit).values({ + actor: input.actor, + action: input.action, + outcome: input.outcome, + }); +} From 26cfce22a8b16265023b95f0287cce1c3362277e Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 19:24:27 -0400 Subject: [PATCH 10/26] feat(backup): admin backup UI (U7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admin-only /backup screen: export (password + confirm, explicit no-recovery warning, native
POST download so a GB-scale bundle is never buffered client-side) and restore (file + password upload via fetch(), refuse-unless-empty messaging, force-replace behind a type-to-confirm "REPLACE ALL DATA" phrase, distinct copy per discriminated outcome, and session invalidation + redirect to /login on success since the users table was just replaced). - app/(admin)/backup/{page,backup-panel,export-panel,restore-panel,constants}.tsx - components/ui/confirm-dialog.tsx: add confirmDisabled for the type-to-confirm guard - app/api/admin/backup/export/route.ts: accept a form-encoded password body (the real submit) alongside the existing JSON contract - app/(auth)/login/login-form.tsx: show "Instance restored — please sign in" after a force-invalidated session redirect - app/(app)/app-shell.tsx: add the admin "Backup" nav link - next.config.ts: externalize sodium-native so Turbopack's build tracing doesn't break its native addon's runtime path resolution (ADDON_NOT_FOUND) — this was blocking every build, not just U7 - e2e/backup.spec.ts: export/restore flows, no-recovery warning, refuse-unless-empty (real bundle), force-replace phrase gating, mocked outcome messaging, and post-restore redirect Signed-off-by: UncleSp1d3r --- app/(admin)/backup/backup-panel.tsx | 18 ++ app/(admin)/backup/constants.ts | 59 ++++++ app/(admin)/backup/export-panel.tsx | 140 ++++++++++++++ app/(admin)/backup/page.tsx | 26 +++ app/(admin)/backup/restore-panel.tsx | 275 +++++++++++++++++++++++++++ app/(app)/app-shell.tsx | 6 +- app/(auth)/login/login-form.tsx | 8 + app/api/admin/backup/export/route.ts | 37 +++- components/ui/confirm-dialog.tsx | 14 +- e2e/backup.spec.ts | 275 +++++++++++++++++++++++++++ next.config.ts | 8 + src/backup/__tests__/routes.test.ts | 25 +++ 12 files changed, 879 insertions(+), 12 deletions(-) create mode 100644 app/(admin)/backup/backup-panel.tsx create mode 100644 app/(admin)/backup/constants.ts create mode 100644 app/(admin)/backup/export-panel.tsx create mode 100644 app/(admin)/backup/page.tsx create mode 100644 app/(admin)/backup/restore-panel.tsx create mode 100644 e2e/backup.spec.ts diff --git a/app/(admin)/backup/backup-panel.tsx b/app/(admin)/backup/backup-panel.tsx new file mode 100644 index 00000000..7756d7aa --- /dev/null +++ b/app/(admin)/backup/backup-panel.tsx @@ -0,0 +1,18 @@ +"use client"; + +import { ExportPanel } from "./export-panel"; +import { RestorePanel } from "./restore-panel"; + +/** + * Admin backup screen (plan Unit U7). Composes the export and restore panels + * side by side on wide viewports, stacked on narrow ones — mirrors the + * `/users` admin surface's create-form + table layout. + */ +export function BackupPanel() { + return ( +
+ + +
+ ); +} diff --git a/app/(admin)/backup/constants.ts b/app/(admin)/backup/constants.ts new file mode 100644 index 00000000..6f2bdd8e --- /dev/null +++ b/app/(admin)/backup/constants.ts @@ -0,0 +1,59 @@ +import type { RestoreOutcome } from "@/src/backup/restore-service"; + +/** + * Admin backup UI constants (plan Unit U7, R6/R7/R12). + */ + +/** + * The exact type-to-confirm phrase gating force-replace restore (R7/AE2). The + * operator must type this literally — no case-insensitive or partial match — + * before the confirm control in the force-replace dialog enables. A fixed + * sentinel (rather than the instance hostname the plan floats as an + * alternative) keeps this simple and works identically across every + * self-hosted deployment, with no dependency on an "instance identity" + * concept this app doesn't otherwise have. + */ +export const FORCE_REPLACE_PHRASE = "REPLACE ALL DATA"; + +/** + * Distinct, actionable copy for each of U6's discriminated restore outcomes + * (R6/R7/R9/AE1-AE4) plus a client-side fallback for a network/parse failure + * the route itself can't produce. Keyed by `RestoreOutcome["kind"]` so a new + * outcome kind fails to compile here until this map is updated. + */ +export const RESTORE_OUTCOME_COPY: Record< + RestoreOutcome["kind"], + { readonly title: string; readonly detail: string } +> = { + ok: { + title: "Instance restored", + detail: "Please sign in again.", + }, + refused_not_empty: { + title: "Restore refused — instance is not empty", + detail: + "This instance already holds inventory data. Use force-replace below to wipe it and apply this backup.", + }, + wrong_password_or_tampered: { + title: "Restore refused — could not authenticate the backup", + detail: + "Check the password, or the bundle may be corrupt or tampered with. No data was changed.", + }, + version_mismatch: { + title: "Restore refused — incompatible backup version", + detail: + "This bundle was produced by an incompatible version of MagStacker and cannot be restored here. No data was changed.", + }, + rolled_back: { + title: "Restore failed — rolled back", + detail: + "The force-replace restore failed partway through and your previous data was automatically rolled back. No data was lost.", + }, +}; + +/** Shown when the request itself fails (network error, unexpected status, or + * a non-JSON response) — a case U6's `RestoreOutcome` union doesn't model. */ +export const RESTORE_UNEXPECTED_ERROR = { + title: "Restore failed unexpectedly", + detail: "Please try again. No confirmation of a data change was received.", +} as const; diff --git a/app/(admin)/backup/export-panel.tsx b/app/(admin)/backup/export-panel.tsx new file mode 100644 index 00000000..cbfa405c --- /dev/null +++ b/app/(admin)/backup/export-panel.tsx @@ -0,0 +1,140 @@ +"use client"; + +import { useId, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Callout, Spinner } from "@/components/ui/feedback"; +import { Field } from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { Card } from "@/components/ui/surface"; + +/** + * Export panel (plan Unit U7, R1/R3/R4/R12/R13). + * + * The download is a real `` + * submit — deliberately NOT a client-side `fetch()` + blob, which would + * re-buffer a GB-scale bundle in the browser and undercut the export route's + * R13 streaming guarantee. Submitting the form is a navigation-triggered + * request: when the response carries `Content-Disposition: attachment` (which + * it always does here), the browser downloads the file and the current page + * is never actually unloaded/replaced — so the in-page pending/success state + * below stays intact for the whole flow. The one edge case this trades away is + * a genuine 401/403/500 from the route, which (having no attachment + * disposition) WOULD navigate the tab to a raw response — client-side + * validation below (password required, confirm match, warning acknowledged) + * keeps every reachable submission a 200, so that path is not expected in + * practice. + */ + +const NO_RECOVERY_WARNING = + "There is no password recovery. If this password is lost, this backup cannot be decrypted — by anyone, ever. Store it somewhere safe before you continue."; + +/** How long after submit to optimistically report the download as started. + * There's no JS completion signal for a plain form-triggered download; by + * this point the browser has received headers and begun streaming the + * response to disk even for a very large bundle (streaming starts + * immediately server-side), so "started" is an honest claim at this delay. */ +const ASSUME_STARTED_MS = 1200; + +export function ExportPanel() { + const passwordId = useId(); + const confirmId = useId(); + const acknowledgeId = useId(); + + const [password, setPassword] = useState(""); + const [confirmPassword, setConfirmPassword] = useState(""); + const [acknowledged, setAcknowledged] = useState(false); + const [status, setStatus] = useState<"idle" | "pending" | "started">("idle"); + + const passwordsMatch = password.length > 0 && password === confirmPassword; + const canExport = passwordsMatch && acknowledged && status !== "pending"; + + function onSubmit() { + if (!canExport) return; + setStatus("pending"); + // The native form submission proceeds after this handler returns (no + // preventDefault) — this just drives the in-page status readout. + window.setTimeout(() => setStatus("started"), ASSUME_STARTED_MS); + } + + return ( + +

Export a backup

+

+ Downloads one encrypted file containing the entire instance — every + user, grant, and inventory item, including firearm documents. +

+ + {NO_RECOVERY_WARNING} + + + + setPassword(event.target.value)} + /> + + 0 && !passwordsMatch + ? "Passwords don't match." + : undefined + } + > + 0 && !passwordsMatch} + value={confirmPassword} + onChange={(event) => setConfirmPassword(event.target.value)} + /> + + + + +

+ {status === "pending" ? "Preparing your download…" : ""} +

+ {status === "started" ? ( + + Export started — check your browser's downloads. + + ) : null} + + + + + ); +} diff --git a/app/(admin)/backup/page.tsx b/app/(admin)/backup/page.tsx new file mode 100644 index 00000000..cfb0796a --- /dev/null +++ b/app/(admin)/backup/page.tsx @@ -0,0 +1,26 @@ +import { redirect } from "next/navigation"; +import { PageHeader } from "@/components/ui/surface"; +import { getCurrentUser } from "@/src/auth/session"; +import { BackupPanel } from "./backup-panel"; + +/** + * Admin backup screen (plan Unit U7, R1/R5/R14). The `(admin)` layout already + * gates non-admins (redirecting to `/magazines`), but this page re-asserts the + * same check as defense-in-depth (KTD6's convention — every admin surface + * re-checks, not just the layout), matching how U6's routes re-assert admin + * even though the layout and the route both gate independently. + */ +export default async function BackupPage() { + const user = await getCurrentUser(); + if (user?.role !== "admin") redirect("/magazines"); + + return ( +
+ + +
+ ); +} diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx new file mode 100644 index 00000000..a0afa846 --- /dev/null +++ b/app/(admin)/backup/restore-panel.tsx @@ -0,0 +1,275 @@ +"use client"; + +import type { ChangeEvent, FormEvent } from "react"; +import { useEffect, useId, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { ConfirmDialog } from "@/components/ui/confirm-dialog"; +import { Callout, Spinner } from "@/components/ui/feedback"; +import { Field } from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { Card } from "@/components/ui/surface"; +import { signOut } from "@/lib/auth-client"; +import type { RestoreOutcome } from "@/src/backup/restore-service"; +import { + FORCE_REPLACE_PHRASE, + RESTORE_OUTCOME_COPY, + RESTORE_UNEXPECTED_ERROR, +} from "./constants"; + +/** + * Restore panel (plan Unit U7, R5/R6/R7/R9/R10, AE1-AE4). + * + * The upload is sent as the raw request body via `fetch()` — the bundle file + * itself is passed as `body` (a `File`, which the browser streams from disk + * without the JS heap materializing it, R13), with the password and + * force-replace intent carried in headers (`X-Backup-Password`, + * `X-Backup-Force`) so the body stays exactly the encrypted bytes, matching + * U6's route contract. + */ + +/** How long a restore runs before the progress label switches from + * "verifying" to "applying" (R7/U7: "so a static screen can't be mistaken for + * a hang"). The route's single JSON response gives no real intermediate + * progress, so this is a readable approximation, not a literal signal. */ +const ASSUME_APPLYING_AFTER_MS = 4000; + +type Phase = "idle" | "verifying" | "applying"; + +/** `RestoreOutcome` plus a client-only kind for a request that never got a + * discriminated response at all (network failure, non-JSON body). */ +type DisplayOutcome = + | RestoreOutcome + | { readonly kind: "client_error"; readonly message: string }; + +async function postRestore( + file: File, + password: string, + force: boolean, +): Promise { + try { + const response = await fetch("/api/admin/backup/restore", { + method: "POST", + headers: { + "Content-Type": "application/octet-stream", + "X-Backup-Password": password, + ...(force ? { "X-Backup-Force": "true" } : {}), + }, + body: file, + }); + const body = (await response.json()) as { + outcome?: string; + message?: string; + }; + if (!body.outcome) { + return { kind: "client_error", message: "Malformed response." }; + } + return { + kind: body.outcome, + message: body.message ?? "", + } as DisplayOutcome; + } catch (error) { + return { + kind: "client_error", + message: error instanceof Error ? error.message : String(error), + }; + } +} + +export function RestorePanel() { + const fileInputId = useId(); + const passwordId = useId(); + const phraseId = useId(); + + const [file, setFile] = useState(null); + const [password, setPassword] = useState(""); + const [phase, setPhase] = useState("idle"); + const [outcome, setOutcome] = useState(null); + const [forceDialogOpen, setForceDialogOpen] = useState(false); + const [forcePhrase, setForcePhrase] = useState(""); + const [signingOut, setSigningOut] = useState(false); + const applyingTimer = useRef | null>(null); + + useEffect( + () => () => { + if (applyingTimer.current) clearTimeout(applyingTimer.current); + }, + [], + ); + + const restoring = phase !== "idle"; + const canSubmit = file !== null && password.length > 0 && !restoring; + + function onFileChange(event: ChangeEvent) { + setFile(event.currentTarget.files?.[0] ?? null); + } + + async function runRestore(force: boolean) { + if (!file) return; + setOutcome(null); + setPhase("verifying"); + applyingTimer.current = setTimeout( + () => setPhase("applying"), + ASSUME_APPLYING_AFTER_MS, + ); + + const result = await postRestore(file, password, force); + + if (applyingTimer.current) clearTimeout(applyingTimer.current); + setPhase("idle"); + setForceDialogOpen(false); + + if (result.kind === "ok") { + await handleSuccessfulRestore(); + return; + } + setOutcome(result); + } + + async function handleSuccessfulRestore() { + setSigningOut(true); + // The users table (including the acting admin's own row) was just + // replaced (R10), so the current session is no longer trustworthy — + // best-effort clear it, then hard-navigate so no stale client state + // survives. signOut() may itself fail if the session row is already + // gone; that's fine, the redirect below still happens. + try { + await signOut(); + } catch { + // Ignored — proceeding to the hard redirect regardless. + } + window.location.assign("/login?restored=1"); + } + + function onSubmit(event: FormEvent) { + event.preventDefault(); + void runRestore(false); + } + + function onConfirmForce() { + void runRestore(true); + } + + const isRefusedNotEmpty = outcome?.kind === "refused_not_empty"; + const phraseMatches = forcePhrase === FORCE_REPLACE_PHRASE; + + return ( + +

+ Restore from a backup +

+

+ Upload an encrypted backup file and its password. A plain restore only + applies to an empty instance — force-replace is required to overwrite + existing data. +

+ +
+ + + + + setPassword(event.target.value)} + /> + + +

+ {phase === "verifying" ? "Verifying backup…" : ""} + {phase === "applying" + ? "Applying changes — this can take a while for large backups. Do not close this window." + : ""} +

+ + {outcome && outcome.kind !== "ok" ? ( + +

+ {outcome.kind === "client_error" + ? RESTORE_UNEXPECTED_ERROR.title + : RESTORE_OUTCOME_COPY[outcome.kind].title} +

+

+ {outcome.kind === "client_error" + ? RESTORE_UNEXPECTED_ERROR.detail + : RESTORE_OUTCOME_COPY[outcome.kind].detail} +

+ + ) : null} + +
+ + {isRefusedNotEmpty ? ( + + ) : null} +
+
+ + {signingOut ? ( +

+ Instance restored — please sign in. +

+ ) : null} + + + + This permanently deletes every user, grant, and inventory item + currently on this instance and replaces them with the uploaded + backup. This cannot be undone. + + + Type {FORCE_REPLACE_PHRASE}{" "} + to confirm. + + setForcePhrase(event.target.value)} + /> + + } + confirmLabel="Force replace" + pending={restoring} + pendingLabel="Restoring…" + confirmDisabled={!phraseMatches} + onConfirm={onConfirmForce} + onCancel={() => setForceDialogOpen(false)} + /> + + ); +} diff --git a/app/(app)/app-shell.tsx b/app/(app)/app-shell.tsx index 0dd58ae4..e710a571 100644 --- a/app/(app)/app-shell.tsx +++ b/app/(app)/app-shell.tsx @@ -41,7 +41,11 @@ export function AppShell({ const router = useRouter(); const nav = user.role === "admin" - ? [...NAV, { href: "/users", label: "Accounts" }] + ? [ + ...NAV, + { href: "/users", label: "Accounts" }, + { href: "/backup", label: "Backup" }, + ] : NAV; function isActive(href: string): boolean { diff --git a/app/(auth)/login/login-form.tsx b/app/(auth)/login/login-form.tsx index 010a2ee2..17527307 100644 --- a/app/(auth)/login/login-form.tsx +++ b/app/(auth)/login/login-form.tsx @@ -17,6 +17,11 @@ export function LoginForm() { const router = useRouter(); const params = useSearchParams(); const redirectTo = params.get("redirectTo") || "/magazines"; + // Set by the backup screen's restore-panel (plan Unit U7, R10) after a + // successful restore: the `users` table (including whoever was signed in) + // was just replaced, so the prior session was force-invalidated and the + // operator is hard-redirected here to sign in fresh. + const restored = params.get("restored") === "1"; const emailId = useId(); const passwordId = useId(); const [error, setError] = useState(null); @@ -56,6 +61,9 @@ export function LoginForm() { return (
+ {restored ? ( + Instance restored — please sign in. + ) : null} {error ? {error} : null} ` submit, so the browser performs a + * genuine navigation-triggered download instead of a client-side `fetch()` + + * blob (which would re-buffer a GB-scale bundle in the browser and undercut + * R13). Either way the request side is small (just a password), so it's read + * fully (`request.json()` / `request.formData()`) rather than streamed; + * R13's streaming guarantee is about the response, which IS streamed + * straight through from U4's `createBackup()` (a Node `Readable`, bridged to + * the Web `ReadableStream` `Response` expects via `Readable.toWeb`) with no + * buffering and nothing written server-side (KTD8). * * Gating mirrors `app/(admin)/users/actions.ts`'s inline `requireAdmin()` * convention (KTD6): an unauthenticated caller gets 401, an authenticated @@ -79,12 +84,24 @@ export async function POST(request: Request): Promise { }); } +/** + * Reads the password from either an `application/x-www-form-urlencoded` body + * (the real `` submit U7's export UI sends) or a JSON body (the + * original contract, still supported so existing callers/tests keep + * working). `Request.formData()` parses both `multipart/form-data` and + * `application/x-www-form-urlencoded` per the Fetch spec, so form-encoded + * bodies are routed there; anything else falls back to `request.json()`. + */ async function readPassword(request: Request): Promise { - const body = (await request.json()) as { password?: unknown }; - if (typeof body.password !== "string" || body.password.length === 0) { + const contentType = request.headers.get("content-type") ?? ""; + const password = contentType.includes("application/x-www-form-urlencoded") + ? (await request.formData()).get("password") + : ((await request.json()) as { password?: unknown }).password; + + if (typeof password !== "string" || password.length === 0) { throw new Error("password is required"); } - return body.password; + return password; } function errorMessage(error: unknown): string { diff --git a/components/ui/confirm-dialog.tsx b/components/ui/confirm-dialog.tsx index 634a4564..34befd0f 100644 --- a/components/ui/confirm-dialog.tsx +++ b/components/ui/confirm-dialog.tsx @@ -20,6 +20,13 @@ interface ConfirmDialogProps { cancelLabel?: string; pending?: boolean; pendingLabel?: string; + /** + * Disables the confirm control independent of `pending` — for a type-to-confirm + * guard (e.g. the backup force-replace phrase, R7) where confirmation should stay + * unavailable until the operator has proven intent, not just while an operation + * is in flight. + */ + confirmDisabled?: boolean; onConfirm: () => void; onCancel: () => void; } @@ -35,6 +42,7 @@ export function ConfirmDialog({ cancelLabel = "Cancel", pending = false, pendingLabel = "Deleting…", + confirmDisabled = false, onConfirm, onCancel, }: ConfirmDialogProps) { @@ -118,7 +126,11 @@ export function ConfirmDialog({ > {cancelLabel} - diff --git a/e2e/backup.spec.ts b/e2e/backup.spec.ts new file mode 100644 index 00000000..539a4ce4 --- /dev/null +++ b/e2e/backup.spec.ts @@ -0,0 +1,275 @@ +import { expect, type Page, test } from "@playwright/test"; +import { readArtifact, storageStateFor } from "./fixtures/auth"; + +/** + * Admin backup screen e2e coverage (encryption-at-rest plan, U7 — R1/R3/R4/ + * R5/R6/R7/R9/R10/R12/R14, AE1-AE4). ARIA roles / accessible names / visible + * text only — no `data-testid` (AGENTS.md). + * + * `/backup` is admin-gated, so this spec drives the real login form with the + * seeded admin (mirrors `table-view-controls.spec.ts`) rather than the + * per-spec pool (all non-admin). Exactly ONE live admin sign-in for the whole + * file, kept well under the 5/60s `/sign-in/email` rate limit alongside + * `auth.spec.ts`'s 2 and `table-view-controls.spec.ts`'s 1. + * + * Test-design note on real vs. mocked restore outcomes: the export flow and + * the refuse-unless-empty restore flow (AE1) are exercised for REAL against + * this run's ephemeral Postgres — export never mutates data, and a plain + * restore on a non-empty instance is refused before touching anything, so + * both are safe inside the shared, serialized (`workers: 1`) e2e suite. The + * genuinely destructive force-replace path is proven only up to (and + * excluding) the final confirm click — actually executing it here would wipe + * every other spec's seeded users/inventory for the rest of the run. The + * wrong-password / version-mismatch / rollback / success outcomes are UI + * branching concerns already covered end-to-end at the service/route level + * by U5's `restore-service.test.ts` and U6's `routes.test.ts` + * (`src/backup/__tests__/`); here they're verified by intercepting the + * restore route's response (`page.route`) and asserting the UI renders each + * outcome's distinct, actionable copy — never by fabricating a real corrupt + * bundle or performing a real destructive restore. + */ +test.describe.configure({ retries: 0 }); + +const ADMIN_STORAGE_STATE = "e2e/.artifacts/backup-admin-storage-state.json"; +const RESTORE_ROUTE = "**/api/admin/backup/restore"; +const EXPORT_PASSWORD = "correct horse battery staple 42"; +const FORCE_REPLACE_PHRASE = "REPLACE ALL DATA"; + +test.beforeAll(async ({ browser }) => { + const { admin, baseURL } = readArtifact(); + const context = await browser.newContext({ + baseURL, + storageState: undefined, + }); + const page = await context.newPage(); + await page.goto("/login"); + await page.getByLabel("Email").fill(admin.email); + await page.getByLabel("Password").fill(admin.password); + await page.getByRole("button", { name: "Sign in" }).click(); + await expect(page).toHaveURL(/\/magazines/); + await context.storageState({ path: ADMIN_STORAGE_STATE }); + await context.close(); +}); + +test.use({ storageState: ADMIN_STORAGE_STATE }); + +/** Fills the restore panel's file + password fields with an inert dummy + * payload (the byte content is irrelevant for the mocked-outcome cases below — + * `page.route` answers before the request ever reaches the real server). */ +async function fillRestoreDummyFile(page: Page): Promise { + await page.getByLabel("Backup file").setInputFiles({ + name: "dummy.magstacker-backup", + mimeType: "application/octet-stream", + buffer: Buffer.from("not a real bundle"), + }); + await page.getByLabel("Restore password").fill("whatever"); +} + +/** Mocks the restore route's discriminated JSON outcome for one request, then + * removes the mock so later steps hit the real route again. */ +async function mockRestoreOutcome( + page: Page, + status: number, + outcome: string, + message: string, +): Promise { + await page.route(RESTORE_ROUTE, async (route) => { + await route.fulfill({ + status, + contentType: "application/json", + body: JSON.stringify({ outcome, message }), + }); + }); +} + +test("export, refuse-unless-empty restore, force-replace guard, and outcome messaging", async ({ + page, + browser, +}) => { + await test.step("non-admin cannot reach /backup or see its nav link (R14)", async () => { + const nonAdminContext = await browser.newContext({ + storageState: storageStateFor("onboarding"), + }); + try { + const nonAdminPage = await nonAdminContext.newPage(); + await nonAdminPage.goto("/magazines"); + await expect( + nonAdminPage.getByRole("link", { name: "Backup" }), + ).toHaveCount(0); + + await nonAdminPage.goto("/backup"); + await expect(nonAdminPage).toHaveURL(/\/magazines/); + await expect( + nonAdminPage.getByRole("heading", { level: 1, name: "Backup" }), + ).toHaveCount(0); + } finally { + await nonAdminContext.close(); + } + }); + + await test.step("seed one firearm as this admin (guarantees a non-empty instance for the refuse-unless-empty check below, independent of other specs' run order)", async () => { + await page.goto("/firearms"); + const coldStart = page.getByRole("button", { + name: "Add your first firearm", + }); + if (await coldStart.isVisible()) { + await coldStart.click(); + } else { + await page.getByRole("button", { name: "Add firearm" }).click(); + } + const form = page.locator("form"); + await form.getByLabel(/^Name/).fill("Backup Spec Rifle"); + await form.getByLabel("Caliber").fill("5.56"); + await form.getByLabel(/^Type/).selectOption("rifle"); + await form.getByLabel("Action").selectOption("semi-auto"); + await page.getByRole("button", { name: "Add firearm" }).click(); + await expect(page.getByText("Firearm logged").first()).toBeVisible(); + }); + + await test.step("the no-recovery warning is visible and Export stays disabled until password, confirmation, and acknowledgement are all satisfied (R12)", async () => { + await page.goto("/backup"); + await expect( + page.getByRole("heading", { level: 1, name: "Backup" }), + ).toBeVisible(); + + await expect(page.getByText(/no password recovery/i)).toBeVisible(); + + const exportButton = page.getByRole("button", { name: "Export backup" }); + await expect(exportButton).toBeDisabled(); + + await page.getByLabel("Export password").fill(EXPORT_PASSWORD); + await page.getByLabel("Confirm password").fill("a different password"); + await expect(exportButton).toBeDisabled(); + + await page.getByLabel("Confirm password").fill(EXPORT_PASSWORD); + await expect(exportButton).toBeDisabled(); // acknowledgement checkbox still unticked + + await page + .getByLabel( + "I understand this backup cannot be recovered without this password.", + ) + .check(); + await expect(exportButton).toBeEnabled(); + }); + + let bundlePath: string; + + await test.step("export downloads a real, non-empty encrypted bundle via a navigation-triggered download, not fetch()+blob (R1/R3/R4/R13)", async () => { + const downloadPromise = page.waitForEvent("download"); + await page.getByRole("button", { name: "Export backup" }).click(); + const download = await downloadPromise; + + expect(download.suggestedFilename()).toMatch( + /^magstacker-backup-.*\.magstacker-backup$/, + ); + const path = await download.path(); + if (!path) + throw new Error("export download did not resolve to a local path"); + bundlePath = path; + + await expect( + page.getByText("Export started — check your browser's downloads."), + ).toBeVisible(); + }); + + await test.step("a plain restore of the real bundle on this non-empty instance is refused, with no data changed (R6/AE1)", async () => { + await page.getByLabel("Backup file").setInputFiles(bundlePath); + await page.getByLabel("Restore password").fill(EXPORT_PASSWORD); + await page.getByRole("button", { name: "Restore" }).click(); + + await expect( + page.getByText("Restore refused — instance is not empty"), + ).toBeVisible(); + await expect(page.getByText(/already holds inventory data/i)).toBeVisible(); + + // No data changed: the seeded firearm from the earlier step is still there. + await page.goto("/firearms"); + await expect(page.getByText("Backup Spec Rifle")).toBeVisible(); + }); + + await test.step("force-replace stays disabled until the exact phrase is typed, and is never actually confirmed here (R7/AE2)", async () => { + await page.goto("/backup"); + await page.getByLabel("Backup file").setInputFiles(bundlePath); + await page.getByLabel("Restore password").fill(EXPORT_PASSWORD); + await page.getByRole("button", { name: "Restore" }).click(); + await expect( + page.getByText("Restore refused — instance is not empty"), + ).toBeVisible(); + + await page.getByRole("button", { name: "Force replace…" }).click(); + const dialog = page.getByRole("alertdialog"); + await expect(dialog).toBeVisible(); + const confirmButton = dialog.getByRole("button", { + name: "Force replace", + }); + await expect(confirmButton).toBeDisabled(); + + const phraseInput = dialog.getByLabel( + `Type ${FORCE_REPLACE_PHRASE} to confirm`, + ); + await phraseInput.fill("replace all data"); // wrong case — must not match + await expect(confirmButton).toBeDisabled(); + await phraseInput.fill(FORCE_REPLACE_PHRASE.slice(0, -1)); // partial + await expect(confirmButton).toBeDisabled(); + + await phraseInput.fill(FORCE_REPLACE_PHRASE); + await expect(confirmButton).toBeEnabled(); + + // Deliberately cancel rather than confirm — a real force-replace would + // wipe the whole shared e2e instance for every other spec in this run. + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect(dialog).toBeHidden(); + }); + + await test.step("wrong-password, version-mismatch, and rollback outcomes each render distinct, actionable messages (AE3/AE4, R7 rollback)", async () => { + await mockRestoreOutcome( + page, + 400, + "wrong_password_or_tampered", + "bundle failed to authenticate: wrong password", + ); + await fillRestoreDummyFile(page); + await page.getByRole("button", { name: "Restore" }).click(); + await expect( + page.getByText("Restore refused — could not authenticate the backup"), + ).toBeVisible(); + await expect(page.getByText(/check the password/i)).toBeVisible(); + await page.unroute(RESTORE_ROUTE); + + await mockRestoreOutcome( + page, + 409, + "version_mismatch", + "bundle backupFormatVersion 99 is incompatible", + ); + await fillRestoreDummyFile(page); + await page.getByRole("button", { name: "Restore" }).click(); + await expect( + page.getByText("Restore refused — incompatible backup version"), + ).toBeVisible(); + await expect(page.getByText(/incompatible version/i)).toBeVisible(); + await page.unroute(RESTORE_ROUTE); + + await mockRestoreOutcome( + page, + 500, + "rolled_back", + "force-restore promotion failed and was rolled back", + ); + await fillRestoreDummyFile(page); + await page.getByRole("button", { name: "Restore" }).click(); + await expect(page.getByText("Restore failed — rolled back")).toBeVisible(); + await expect(page.getByText(/automatically rolled back/i)).toBeVisible(); + await page.unroute(RESTORE_ROUTE); + }); + + await test.step("a successful restore invalidates the session and redirects to login (R10) — run LAST, this signs the browser out", async () => { + await mockRestoreOutcome(page, 200, "ok", "restore completed successfully"); + await fillRestoreDummyFile(page); + await page.getByRole("button", { name: "Restore" }).click(); + + await expect(page).toHaveURL(/\/login\?restored=1/); + await expect(page.getByText("Instance restored")).toBeVisible(); + await page.unroute(RESTORE_ROUTE); + }); +}); diff --git a/next.config.ts b/next.config.ts index f65139b7..2a4fa0ae 100644 --- a/next.config.ts +++ b/next.config.ts @@ -3,6 +3,14 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ reactCompiler: true, + // `sodium-native` (src/backup/crypto.ts, U1) ships a native `.node` addon + // loaded via `node-gyp-build`'s runtime path resolution. Turbopack's build + // tracing otherwise bundles/relocates `binding.js` into `.next/server/chunks`, + // which breaks that relative-path resolution and fails page-data collection + // for every route that imports it (`ADDON_NOT_FOUND`) — both backup API + // routes, transitively. Excluding it from bundling keeps it a normal + // `require()` resolved from `node_modules` at runtime instead. + serverExternalPackages: ["sodium-native"], experimental: { serverActions: { // Photo AND document uploads go through Server Actions as multipart diff --git a/src/backup/__tests__/routes.test.ts b/src/backup/__tests__/routes.test.ts index 92911415..21440c99 100644 --- a/src/backup/__tests__/routes.test.ts +++ b/src/backup/__tests__/routes.test.ts @@ -90,6 +90,17 @@ function exportRequest(password: unknown): Request { }); } +/** Mirrors U7's real `` export submit — form-encoded, + * not JSON (R13: a navigation-triggered download, not client `fetch()`+blob). */ +function exportFormRequest(password: string): Request { + const body = new URLSearchParams({ password }); + return new Request("http://localhost/api/admin/backup/export", { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: body.toString(), + }); +} + function restoreRequest(options: { password?: string; force?: boolean; @@ -291,6 +302,20 @@ describe("admin backup API routes (U6)", () => { expect(json.outcome).toBe("ok"); }); + test("admin export accepts a form-encoded password (U7's real submit)", async () => { + currentUser = ADMIN; + const res = await exportPost(exportFormRequest(PASSWORD)); + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("application/octet-stream"); + const bytes = await res.arrayBuffer(); + expect(bytes.byteLength).toBeGreaterThan(0); + + const rows = await auditRowsFor("export"); + expect(rows).toHaveLength(1); + expect(rows[0]?.outcome).toBe("success"); + }); + test("restore refuses with 400 when no password header is supplied", async () => { currentUser = ADMIN; const res = await restorePost( From 10272fa70ab8de797dd11c8a01725dfa255621e7 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 20:12:12 -0400 Subject: [PATCH 11/26] refactor(backup): apply simplify + code-review fixes Simplify: bufferDbExport tallies rows in-loop and returns a Buffer (drop a full-payload string split + round-trip); parallelize blob stats; mkdir staging dir once per dir. Restore UI: validate the route outcome against the copy map and fall back to the unexpected-error display instead of crashing on an unmodeled (bad_request/error) outcome. Review fixes: F2 empty-instance restore now wipes before promote inside its transaction so restoring onto an instance holding the bootstrap admin no longer collides on user.email; bound untrusted KDF params from the bundle header before deriveKey to prevent a pre-auth Argon2id resource blowup. Test: gating.test.ts self-provisions its documented seeded admin in beforeAll, fixing a pre-existing full-suite flake where a sibling test deleting the shared ambient-DB admin made these assertions order-dependent. Signed-off-by: UncleSp1d3r --- app/(admin)/backup/restore-panel.tsx | 14 ++++++++---- src/auth/__tests__/gating.test.ts | 25 +++++++++++++++++++- src/backup/bundle.ts | 7 +++++- src/backup/crypto.ts | 21 +++++++++++++++++ src/backup/export-service.ts | 34 +++++++++++++++------------- src/backup/restore-service.ts | 15 ++++++++++-- 6 files changed, 92 insertions(+), 24 deletions(-) diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx index a0afa846..ce600660 100644 --- a/app/(admin)/backup/restore-panel.tsx +++ b/app/(admin)/backup/restore-panel.tsx @@ -60,13 +60,19 @@ async function postRestore( outcome?: string; message?: string; }; - if (!body.outcome) { - return { kind: "client_error", message: "Malformed response." }; + if (!body.outcome || !(body.outcome in RESTORE_OUTCOME_COPY)) { + // The route can also return outcomes the UI doesn't model (`bad_request`, + // `error`); render those through the unexpected-error fallback rather + // than indexing the copy map with an unknown key. + return { + kind: "client_error", + message: body.message ?? "Malformed response.", + }; } return { - kind: body.outcome, + kind: body.outcome as RestoreOutcome["kind"], message: body.message ?? "", - } as DisplayOutcome; + }; } catch (error) { return { kind: "client_error", diff --git a/src/auth/__tests__/gating.test.ts b/src/auth/__tests__/gating.test.ts index b9f3b04f..c53fbcbc 100644 --- a/src/auth/__tests__/gating.test.ts +++ b/src/auth/__tests__/gating.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { beforeAll, describe, expect, test } from "bun:test"; import { NextRequest } from "next/server"; import { proxy } from "@/proxy"; @@ -44,6 +44,29 @@ liveAuth("Better Auth HTTP surface", () => { const ADMIN_EMAIL = process.env.ADMIN_EMAIL ?? "admin@example.com"; const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD ?? "localadminpassword"; + // Self-provision the seeded admin this suite documents it needs. bun runs a + // file's beforeAll + tests as one contiguous block, so re-creating the admin + // here makes these assertions robust to a sibling test file that deletes the + // shared ambient-DB admin — the source of the pre-existing full-suite flake. + // Idempotent, mirroring scripts/seed-admin.ts. + beforeAll(async () => { + const { auth } = await import("@/auth"); + const { db } = await import("@/src/db/client"); + const existing = await db.query.user.findFirst({ + where: (u, { eq }) => eq(u.email, ADMIN_EMAIL), + }); + if (!existing) { + await auth.api.createUser({ + body: { + email: ADMIN_EMAIL, + password: ADMIN_PASSWORD, + name: "Administrator", + role: "admin", + }, + }); + } + }); + // A fresh client IP per test isolates each from the DB-backed rate-limit // buckets so a re-run within the 60s window does not interfere. let ipCounter = 0; diff --git a/src/backup/bundle.ts b/src/backup/bundle.ts index ac06d50d..eef1fea1 100644 --- a/src/backup/bundle.ts +++ b/src/backup/bundle.ts @@ -294,6 +294,7 @@ export async function* readBundle( let entryCount = 0; let blobsSeen = 0; let blobBytesSeen = 0; + const ensuredDirs = new Set(); for await (const entry of extract) { entryCount++; @@ -368,7 +369,11 @@ export async function* readBundle( ); } - await mkdir(dirname(destPath), { recursive: true, mode: 0o700 }); + const destDir = dirname(destPath); + if (!ensuredDirs.has(destDir)) { + await mkdir(destDir, { recursive: true, mode: 0o700 }); + ensuredDirs.add(destDir); + } const remainingBudget = manifest.counts.totalBlobBytes - blobBytesSeen; const writtenSize = await writeEntryToFile( entry, diff --git a/src/backup/crypto.ts b/src/backup/crypto.ts index e0af09a9..397e1de4 100644 --- a/src/backup/crypto.ts +++ b/src/backup/crypto.ts @@ -69,6 +69,15 @@ export const DEFAULT_KDF_PARAMS: KdfParams = { alg: sodium.crypto_pwhash_ALG_ARGON2ID13, }; +/** + * Upper bounds accepted for KDF parameters read from an untrusted bundle + * header (see `readHeader`). Generous relative to MODERATE (3 / 256 MiB) so a + * future SENSITIVE-tier bundle still validates, but finite so a hostile header + * cannot force an unbounded pre-authentication Argon2id allocation. + */ +const MAX_KDF_OPSLIMIT = 10; +const MAX_KDF_MEMLIMIT = 1024 * 1024 * 1024; // 1 GiB + /** The bundle's unencrypted crypto header — see module doc comment. */ export interface CryptoHeader { readonly version: number; @@ -221,6 +230,18 @@ export function readHeader(buf: Buffer): CryptoHeader { const alg = buf.readUInt8(offset); offset += 1; + // The KDF cost parameters come straight off an untrusted bundle header and + // are fed to Argon2id (`deriveKey`) BEFORE any ciphertext byte is + // authenticated. Without an upper bound, a crafted bundle could set + // memlimit/opslimit arbitrarily high and exhaust memory/CPU on an admin's + // restore attempt. Legitimate bundles use OWASP MODERATE (opslimit 3, + // memlimit 256 MiB), well within these ceilings. + if (opslimit > MAX_KDF_OPSLIMIT || memlimit > MAX_KDF_MEMLIMIT) { + throw new InvalidHeaderError( + `crypto header KDF parameters exceed the accepted maximum (opslimit=${opslimit}, memlimit=${memlimit})`, + ); + } + const secretstreamHeader = Buffer.from( buf.subarray(offset, offset + SECRETSTREAM_HEADER_BYTES), ); diff --git a/src/backup/export-service.ts b/src/backup/export-service.ts index 49a32255..a48d1f38 100644 --- a/src/backup/export-service.ts +++ b/src/backup/export-service.ts @@ -62,12 +62,12 @@ async function listUploadBlobs(): Promise { ); const fileNames = entries.filter((entry) => entry.isFile()); - const infos: BlobFileInfo[] = []; - for (const entry of fileNames) { - const info = await stat(join(uploadDir, entry.name)); - infos.push({ storageKey: entry.name, size: info.size }); - } - return infos; + return Promise.all( + fileNames.map(async (entry) => ({ + storageKey: entry.name, + size: (await stat(join(uploadDir, entry.name))).size, + })), + ); } /** @@ -97,21 +97,23 @@ async function* blobEntriesFor( * `bundle.ts`'s own documented exception: `db.ndjson` is JSON-per-row text, * not the binary attachments R13/KTD3 are actually concerned with, and * `writeBundle` buffers it internally regardless (it needs an exact byte - * length up front for the tar header). Buffering it once here — instead of - * once here and again inside `writeBundle` — would require re-plumbing - * `writeBundle`'s API, which U4 does not own; the double-buffer cost is a - * small NDJSON payload, not the large blob set R13 is about. + * length up front for the tar header); the double-buffer cost is a small + * NDJSON payload, not the large blob set R13 is about. + * + * `exportDatabase` yields exactly one NDJSON line per row, so the row count is + * tallied in the same pass that buffers the bytes — no second scan of the + * payload. The `Buffer` is returned as-is (no `toString`/re-encode round trip). */ async function bufferDbExport( db: DbOrTx, -): Promise<{ text: string; rowCount: number }> { +): Promise<{ buffer: Buffer; rowCount: number }> { const chunks: Buffer[] = []; + let rowCount = 0; for await (const chunk of exportDatabase(db)) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + rowCount += 1; } - const text = Buffer.concat(chunks).toString("utf8"); - const rowCount = text.split("\n").filter((line) => line.trim() !== "").length; - return { text, rowCount }; + return { buffer: Buffer.concat(chunks), rowCount }; } export interface CreateBackupOptions { @@ -141,7 +143,7 @@ export async function createBackup( ): Promise { await requireAdmin(); - const [{ text: dbText, rowCount }, blobInfos] = await Promise.all([ + const [{ buffer: dbBuffer, rowCount }, blobInfos] = await Promise.all([ bufferDbExport(options.db), listUploadBlobs(), ]); @@ -161,7 +163,7 @@ export async function createBackup( return writeBundle( { manifest, - dbStream: Readable.from([dbText]), + dbStream: Readable.from([dbBuffer]), blobEntries: blobEntriesFor(blobInfos), }, createEncryptStream(key, salt), diff --git a/src/backup/restore-service.ts b/src/backup/restore-service.ts index 1df855a2..9ca1c61b 100644 --- a/src/backup/restore-service.ts +++ b/src/backup/restore-service.ts @@ -387,8 +387,18 @@ async function wipeLive(tx: Transaction): Promise { /** * F2 (empty instance) promote: swap the staged blob directory in, then - * promote staging rows into live in one transaction. Either step failing - * undoes the other — no live change survives a partial failure. + * wipe-and-promote staging rows into live in one transaction. Either step + * failing undoes the other — no live change survives a partial failure. + * + * "Empty" here means no *inventory* data (`instanceHasInventoryData`); the + * instance can still hold bootstrap auth rows (the admin who is performing the + * restore always exists in `public.user`). Those must be replaced, not merged + * into — a plain `INSERT` of the backup's users would collide with the live + * admin on `user.email`'s UNIQUE constraint and roll the whole restore back. + * So this wipes live tables before copying, exactly like force-replace, and + * relies on the single wrapping transaction for atomic DB rollback (no + * separate snapshot schema is needed: a failed transaction reverts the wipe + * too, and the blob swap happened first and is undone on failure). */ async function emptyInstancePromote(ctx: { db: Database; @@ -408,6 +418,7 @@ async function emptyInstancePromote(ctx: { try { await ctx.db.transaction(async (tx) => { + await wipeLive(tx); await copySchemaToLive(tx, STAGING_SCHEMA); }); } catch (err) { From 6aa533372548a0740fdc674d51ed7909e707d04a Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 23:00:19 -0400 Subject: [PATCH 12/26] feat(backup): same-origin CSRF check + minimum backup-password length Add an explicit same-origin guard (src/backup/same-origin.ts) to both admin backup routes, checked right after the admin gate: they are plain Route Handlers never routed through Better Auth's own handler, so Better Auth's origin checks never applied to them, leaving only the session cookie's sameSite attribute as implicit CSRF protection on the app's highest-blast- radius endpoints. The guard checks Origin, falling back to Referer then Sec-Fetch-Site, against the request's own origin and BETTER_AUTH_URL. Enforce a 12-character minimum on the export password (src/backup/password-policy.ts), server-side in the export route and client-side in the export panel; restore intentionally keeps no such minimum since its password must match whatever encrypted the given bundle. Signed-off-by: UncleSp1d3r --- app/(admin)/backup/export-panel.tsx | 47 ++++++- app/api/admin/backup/export/route.ts | 35 +++++- app/api/admin/backup/restore/route.ts | 10 +- src/backup/__tests__/routes.test.ts | 173 ++++++++++++++++++++++++-- src/backup/password-policy.ts | 13 ++ src/backup/same-origin.ts | 88 +++++++++++++ 6 files changed, 351 insertions(+), 15 deletions(-) create mode 100644 src/backup/password-policy.ts create mode 100644 src/backup/same-origin.ts diff --git a/app/(admin)/backup/export-panel.tsx b/app/(admin)/backup/export-panel.tsx index cbfa405c..18f615c0 100644 --- a/app/(admin)/backup/export-panel.tsx +++ b/app/(admin)/backup/export-panel.tsx @@ -6,6 +6,7 @@ import { Callout, Spinner } from "@/components/ui/feedback"; import { Field } from "@/components/ui/field"; import { Input } from "@/components/ui/input"; import { Card } from "@/components/ui/surface"; +import { MIN_BACKUP_PASSWORD_LENGTH } from "@/src/backup/password-policy"; /** * Export panel (plan Unit U7, R1/R3/R4/R12/R13). @@ -35,6 +36,31 @@ const NO_RECOVERY_WARNING = * immediately server-side), so "started" is an honest claim at this delay. */ const ASSUME_STARTED_MS = 1200; +export interface ExportGateState { + readonly password: string; + readonly confirmPassword: string; + readonly acknowledged: boolean; + readonly pending: boolean; +} + +/** + * Whether the export trigger should be enabled (hardening pass, mirrors + * `MIN_BACKUP_PASSWORD_LENGTH`/`readPassword` in the export route): the + * password meets the minimum length, both password fields match, the + * no-recovery warning is acknowledged, and no export is already in flight. + * Exported as a pure function so the exact gating logic backing the + * rendered button's `disabled` state is unit-testable without a DOM. + */ +export function canExportBackup(state: ExportGateState): boolean { + const passwordLongEnough = + state.password.length >= MIN_BACKUP_PASSWORD_LENGTH; + const passwordsMatch = + state.password.length > 0 && state.password === state.confirmPassword; + return ( + passwordLongEnough && passwordsMatch && state.acknowledged && !state.pending + ); +} + export function ExportPanel() { const passwordId = useId(); const confirmId = useId(); @@ -45,8 +71,14 @@ export function ExportPanel() { const [acknowledged, setAcknowledged] = useState(false); const [status, setStatus] = useState<"idle" | "pending" | "started">("idle"); + const passwordLongEnough = password.length >= MIN_BACKUP_PASSWORD_LENGTH; const passwordsMatch = password.length > 0 && password === confirmPassword; - const canExport = passwordsMatch && acknowledged && status !== "pending"; + const canExport = canExportBackup({ + password, + confirmPassword, + acknowledged, + pending: status === "pending", + }); function onSubmit() { if (!canExport) return; @@ -73,13 +105,24 @@ export function ExportPanel() { className="mt-4 flex flex-col gap-3" noValidate > - + 0 && !passwordLongEnough + ? `Password must be at least ${MIN_BACKUP_PASSWORD_LENGTH} characters.` + : undefined + } + > 0 && !passwordLongEnough} value={password} onChange={(event) => setPassword(event.target.value)} /> diff --git a/app/api/admin/backup/export/route.ts b/app/api/admin/backup/export/route.ts index d15784e0..b00d92da 100644 --- a/app/api/admin/backup/export/route.ts +++ b/app/api/admin/backup/export/route.ts @@ -2,6 +2,8 @@ import { Readable } from "node:stream"; import { getCurrentUser } from "@/src/auth/session"; import { recordOperatorEvent } from "@/src/backup/audit"; import { createBackup } from "@/src/backup/export-service"; +import { MIN_BACKUP_PASSWORD_LENGTH } from "@/src/backup/password-policy"; +import { sameOriginError } from "@/src/backup/same-origin"; import { db } from "@/src/db/client"; /** @@ -26,6 +28,19 @@ import { db } from "@/src/db/client"; * existence to hide here (unlike `app/api/documents/[id]/route.ts`'s 404 * collapse), only a whole admin feature to gate. * + * Right after the admin gate, `sameOriginError` (`src/backup/same-origin.ts`) + * refuses a cross-origin POST with a bodyless 403 (hardening pass): this + * route is a plain Route Handler, never routed through Better Auth's own + * handler, so Better Auth's origin checks never apply to it — the session + * cookie's `sameSite` attribute alone isn't a substitute for an explicit + * check on the highest-blast-radius endpoint in the app. + * + * The export password must be at least `MIN_BACKUP_PASSWORD_LENGTH` + * characters (`src/backup/password-policy.ts`, hardening pass) — enforced + * here AND client-side in the export panel; restore deliberately has no such + * minimum (a restore's password must match whatever encrypted that specific + * bundle, including older, shorter passwords). + * * Every attempt that reaches the admin gate is recorded to `operator_audit` * (R15), success or failure. "Success" is recorded once the encrypted * stream has been composed and handed to the platform for delivery — the @@ -38,6 +53,9 @@ export async function POST(request: Request): Promise { if (!user) return new Response(null, { status: 401 }); if (user.role !== "admin") return new Response(null, { status: 403 }); + const originError = sameOriginError(request); + if (originError) return originError; + let password: string; try { password = await readPassword(request); @@ -47,10 +65,7 @@ export async function POST(request: Request): Promise { action: "export", outcome: `failure: ${errorMessage(error)}`, }).catch(() => {}); - return Response.json( - { error: "a non-empty password is required" }, - { status: 400 }, - ); + return Response.json({ error: errorMessage(error) }, { status: 400 }); } let bundle: Readable; @@ -91,6 +106,11 @@ export async function POST(request: Request): Promise { * working). `Request.formData()` parses both `multipart/form-data` and * `application/x-www-form-urlencoded` per the Fetch spec, so form-encoded * bodies are routed there; anything else falls back to `request.json()`. + * + * Also enforces `MIN_BACKUP_PASSWORD_LENGTH` (hardening pass): a short + * export password is easy to brute-force offline against the encrypted + * bundle, so it's rejected here with a distinct, actionable message rather + * than silently accepted. */ async function readPassword(request: Request): Promise { const contentType = request.headers.get("content-type") ?? ""; @@ -99,7 +119,12 @@ async function readPassword(request: Request): Promise { : ((await request.json()) as { password?: unknown }).password; if (typeof password !== "string" || password.length === 0) { - throw new Error("password is required"); + throw new Error("a non-empty password is required"); + } + if (password.length < MIN_BACKUP_PASSWORD_LENGTH) { + throw new Error( + `a password of at least ${MIN_BACKUP_PASSWORD_LENGTH} characters is required`, + ); } return password; } diff --git a/app/api/admin/backup/restore/route.ts b/app/api/admin/backup/restore/route.ts index c8073d45..6a0f0cf3 100644 --- a/app/api/admin/backup/restore/route.ts +++ b/app/api/admin/backup/restore/route.ts @@ -3,6 +3,7 @@ import type { ReadableStream as NodeWebReadableStream } from "node:stream/web"; import { getCurrentUser } from "@/src/auth/session"; import { recordOperatorEvent } from "@/src/backup/audit"; import { type RestoreOutcome, restore } from "@/src/backup/restore-service"; +import { sameOriginError } from "@/src/backup/same-origin"; /** * Admin backup restore (plan Unit U6, R5/R14/R15). @@ -26,7 +27,11 @@ import { type RestoreOutcome, restore } from "@/src/backup/restore-service"; * never lands in a URL (server access logs, browser history, proxies). * * Gating mirrors the export route: 401 unauthenticated, 403 non-admin, both - * with no body (KTD6). The response is a discriminated JSON outcome the UI + * with no body (KTD6), followed immediately by `sameOriginError` + * (`src/backup/same-origin.ts`, hardening pass) refusing a cross-origin POST + * with a bodyless 403 — this route is a plain Route Handler, never routed + * through Better Auth's own handler, so Better Auth's origin checks never + * apply to it. The response is a discriminated JSON outcome the UI * can branch on (R6/R7/R8/R9/AE1-AE4): `{ outcome, message }`, `outcome` * mirroring U5's `RestoreOutcome["kind"]` one-for-one, mapped to the HTTP * status below. Every attempt that reaches the admin gate is recorded to @@ -45,6 +50,9 @@ export async function POST(request: Request): Promise { if (!user) return new Response(null, { status: 401 }); if (user.role !== "admin") return new Response(null, { status: 403 }); + const originError = sameOriginError(request); + if (originError) return originError; + const password = request.headers.get("x-backup-password"); if (!password) { return Response.json( diff --git a/src/backup/__tests__/routes.test.ts b/src/backup/__tests__/routes.test.ts index 21440c99..dfef70c0 100644 --- a/src/backup/__tests__/routes.test.ts +++ b/src/backup/__tests__/routes.test.ts @@ -25,8 +25,13 @@ import { } from "@testcontainers/postgresql"; import { eq } from "drizzle-orm"; import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { + canExportBackup, + type ExportGateState, +} from "@/app/(admin)/backup/export-panel"; import { wipeDatabase } from "@/src/backup/db-import"; import { createBackup } from "@/src/backup/export-service"; +import { MIN_BACKUP_PASSWORD_LENGTH } from "@/src/backup/password-policy"; import { closePool, db } from "@/src/db/client"; import { operatorAudit } from "@/src/db/operator-audit-schema"; @@ -82,21 +87,56 @@ const NON_ADMIN: MockUser = { role: "user", }; -function exportRequest(password: unknown): Request { +/** Origin the routes' `sameOriginError` guard treats as same-origin for + * every request built below (matches the `http://localhost/...` request URL + * these `new Request()` calls use). */ +const SAME_ORIGIN = "http://localhost"; + +/** Merges `overrides` onto `base`; a `null` override value deletes that + * header entirely (used to test the Origin-absent / Referer-fallback path, + * which plain object spread can't express). */ +function mergeHeaders( + base: Record, + overrides: Record = {}, +): Headers { + const headers = new Headers(base); + for (const [key, value] of Object.entries(overrides)) { + if (value === null) headers.delete(key); + else headers.set(key, value); + } + return headers; +} + +function exportRequest( + password: unknown, + headerOverrides: Record = {}, +): Request { return new Request("http://localhost/api/admin/backup/export", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: mergeHeaders( + { "Content-Type": "application/json", Origin: SAME_ORIGIN }, + headerOverrides, + ), body: JSON.stringify({ password }), }); } /** Mirrors U7's real `` export submit — form-encoded, * not JSON (R13: a navigation-triggered download, not client `fetch()`+blob). */ -function exportFormRequest(password: string): Request { +function exportFormRequest( + password: string, + headerOverrides: Record = {}, +): Request { const body = new URLSearchParams({ password }); return new Request("http://localhost/api/admin/backup/export", { method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, + headers: mergeHeaders( + { + "Content-Type": "application/x-www-form-urlencoded", + Origin: SAME_ORIGIN, + }, + headerOverrides, + ), body: body.toString(), }); } @@ -105,11 +145,15 @@ function restoreRequest(options: { password?: string; force?: boolean; body?: Readable | null; + headerOverrides?: Record; }): Request { - const headers: Record = {}; + const headers = mergeHeaders( + { Origin: SAME_ORIGIN }, + options.headerOverrides, + ); if (options.password !== undefined) - headers["x-backup-password"] = options.password; - if (options.force) headers["x-backup-force"] = "true"; + headers.set("x-backup-password", options.password); + if (options.force) headers.set("x-backup-force", "true"); const init: RequestInit & { duplex?: "half" } = { method: "POST", @@ -323,4 +367,119 @@ describe("admin backup API routes (U6)", () => { ); expect(res.status).toBe(400); }); + + // --- Same-origin (CSRF) guard (hardening pass) ---------------------------- + + test("a cross-origin export POST is refused with 403 and records no backup", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportRequest(PASSWORD, { Origin: "http://evil.example" }), + ); + expect(res.status).toBe(403); + expect(await auditRowsFor("export")).toHaveLength(0); + }); + + test("a cross-origin restore POST is refused with 403 and records no attempt", async () => { + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ + password: PASSWORD, + body: Readable.from([Buffer.alloc(0)]), + headerOverrides: { Origin: "http://evil.example" }, + }), + ); + expect(res.status).toBe(403); + expect(await auditRowsFor("restore")).toHaveLength(0); + }); + + test("a same-origin export POST is allowed (Origin header matches the app's own origin)", async () => { + currentUser = ADMIN; + const res = await exportPost(exportRequest(PASSWORD)); + expect(res.status).toBe(200); + }); + + test("an export POST with no Origin header but a matching Referer succeeds (export-form fallback)", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportFormRequest(PASSWORD, { + Origin: null, + Referer: `${SAME_ORIGIN}/backup`, + }), + ); + expect(res.status).toBe(200); + }); + + test("an export POST with no Origin header and a cross-origin Referer is refused with 403", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportFormRequest(PASSWORD, { + Origin: null, + Referer: "http://evil.example/backup", + }), + ); + expect(res.status).toBe(403); + }); + + // --- Minimum export password length (hardening pass) ----------------------- + + test("an export with a too-short password returns 400 and records no backup", async () => { + currentUser = ADMIN; + const shortPassword = "a".repeat(MIN_BACKUP_PASSWORD_LENGTH - 1); + const res = await exportPost(exportRequest(shortPassword)); + const json = (await res.json()) as { error: string }; + + expect(res.status).toBe(400); + expect(json.error).toContain(String(MIN_BACKUP_PASSWORD_LENGTH)); + + const rows = await auditRowsFor("export"); + expect(rows).toHaveLength(1); + expect(rows[0]?.outcome).toContain("failure"); + }); + + test("an export with a password exactly at the minimum length succeeds", async () => { + currentUser = ADMIN; + const exactPassword = "a".repeat(MIN_BACKUP_PASSWORD_LENGTH); + const res = await exportPost(exportRequest(exactPassword)); + expect(res.status).toBe(200); + }); +}); + +// --- Export panel gating logic (component-level, no DOM needed) ------------ + +describe("export panel `canExportBackup` gate (U7 hardening)", () => { + const base: ExportGateState = { + password: "a".repeat(MIN_BACKUP_PASSWORD_LENGTH), + confirmPassword: "a".repeat(MIN_BACKUP_PASSWORD_LENGTH), + acknowledged: true, + pending: false, + }; + + test("enabled once password meets the minimum length, matches, and is acknowledged", () => { + expect(canExportBackup(base)).toBe(true); + }); + + test("disabled when the password is shorter than the minimum length", () => { + const shortPassword = "a".repeat(MIN_BACKUP_PASSWORD_LENGTH - 1); + expect( + canExportBackup({ + ...base, + password: shortPassword, + confirmPassword: shortPassword, + }), + ).toBe(false); + }); + + test("disabled when the confirm password doesn't match", () => { + expect( + canExportBackup({ ...base, confirmPassword: `${base.confirmPassword}x` }), + ).toBe(false); + }); + + test("disabled when the no-recovery warning isn't acknowledged", () => { + expect(canExportBackup({ ...base, acknowledged: false })).toBe(false); + }); + + test("disabled while an export is already pending", () => { + expect(canExportBackup({ ...base, pending: true })).toBe(false); + }); }); diff --git a/src/backup/password-policy.ts b/src/backup/password-policy.ts new file mode 100644 index 00000000..384ed42c --- /dev/null +++ b/src/backup/password-policy.ts @@ -0,0 +1,13 @@ +/** + * Minimum length for a backup EXPORT password (hardening pass on plan Unit + * U6/U7). Shared between the export route (`app/api/admin/backup/export/route.ts`, + * server-side enforcement) and the export panel + * (`app/(admin)/backup/export-panel.tsx`, client-side gating) so the two + * never drift. + * + * Deliberately NOT applied to restore: a restore's password must match + * whatever password encrypted the specific bundle being restored, including + * bundles produced before this minimum existed — enforcing it there would + * reject a legitimately-short older password. See `restore/route.ts`. + */ +export const MIN_BACKUP_PASSWORD_LENGTH = 12; diff --git a/src/backup/same-origin.ts b/src/backup/same-origin.ts new file mode 100644 index 00000000..07fcfcf7 --- /dev/null +++ b/src/backup/same-origin.ts @@ -0,0 +1,88 @@ +/** + * Explicit same-origin (CSRF) guard for the admin backup routes (hardening + * pass on plan Unit U6). + * + * `app/api/admin/backup/export/route.ts` and `.../restore/route.ts` are the + * highest-blast-radius endpoints in the app (whole-instance export/restore). + * They are plain Next.js Route Handlers, never routed through Better Auth's + * own request handler — so Better Auth's origin handling (`BETTER_AUTH_URL` + * / `trustedOrigins`) never sees them, and the session cookie's `sameSite` + * attribute is the only implicit CSRF protection they had. `sameSite` alone + * is not a substitute for an explicit check on a route this sensitive + * (older browsers, subdomain edge cases, misconfigured proxies), so both + * routes call {@link sameOriginError} right after their admin gate and + * return its 403 verbatim when it is non-null. + * + * Same-origin is decided from, in order: + * 1. `Origin` — present on essentially every modern-browser POST, same- or + * cross-origin (the Fetch spec has required it on state-changing + * requests for years). Covers both the restore panel's `fetch()` and, + * in practice, the export panel's real `` submit too. + * 2. `Referer` — a fallback for the rare navigation that omits `Origin` + * (some older/`Referrer-Policy`-restricted same-origin form + * navigations). This is the case the export route specifically needs the + * fallback for. + * 3. `Sec-Fetch-Site` — a Fetch-Metadata header some browsers attach even + * when neither of the above is present; `"same-origin"`/`"none"` (a + * direct, non-navigational request) are accepted, anything else refused. + * + * A request carrying none of the three is refused outright rather than + * assumed same-origin — every legitimate same-origin POST from this app's + * own export form or restore panel supplies at least one. + */ +export function sameOriginError(request: Request): Response | null { + const acceptable = acceptableOrigins(request); + + const origin = request.headers.get("origin"); + if (origin !== null) { + return acceptable.has(origin) ? null : mismatch(); + } + + const referer = request.headers.get("referer"); + if (referer !== null) { + const refererOrigin = safeOrigin(referer); + return refererOrigin !== null && acceptable.has(refererOrigin) + ? null + : mismatch(); + } + + const secFetchSite = request.headers.get("sec-fetch-site"); + if (secFetchSite !== null) { + return secFetchSite === "same-origin" || secFetchSite === "none" + ? null + : mismatch(); + } + + return mismatch(); +} + +/** + * The set of origins this request may legitimately have come from: the + * origin the request itself arrived on (derived from `request.url`, i.e. + * the `Host` Next.js resolved the request against), plus `BETTER_AUTH_URL`'s + * origin when configured — the same env var `auth.ts`/Better Auth treats as + * this deployment's canonical public origin (see `AGENTS.md`: "`BETTER_AUTH_URL` + * must equal the request origin"). Accepting both, rather than only one, + * keeps this correct whether or not a reverse proxy rewrites `Host`. + */ +function acceptableOrigins(request: Request): ReadonlySet { + const origins = new Set([new URL(request.url).origin]); + const configured = process.env.BETTER_AUTH_URL; + if (configured) { + const configuredOrigin = safeOrigin(configured); + if (configuredOrigin !== null) origins.add(configuredOrigin); + } + return origins; +} + +function safeOrigin(url: string): string | null { + try { + return new URL(url).origin; + } catch { + return null; + } +} + +function mismatch(): Response { + return new Response(null, { status: 403 }); +} From 8591cd8c79a611886eea691b9beea7b9faa92454 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 23:06:12 -0400 Subject: [PATCH 13/26] feat(backup): snapshot-isolate export + cap NDJSON import line length Wrap exportDatabase's per-table SELECTs in one repeatable-read, read-only transaction (export-service.ts's bufferDbExport) so a write landing mid-export can no longer produce a torn bundle mixing pre/post rows. Cap a single db.ndjson line at 8 MiB during import (db-import.ts) and reject the bundle before the oversized line is ever buffered or parsed, closing a memory-exhaustion vector in a crafted/corrupted restore bundle. Signed-off-by: UncleSp1d3r --- src/backup/__tests__/db-roundtrip.test.ts | 70 ++++++++++++++- src/backup/__tests__/export-service.test.ts | 72 ++++++++++++++++ src/backup/db-import.ts | 95 ++++++++++++++++++--- src/backup/export-service.ts | 41 ++++++--- 4 files changed, 253 insertions(+), 25 deletions(-) diff --git a/src/backup/__tests__/db-roundtrip.test.ts b/src/backup/__tests__/db-roundtrip.test.ts index fc4f69e3..fb8c07dc 100644 --- a/src/backup/__tests__/db-roundtrip.test.ts +++ b/src/backup/__tests__/db-roundtrip.test.ts @@ -38,7 +38,11 @@ import { verification, } from "../../db/schema"; import { type ExportedRow, exportDatabase } from "../db-export"; -import { importDatabase, wipeDatabase } from "../db-import"; +import { + importDatabase, + MAX_NDJSON_LINE_BYTES, + wipeDatabase, +} from "../db-import"; import { EPHEMERAL_TABLE_NAMES, EXPORT_TABLE_ORDER } from "../table-order"; /** @@ -375,4 +379,68 @@ describe("DB export/import round trip (U3)", () => { expect(afterMagazine.baseCapacity).toBe(beforeMagazine.baseCapacity); expect(afterMagazine.baseCapacity).toBe(17); }); + + test("import rejects a db.ndjson line larger than the cap, without inserting any rows (oversized-line DoS guard)", async () => { + const validLine = `${JSON.stringify({ + table: "user", + row: { + id: randomUUID(), + name: "Should Not Persist", + email: `${randomUUID()}@example.test`, + }, + } satisfies ExportedRow)}\n`; + // Deliberately no trailing newline — the reader must reject this before + // ever handing it to JSON.parse, not just at end-of-stream. + const oversizedLine = JSON.stringify({ + table: "user", + row: { + id: randomUUID(), + name: "x".repeat(MAX_NDJSON_LINE_BYTES + 1024), + email: `${randomUUID()}@example.test`, + }, + } satisfies ExportedRow); + + let caught: unknown; + try { + await importDatabase(db, Readable.from([validLine + oversizedLine])); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toMatch(/longer than/i); + + // The whole import runs in one transaction — the oversized line's + // rejection must roll back the earlier, otherwise-valid `user` insert + // too, not leave a partially-applied import. + const rows = await db.select().from(user); + expect(rows).toHaveLength(0); + }); + + test("import still handles a final line with no trailing newline (no regression from switching off the readline-based reader)", async () => { + const rows: ExportedRow[] = [ + { + table: "user", + row: { + id: randomUUID(), + name: "No Trailing Newline A", + email: `${randomUUID()}@example.test`, + }, + }, + { + table: "user", + row: { + id: randomUUID(), + name: "No Trailing Newline B", + email: `${randomUUID()}@example.test`, + }, + }, + ]; + const ndjson = rows.map((row) => JSON.stringify(row)).join("\n"); + + await importDatabase(db, Readable.from([ndjson])); + + const persisted = await db.select().from(user); + expect(persisted).toHaveLength(2); + }); }); diff --git a/src/backup/__tests__/export-service.test.ts b/src/backup/__tests__/export-service.test.ts index 5ba76d2e..d495b959 100644 --- a/src/backup/__tests__/export-service.test.ts +++ b/src/backup/__tests__/export-service.test.ts @@ -192,6 +192,49 @@ function makeStagingDir(): string { return mkdtempSync(join(tmpdir(), "export-service-staging-")); } +/** + * Temporarily wraps `targetPool.connect` so every query text issued by + * whichever client a transaction checks out gets captured — used to prove + * `createBackup` actually opens its DB-export transaction with the + * isolation level and access mode the snapshot-isolation fix mandates. + * Drizzle issues that as a single `begin isolation level ... ` + * statement (see `PgTransaction.getTransactionConfigSQL` / + * `NodePgSession.transaction` in `drizzle-orm/pg-core/session.js` and + * `drizzle-orm/node-postgres/session.js`), passed to `client.query` as a + * `{ text, ... }` config object rather than a bare string. Always call + * `restore()` (a `try/finally` around the exercised code) so the spy doesn't + * leak into other tests sharing the same pooled connections. + */ +function captureBeginStatements(targetPool: Pool): { + statements: string[]; + restore: () => void; +} { + const statements: string[] = []; + const originalConnect = targetPool.connect.bind(targetPool); + // biome-ignore lint/suspicious/noExplicitAny: instrumenting pg's own overloaded connect()/query() for a test-only spy. + (targetPool as any).connect = async (...args: unknown[]) => { + // biome-ignore lint/suspicious/noExplicitAny: see above. + const client = await (originalConnect as any)(...args); + const originalQuery = client.query.bind(client); + client.query = (...queryArgs: unknown[]) => { + const first = queryArgs[0]; + const text = + typeof first === "string" + ? first + : (first as { text?: string } | undefined)?.text; + if (typeof text === "string") statements.push(text); + return originalQuery(...queryArgs); + }; + return client; + }; + return { + statements, + restore: () => { + targetPool.connect = originalConnect; + }, + }; +} + describe("backup export service (U4)", () => { let container: StartedPostgreSqlContainer; let pool: Pool; @@ -321,6 +364,35 @@ describe("backup export service (U4)", () => { expect(afterFiles).toEqual(beforeFiles); }); + test("the DB export runs inside a repeatable-read, read-only snapshot transaction (torn-bundle guard)", async () => { + const ownerId = `owner-${randomUUID()}`; + await db.insert(user).values({ + id: ownerId, + name: "Snapshot Owner", + email: `${ownerId}@example.test`, + }); + + const capture = captureBeginStatements(pool); + try { + const stream = await createBackup(PASSWORD, { db }); + await collectWithStats(stream); + } finally { + capture.restore(); + } + + const beginStatements = capture.statements.filter((statement) => + statement.trim().toLowerCase().startsWith("begin"), + ); + expect(beginStatements.length).toBeGreaterThan(0); + expect( + beginStatements.some( + (statement) => + statement.trim().toLowerCase() === + "begin isolation level repeatable read read only", + ), + ).toBe(true); + }); + test("a non-admin caller is rejected", async () => { currentIsAdmin = false; diff --git a/src/backup/db-import.ts b/src/backup/db-import.ts index 53d8ff33..7369f518 100644 --- a/src/backup/db-import.ts +++ b/src/backup/db-import.ts @@ -1,4 +1,3 @@ -import { createInterface } from "node:readline"; import type { Readable } from "node:stream"; import { getTableColumns, getTableName } from "drizzle-orm"; import type { PgTable } from "drizzle-orm/pg-core"; @@ -10,6 +9,78 @@ const TABLES_BY_NAME: ReadonlyMap = new Map( EXPORT_TABLE_ORDER.map((table) => [getTableName(table), table]), ); +const NEWLINE_BYTE = 0x0a; // "\n" + +/** + * Hard ceiling on a single NDJSON line's byte length before import refuses to + * read any more of it. A restore bundle is admin-uploaded, but its content is + * still untrusted (corrupted download, or a deliberately crafted bundle) — + * without a bound, one enormous unterminated line would buffer without limit + * in memory before `JSON.parse` ever ran (a memory-exhaustion vector). The + * largest legitimate line here is one row of inventory JSON — a handful of + * scalar/UUID/timestamp columns, never a blob (blob content lives as its own + * bundle entry, never inline in `db.ndjson`) — so a real row is a few KiB at + * most. 8 MiB is generous headroom above that while still being a finite, + * enforced ceiling. + */ +export const MAX_NDJSON_LINE_BYTES = 8 * 1024 * 1024; + +/** + * Reads `stream` as NDJSON lines (`\n`-terminated; a trailing `\r` is + * trimmed so CRLF-terminated files still work), bounding how many bytes may + * accumulate for a single line before a newline is seen. Reads whatever + * chunks the underlying stream delivers and checks the running total after + * each one, so a line is rejected — via `MAX_NDJSON_LINE_BYTES` above — as + * soon as it crosses the cap, before the offending line is ever fully + * buffered or handed to `JSON.parse`. Throwing mid-iteration lets + * `for await...of`'s built-in cleanup (calling the async iterator's + * `return()`) close/destroy the source stream. + */ +async function* readBoundedLines( + stream: Readable, + maxLineBytes: number, +): AsyncGenerator { + // Typed as `Buffer` (not the narrower default + // `Buffer`) because `Readable`'s async-iterator chunks and + // `Buffer.from()`'s overload resolution on them both carry that wider, + // more permissive generic. + let pending: Buffer = Buffer.alloc(0); + + for await (const rawChunk of stream) { + const chunk: Buffer = Buffer.isBuffer(rawChunk) + ? rawChunk + : Buffer.from(rawChunk); + pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]); + + let newlineIndex = pending.indexOf(NEWLINE_BYTE); + while (newlineIndex !== -1) { + yield decodeLine(pending.subarray(0, newlineIndex)); + pending = pending.subarray(newlineIndex + 1); + newlineIndex = pending.indexOf(NEWLINE_BYTE); + } + + if (pending.length > maxLineBytes) { + throw new Error( + `Backup's db.ndjson contains a line longer than ${maxLineBytes} bytes (no newline found) — refusing to import; this may be a corrupted or malicious bundle.`, + ); + } + } + + if (pending.length > 0) { + if (pending.length > maxLineBytes) { + throw new Error( + `Backup's db.ndjson ends with a line longer than ${maxLineBytes} bytes — refusing to import; this may be a corrupted or malicious bundle.`, + ); + } + yield decodeLine(pending); + } +} + +function decodeLine(buffer: Buffer): string { + const text = buffer.toString("utf8"); + return text.endsWith("\r") ? text.slice(0, -1) : text; +} + /** * JS property keys (not SQL column names) on `table` whose values are * date/timestamp columns. Cached per table by `importDatabase` since the same @@ -46,13 +117,15 @@ function reviveRow( /** * Import an NDJSON export produced by `exportDatabase` (U3, R5/R10). * - * Reads the stream line by line (never buffering the whole file) and inserts - * each row inside ONE transaction, so a failure partway through — a parse - * error, an unknown table, a constraint violation — rolls back every row - * already inserted rather than leaving a half-restored database. Insert order - * is whatever order the file already carries its rows in: `exportDatabase` - * always writes tables in `EXPORT_TABLE_ORDER` (FK-safe), so import trusts - * that order instead of re-sorting or buffering the file to re-derive it. + * Reads the stream line by line (never buffering the whole file, and never + * buffering a single line past `MAX_NDJSON_LINE_BYTES` — see + * `readBoundedLines`) and inserts each row inside ONE transaction, so a + * failure partway through — a parse error, an unknown table, a constraint + * violation, an oversized line — rolls back every row already inserted + * rather than leaving a half-restored database. Insert order is whatever + * order the file already carries its rows in: `exportDatabase` always writes + * tables in `EXPORT_TABLE_ORDER` (FK-safe), so import trusts that order + * instead of re-sorting or buffering the file to re-derive it. * * Refuse-unless-empty, force-replace, and version-compatibility checks are * the caller's job (a later restore-flow unit) — this function is the raw @@ -64,12 +137,8 @@ export async function importDatabase( ): Promise { await db.transaction(async (tx) => { const dateKeyCache = new Map(); - const lines = createInterface({ - input: stream, - crlfDelay: Number.POSITIVE_INFINITY, - }); - for await (const line of lines) { + for await (const line of readBoundedLines(stream, MAX_NDJSON_LINE_BYTES)) { if (line.trim() === "") continue; const parsed = JSON.parse(line) as ExportedRow; diff --git a/src/backup/export-service.ts b/src/backup/export-service.ts index a48d1f38..5bb7bbfe 100644 --- a/src/backup/export-service.ts +++ b/src/backup/export-service.ts @@ -17,7 +17,7 @@ import { join } from "node:path"; import { Readable } from "node:stream"; import { NotAuthorizedError } from "@/src/auth/errors"; import { isAdmin } from "@/src/auth/session"; -import type { DbOrTx } from "@/src/db/client"; +import type { Database } from "@/src/db/client"; import { activeStorageRoot } from "@/src/storage/index"; import { type BundleBlobEntry, writeBundle } from "./bundle"; import { createEncryptStream, deriveKey, generateSalt } from "./crypto"; @@ -103,24 +103,43 @@ async function* blobEntriesFor( * `exportDatabase` yields exactly one NDJSON line per row, so the row count is * tallied in the same pass that buffers the bytes — no second scan of the * payload. The `Buffer` is returned as-is (no `toString`/re-encode round trip). + * + * Runs the whole export inside one `repeatable read`, `read only` snapshot + * transaction: every per-table `SELECT` `exportDatabase` issues then shares + * a single MVCC snapshot taken at the transaction's first statement. Without + * this, `exportDatabase`'s per-table `SELECT`s ran unscoped against + * whatever the table looked like at the moment each one executed — a write + * landing mid-export (e.g. a new `firearm_document` row FK'ing to a + * `firearm` the export already read) could produce a torn bundle mixing + * pre- and post-write rows, potentially violating referential integrity + * within the bundle itself. `read only` is an extra guarantee that this + * transaction can never itself write, matching what it's actually doing. */ async function bufferDbExport( - db: DbOrTx, + db: Database, ): Promise<{ buffer: Buffer; rowCount: number }> { - const chunks: Buffer[] = []; - let rowCount = 0; - for await (const chunk of exportDatabase(db)) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - rowCount += 1; - } - return { buffer: Buffer.concat(chunks), rowCount }; + return db.transaction( + async (tx) => { + const chunks: Buffer[] = []; + let rowCount = 0; + for await (const chunk of exportDatabase(tx)) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + rowCount += 1; + } + return { buffer: Buffer.concat(chunks), rowCount }; + }, + { isolationLevel: "repeatable read", accessMode: "read only" }, + ); } export interface CreateBackupOptions { /** Drizzle handle to export from. Production callers pass the shared `db` * from `@/src/db/client`; tests pass a handle bound to their own instance - * (e.g. a Testcontainers Postgres). */ - readonly db: DbOrTx; + * (e.g. a Testcontainers Postgres). Must be a top-level `Database`, not an + * already-open `Transaction` — `bufferDbExport` opens its own + * repeatable-read, read-only snapshot transaction around the whole export + * to keep the bundle internally consistent even if writes land mid-export. */ + readonly db: Database; } /** From 9caec0a73adef64c4a7cdd33bd0e29ba04280ee7 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 23:07:18 -0400 Subject: [PATCH 14/26] =?UTF-8?q?feat(backup):=20harden=20restore=20core?= =?UTF-8?q?=20=E2=80=94=20unique=20schemas,=20crash=20recovery,=20maintena?= =?UTF-8?q?nce=20guard,=20error=20classification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes to the restore core from code review: 1. Per-run unique staging/snapshot schema names (restore_staging_ / restore_snapshot_) plus a single advisory lock held for restore()'s ENTIRE body (staging through promote, both empty-instance and force paths) instead of just forcePromote's promote step. Two concurrent restores now fully serialize instead of racing to promote into public at the same time. 2. Boot-time crash recovery: maintenance.ts now records which snapshot schema belongs to an in-progress force-restore, and exports recoverInterruptedRestore(db, uploadDir) to roll the DB back from that snapshot, restore blobs from the newest pre-restore directory, and sweep any leftover restore_staging_*/restore_snapshot_* schemas and restore-staging-* temp dirs. Wired into a new instrumentation.ts register() hook, guarded on NEXT_RUNTIME=nodejs and DATABASE_URL, and never throws out. 3. Exported assertWritesAllowed(db) + MaintenanceModeError from maintenance.ts for the write-path enforcement worker to consume. 4. restore()'s staging-error classification now only maps genuine crypto auth failures (DecryptionAuthError/InvalidHeaderError) to wrong_password_or_tampered; every other staging error (ENOSPC, malformed tar, path traversal, ...) re-throws so the route's own catch-all surfaces a generic failure instead of misleading the operator. Signed-off-by: UncleSp1d3r --- instrumentation.ts | 46 +++ src/backup/__tests__/maintenance.test.ts | 355 ++++++++++++++++++ src/backup/__tests__/restore-service.test.ts | 136 ++++++- src/backup/maintenance.ts | 365 +++++++++++++++++- src/backup/restore-service.ts | 372 +++++++++++-------- 5 files changed, 1108 insertions(+), 166 deletions(-) create mode 100644 instrumentation.ts create mode 100644 src/backup/__tests__/maintenance.test.ts diff --git a/instrumentation.ts b/instrumentation.ts new file mode 100644 index 00000000..e21461f9 --- /dev/null +++ b/instrumentation.ts @@ -0,0 +1,46 @@ +/** + * Next.js server-startup hook (the `register()` file convention — see + * `node_modules/next/dist/docs/01-app/api-reference/file-conventions/instrumentation.md`). + * Next.js calls `register()` once, in every runtime, when a new server + * instance boots — including during `next build`'s page-data collection, + * where no database is configured at all. + * + * This wires the backup/restore crash-recovery sweep + * (`recoverInterruptedRestore`, `src/backup/maintenance.ts`, KTD5 hardening): + * if a force-restore was interrupted mid-flight (a crash between entering + * maintenance and finishing cleanup), the durable maintenance flag is left + * active and the pre-restore snapshot schema/blob directory are orphaned. + * Running this once at boot rolls the DB and blobs back to their pre-restore + * state and sweeps any other leftover restore staging/snapshot schema or + * temp directory, before the server starts handling requests. + * + * Guarded on BOTH conditions the Next.js docs call out for + * runtime-/environment-specific code in `register()`: + * - `NEXT_RUNTIME === "nodejs"` — the DB pool (`pg`) and filesystem access + * this needs don't exist at the Edge, and importing them there would throw. + * - `DATABASE_URL` is set — this file is imported unconditionally by every + * Next.js server instance, so it must never assume a database is + * reachable (mirrors `src/db/client.ts`'s own lazy-construction contract). + * + * A recovery failure is caught and logged, never thrown: this must never + * prevent the server from starting. + */ +export async function register(): Promise { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + if (!process.env.DATABASE_URL) return; + + try { + const [{ db }, { activeStorageRoot }, { recoverInterruptedRestore }] = + await Promise.all([ + import("@/src/db/client"), + import("@/src/storage"), + import("@/src/backup/maintenance"), + ]); + await recoverInterruptedRestore(db, activeStorageRoot()); + } catch (err) { + console.error( + "instrumentation: backup/restore crash-recovery sweep failed to run", + err, + ); + } +} diff --git a/src/backup/__tests__/maintenance.test.ts b/src/backup/__tests__/maintenance.test.ts new file mode 100644 index 00000000..a81e0d1a --- /dev/null +++ b/src/backup/__tests__/maintenance.test.ts @@ -0,0 +1,355 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { getTableName, sql } from "drizzle-orm"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import * as schema from "../../db/schema"; +import { firearm, user } from "../../db/schema"; +import { wipeDatabase } from "../db-import"; +import { + assertWritesAllowed, + enterMaintenance, + exitMaintenance, + isMaintenanceActive, + MaintenanceModeError, + recordMaintenanceSnapshotSchema, + recoverInterruptedRestore, + SNAPSHOT_SCHEMA_PREFIX, + STAGING_SCHEMA_PREFIX, +} from "../maintenance"; +import { EXPORT_TABLE_ORDER } from "../table-order"; + +/** + * Integration tests for the maintenance envelope's crash-recovery and + * write-blocking primitives (KTD5 hardening). Every test runs against an + * ephemeral Testcontainers Postgres (never the ambient dev DB) plus a + * per-test temporary "UPLOAD_DIR" on the real filesystem, matching + * `restore-service.test.ts`'s harness. + */ +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +type Db = NodePgDatabase; + +function qualified(schemaName: string, name: string) { + return sql`${sql.identifier(schemaName)}.${sql.identifier(name)}`; +} + +async function snapshotTables( + db: Db, +): Promise[]>> { + const snapshot: Record[]> = {}; + for (const table of EXPORT_TABLE_ORDER) { + // biome-ignore lint/suspicious/noExplicitAny: EXPORT_TABLE_ORDER is deliberately heterogeneous. + const rows = (await db.select().from(table as any)) as Record< + string, + unknown + >[]; + snapshot[getTableName(table)] = rows.sort((a, b) => + JSON.stringify(a).localeCompare(JSON.stringify(b)), + ); + } + return snapshot; +} + +/** Seeds one user + one firearm under a fresh random owner; returns the owner label so callers can tell datasets apart. */ +async function seedOwner(db: Db, label: string): Promise { + const ownerId = `owner-${randomUUID()}`; + await db + .insert(user) + .values({ id: ownerId, name: label, email: `${ownerId}@example.test` }); + await db + .insert(firearm) + .values({ ownerId, name: `${label} FA`, caliber: "9mm" }); + return ownerId; +} + +/** Mimics `restore-service.ts`'s `forcePromote` snapshot creation: a committed `CREATE TABLE ... AS TABLE` copy of every `public` table under a fresh schema. */ +async function createSnapshotSchemaFromCurrentState( + db: Db, + snapshotSchema: string, +): Promise { + await db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(snapshotSchema)} CASCADE`, + ); + await db.execute(sql`CREATE SCHEMA ${sql.identifier(snapshotSchema)}`); + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await db.execute(sql` + CREATE TABLE ${qualified(snapshotSchema, name)} + AS TABLE ${qualified("public", name)} + `); + } +} + +async function schemaExists(db: Db, schemaName: string): Promise { + const result = await db.execute<{ present: boolean }>(sql` + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = ${schemaName} + ) AS present + `); + return result.rows[0]?.present ?? false; +} + +async function listRestoreSchemas(db: Db): Promise { + const result = await db.execute<{ nspname: string }>(sql` + SELECT nspname FROM pg_catalog.pg_namespace + WHERE nspname ~ '^restore_(staging|snapshot)_' + ORDER BY nspname + `); + return result.rows.map((r) => r.nspname); +} + +async function readUploadDirKeys(uploadDir: string): Promise { + try { + return (await readdir(uploadDir)).sort(); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return []; + throw err; + } +} + +describe("maintenance envelope (KTD5 hardening)", () => { + let container: StartedPostgreSqlContainer; + let pool: Pool; + let db: Db; + let uploadDir: string; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_maintenance_test") + .start(); + pool = new Pool({ connectionString: container.getConnectionUri() }); + db = drizzle(pool, { schema }); + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + }, 120_000); + + afterAll(async () => { + await pool?.end(); + await container?.stop(); + }); + + beforeEach(async () => { + await wipeDatabase(db); + await exitMaintenance(db); + for (const schemaName of await listRestoreSchemas(db)) { + await db + .execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(schemaName)} CASCADE`, + ) + .catch(() => {}); + } + uploadDir = await mkdtemp(join(tmpdir(), "magstacker-maintenance-upload-")); + }); + + afterEach(async () => { + const parentDir = join(uploadDir, ".."); + const entries = await readdir(parentDir).catch(() => [] as string[]); + const uploadBase = uploadDir.slice(parentDir.length + 1); + for (const entry of entries) { + if ( + entry === uploadBase || + entry.startsWith(`${uploadBase}.pre-restore-`) || + entry.startsWith("restore-staging-") + ) { + await rm(join(parentDir, entry), { + recursive: true, + force: true, + }).catch(() => {}); + } + } + }); + + describe("assertWritesAllowed", () => { + test("resolves without throwing when maintenance is not active", async () => { + expect(await isMaintenanceActive(db)).toBe(false); + await expect(assertWritesAllowed(db)).resolves.toBeUndefined(); + }); + + test("throws MaintenanceModeError while maintenance is active", async () => { + await enterMaintenance(db, "force-restore"); + + let thrown: unknown; + try { + await assertWritesAllowed(db); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(MaintenanceModeError); + expect((thrown as Error).message).toMatch(/maintenance/i); + + await exitMaintenance(db); + await expect(assertWritesAllowed(db)).resolves.toBeUndefined(); + }); + }); + + describe("recoverInterruptedRestore", () => { + test("is a no-op when maintenance was never active", async () => { + await seedOwner(db, "Untouched Owner"); + const before = await snapshotTables(db); + + await recoverInterruptedRestore(db, uploadDir); + + expect(await snapshotTables(db)).toEqual(before); + expect(await isMaintenanceActive(db)).toBe(false); + }); + + test("rolls back the DB from the recorded snapshot schema and clears the flag when a force-restore was interrupted mid-promote", async () => { + // Simulate `forcePromote`'s sequence up to the crash point: the + // pre-restore data is snapshotted, the flag records that snapshot, + // then the wipe+promote transaction commits NEW data — and the + // process "crashes" right there, before cleanup ever runs. + await seedOwner(db, "Pre-Restore Owner"); + const expectedSnapshot = await snapshotTables(db); + + const snapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + await createSnapshotSchemaFromCurrentState(db, snapshotSchema); + + await wipeDatabase(db); + await seedOwner(db, "Half-Promoted New Owner"); + + await enterMaintenance(db, "force-restore"); + await recordMaintenanceSnapshotSchema(db, snapshotSchema); + // No `exitMaintenance` call — this is the "crash before cleanup" state. + + expect(await isMaintenanceActive(db)).toBe(true); + + await recoverInterruptedRestore(db, uploadDir); + + expect(await snapshotTables(db)).toEqual(expectedSnapshot); + expect(await isMaintenanceActive(db)).toBe(false); + expect(await schemaExists(db, snapshotSchema)).toBe(false); + }); + + test("restores blobs from the newest pre-restore directory and discards stale ones during an interrupted-restore rollback", async () => { + await seedOwner(db, "Pre-Restore Owner"); + const expectedSnapshot = await snapshotTables(db); + const snapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + await createSnapshotSchemaFromCurrentState(db, snapshotSchema); + await wipeDatabase(db); + await seedOwner(db, "Half-Promoted New Owner"); + await enterMaintenance(db, "force-restore"); + await recordMaintenanceSnapshotSchema(db, snapshotSchema); + + // The pre-restore blob directory `beginBlobSwap` would have moved + // aside — holds the ORIGINAL blobs, which is what recovery must + // restore `uploadDir` back to. + const staleDir = `${uploadDir}.pre-restore-${randomUUID()}`; + await mkdir(staleDir, { recursive: true }); + await writeFile( + join(staleDir, "stale.txt"), + "stale — from an even older crash", + ); + + const originalDir = `${uploadDir}.pre-restore-${randomUUID()}`; + await mkdir(originalDir, { recursive: true }); + await writeFile( + join(originalDir, "original.txt"), + "original pre-restore blob", + ); + // The newest-by-mtime directory is the one that should win — force a + // detectable ordering by writing the "original" (newest) dir last. + + // `uploadDir` itself holds whatever the half-finished force-restore + // had already swapped in — new, half-promoted blobs. + await rm(uploadDir, { recursive: true, force: true }).catch(() => {}); + await mkdir(uploadDir, { recursive: true }); + await writeFile(join(uploadDir, "half-promoted.txt"), "new blob"); + + await recoverInterruptedRestore(db, uploadDir); + + expect(await snapshotTables(db)).toEqual(expectedSnapshot); + expect(await isMaintenanceActive(db)).toBe(false); + + const files = await readUploadDirKeys(uploadDir); + expect(files).toEqual(["original.txt"]); + const content = await readFile(join(uploadDir, "original.txt"), "utf8"); + expect(content).toBe("original pre-restore blob"); + + // Both pre-restore directories (the newest one that was consumed, and + // the stale older one) must be gone afterward — nothing left orphaned. + const parentDir = join(uploadDir, ".."); + const remaining = await readdir(parentDir); + expect(remaining.some((name) => name.includes(".pre-restore-"))).toBe( + false, + ); + }); + + test("sweeps leftover restore_staging_*/restore_snapshot_* schemas and restore-staging-* temp directories regardless of flag state", async () => { + const leftoverStagingSchema = `${STAGING_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + const leftoverSnapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + await db.execute( + sql`CREATE SCHEMA ${sql.identifier(leftoverStagingSchema)}`, + ); + await db.execute( + sql`CREATE SCHEMA ${sql.identifier(leftoverSnapshotSchema)}`, + ); + + const leftoverStagingDir = join( + join(uploadDir, ".."), + `restore-staging-${randomUUID()}`, + ); + await mkdir(leftoverStagingDir, { recursive: true }); + await writeFile(join(leftoverStagingDir, "orphan.bin"), "orphan"); + + expect(await isMaintenanceActive(db)).toBe(false); + + await recoverInterruptedRestore(db, uploadDir); + + expect(await schemaExists(db, leftoverStagingSchema)).toBe(false); + expect(await schemaExists(db, leftoverSnapshotSchema)).toBe(false); + expect(await listRestoreSchemas(db)).toEqual([]); + + const parentDir = join(uploadDir, ".."); + const remaining = await readdir(parentDir); + expect( + remaining.some((name) => name.startsWith("restore-staging-")), + ).toBe(false); + }); + + test("is idempotent — calling it twice in a row after a rollback is a harmless no-op", async () => { + await seedOwner(db, "Pre-Restore Owner"); + const expectedSnapshot = await snapshotTables(db); + const snapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + await createSnapshotSchemaFromCurrentState(db, snapshotSchema); + await wipeDatabase(db); + await seedOwner(db, "Half-Promoted New Owner"); + await enterMaintenance(db, "force-restore"); + await recordMaintenanceSnapshotSchema(db, snapshotSchema); + + await recoverInterruptedRestore(db, uploadDir); + const afterFirst = await snapshotTables(db); + expect(afterFirst).toEqual(expectedSnapshot); + + // Second call: flag is already clear and the snapshot schema is + // already gone — must not throw, and must not change anything. + await recoverInterruptedRestore(db, uploadDir); + expect(await snapshotTables(db)).toEqual(afterFirst); + expect(await isMaintenanceActive(db)).toBe(false); + }); + }); +}); diff --git a/src/backup/__tests__/restore-service.test.ts b/src/backup/__tests__/restore-service.test.ts index 8382a77d..e82fd0e4 100644 --- a/src/backup/__tests__/restore-service.test.ts +++ b/src/backup/__tests__/restore-service.test.ts @@ -31,6 +31,7 @@ import * as tar from "tar-stream"; import { NotAuthorizedError } from "../../auth/errors"; import * as schema from "../../db/schema"; import { firearm, firearmDocument, user } from "../../db/schema"; +import { PathTraversalError } from "../../storage"; import { type BundleBlobEntry, writeBundle } from "../bundle"; import { createEncryptStream, deriveKey, generateSalt } from "../crypto"; import { exportDatabase } from "../db-export"; @@ -40,7 +41,7 @@ import { type BackupManifest, buildManifest, } from "../manifest"; -import { restore } from "../restore-service"; +import { type RestoreOutcome, restore } from "../restore-service"; import { EXPORT_TABLE_ORDER } from "../table-order"; /** @@ -596,7 +597,14 @@ describe("restore service (U5)", () => { ).toBe(true); }); - test("a path-traversal blob entry is refused; nothing is written outside staging (KTD11)", async () => { + test("a path-traversal blob entry is refused; nothing is written outside staging (KTD11) — and surfaces as a generic thrown error, not wrong_password_or_tampered", async () => { + // Reclassification fix: a path-traversal entry is a bundle-content + // safety violation, not a cryptographic authentication failure — it + // must NOT be reported to the operator as "wrong password or tampered + // bundle" (that would send them chasing the wrong problem). `restore()` + // re-throws it instead, so the route's own catch-all surfaces its + // generic "restore failed unexpectedly" outcome (see + // `app/api/admin/backup/restore/route.ts`). const beforeSnapshot = await snapshotTables(db); const manifest = buildManifest({ counts: { rows: 0, blobs: 1, totalBlobBytes: 10 }, @@ -610,17 +618,45 @@ describe("restore service (U5)", () => { PASSWORD, ); - const outcome = await restore( - Readable.from([bundle]), - PASSWORD, - restoreOptions(), - ); + let thrown: unknown; + try { + await restore(Readable.from([bundle]), PASSWORD, restoreOptions()); + } catch (err) { + thrown = err; + } - expect(outcome.kind).toBe("wrong_password_or_tampered"); + expect(thrown).toBeInstanceOf(PathTraversalError); expect(await snapshotTables(db)).toEqual(beforeSnapshot); expect(await readUploadDirKeys(uploadDir)).toEqual([]); }); + test("a genuine wrong password still maps to wrong_password_or_tampered, not a generic thrown error", async () => { + await seedInventory(db, uploadDir); + const beforeSnapshot = await snapshotTables(db); + const beforeFiles = await readUploadDirKeys(uploadDir); + + const bundle = await buildEncryptedBundle(db, { + password: "the-real-password", + }); + + let thrown: unknown; + let outcome: RestoreOutcome | undefined; + try { + outcome = await restore( + Readable.from([bundle]), + "definitely-wrong-password", + restoreOptions(), + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeUndefined(); + expect(outcome?.kind).toBe("wrong_password_or_tampered"); + expect(await snapshotTables(db)).toEqual(beforeSnapshot); + expect(await readUploadDirKeys(uploadDir)).toEqual(beforeFiles); + }); + test("a non-admin caller is refused inside the service, independent of any route gate (R14)", async () => { const beforeSnapshot = await snapshotTables(db); const bundle = await buildEncryptedBundle(db, {}); @@ -639,4 +675,88 @@ describe("restore service (U5)", () => { expect(thrown).toBeInstanceOf(NotAuthorizedError); expect(await snapshotTables(db)).toEqual(beforeSnapshot); }); + + test("two concurrent restore() calls against the same instance serialize via the per-run advisory lock; the final state is exactly one bundle's, not a mix (KTD5 hardening)", async () => { + /** Seeds a distinct one-owner/one-firearm/one-blob dataset, builds an encrypted bundle from it, snapshots the expected post-restore DB state, then wipes back to empty — leaving `db` empty again for the next call/the actual concurrent test below. */ + async function buildBundleWithExpectedSnapshot( + ownerLabel: string, + blobKey: string, + blobContent: Buffer, + ) { + const ownerId = `owner-${randomUUID()}`; + await db.insert(user).values({ + id: ownerId, + name: ownerLabel, + email: `${ownerId}@example.test`, + }); + const [firearmRow] = await db + .insert(firearm) + .values({ ownerId, name: `${ownerLabel} FA`, caliber: "9mm" }) + .returning(); + if (!firearmRow) throw new Error("seed failed"); + await db.insert(firearmDocument).values({ + firearmId: firearmRow.id, + storageKey: blobKey, + filename: "doc.pdf", + mimeType: "application/pdf", + sizeBytes: blobContent.byteLength, + docType: "receipt", + }); + + const bundle = await buildEncryptedBundle(db, { + blobs: [{ key: blobKey, content: blobContent }], + }); + const expectedSnapshot = await snapshotTables(db); + await wipeDatabase(db); + + return { bundle, expectedSnapshot, blobKey }; + } + + const a = await buildBundleWithExpectedSnapshot( + "Owner A", + "a-doc.pdf", + Buffer.from("bundle A"), + ); + const b = await buildBundleWithExpectedSnapshot( + "Owner B", + "b-doc.pdf", + Buffer.from("bundle B"), + ); + // Sanity: the two bundles really are different datasets — otherwise a + // "which one won" assertion below would be meaningless. + expect(a.expectedSnapshot).not.toEqual(b.expectedSnapshot); + + // Both calls target the same empty instance with `force: false`: without + // the fix, unsynchronized concurrent staging+promote could interleave + // and leave `public` a mix of both bundles. With the fix, the advisory + // lock (held for `restore()`'s ENTIRE body, not just the promote step) + // fully serializes them — whichever call acquires the lock first runs + // to completion before the second one's own emptiness check even runs, + // so the second is refused as `NotEmptySignal` sees a non-empty instance. + const [outcomeA, outcomeB] = await Promise.all([ + restore(Readable.from([a.bundle]), PASSWORD, restoreOptions()), + restore(Readable.from([b.bundle]), PASSWORD, restoreOptions()), + ]); + const outcomes = [outcomeA, outcomeB]; + + expect(outcomes.filter((o) => o.kind === "ok")).toHaveLength(1); + expect(outcomes.filter((o) => o.kind === "refused_not_empty")).toHaveLength( + 1, + ); + + const finalSnapshot = await snapshotTables(db); + const matchesA = + JSON.stringify(finalSnapshot) === JSON.stringify(a.expectedSnapshot); + const matchesB = + JSON.stringify(finalSnapshot) === JSON.stringify(b.expectedSnapshot); + // Exactly one bundle's data won — never neither (a broken restore) and + // never both/a mix (the corruption race this fix closes). + expect(matchesA !== matchesB).toBe(true); + + const winningBlobKey = matchesA ? a.blobKey : b.blobKey; + const losingBlobKey = matchesA ? b.blobKey : a.blobKey; + const files = await readUploadDirKeys(uploadDir); + expect(files).toEqual([winningBlobKey]); + expect(files).not.toContain(losingBlobKey); + }); }); diff --git a/src/backup/maintenance.ts b/src/backup/maintenance.ts index a033941b..274763aa 100644 --- a/src/backup/maintenance.ts +++ b/src/backup/maintenance.ts @@ -9,13 +9,13 @@ * * **Durable flag.** A single-row table (`restore_ops.maintenance_flag`) * rather than an in-memory flag: the flag must survive a process restart so - * a crash mid-force-restore is still visible afterward. Crash-recovery - * contract (consumed by future tooling, not built here — U5 only owns the - * flag primitive): `active = true` together with a still-present - * `restore_snapshot` schema (see `restore-service.ts`) signals an - * interrupted force-restore; `active = true` with no snapshot schema means - * the crash happened before the risky section began and nothing live was - * touched. + * a crash mid-force-restore is still visible afterward. The row also records + * which `restore_snapshot_*` schema (if any) belongs to the in-progress + * force-restore (`recordMaintenanceSnapshotSchema`) — this is what lets + * `recoverInterruptedRestore` tell "crashed before the risky section began, + * nothing live touched" (`active = true`, no recorded snapshot) apart from + * "crashed mid wipe+promote, live data may be in the new OR old state" + * (`active = true`, a recorded snapshot schema that still exists). * * **Pool-safe advisory lock.** `withRestoreAdvisoryLock` holds ONE * `pool.connect()`-checked-out client for its entire duration and issues a @@ -27,16 +27,53 @@ * held by a connection this code no longer has a handle on, and releasing * that connection back to the pool without unlocking would leak the lock for * the connection's lifetime. Dedicating and holding a single connection for - * the whole envelope avoids that. + * the whole envelope avoids that. `restore()` (`restore-service.ts`) now + * holds this lock for its ENTIRE body — staging through promote, for both + * the empty-instance and force paths — so two concurrent restore attempts + * fully serialize instead of racing to promote into `public` at the same + * time. Nothing else in this module re-acquires the lock (it is NOT + * reentrant across connections: a second `pg_advisory_lock` call for the + * same key on a different session blocks until the first session releases + * it, so calling this from inside an already-locked section would deadlock). + * + * **Write-blocking guard.** `assertWritesAllowed` lets the ordinary + * (non-restore) write path refuse writes for as long as the flag is active, + * so a request that lands mid-restore fails fast with a clear + * `MaintenanceModeError` instead of racing the restore's own wipe+promote. */ -import { sql } from "drizzle-orm"; +import type { Dirent } from "node:fs"; +import { readdir, rename, rm, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { getTableName, sql } from "drizzle-orm"; import type { Pool, PoolClient } from "pg"; import type { DbOrTx } from "@/src/db/client"; +import { EXPORT_TABLE_ORDER, WIPE_TABLE_ORDER } from "./table-order"; const MAINTENANCE_SCHEMA = "restore_ops"; const MAINTENANCE_TABLE = "maintenance_flag"; +/** + * Prefix every per-run staging schema `restore-service.ts` creates is named + * with (`${STAGING_SCHEMA_PREFIX}`). Exported so + * `recoverInterruptedRestore`'s leftover-schema sweep and + * `restore-service.ts`'s own per-run naming share one source of truth. + */ +export const STAGING_SCHEMA_PREFIX = "restore_staging_"; + +/** + * Prefix every per-run pre-restore snapshot schema `restore-service.ts` + * creates is named with (`${SNAPSHOT_SCHEMA_PREFIX}`). See + * {@link STAGING_SCHEMA_PREFIX}. + */ +export const SNAPSHOT_SCHEMA_PREFIX = "restore_snapshot_"; + +/** Prefix every staging blob directory `restore-service.ts` creates (a sibling of the upload dir) is named with. */ +const STAGING_BLOB_DIR_PREFIX = "restore-staging-"; + +/** Infix (before a random id) every moved-aside pre-restore blob directory is named with — a sibling of the upload dir. */ +const PRE_RESTORE_BLOB_DIR_INFIX = ".pre-restore-"; + /** * Fixed application-specific advisory-lock key for the force-restore * envelope. Arbitrary but must stay stable and must not collide with any @@ -48,6 +85,23 @@ function qualified(schemaName: string, name: string) { return sql`${sql.identifier(schemaName)}.${sql.identifier(name)}`; } +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return false; + throw err; + } +} + +function logRecoveryFailure(step: string, err: unknown): void { + // Crash recovery must never throw out of `recoverInterruptedRestore` (it + // runs from `instrumentation.ts`'s `register()`, and a thrown error there + // would fail the whole server's boot) — every failure is logged instead. + console.error(`backup/maintenance: recovery step "${step}" failed`, err); +} + /** * Creates the maintenance schema/table/singleton-row if they don't already * exist. Idempotent and cheap (`IF NOT EXISTS` / `ON CONFLICT DO NOTHING`) — @@ -67,6 +121,13 @@ async function ensureMaintenanceInfrastructure(db: DbOrTx): Promise { CONSTRAINT maintenance_flag_singleton CHECK (id) ) `); + // Added after the table's original shape (KTD5 hardening follow-up) — + // `IF NOT EXISTS` keeps this safe to re-run against a table created by an + // older version of this module. + await db.execute(sql` + ALTER TABLE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + ADD COLUMN IF NOT EXISTS snapshot_schema text + `); await db.execute(sql` INSERT INTO ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} (id, active) VALUES (true, false) @@ -84,7 +145,7 @@ export async function isMaintenanceActive(db: DbOrTx): Promise { return result.rows[0]?.active ?? false; } -/** Sets the durable flag active. Must be called before the risky section of a force-restore begins. */ +/** Sets the durable flag active. Must be called before the risky section of a force-restore begins. Clears any previously-recorded snapshot schema, so a fresh attempt always starts clean. */ export async function enterMaintenance( db: DbOrTx, reason: string, @@ -92,21 +153,63 @@ export async function enterMaintenance( await ensureMaintenanceInfrastructure(db); await db.execute(sql` UPDATE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} - SET active = true, reason = ${reason}, started_at = now() + SET active = true, reason = ${reason}, started_at = now(), snapshot_schema = NULL + WHERE id = true + `); +} + +/** + * Records which `restore_snapshot_*` schema belongs to the currently + * in-progress force-restore. Must only be called once that schema has been + * fully created and committed (i.e. it is genuinely safe to roll back from) + * — this is the single durable signal `recoverInterruptedRestore` uses to + * decide whether a crash happened before or during the risky wipe+promote + * section. + */ +export async function recordMaintenanceSnapshotSchema( + db: DbOrTx, + snapshotSchema: string, +): Promise { + await ensureMaintenanceInfrastructure(db); + await db.execute(sql` + UPDATE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + SET snapshot_schema = ${snapshotSchema} WHERE id = true `); } -/** Clears the durable flag. Always called from a `finally`, on both success and rollback. */ +/** Clears the durable flag (including the recorded snapshot schema). Always called from a `finally`, on both success and rollback. */ export async function exitMaintenance(db: DbOrTx): Promise { await ensureMaintenanceInfrastructure(db); await db.execute(sql` UPDATE ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} - SET active = false, reason = NULL, started_at = NULL + SET active = false, reason = NULL, started_at = NULL, snapshot_schema = NULL WHERE id = true `); } +interface MaintenanceFlagState { + readonly active: boolean; + readonly snapshotSchema: string | null; +} + +async function readMaintenanceFlag(db: DbOrTx): Promise { + await ensureMaintenanceInfrastructure(db); + const result = await db.execute<{ + active: boolean; + snapshot_schema: string | null; + }>(sql` + SELECT active, snapshot_schema + FROM ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + WHERE id = true + `); + const row = result.rows[0]; + return { + active: row?.active ?? false, + snapshotSchema: row?.snapshot_schema ?? null, + }; +} + /** * Runs `fn` with the force-restore advisory lock held on a single dedicated * connection for `fn`'s entire duration (see module doc comment for why this @@ -135,3 +238,239 @@ export async function withRestoreAdvisoryLock( client.release(); } } + +/** + * Thrown by {@link assertWritesAllowed} when the instance is under a + * force-restore's write-blocking maintenance window. + */ +export class MaintenanceModeError extends Error { + constructor( + message = "instance is under maintenance (restore in progress); try again shortly", + ) { + super(message); + this.name = "MaintenanceModeError"; + } +} + +/** + * Write-blocking guard for the ordinary (non-restore) write path: throws + * {@link MaintenanceModeError} while a force-restore's maintenance window is + * active, otherwise resolves normally. Callers should call this immediately + * before performing a write, not cache the result — the window can open or + * close at any time. + */ +export async function assertWritesAllowed(db: DbOrTx): Promise { + if (await isMaintenanceActive(db)) { + throw new MaintenanceModeError(); + } +} + +async function schemaExists(db: DbOrTx, schemaName: string): Promise { + const result = await db.execute<{ present: boolean }>(sql` + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = ${schemaName} + ) AS present + `); + return result.rows[0]?.present ?? false; +} + +async function dropSchemaIfExists( + db: DbOrTx, + schemaName: string, +): Promise { + await db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(schemaName)} CASCADE`, + ); +} + +/** + * Wipes `public`'s tables and copies `snapshotSchema`'s rows back in, + * FK-safe order (mirrors `restore-service.ts`'s own wipe+promote, kept as an + * independent implementation here rather than imported so this module has no + * dependency on `restore-service.ts`). Deliberately not wrapped in a single + * transaction: this only ever runs from best-effort, run-once boot-time + * recovery (`recoverInterruptedRestore`), and a partial failure here just + * means a future `recoverInterruptedRestore` call can pick up where it left + * off — the flag/snapshot stay in place until the whole recovery finishes. + */ +async function rollbackLiveFromSnapshot( + db: DbOrTx, + snapshotSchema: string, +): Promise { + for (const table of WIPE_TABLE_ORDER) { + const name = getTableName(table); + await db.execute(sql`DELETE FROM ${qualified("public", name)}`); + } + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await db.execute(sql` + INSERT INTO ${qualified("public", name)} + SELECT * FROM ${qualified(snapshotSchema, name)} + `); + } +} + +/** + * Restores blobs from the newest `${uploadDir}${PRE_RESTORE_BLOB_DIR_INFIX}*` + * sibling directory (the directory `restore-service.ts`'s `beginBlobSwap` + * moves the pre-restore blob store aside to) back into `uploadDir`, mirroring + * `restore-service.ts`'s own `undoBlobSwap`. If more than one such directory + * exists (e.g. from more than one interrupted attempt), the newest by mtime + * wins and every other one is discarded — they're stale artifacts of earlier + * crashes, not independently recoverable state. A no-op if no such directory + * exists (the crash happened before any pre-restore directory was created, + * or blob promotion had already fully completed and cleaned up). + */ +async function restoreBlobsFromNewestPreRestoreDir( + uploadDir: string, +): Promise { + const parentDir = dirname(uploadDir); + const baseName = uploadDir.slice(parentDir.length + 1); + const prefix = `${baseName}${PRE_RESTORE_BLOB_DIR_INFIX}`; + + let entries: Dirent[]; + try { + entries = await readdir(parentDir, { withFileTypes: true }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return; + throw err; + } + + const candidates = entries + .filter((entry) => entry.isDirectory() && entry.name.startsWith(prefix)) + .map((entry) => join(parentDir, entry.name)); + if (candidates.length === 0) return; + + const withMtimes = await Promise.all( + candidates.map(async (path) => ({ + path, + mtimeMs: (await stat(path)).mtimeMs, + })), + ); + withMtimes.sort((a, b) => b.mtimeMs - a.mtimeMs); + const [newest, ...stale] = withMtimes; + if (!newest) return; + + await rm(uploadDir, { recursive: true, force: true }).catch(() => {}); + if (await pathExists(newest.path)) { + await rename(newest.path, uploadDir); + } + + for (const { path } of stale) { + await rm(path, { recursive: true, force: true }).catch(() => {}); + } +} + +/** Drops every leftover `restore_staging_*`/`restore_snapshot_*` schema found in the database — orphans from a restore that never reached its own cleanup. Uses a regex match (not `LIKE`) so the prefixes' literal underscores aren't treated as single-character wildcards. */ +async function sweepLeftoverSchemas(db: DbOrTx): Promise { + const result = await db.execute<{ nspname: string }>(sql` + SELECT nspname FROM pg_catalog.pg_namespace + WHERE nspname ~ '^restore_(staging|snapshot)_' + `); + for (const row of result.rows) { + try { + await dropSchemaIfExists(db, row.nspname); + } catch (err) { + logRecoveryFailure(`sweep leftover schema "${row.nspname}"`, err); + } + } +} + +/** Removes every leftover `restore-staging-*` temp directory (siblings of `uploadDir`) — orphans from a restore whose own `finally` cleanup never ran. */ +async function sweepLeftoverStagingDirs(uploadDir: string): Promise { + const parentDir = dirname(uploadDir); + let entries: Dirent[]; + try { + entries = await readdir(parentDir, { withFileTypes: true }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return; + throw err; + } + for (const entry of entries) { + if ( + !entry.isDirectory() || + !entry.name.startsWith(STAGING_BLOB_DIR_PREFIX) + ) { + continue; + } + const path = join(parentDir, entry.name); + try { + await rm(path, { recursive: true, force: true }); + } catch (err) { + logRecoveryFailure(`sweep leftover staging directory "${path}"`, err); + } + } +} + +/** + * Boot-time crash recovery for an interrupted force-restore (KTD5 fix): a + * process that dies mid force-restore (`restore-service.ts`'s + * `forcePromote`) can leave the durable maintenance flag stuck active, its + * `restore_snapshot_*` schema orphaned, the pre-restore blob directory never + * swapped back in, and staging schemas/directories leaked. Called once from + * `instrumentation.ts`'s `register()` on every server boot. + * + * Recovery contract: `active = true` together with a recorded + * `snapshot_schema` that still exists in the database means the crash + * happened inside (or after) the risky wipe+promote section — the DB is + * rolled back from that snapshot and blobs are restored from the newest + * pre-restore directory, then the snapshot is dropped. `active = true` with + * no recorded snapshot (or one that no longer exists) means the crash + * happened before the risky section began — nothing live was touched, so no + * rollback is needed. Either way the flag is always cleared afterward, and a + * general sweep removes any other leftover restore staging/snapshot schema + * or temp directory regardless of what the flag says (they can only be + * orphans by the time this runs — nothing should be actively restoring at + * boot). + * + * Idempotent (safe to call multiple times, e.g. across restarts) and + * defensive: every step is individually caught and logged, so a failure in + * one step never prevents the rest from running and this function never + * throws — a recovery failure must not crash server boot. + */ +export async function recoverInterruptedRestore( + db: DbOrTx, + uploadDir: string, +): Promise { + try { + const flag = await readMaintenanceFlag(db); + if (flag.active && flag.snapshotSchema) { + const snapshotSchema = flag.snapshotSchema; + try { + if (await schemaExists(db, snapshotSchema)) { + await rollbackLiveFromSnapshot(db, snapshotSchema); + try { + await restoreBlobsFromNewestPreRestoreDir(uploadDir); + } catch (err) { + logRecoveryFailure("restore blobs from pre-restore directory", err); + } + await dropSchemaIfExists(db, snapshotSchema); + } + } catch (err) { + logRecoveryFailure( + `roll back interrupted force-restore from snapshot "${snapshotSchema}"`, + err, + ); + } + } + } catch (err) { + logRecoveryFailure("read maintenance flag", err); + } finally { + try { + await exitMaintenance(db); + } catch (err) { + logRecoveryFailure("clear maintenance flag", err); + } + } + + try { + await sweepLeftoverSchemas(db); + } catch (err) { + logRecoveryFailure("sweep leftover schemas", err); + } + try { + await sweepLeftoverStagingDirs(uploadDir); + } catch (err) { + logRecoveryFailure("sweep leftover staging directories", err); + } +} diff --git a/src/backup/restore-service.ts b/src/backup/restore-service.ts index 9ca1c61b..1793b54c 100644 --- a/src/backup/restore-service.ts +++ b/src/backup/restore-service.ts @@ -7,15 +7,15 @@ * * 1. Re-assert admin (defense-in-depth — the route also gates this). * 2. Decrypt the upload with `createDecryptStreamFromPassword` and drive - * `readBundle` over an isolated staging area: DB rows land in a Postgres - * `restore_staging` schema (via U3's `importDatabase`, redirected there - * with a `search_path` trick rather than a modified copy — U3 is - * consumed as-is); blobs land in a staging directory that `readBundle` - * itself path-validates (KTD11). A wrong password, a tampered byte - * ANYWHERE (including the last secretstream chunk), or a truncated - * stream throws before staging completes, or is only discovered once - * staging finishes (secretstream authenticates the final chunk at - * stream-end) — either way, nothing live has been touched yet. + * `readBundle` over an isolated staging area: DB rows land in a per-run + * Postgres `restore_staging_` schema (via U3's `importDatabase`, + * redirected there with a `search_path` trick rather than a modified copy + * — U3 is consumed as-is); blobs land in a staging directory that + * `readBundle` itself path-validates (KTD11). A wrong password, a + * tampered byte ANYWHERE (including the last secretstream chunk), or a + * truncated stream throws before staging completes, or is only + * discovered once staging finishes (secretstream authenticates the final + * chunk at stream-end) — either way, nothing live has been touched yet. * 3. The manifest's `backupFormatVersion` is checked the moment it's read * (the manifest is always the bundle's first entry), before any DB rows * or blobs are staged (R8/AE4). @@ -23,12 +23,24 @@ * instance emptiness (R6/AE1) and, if empty (or `force`), promote staging * to live. * 5. A `force` restore additionally runs the KTD5 envelope (`maintenance.ts`): - * durable maintenance flag, pool-safe advisory lock, a committed - * `restore_snapshot` schema of the pre-restore DB, and the pre-restore - * blob directory moved aside — so a failure at ANY point in the - * wipe+promote step (including after the DB side has already committed, - * but before the blob directory has been swapped in) rolls both stores - * back together. + * durable maintenance flag, a committed `restore_snapshot_` schema + * of the pre-restore DB, and the pre-restore blob directory moved aside — + * so a failure at ANY point in the wipe+promote step (including after the + * DB side has already committed, but before the blob directory has been + * swapped in) rolls both stores back together. + * + * **Concurrency (KTD5 hardening).** Every per-run schema (`restore_staging_*` + * and, for `force`, `restore_snapshot_*`) is named with a fresh random suffix + * per call, so two overlapping restores never collide on schema names. That + * alone isn't enough to prevent corruption, though: both restores still + * write to the shared `public` schema during promote, and interleaved + * wipe+promote transactions from two different restores could each commit + * different tables' worth of data, leaving `public` a genuine mix of both + * bundles. `restore()` therefore holds `withRestoreAdvisoryLock` for its + * ENTIRE body — staging through promote, both the empty-instance and force + * paths — so only one restore attempt is ever inside the risky section at a + * time; a second concurrent call blocks until the first fully finishes + * (commit or rollback) before it even begins staging. */ import { randomUUID } from "node:crypto"; @@ -50,23 +62,24 @@ import * as schema from "@/src/db/schema"; import { accessory, ammo, firearm, magazine } from "@/src/db/schema"; import { activeStorageRoot } from "@/src/storage"; import { readBundle } from "./bundle"; -import { createDecryptStreamFromPassword } from "./crypto"; +import { + createDecryptStreamFromPassword, + DecryptionAuthError, + InvalidHeaderError, +} from "./crypto"; import { importDatabase } from "./db-import"; import { enterMaintenance, exitMaintenance, + recordMaintenanceSnapshotSchema, + SNAPSHOT_SCHEMA_PREFIX, + STAGING_SCHEMA_PREFIX, withRestoreAdvisoryLock, } from "./maintenance"; import { BACKUP_FORMAT_VERSION, type BackupManifest } from "./manifest"; import { EXPORT_TABLE_ORDER, WIPE_TABLE_ORDER } from "./table-order"; -/** Postgres schema DB rows are staged into before the whole bundle authenticates (KTD10). Recreated fresh on every restore attempt. */ -const STAGING_SCHEMA = "restore_staging"; - -/** Postgres schema a force-restore's pre-restore live data is copied into before the wipe (KTD5) — the DB-side rollback source if promote fails after committing. */ -const SNAPSHOT_SCHEMA = "restore_snapshot"; - -/** Discriminated outcome of a restore attempt. Every branch carries an operator-facing `message`; none of them throw for expected restore-flow refusals — only a genuine programming/authorization error (see `restore`'s admin check) throws. */ +/** Discriminated outcome of a restore attempt. Every branch carries an operator-facing `message`; none of them throw for expected restore-flow refusals — only a genuine programming/authorization error (see `restore`'s admin check) or an unclassified staging failure (see the module doc comment on error classification) throws. */ export type RestoreOutcome = | { readonly kind: "ok"; readonly message: string } | { readonly kind: "refused_not_empty"; readonly message: string } @@ -85,7 +98,7 @@ class RestoreRolledBackError extends Error { export interface RestoreOptions { /** Force-replace an already-populated instance (R7/F3). Defaults to false (refuse-unless-empty, R6/F2). */ readonly force?: boolean; - /** DI seam for tests — defaults to the shared singleton (`src/db/client.ts`). */ + /** DI seam for tests — defaults to the shared singleton (`src/db/client.ts`). Used for reads (the emptiness check) that don't need to run on the locked connection; every schema-level write runs on a connection bound to `withRestoreAdvisoryLock`'s checked-out client instead (see the module doc comment). */ readonly db?: Database; /** DI seam for tests — defaults to the shared singleton pool (`src/db/client.ts`). Must be the same pool `db` is bound to. */ readonly pool?: Pool; @@ -124,9 +137,16 @@ async function pathExists(path: string): Promise { } } +/** Generates a fresh per-run hex id so two overlapping restore attempts never share a staging/snapshot schema name. */ +function generateRunId(): string { + return randomUUID().replace(/-/g, ""); +} + /** * Restore the whole instance from an encrypted backup bundle (F2/F3). - * See the module doc comment for the full stage-then-promote sequence. + * See the module doc comment for the full stage-then-promote sequence and + * the concurrency contract (the whole body runs under the restore advisory + * lock, keyed by a fresh per-run schema suffix). */ export async function restore( stream: Readable, @@ -146,84 +166,127 @@ export async function restore( const uploadDir = options.uploadDir ?? activeStorageRoot(); const force = options.force ?? false; - await recreateStagingSchema(db); - const stagingBlobDir = await mkdtemp( - join(dirname(uploadDir), "restore-staging-"), - ); - - try { - const decryptStream = createDecryptStreamFromPassword(password); - stream.on("error", (err) => decryptStream.destroy(err)); - stream.pipe(decryptStream); + const runId = generateRunId(); + const stagingSchema = `${STAGING_SCHEMA_PREFIX}${runId}`; + const snapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${runId}`; + + // The ENTIRE staging+promote body runs under one advisory lock, held on a + // single dedicated connection (`opsDb`) for the whole call — see the + // module doc comment. `forcePromote`/`emptyInstancePromote` therefore MUST + // NOT acquire this lock themselves (it isn't reentrant across connections; + // doing so would deadlock this very call). + return await withRestoreAdvisoryLock(pool, async (client) => { + const opsDb = drizzle(client, { schema }); + + await recreateStagingSchema(opsDb, stagingSchema); + const stagingBlobDir = await mkdtemp( + join(dirname(uploadDir), "restore-staging-"), + ); try { - // Version and emptiness are both checked inside stageBundle, the - // moment the manifest (always the bundle's first entry) is read — - // before any row/blob is staged (R8/R6, AE4/AE1). Only once both pass - // does stageBundle continue on to actually stage the bundle's rows and - // blobs, so a full authentication check of the whole stream (including - // a tamper in the final chunk, KTD10) only happens for restores that - // could otherwise proceed. - await stageBundle(decryptStream, stagingBlobDir, pool, { - checkEmptiness: !force, - instanceHasInventoryData: () => instanceHasInventoryData(db), - }); - } catch (err) { - if (err instanceof VersionMismatchSignal) { - return { kind: "version_mismatch", message: err.message }; - } - if (err instanceof NotEmptySignal) { - return { kind: "refused_not_empty", message: err.message }; - } - return { - kind: "wrong_password_or_tampered", - message: `bundle failed to authenticate: ${toError(err).message}`, - }; - } + const decryptStream = createDecryptStreamFromPassword(password); + stream.on("error", (err) => decryptStream.destroy(err)); + stream.pipe(decryptStream); - try { - if (force) { - await forcePromote({ - db, - pool, - uploadDir, + try { + // Version and emptiness are both checked inside stageBundle, the + // moment the manifest (always the bundle's first entry) is read — + // before any row/blob is staged (R8/R6, AE4/AE1). Only once both + // pass does stageBundle continue on to actually stage the bundle's + // rows and blobs, so a full authentication check of the whole + // stream (including a tamper in the final chunk, KTD10) only + // happens for restores that could otherwise proceed. + await stageBundle( + decryptStream, stagingBlobDir, - testFaultInjection: options._testFaultInjection, - }); - } else { - await emptyInstancePromote({ db, uploadDir, stagingBlobDir }); + pool, + { + checkEmptiness: !force, + instanceHasInventoryData: () => instanceHasInventoryData(db), + }, + stagingSchema, + ); + } catch (err) { + if (err instanceof VersionMismatchSignal) { + return { kind: "version_mismatch", message: err.message }; + } + if (err instanceof NotEmptySignal) { + return { kind: "refused_not_empty", message: err.message }; + } + // Only a genuine cryptographic authentication failure means "wrong + // password or a tampered bundle" — every other staging failure + // (ENOSPC, a malformed tar, an unknown table in the NDJSON export, + // a path-traversal blob entry, ...) is a distinct, unrelated + // problem and must not be misreported as a password/tamper issue. + // Re-throwing here lets the route's own catch-all surface its + // generic "restore failed unexpectedly" outcome instead. + if ( + err instanceof DecryptionAuthError || + err instanceof InvalidHeaderError + ) { + return { + kind: "wrong_password_or_tampered", + message: `bundle failed to authenticate: ${toError(err).message}`, + }; + } + throw err; } - } catch (err) { - if (err instanceof RestoreRolledBackError) { - return { kind: "rolled_back", message: err.message }; + + try { + if (force) { + await forcePromote({ + db: opsDb, + uploadDir, + stagingBlobDir, + stagingSchema, + snapshotSchema, + testFaultInjection: options._testFaultInjection, + }); + } else { + await emptyInstancePromote({ + db: opsDb, + uploadDir, + stagingBlobDir, + stagingSchema, + }); + } + } catch (err) { + if (err instanceof RestoreRolledBackError) { + return { kind: "rolled_back", message: err.message }; + } + throw err; } - throw err; - } - return { kind: "ok", message: "restore completed successfully" }; - } finally { - await db - .execute( - sql`DROP SCHEMA IF EXISTS ${sql.identifier(STAGING_SCHEMA)} CASCADE`, - ) - .catch(() => {}); - await rm(stagingBlobDir, { recursive: true, force: true }).catch(() => {}); - } + return { kind: "ok", message: "restore completed successfully" }; + } finally { + await opsDb + .execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(stagingSchema)} CASCADE`, + ) + .catch(() => {}); + await rm(stagingBlobDir, { recursive: true, force: true }).catch( + () => {}, + ); + } + }); } class VersionMismatchSignal extends Error {} class NotEmptySignal extends Error {} -/** (Re)creates an empty `restore_staging` schema with one table per `EXPORT_TABLE_ORDER` entry, structurally mirroring `public` (KTD10 staging area). */ -async function recreateStagingSchema(db: Database): Promise { +/** (Re)creates an empty per-run staging schema with one table per `EXPORT_TABLE_ORDER` entry, structurally mirroring `public` (KTD10 staging area). */ +async function recreateStagingSchema( + db: Database, + stagingSchema: string, +): Promise { await db.execute( - sql`DROP SCHEMA IF EXISTS ${sql.identifier(STAGING_SCHEMA)} CASCADE`, + sql`DROP SCHEMA IF EXISTS ${sql.identifier(stagingSchema)} CASCADE`, ); - await db.execute(sql`CREATE SCHEMA ${sql.identifier(STAGING_SCHEMA)}`); + await db.execute(sql`CREATE SCHEMA ${sql.identifier(stagingSchema)}`); for (const table of EXPORT_TABLE_ORDER) { const name = getTableName(table); await db.execute(sql` - CREATE TABLE ${qualified(STAGING_SCHEMA, name)} + CREATE TABLE ${qualified(stagingSchema, name)} (LIKE ${qualified("public", name)} INCLUDING ALL) `); } @@ -252,6 +315,7 @@ async function stageBundle( stagingBlobDir: string, pool: Pool, options: StageBundleOptions, + stagingSchema: string, ): Promise<{ manifest: BackupManifest }> { const generator = readBundle(decryptStream, { stagingDir: stagingBlobDir }); @@ -273,7 +337,7 @@ async function stageBundle( ); } } else if (event.kind === "db") { - await importIntoStaging(pool, event.stream); + await importIntoStaging(pool, event.stream, stagingSchema); } // "blob" events: readBundle has already written + path-validated the // file under stagingBlobDir (KTD11) — nothing further to do here. @@ -288,16 +352,22 @@ async function stageBundle( /** * Imports `dbStream` into the staging schema by reusing U3's * `importDatabase` UNMODIFIED: a dedicated connection has its `search_path` - * redirected to `restore_staging` first, so `importDatabase`'s unqualified - * `INSERT INTO "tablename"` statements land there instead of `public`. + * redirected to `stagingSchema` first, so `importDatabase`'s unqualified + * `INSERT INTO "tablename"` statements land there instead of `public`. This + * deliberately uses its own `pool.connect()`-checked-out connection rather + * than the outer restore's locked connection — the two don't need to share a + * connection (the advisory lock already serializes concurrent restore + * attempts at the JS level), and isolating the `search_path` change here + * keeps it from leaking onto any other connection. */ async function importIntoStaging( pool: Pool, dbStream: Readable, + stagingSchema: string, ): Promise { const client = await pool.connect(); try { - await client.query(`SET search_path TO "${STAGING_SCHEMA}", public`); + await client.query(`SET search_path TO "${stagingSchema}", public`); const stagingDb = drizzle(client, { schema }); await importDatabase(stagingDb, dbStream); } finally { @@ -399,11 +469,16 @@ async function wipeLive(tx: Transaction): Promise { * relies on the single wrapping transaction for atomic DB rollback (no * separate snapshot schema is needed: a failed transaction reverts the wipe * too, and the blob swap happened first and is undone on failure). + * + * `ctx.db` is bound to the SAME locked connection `restore()` holds for its + * entire body (see the module doc comment) — this doesn't need its own + * advisory lock. */ async function emptyInstancePromote(ctx: { db: Database; uploadDir: string; stagingBlobDir: string; + stagingSchema: string; }): Promise { const swap = await beginBlobSwap(ctx.uploadDir); try { @@ -419,7 +494,7 @@ async function emptyInstancePromote(ctx: { try { await ctx.db.transaction(async (tx) => { await wipeLive(tx); - await copySchemaToLive(tx, STAGING_SCHEMA); + await copySchemaToLive(tx, ctx.stagingSchema); }); } catch (err) { await undoBlobSwap(swap, ctx.uploadDir); @@ -433,79 +508,86 @@ async function emptyInstancePromote(ctx: { } /** - * F3 (force-replace) promote — the KTD5 envelope: maintenance flag, pool-safe - * advisory lock, a committed pre-restore snapshot schema, wipe+promote in one - * transaction, and a blob-directory swap. A failure anywhere after the - * snapshot is committed rolls BOTH the DB (restored from the snapshot, if the + * F3 (force-replace) promote — the KTD5 envelope: durable maintenance flag, + * a committed pre-restore snapshot schema, wipe+promote in one transaction, + * and a blob-directory swap. A failure anywhere after the snapshot is + * committed rolls BOTH the DB (restored from the snapshot, if the * wipe+promote transaction had already committed) and the blobs (restored * from the moved-aside directory) back together, then always exits * maintenance. + * + * `ctx.db` is bound to the SAME locked connection `restore()` holds for its + * entire body (see the module doc comment) — this function does NOT acquire + * its own advisory lock (doing so would deadlock: a second + * `pg_advisory_lock` call for the same key on a different connection blocks + * until the first is released, and the first is this very call). */ async function forcePromote(ctx: { db: Database; - pool: Pool; uploadDir: string; stagingBlobDir: string; + stagingSchema: string; + snapshotSchema: string; testFaultInjection?: RestoreOptions["_testFaultInjection"]; }): Promise { await enterMaintenance(ctx.db, "force-restore"); try { - await withRestoreAdvisoryLock(ctx.pool, async (client) => { - const opsDb = drizzle(client, { schema }); + await ctx.db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(ctx.snapshotSchema)} CASCADE`, + ); + await ctx.db.execute( + sql`CREATE SCHEMA ${sql.identifier(ctx.snapshotSchema)}`, + ); + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + await ctx.db.execute(sql` + CREATE TABLE ${qualified(ctx.snapshotSchema, name)} + AS TABLE ${qualified("public", name)} + `); + } + // Only recorded once the snapshot schema is fully built and committed — + // this is the durable signal `recoverInterruptedRestore` uses to decide + // a crash happened during (or after) the risky section, not before it. + await recordMaintenanceSnapshotSchema(ctx.db, ctx.snapshotSchema); - await opsDb.execute( - sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, - ); - await opsDb.execute( - sql`CREATE SCHEMA ${sql.identifier(SNAPSHOT_SCHEMA)}`, - ); - for (const table of EXPORT_TABLE_ORDER) { - const name = getTableName(table); - await opsDb.execute(sql` - CREATE TABLE ${qualified(SNAPSHOT_SCHEMA, name)} - AS TABLE ${qualified("public", name)} - `); - } + const swap = await beginBlobSwap(ctx.uploadDir); - const swap = await beginBlobSwap(ctx.uploadDir); + let dbCommitted = false; + try { + await ctx.db.transaction(async (tx) => { + await wipeLive(tx); + await copySchemaToLive(tx, ctx.stagingSchema); + await ctx.testFaultInjection?.("pre-commit"); + }); + dbCommitted = true; - let dbCommitted = false; - try { - await opsDb.transaction(async (tx) => { + await ctx.testFaultInjection?.("post-commit-pre-blob-swap"); + await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); + } catch (err) { + if (dbCommitted) { + // The wipe+promote transaction already committed new data — the + // only way back is to explicitly restore from the snapshot. + await ctx.db.transaction(async (tx) => { await wipeLive(tx); - await copySchemaToLive(tx, STAGING_SCHEMA); - await ctx.testFaultInjection?.("pre-commit"); + await copySchemaToLive(tx, ctx.snapshotSchema); }); - dbCommitted = true; - - await ctx.testFaultInjection?.("post-commit-pre-blob-swap"); - await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); - } catch (err) { - if (dbCommitted) { - // The wipe+promote transaction already committed new data — the - // only way back is to explicitly restore from the snapshot. - await opsDb.transaction(async (tx) => { - await wipeLive(tx); - await copySchemaToLive(tx, SNAPSHOT_SCHEMA); - }); - } - await undoBlobSwap(swap, ctx.uploadDir); - await opsDb - .execute( - sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, - ) - .catch(() => {}); - throw new RestoreRolledBackError( - `force-restore promotion failed and was rolled back: ${toError(err).message}`, - { cause: err }, - ); } - - await opsDb.execute( - sql`DROP SCHEMA IF EXISTS ${sql.identifier(SNAPSHOT_SCHEMA)} CASCADE`, + await undoBlobSwap(swap, ctx.uploadDir); + await ctx.db + .execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(ctx.snapshotSchema)} CASCADE`, + ) + .catch(() => {}); + throw new RestoreRolledBackError( + `force-restore promotion failed and was rolled back: ${toError(err).message}`, + { cause: err }, ); - await commitBlobSwap(swap); - }); + } + + await ctx.db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(ctx.snapshotSchema)} CASCADE`, + ); + await commitBlobSwap(swap); } finally { await exitMaintenance(ctx.db); } From 19fc7a911e6cc38bd6d055393539f78048fce43a Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Sun, 12 Jul 2026 23:34:10 -0400 Subject: [PATCH 15/26] feat(backup): enforce maintenance mode on the write path A force-restore sets the durable maintenance flag, but nothing outside src/backup/maintenance.ts read it, so ordinary writes proceeded unguarded during the restore's wipe+promote window. Wire assertWritesAllowed into every write-authorization entry point (authorize.ts's resolveCreateOwner/authorizeUpdate/ authorizeOwnerOnlyUpdate/authorizeDelete/authorizeAndDeleteParent, accessory-visibility.ts's authorizeMount, accessories/service.ts's bespoke requireEditPermission gate, grants.ts's createGrant/ revokeGrant, and the settings/admin-user actions that bypass those helpers) so every create/update/delete/grant/settings-change/ admin-user-op is blocked while a restore is active. Reads are untouched. Made the check itself cheap: assertWritesAllowed now does a single SELECT instead of also running the restore path's CREATE SCHEMA/TABLE IF NOT EXISTS ensure-step on every call, and fails open (allows writes) when Postgres reports the maintenance relation doesn't exist yet (42P01) - i.e. no restore has ever run on this instance. The SELECT runs inside db.transaction(...) rather than a bare db.execute(...): most callers pass an already-open transaction, and a caught 42P01 from a bare execute would still leave that transaction aborted at the protocol level (25P02), poisoning every later statement in it even though the JS exception was swallowed. Wrapping it lets drizzle's nested-transaction support use a SAVEPOINT/ROLLBACK TO SAVEPOINT instead, so the caller's transaction stays usable. Signed-off-by: UncleSp1d3r --- app/(admin)/users/__tests__/actions.test.ts | 131 ++++++++ app/(admin)/users/actions.ts | 4 + app/(app)/settings/__tests__/actions.test.ts | 87 +++++ app/(app)/settings/actions.ts | 2 + src/auth/accessory-visibility.ts | 3 + src/auth/authorize.ts | 9 + src/auth/grants.ts | 5 + src/backup/__tests__/maintenance.test.ts | 15 + .../write-path-maintenance-guard.test.ts | 297 ++++++++++++++++++ src/backup/maintenance.ts | 73 ++++- src/domain/accessories/service.ts | 6 + src/domain/action-result.ts | 3 + 12 files changed, 634 insertions(+), 1 deletion(-) create mode 100644 app/(admin)/users/__tests__/actions.test.ts create mode 100644 app/(app)/settings/__tests__/actions.test.ts create mode 100644 src/backup/__tests__/write-path-maintenance-guard.test.ts diff --git a/app/(admin)/users/__tests__/actions.test.ts b/app/(admin)/users/__tests__/actions.test.ts new file mode 100644 index 00000000..5a95e0c9 --- /dev/null +++ b/app/(admin)/users/__tests__/actions.test.ts @@ -0,0 +1,131 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +/** + * Server-action unit test for `users/actions.ts`'s maintenance-mode wiring + * (KTD5 write-path enforcement gap fix). These admin operations go through + * Better Auth's admin API (`auth.api.createUser`/`banUser`/`unbanUser`), not + * the shared `authorize.ts` gates, so they needed their own explicit + * `assertWritesAllowed` call — this proves it's actually wired in and blocks + * before the Better Auth call runs. Mocks the session, Better Auth, headers, + * and the maintenance guard (`mock.module`, mirroring + * `app/(app)/firearms/__tests__/documents-actions.test.ts`'s approach) rather + * than exercising real Better Auth/DB. + */ + +let currentRole: string | null = "admin"; +mock.module("@/src/auth/session", () => ({ + getCurrentUser: async () => + currentRole ? { id: "admin-1", role: currentRole } : null, +})); + +let assertWritesAllowedThrows: unknown = null; +let assertWritesAllowedCalls = 0; +class FakeMaintenanceModeError extends Error { + constructor() { + super( + "instance is under maintenance (restore in progress); try again shortly", + ); + this.name = "MaintenanceModeError"; + } +} +mock.module("@/src/backup/maintenance", () => ({ + assertWritesAllowed: async () => { + assertWritesAllowedCalls += 1; + if (assertWritesAllowedThrows) throw assertWritesAllowedThrows; + }, + MaintenanceModeError: FakeMaintenanceModeError, +})); + +let createUserCalls = 0; +let banUserCalls = 0; +let unbanUserCalls = 0; +mock.module("@/auth", () => ({ + auth: { + api: { + createUser: async () => { + createUserCalls += 1; + return {}; + }, + banUser: async () => { + banUserCalls += 1; + return {}; + }, + unbanUser: async () => { + unbanUserCalls += 1; + return {}; + }, + }, + }, +})); + +mock.module("next/headers", () => ({ + headers: async () => new Headers(), +})); + +mock.module("next/cache", () => ({ + revalidatePath: () => {}, +})); + +const { createAccountAction, setAccountDisabledAction } = await import( + "../actions" +); + +beforeEach(() => { + currentRole = "admin"; + assertWritesAllowedThrows = null; + assertWritesAllowedCalls = 0; + createUserCalls = 0; + banUserCalls = 0; + unbanUserCalls = 0; +}); + +function accountFormData(): FormData { + const formData = new FormData(); + formData.set("email", "new-user@example.test"); + formData.set("name", "New User"); + formData.set("password", "password123"); + return formData; +} + +describe("createAccountAction", () => { + test("checks maintenance before calling Better Auth and succeeds when writes are allowed", async () => { + const result = await createAccountAction(accountFormData()); + + expect(assertWritesAllowedCalls).toBe(1); + expect(createUserCalls).toBe(1); + expect(result.ok).toBe(true); + }); + + test("does not call Better Auth and returns a failed, non-throwing ActionResult while maintenance is active", async () => { + assertWritesAllowedThrows = new FakeMaintenanceModeError(); + + const result = await createAccountAction(accountFormData()); + + expect(assertWritesAllowedCalls).toBe(1); + expect(createUserCalls).toBe(0); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/maintenance/i); + }); +}); + +describe("setAccountDisabledAction", () => { + test("checks maintenance before calling Better Auth and succeeds when writes are allowed", async () => { + const result = await setAccountDisabledAction("user-2", true); + + expect(assertWritesAllowedCalls).toBe(1); + expect(banUserCalls).toBe(1); + expect(result.ok).toBe(true); + }); + + test("does not call Better Auth and returns a failed, non-throwing ActionResult while maintenance is active", async () => { + assertWritesAllowedThrows = new FakeMaintenanceModeError(); + + const result = await setAccountDisabledAction("user-2", false); + + expect(assertWritesAllowedCalls).toBe(1); + expect(banUserCalls).toBe(0); + expect(unbanUserCalls).toBe(0); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/maintenance/i); + }); +}); diff --git a/app/(admin)/users/actions.ts b/app/(admin)/users/actions.ts index ad72af87..7f1c874b 100644 --- a/app/(admin)/users/actions.ts +++ b/app/(admin)/users/actions.ts @@ -4,6 +4,8 @@ import { revalidatePath } from "next/cache"; import { headers } from "next/headers"; import { auth } from "@/auth"; import { getCurrentUser } from "@/src/auth/session"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; +import { db } from "@/src/db/client"; /** * Operator account-management actions (U13, R7). Each re-resolves the session @@ -37,6 +39,7 @@ export async function createAccountAction( }; } try { + await assertWritesAllowed(db); await auth.api.createUser({ body: { email, password, name, role: "user" }, headers: await headers(), @@ -58,6 +61,7 @@ export async function setAccountDisabledAction( ): Promise { await requireAdmin(); try { + await assertWritesAllowed(db); const h = await headers(); if (disabled) { await auth.api.banUser({ body: { userId }, headers: h }); diff --git a/app/(app)/settings/__tests__/actions.test.ts b/app/(app)/settings/__tests__/actions.test.ts new file mode 100644 index 00000000..807d9836 --- /dev/null +++ b/app/(app)/settings/__tests__/actions.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +/** + * Server-action unit test for `settings/actions.ts`'s maintenance-mode wiring + * (KTD5 write-path enforcement gap fix). Mocks the session, the maintenance + * guard, and the DB update chain (`mock.module`, mirroring + * `app/(app)/firearms/__tests__/documents-actions.test.ts`'s approach) rather + * than hitting a real DB — this only needs to prove `updateMagpulModeAction` + * calls `assertWritesAllowed` before writing and propagates a + * `MaintenanceModeError` as a failed, non-throwing `ActionResult`, and that a + * successful check still reaches the update. + */ + +let currentUserId: string | null = "user-1"; +mock.module("@/src/auth/session", () => ({ + getCurrentUser: async () => (currentUserId ? { id: currentUserId } : null), +})); + +let assertWritesAllowedThrows: unknown = null; +let assertWritesAllowedCalls = 0; +class FakeMaintenanceModeError extends Error { + constructor() { + super( + "instance is under maintenance (restore in progress); try again shortly", + ); + this.name = "MaintenanceModeError"; + } +} +mock.module("@/src/backup/maintenance", () => ({ + assertWritesAllowed: async () => { + assertWritesAllowedCalls += 1; + if (assertWritesAllowedThrows) throw assertWritesAllowedThrows; + }, + MaintenanceModeError: FakeMaintenanceModeError, +})); + +let updateCalls = 0; +let updateResult: { id: string }[] = [{ id: "user-1" }]; +mock.module("@/src/db/client", () => ({ + db: { + update: () => ({ + set: () => ({ + where: () => ({ + returning: async () => { + updateCalls += 1; + return updateResult; + }, + }), + }), + }), + }, +})); + +mock.module("next/cache", () => ({ + revalidatePath: () => {}, +})); + +const { updateMagpulModeAction } = await import("../actions"); + +beforeEach(() => { + currentUserId = "user-1"; + assertWritesAllowedThrows = null; + assertWritesAllowedCalls = 0; + updateCalls = 0; + updateResult = [{ id: "user-1" }]; +}); + +describe("updateMagpulModeAction", () => { + test("checks maintenance before writing and succeeds when writes are allowed", async () => { + const result = await updateMagpulModeAction(true); + + expect(assertWritesAllowedCalls).toBe(1); + expect(updateCalls).toBe(1); + expect(result.ok).toBe(true); + }); + + test("does not write and returns a failed, non-throwing ActionResult while maintenance is active", async () => { + assertWritesAllowedThrows = new FakeMaintenanceModeError(); + + const result = await updateMagpulModeAction(true); + + expect(assertWritesAllowedCalls).toBe(1); + expect(updateCalls).toBe(0); + expect(result.ok).toBe(false); + expect(result.ok === false && result.error).toMatch(/maintenance/i); + }); +}); diff --git a/app/(app)/settings/actions.ts b/app/(app)/settings/actions.ts index f7863743..f8cf0e79 100644 --- a/app/(app)/settings/actions.ts +++ b/app/(app)/settings/actions.ts @@ -3,6 +3,7 @@ import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { getCurrentUser } from "@/src/auth/session"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; import { db } from "@/src/db/client"; import { user as userTable } from "@/src/db/schema"; import { type ActionResult, toActionError } from "@/src/domain/action-result"; @@ -30,6 +31,7 @@ export async function updateMagpulModeAction( return { ok: false, error: "Invalid setting value." }; } const userId = await requireUserId(); + await assertWritesAllowed(db); const [updated] = await db .update(userTable) .set({ magpulMode: enabled }) diff --git a/src/auth/accessory-visibility.ts b/src/auth/accessory-visibility.ts index 48ca2960..192fd49a 100644 --- a/src/auth/accessory-visibility.ts +++ b/src/auth/accessory-visibility.ts @@ -1,4 +1,5 @@ import { eq, inArray } from "drizzle-orm"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; import type { DbOrTx } from "@/src/db/client"; import { accessory, firearm } from "@/src/db/schema"; import { authorizeUpdate } from "./authorize"; @@ -90,6 +91,8 @@ export async function authorizeMount( accessoryId: string, targetFirearmId: string | null, ): Promise { + await assertWritesAllowed(tx); + const perm = await resolveAccessoryPermission(tx, actorId, accessoryId); if (perm !== "owner" && perm !== "edit") { if (perm === "view") { diff --git a/src/auth/authorize.ts b/src/auth/authorize.ts index f57d1fcf..32ff2f4d 100644 --- a/src/auth/authorize.ts +++ b/src/auth/authorize.ts @@ -1,4 +1,5 @@ import { and, eq } from "drizzle-orm"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; import { type DbOrTx, db as defaultDb } from "@/src/db/client"; import { grant } from "@/src/db/schema"; import { NotAuthorizedError, NotFoundError } from "./errors"; @@ -21,6 +22,8 @@ export async function resolveCreateOwner( actorId: string, targetOwnerId?: string | null, ): Promise { + await assertWritesAllowed(tx); + if (!targetOwnerId || targetOwnerId === actorId) return actorId; const rows = await tx @@ -54,6 +57,8 @@ export async function authorizeUpdate( parentType: ParentType, parentId: string, ): Promise { + await assertWritesAllowed(tx); + const perm = await resolvePermission(tx, actorId, parentType, parentId); if (perm === "owner" || perm === "edit") return; if (perm === "view") { @@ -93,6 +98,7 @@ export async function authorizeOwnerOnlyUpdate( parentType: ParentType, parentId: string, ): Promise { + await assertWritesAllowed(tx); return authorizeOwnerOnly(tx, actorId, parentType, parentId, "modify"); } @@ -106,6 +112,7 @@ export async function authorizeDelete( parentType: ParentType, parentId: string, ): Promise { + await assertWritesAllowed(tx); return authorizeOwnerOnly(tx, actorId, parentType, parentId, "delete"); } @@ -152,6 +159,8 @@ export async function authorizeAndDeleteParent( database: DbOrTx = defaultDb, onBeforeDelete?: PreDeleteHook, ): Promise { + await assertWritesAllowed(database); + const runner = "transaction" in database ? database : null; const run = async (tx: DbOrTx) => { await authorizeDelete(tx, actorId, parentType, parentId); diff --git a/src/auth/grants.ts b/src/auth/grants.ts index 6d0db7dd..650dee70 100644 --- a/src/auth/grants.ts +++ b/src/auth/grants.ts @@ -1,4 +1,5 @@ import { and, eq } from "drizzle-orm"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; import type { DbOrTx } from "@/src/db/client"; import { grant } from "@/src/db/schema"; import { NotAuthorizedError } from "./errors"; @@ -40,6 +41,8 @@ export async function createGrant( db: DbOrTx, input: CreateGrantInput, ): Promise { + await assertWritesAllowed(db); + const { actorId, granteeId, parentType, parentId, permission } = input; if (granteeId === actorId) { throw new NotAuthorizedError("cannot grant an item to its owner"); @@ -81,6 +84,8 @@ export async function revokeGrant( db: DbOrTx, input: RevokeGrantInput, ): Promise { + await assertWritesAllowed(db); + const { actorId, granteeId, parentType, parentId } = input; const perm = await resolvePermission(db, actorId, parentType, parentId); if (perm !== "owner") { diff --git a/src/backup/__tests__/maintenance.test.ts b/src/backup/__tests__/maintenance.test.ts index a81e0d1a..c3dfdd3d 100644 --- a/src/backup/__tests__/maintenance.test.ts +++ b/src/backup/__tests__/maintenance.test.ts @@ -205,6 +205,21 @@ describe("maintenance envelope (KTD5 hardening)", () => { await exitMaintenance(db); await expect(assertWritesAllowed(db)).resolves.toBeUndefined(); }); + + test("fails open (resolves without throwing) when the maintenance infra has never been created — no restore has ever run", async () => { + // `beforeEach` calls `exitMaintenance`, which (like every other + // maintenance.ts function except `assertWritesAllowed`) ensures the + // infra exists — drop it here to simulate a fresh instance that has + // never entered maintenance, so `assertWritesAllowed`'s single SELECT + // hits Postgres's 42P01 (undefined_table) and must fail open rather + // than surface that as an error. + await db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier("restore_ops")} CASCADE`, + ); + expect(await schemaExists(db, "restore_ops")).toBe(false); + + await expect(assertWritesAllowed(db)).resolves.toBeUndefined(); + }); }); describe("recoverInterruptedRestore", () => { diff --git a/src/backup/__tests__/write-path-maintenance-guard.test.ts b/src/backup/__tests__/write-path-maintenance-guard.test.ts new file mode 100644 index 00000000..56d77fa0 --- /dev/null +++ b/src/backup/__tests__/write-path-maintenance-guard.test.ts @@ -0,0 +1,297 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from "@testcontainers/postgresql"; +import { eq } from "drizzle-orm"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import { authorizeMount } from "../../auth/accessory-visibility"; +import { + authorizeAndDeleteParent, + authorizeDelete, + authorizeOwnerOnlyRead, + authorizeOwnerOnlyUpdate, + authorizeUpdate, + resolveCreateOwner, +} from "../../auth/authorize"; +import { NotFoundError } from "../../auth/errors"; +import { createGrant, revokeGrant } from "../../auth/grants"; +import * as schema from "../../db/schema"; +import { accessory, firearm, grant, magazine, user } from "../../db/schema"; +import { wipeDatabase } from "../db-import"; +import { + enterMaintenance, + exitMaintenance, + MaintenanceModeError, +} from "../maintenance"; + +/** + * Integration coverage for the KTD5 write-path enforcement gap: a + * force-restore's durable maintenance flag must block every ordinary write — + * not just be checkable from `maintenance.ts`. Exercises the shared + * write-authorization gates (`authorize.ts`, `accessory-visibility.ts`, + * `grants.ts`) directly against an isolated Testcontainers Postgres, mirroring + * `maintenance.test.ts`'s harness. These take `db`/`tx: DbOrTx` as an + * explicit parameter, so this runs against a dedicated container rather than + * the app's shared `@/src/db/client` singleton — no risk of leaving the + * ambient dev DB's maintenance flag stuck active for every other test file in + * the same `bun test src` run. + */ +const POSTGRES_IMAGE = + "public.ecr.aws/docker/library/postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e"; + +type Db = NodePgDatabase; + +async function seedOwner(db: Db, label: string): Promise { + const ownerId = `owner-${randomUUID()}`; + await db + .insert(user) + .values({ id: ownerId, name: label, email: `${ownerId}@example.test` }); + return ownerId; +} + +async function seedFirearm(db: Db, ownerId: string): Promise { + const [row] = await db + .insert(firearm) + .values({ ownerId, name: "Test FA", caliber: "9mm" }) + .returning({ id: firearm.id }); + return row.id; +} + +async function seedMagazine(db: Db, ownerId: string): Promise { + const [row] = await db + .insert(magazine) + .values({ + ownerId, + brandModel: "Test Mag", + caliber: "9mm", + baseCapacity: 15, + }) + .returning({ id: magazine.id }); + return row.id; +} + +async function seedAccessory(db: Db, ownerId: string): Promise { + const [row] = await db + .insert(accessory) + .values({ ownerId, category: "optic" }) + .returning({ id: accessory.id }); + return row.id; +} + +describe("write path is blocked during maintenance (KTD5 gap fix)", () => { + let container: StartedPostgreSqlContainer; + let pool: Pool; + let db: Db; + let ownerId: string; + + beforeAll(async () => { + container = await new PostgreSqlContainer(POSTGRES_IMAGE) + .withDatabase("magstacker_write_guard_test") + .start(); + pool = new Pool({ connectionString: container.getConnectionUri() }); + db = drizzle(pool, { schema }); + await migrate(db, { migrationsFolder: "./src/db/migrations" }); + }, 120_000); + + afterAll(async () => { + await pool?.end(); + await container?.stop(); + }); + + beforeEach(async () => { + await wipeDatabase(db); + await exitMaintenance(db); + ownerId = await seedOwner(db, "Owner"); + }); + + afterEach(async () => { + await exitMaintenance(db); + }); + + describe("with maintenance inactive (or the infra never created)", () => { + test("create (resolveCreateOwner) proceeds normally", async () => { + await expect(resolveCreateOwner(db, ownerId, undefined)).resolves.toBe( + ownerId, + ); + }); + + test("update (authorizeUpdate / authorizeOwnerOnlyUpdate) proceeds normally", async () => { + const firearmId = await seedFirearm(db, ownerId); + await expect( + authorizeUpdate(db, ownerId, "firearm", firearmId), + ).resolves.toBeUndefined(); + + const magazineId = await seedMagazine(db, ownerId); + await expect( + authorizeOwnerOnlyUpdate(db, ownerId, "magazine", magazineId), + ).resolves.toBeUndefined(); + }); + + test("delete (authorizeDelete / authorizeAndDeleteParent) proceeds normally", async () => { + const firearmId = await seedFirearm(db, ownerId); + await authorizeAndDeleteParent(ownerId, "firearm", firearmId, db); + const rows = await db + .select() + .from(firearm) + .where(eq(firearm.id, firearmId)); + expect(rows).toHaveLength(0); + }); + + test("grant create/revoke proceeds normally", async () => { + const firearmId = await seedFirearm(db, ownerId); + const granteeId = await seedOwner(db, "Grantee"); + await createGrant(db, { + actorId: ownerId, + granteeId, + parentType: "firearm", + parentId: firearmId, + permission: "view", + }); + const rows = await db + .select() + .from(grant) + .where(eq(grant.granteeId, granteeId)); + expect(rows).toHaveLength(1); + + await revokeGrant(db, { + actorId: ownerId, + granteeId, + parentType: "firearm", + parentId: firearmId, + }); + const after = await db + .select() + .from(grant) + .where(eq(grant.granteeId, granteeId)); + expect(after).toHaveLength(0); + }); + + test("accessory mount (authorizeMount) proceeds normally", async () => { + const firearmId = await seedFirearm(db, ownerId); + const accessoryId = await seedAccessory(db, ownerId); + await expect( + authorizeMount(db, ownerId, accessoryId, firearmId), + ).resolves.toBeUndefined(); + }); + }); + + describe("with maintenance active", () => { + beforeEach(async () => { + await enterMaintenance(db, "force-restore"); + }); + + test("create (resolveCreateOwner) throws MaintenanceModeError", async () => { + await expect( + resolveCreateOwner(db, ownerId, undefined), + ).rejects.toBeInstanceOf(MaintenanceModeError); + }); + + test("update (authorizeUpdate / authorizeOwnerOnlyUpdate) throws MaintenanceModeError", async () => { + // Seed BEFORE entering maintenance would also work, but seeding a + // parent here (with maintenance already active on this connection) + // proves the guard doesn't accidentally block the *test's* setup + // writes — only calls that go through the guarded helpers. + await exitMaintenance(db); + const firearmId = await seedFirearm(db, ownerId); + const magazineId = await seedMagazine(db, ownerId); + await enterMaintenance(db, "force-restore"); + + await expect( + authorizeUpdate(db, ownerId, "firearm", firearmId), + ).rejects.toBeInstanceOf(MaintenanceModeError); + await expect( + authorizeOwnerOnlyUpdate(db, ownerId, "magazine", magazineId), + ).rejects.toBeInstanceOf(MaintenanceModeError); + }); + + test("delete (authorizeDelete / authorizeAndDeleteParent) throws MaintenanceModeError and leaves the row untouched", async () => { + await exitMaintenance(db); + const firearmId = await seedFirearm(db, ownerId); + await enterMaintenance(db, "force-restore"); + + await expect( + authorizeDelete(db, ownerId, "firearm", firearmId), + ).rejects.toBeInstanceOf(MaintenanceModeError); + await expect( + authorizeAndDeleteParent(ownerId, "firearm", firearmId, db), + ).rejects.toBeInstanceOf(MaintenanceModeError); + + await exitMaintenance(db); + const rows = await db + .select() + .from(firearm) + .where(eq(firearm.id, firearmId)); + expect(rows).toHaveLength(1); // untouched by the blocked delete + }); + + test("grant create/revoke throws MaintenanceModeError", async () => { + await exitMaintenance(db); + const firearmId = await seedFirearm(db, ownerId); + const granteeId = await seedOwner(db, "Grantee"); + await enterMaintenance(db, "force-restore"); + + await expect( + createGrant(db, { + actorId: ownerId, + granteeId, + parentType: "firearm", + parentId: firearmId, + permission: "view", + }), + ).rejects.toBeInstanceOf(MaintenanceModeError); + + await expect( + revokeGrant(db, { + actorId: ownerId, + granteeId, + parentType: "firearm", + parentId: firearmId, + }), + ).rejects.toBeInstanceOf(MaintenanceModeError); + }); + + test("accessory mount (authorizeMount) throws MaintenanceModeError", async () => { + await exitMaintenance(db); + const firearmId = await seedFirearm(db, ownerId); + const accessoryId = await seedAccessory(db, ownerId); + await enterMaintenance(db, "force-restore"); + + await expect( + authorizeMount(db, ownerId, accessoryId, firearmId), + ).rejects.toBeInstanceOf(MaintenanceModeError); + }); + + test("a READ (authorizeOwnerOnlyRead) is NOT blocked during maintenance", async () => { + await exitMaintenance(db); + const firearmId = await seedFirearm(db, ownerId); + await enterMaintenance(db, "force-restore"); + + await expect( + authorizeOwnerOnlyRead(db, ownerId, "firearm", firearmId), + ).resolves.toBeUndefined(); + }); + + test("a not-found delete during maintenance still reports MaintenanceModeError, not NotFoundError (write-blocking checked first)", async () => { + await expect( + authorizeAndDeleteParent(ownerId, "firearm", randomUUID(), db), + ).rejects.toBeInstanceOf(MaintenanceModeError); + // Sanity: outside maintenance, the same call is a NotFoundError. + await exitMaintenance(db); + await expect( + authorizeAndDeleteParent(ownerId, "firearm", randomUUID(), db), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); +}); diff --git a/src/backup/maintenance.ts b/src/backup/maintenance.ts index 274763aa..c479aae8 100644 --- a/src/backup/maintenance.ts +++ b/src/backup/maintenance.ts @@ -102,6 +102,37 @@ function logRecoveryFailure(step: string, err: unknown): void { console.error(`backup/maintenance: recovery step "${step}" failed`, err); } +/** Postgres SQLSTATE for "undefined_table" (a relation referenced in a query doesn't exist). */ +const POSTGRES_UNDEFINED_TABLE = "42P01"; + +function hasUndefinedTableCode(err: unknown): boolean { + return ( + typeof err === "object" && + err !== null && + "code" in err && + (err as { code?: unknown }).code === POSTGRES_UNDEFINED_TABLE + ); +} + +/** + * True when `err` is a Postgres error whose SQLSTATE is `undefined_table` + * (`42P01`) — i.e. a query referenced a relation that doesn't exist. Used by + * {@link assertWritesAllowed} to distinguish "the maintenance infra was never + * created" (fail open) from any other, genuine failure (propagate). + * + * Drizzle's node-postgres driver wraps the raw `pg` error (which carries + * `.code`) in its own `DrizzleQueryError`, with the original error on + * `.cause` — the SQLSTATE isn't on the outer error, so both layers must be + * checked. + */ +function isUndefinedTableError(err: unknown): boolean { + if (hasUndefinedTableCode(err)) return true; + if (err instanceof Error && err.cause) { + return hasUndefinedTableCode(err.cause); + } + return false; +} + /** * Creates the maintenance schema/table/singleton-row if they don't already * exist. Idempotent and cheap (`IF NOT EXISTS` / `ON CONFLICT DO NOTHING`) — @@ -258,9 +289,49 @@ export class MaintenanceModeError extends Error { * active, otherwise resolves normally. Callers should call this immediately * before performing a write, not cache the result — the window can open or * close at any time. + * + * Deliberately NOT built on {@link isMaintenanceActive}: this runs on every + * ordinary write across the app (every create/update/delete/grant/settings- + * change/admin-user-op), not just the restore path, so it must stay a single + * cheap `SELECT` rather than also paying for `ensureMaintenanceInfrastructure`'s + * `CREATE SCHEMA/TABLE IF NOT EXISTS` + `ALTER TABLE ADD COLUMN IF NOT EXISTS` + * + `INSERT ... ON CONFLICT` on every call. If the maintenance schema/table + * don't exist yet — no force-restore has ever run against this instance, so + * nothing ever created them — Postgres raises `42P01` (undefined_table) for + * the `SELECT`; that is treated as "maintenance was never entered" and writes + * are allowed (fail open), not surfaced as an error. Any other error + * propagates. + * + * The `SELECT` runs inside `db.transaction(...)` rather than as a bare + * `db.execute(...)` — NOT for atomicity, but because most callers pass an + * already-open transaction (`tx`) they're about to keep using for the write + * itself. A statement that errors inside a Postgres transaction aborts that + * whole transaction at the protocol level (`25P02`, "current transaction is + * aborted") — catching the JS exception does not undo that, so a bare + * `db.execute` here would poison the caller's transaction on the very 42P01 + * this function means to swallow, breaking every subsequent statement in it. + * `DbOrTx.transaction()` avoids that politely: called on the top-level + * `Database` it's a real `BEGIN`/`COMMIT`/`ROLLBACK`; called on an + * already-open `Transaction` (drizzle's nested-transaction support) it's a + * `SAVEPOINT`/`RELEASE SAVEPOINT`/`ROLLBACK TO SAVEPOINT` instead — either + * way, a caught error here leaves the caller's connection in a clean, usable + * state. */ export async function assertWritesAllowed(db: DbOrTx): Promise { - if (await isMaintenanceActive(db)) { + let active: boolean; + try { + active = await db.transaction(async (tx) => { + const result = await tx.execute<{ active: boolean }>(sql` + SELECT active FROM ${qualified(MAINTENANCE_SCHEMA, MAINTENANCE_TABLE)} + WHERE id = true + `); + return result.rows[0]?.active ?? false; + }); + } catch (err) { + if (isUndefinedTableError(err)) return; + throw err; + } + if (active) { throw new MaintenanceModeError(); } } diff --git a/src/domain/accessories/service.ts b/src/domain/accessories/service.ts index 0760b5ce..6a5be79a 100644 --- a/src/domain/accessories/service.ts +++ b/src/domain/accessories/service.ts @@ -7,6 +7,7 @@ import { import { authorizeUpdate, resolveCreateOwner } from "@/src/auth/authorize"; import { NotAuthorizedError, NotFoundError } from "@/src/auth/errors"; import type { Permission } from "@/src/auth/visibility"; +import { assertWritesAllowed } from "@/src/backup/maintenance"; import { type DbOrTx, db } from "@/src/db/client"; import { accessory, firearm } from "@/src/db/schema"; import { ValidationError } from "../errors"; @@ -112,6 +113,11 @@ async function requireEditPermission( actorId: string, id: string, ): Promise { + // Bespoke write gate (accessories aren't a grant `ParentType`, so this + // doesn't route through `authorize.ts`'s helpers) — guard it directly so + // `updateAccessory`/`deleteAccessory` are blocked during maintenance too. + await assertWritesAllowed(tx); + const permission = await resolveAccessoryPermission(tx, actorId, id); if (permission === "owner" || permission === "edit") return permission; if (permission === "view") { diff --git a/src/domain/action-result.ts b/src/domain/action-result.ts index c7f72d78..58b1571d 100644 --- a/src/domain/action-result.ts +++ b/src/domain/action-result.ts @@ -1,5 +1,6 @@ import { NotAuthorizedError, NotFoundError } from "@/src/auth/errors"; import { RateLimitError } from "@/src/auth/rate-limit"; +import { MaintenanceModeError } from "@/src/backup/maintenance"; import { DatabaseUnavailableError } from "@/src/db/health"; import { ValidationError } from "./errors"; @@ -24,6 +25,8 @@ export function toActionError(error: unknown): ActionResult { } if (error instanceof DatabaseUnavailableError) return { ok: false, error: error.message }; + if (error instanceof MaintenanceModeError) + return { ok: false, error: error.message }; // Anything that reaches here is unmapped — a bug, a DB deadlock, an unexpected // storage failure. The user gets a generic message; log the real error so // there is a server-side trail (every mapped case above is expected and From 32c4922c4811157f98c168fd860f66caeb51611f Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 00:14:21 -0400 Subject: [PATCH 16/26] fix(backup): validate untrusted NDJSON rows + behavioral snapshot-isolation & unknown-table tests Signed-off-by: UncleSp1d3r --- src/backup/__tests__/db-roundtrip.test.ts | 69 +++++++++++++++++++++ src/backup/__tests__/export-service.test.ts | 64 +++++++++++++++++++ src/backup/db-import.ts | 41 +++++++++++- 3 files changed, 173 insertions(+), 1 deletion(-) diff --git a/src/backup/__tests__/db-roundtrip.test.ts b/src/backup/__tests__/db-roundtrip.test.ts index fb8c07dc..1aacf2fb 100644 --- a/src/backup/__tests__/db-roundtrip.test.ts +++ b/src/backup/__tests__/db-roundtrip.test.ts @@ -39,6 +39,7 @@ import { } from "../../db/schema"; import { type ExportedRow, exportDatabase } from "../db-export"; import { + InvalidExportRowError, importDatabase, MAX_NDJSON_LINE_BYTES, wipeDatabase, @@ -417,6 +418,74 @@ describe("DB export/import round trip (U3)", () => { expect(rows).toHaveLength(0); }); + test("import rejects a line referencing an unknown table, without inserting any rows from earlier in the same stream (rollback guard)", async () => { + const validLine = `${JSON.stringify({ + table: "user", + row: { + id: randomUUID(), + name: "Should Not Persist", + email: `${randomUUID()}@example.test`, + }, + } satisfies ExportedRow)}\n`; + const unknownTableLine = JSON.stringify({ + table: "not_a_real_table", + row: {}, + } satisfies ExportedRow); + + let caught: unknown; + try { + await importDatabase(db, Readable.from([validLine + unknownTableLine])); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toMatch(/unknown table/i); + + // The whole import runs in one transaction — the unknown-table line's + // rejection must roll back the earlier, otherwise-valid `user` insert + // too, not leave a partially-applied import. + const rows = await db.select().from(user); + expect(rows).toHaveLength(0); + }); + + test("import rejects a malformed row (null, string, or array `row`) via a named InvalidExportRowError, without inserting any rows from earlier in the same stream", async () => { + const validLine = `${JSON.stringify({ + table: "user", + row: { + id: randomUUID(), + name: "Should Not Persist Either", + email: `${randomUUID()}@example.test`, + }, + } satisfies ExportedRow)}\n`; + + const malformedRowLines = [ + `{"table":"user","row":null}`, + `{"table":"user","row":"not-an-object"}`, + `{"table":"user","row":[]}`, + ]; + + for (const malformedLine of malformedRowLines) { + await wipeDatabase(db); + + let caught: unknown; + try { + await importDatabase(db, Readable.from([validLine + malformedLine])); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(InvalidExportRowError); + expect((caught as Error).message).toMatch(/not a well-formed/i); + + // The whole import runs in one transaction — the malformed row's + // rejection must roll back the earlier, otherwise-valid `user` insert + // too, not leave a partially-applied import. + const rows = await db.select().from(user); + expect(rows).toHaveLength(0); + } + }); + test("import still handles a final line with no trailing newline (no regression from switching off the readline-based reader)", async () => { const rows: ExportedRow[] = [ { diff --git a/src/backup/__tests__/export-service.test.ts b/src/backup/__tests__/export-service.test.ts index d495b959..94141fd8 100644 --- a/src/backup/__tests__/export-service.test.ts +++ b/src/backup/__tests__/export-service.test.ts @@ -34,6 +34,7 @@ import { PostgreSqlContainer, type StartedPostgreSqlContainer, } from "@testcontainers/postgresql"; +import { eq } from "drizzle-orm"; import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; import { migrate } from "drizzle-orm/node-postgres/migrator"; import { Pool } from "pg"; @@ -50,6 +51,7 @@ import { HEADER_BYTE_LENGTH, readHeader, } from "../crypto"; +import { type ExportedRow, exportDatabase } from "../db-export"; import { wipeDatabase } from "../db-import"; import { createBackup } from "../export-service"; import { @@ -393,6 +395,68 @@ describe("backup export service (U4)", () => { ).toBe(true); }); + test("the DB export's snapshot is behaviorally consistent: a row inserted by a second, concurrent connection while the export transaction is open never appears in the export (torn-bundle guard, behavioral)", async () => { + // The previous test only proves the mechanism — that `begin isolation + // level repeatable read read only` was issued. This test proves the + // outcome that mechanism is supposed to guarantee: once the transaction's + // snapshot is pinned, a write landing on a second, independent connection + // is invisible to a `SELECT` running inside it. + const ownerId = `owner-${randomUUID()}`; + await db.insert(user).values({ + id: ownerId, + name: "Snapshot Before", + email: `${ownerId}@example.test`, + }); + + const concurrentOwnerId = `owner-${randomUUID()}`; + let dbText = ""; + + await db.transaction( + async (tx) => { + // Repeatable read's MVCC snapshot is pinned at the transaction's + // first statement, so issue one here — before the concurrent write + // below — to make the snapshot boundary deterministic instead of + // racing it. + await tx.select().from(user).where(eq(user.id, ownerId)); + + // A second, fully independent connection writes a brand-new row into + // an exported table WHILE this transaction's snapshot is already + // pinned open. + const otherPool = new Pool({ + connectionString: container.getConnectionUri(), + }); + try { + const otherDb = drizzle(otherPool, { schema }); + await otherDb.insert(user).values({ + id: concurrentOwnerId, + name: "Snapshot After (must not appear in export)", + email: `${concurrentOwnerId}@example.test`, + }); + } finally { + await otherPool.end(); + } + + // exportDatabase reads through tx's pinned snapshot — the concurrent + // insert above must be invisible to it. + const { buffer } = await collectWithStats(exportDatabase(tx)); + dbText = buffer.toString("utf8"); + }, + { isolationLevel: "repeatable read", accessMode: "read only" }, + ); + + const exportedUserIds = new Set( + dbText + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as ExportedRow) + .filter((entry) => entry.table === "user") + .map((entry) => entry.row.id), + ); + + expect(exportedUserIds.has(ownerId)).toBe(true); + expect(exportedUserIds.has(concurrentOwnerId)).toBe(false); + }); + test("a non-admin caller is rejected", async () => { currentIsAdmin = false; diff --git a/src/backup/db-import.ts b/src/backup/db-import.ts index 7369f518..7b9adc68 100644 --- a/src/backup/db-import.ts +++ b/src/backup/db-import.ts @@ -81,6 +81,39 @@ function decodeLine(buffer: Buffer): string { return text.endsWith("\r") ? text.slice(0, -1) : text; } +/** + * Thrown when a parsed `db.ndjson` line is not a well-formed + * `{ table, row }` entry. A restore bundle is admin-uploaded but its content + * is still untrusted (corrupted download, or a deliberately crafted bundle — + * KTD11), so a malformed line must fail with a clear, attributable error + * before it is ever spread into `tx.insert(...).values(row)`, rather than + * surfacing as a generic downstream Postgres error. + */ +export class InvalidExportRowError extends Error { + constructor(message: string) { + super(message); + this.name = "InvalidExportRowError"; + } +} + +/** + * Shape guard for one `JSON.parse`d NDJSON line, mirroring `manifest.ts`'s + * `parseManifest` validation pattern rather than trusting the parse result + * via a bare `as ExportedRow` cast: `table` must be a string, and `row` must + * be a plain object — not `null`, and not an array (both of which pass + * `typeof x === "object"`). + */ +export function isExportedRow(value: unknown): value is ExportedRow { + if (typeof value !== "object" || value === null) return false; + const { table, row } = value as Record; + return ( + typeof table === "string" && + typeof row === "object" && + row !== null && + !Array.isArray(row) + ); +} + /** * JS property keys (not SQL column names) on `table` whose values are * date/timestamp columns. Cached per table by `importDatabase` since the same @@ -141,7 +174,13 @@ export async function importDatabase( for await (const line of readBoundedLines(stream, MAX_NDJSON_LINE_BYTES)) { if (line.trim() === "") continue; - const parsed = JSON.parse(line) as ExportedRow; + const parsed: unknown = JSON.parse(line); + if (!isExportedRow(parsed)) { + throw new InvalidExportRowError( + "Backup's db.ndjson contains a line that is not a well-formed { table, row } entry — refusing to import; this may be a corrupted or malicious bundle.", + ); + } + const table = TABLES_BY_NAME.get(parsed.table); if (!table) { throw new Error( From 5b399dac663fdab84c8230fb2d442f93522edc51 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 00:15:35 -0400 Subject: [PATCH 17/26] fix(backup): validate KDF alg + copy salt on ingest + relocate crypto JSDoc Signed-off-by: UncleSp1d3r --- src/backup/__tests__/crypto.test.ts | 79 +++++++++++++++++++++++++++++ src/backup/crypto.ts | 52 ++++++++++++------- 2 files changed, 114 insertions(+), 17 deletions(-) diff --git a/src/backup/__tests__/crypto.test.ts b/src/backup/__tests__/crypto.test.ts index a1746563..41b8a28b 100644 --- a/src/backup/__tests__/crypto.test.ts +++ b/src/backup/__tests__/crypto.test.ts @@ -11,6 +11,8 @@ import { FORMAT_VERSION, generateSalt, InvalidHeaderError, + MAX_KDF_MEMLIMIT, + MAX_KDF_OPSLIMIT, readHeader, writeHeader, } from "../crypto"; @@ -130,6 +132,63 @@ describe("header round-trip", () => { expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); }); + + test("readHeader rejects an opslimit one above the accepted maximum", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: { ...DEFAULT_KDF_PARAMS, opslimit: MAX_KDF_OPSLIMIT + 1 }, + secretstreamHeader: Buffer.alloc(24), + }); + + expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); + expect(() => readHeader(bytes)).toThrow(/exceed/i); + }); + + test("readHeader rejects a memlimit one above the accepted maximum", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: { ...DEFAULT_KDF_PARAMS, memlimit: MAX_KDF_MEMLIMIT + 1 }, + secretstreamHeader: Buffer.alloc(24), + }); + + expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); + expect(() => readHeader(bytes)).toThrow(/exceed/i); + }); + + test("readHeader accepts opslimit and memlimit exactly at the accepted maximum", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: { + ...DEFAULT_KDF_PARAMS, + opslimit: MAX_KDF_OPSLIMIT, + memlimit: MAX_KDF_MEMLIMIT, + }, + secretstreamHeader: Buffer.alloc(24), + }); + + const header = readHeader(bytes); + + expect(header.kdfParams.opslimit).toBe(MAX_KDF_OPSLIMIT); + expect(header.kdfParams.memlimit).toBe(MAX_KDF_MEMLIMIT); + }); + + test("readHeader rejects a KDF algorithm other than Argon2id", () => { + const salt = generateSalt(); + const bytes = writeHeader({ + version: FORMAT_VERSION, + salt, + kdfParams: { ...DEFAULT_KDF_PARAMS, alg: DEFAULT_KDF_PARAMS.alg + 1 }, + secretstreamHeader: Buffer.alloc(24), + }); + + expect(() => readHeader(bytes)).toThrow(InvalidHeaderError); + }); }); describe("encrypt/decrypt round-trip", () => { @@ -259,6 +318,26 @@ describe("encrypt/decrypt round-trip", () => { createEncryptStream(Buffer.alloc(10), generateSalt()), ).toThrow(); }); + + test("mutating the caller's salt buffer after createEncryptStream does not affect the written header", async () => { + const salt = generateSalt(); + const originalSalt = Buffer.from(salt); + const key = deriveKey("hunter2", salt); + const plaintext = Buffer.from("small payload"); + + const source = Readable.from([plaintext]); + const stream = source.pipe(createEncryptStream(key, salt)); + + // Mutate the caller's salt buffer immediately after creating the stream, + // before any data has necessarily been pushed through it. + salt.fill(0xff); + + const encrypted = await collect(stream); + const header = readHeader(encrypted); + + expect(header.salt.equals(originalSalt)).toBe(true); + expect(header.salt.equals(salt)).toBe(false); + }); }); describe("DecryptionAuthError", () => { diff --git a/src/backup/crypto.ts b/src/backup/crypto.ts index 397e1de4..6573a63d 100644 --- a/src/backup/crypto.ts +++ b/src/backup/crypto.ts @@ -75,8 +75,8 @@ export const DEFAULT_KDF_PARAMS: KdfParams = { * future SENSITIVE-tier bundle still validates, but finite so a hostile header * cannot force an unbounded pre-authentication Argon2id allocation. */ -const MAX_KDF_OPSLIMIT = 10; -const MAX_KDF_MEMLIMIT = 1024 * 1024 * 1024; // 1 GiB +export const MAX_KDF_OPSLIMIT = 10; +export const MAX_KDF_MEMLIMIT = 1024 * 1024 * 1024; // 1 GiB /** The bundle's unencrypted crypto header — see module doc comment. */ export interface CryptoHeader { @@ -242,6 +242,18 @@ export function readHeader(buf: Buffer): CryptoHeader { ); } + // `alg` is the third untrusted KDF field and is fed to `crypto_pwhash` + // (`deriveKey`) just like opslimit/memlimit above. This module only ever + // writes Argon2id (see `DEFAULT_KDF_PARAMS`), so any other value is a + // corrupt or crafted bundle — reject it here as a header problem, rather + // than letting a generic native `crypto_pwhash` error escape unclassified + // past `InvalidHeaderError`/`DecryptionAuthError`. + if (alg !== sodium.crypto_pwhash_ALG_ARGON2ID13) { + throw new InvalidHeaderError( + `crypto header KDF algorithm is not supported: ${alg}`, + ); + } + const secretstreamHeader = Buffer.from( buf.subarray(offset, offset + SECRETSTREAM_HEADER_BYTES), ); @@ -317,6 +329,12 @@ export function createEncryptStream( throw new RangeError(`salt must be ${SALT_BYTES} bytes`); } + // Defensively copy on ingest, matching `readHeader`'s discipline — this + // closure holds `salt` for the lifetime of the stream (it's written into + // every call's header), so a caller mutating its buffer after the call + // must not be able to affect an already-in-flight encryption. + const ingestedSalt = Buffer.from(salt); + const state = Buffer.alloc(SECRETSTREAM_STATE_BYTES); const secretstreamHeader = Buffer.alloc(SECRETSTREAM_HEADER_BYTES); sodium.crypto_secretstream_xchacha20poly1305_init_push( @@ -351,7 +369,7 @@ export function createEncryptStream( this.push( writeHeader({ version: FORMAT_VERSION, - salt, + salt: ingestedSalt, kdfParams, secretstreamHeader, }), @@ -374,7 +392,7 @@ export function createEncryptStream( this.push( writeHeader({ version: FORMAT_VERSION, - salt, + salt: ingestedSalt, kdfParams, secretstreamHeader, }), @@ -394,19 +412,6 @@ export function createEncryptStream( }); } -/** - * Builds a `Transform` that decrypts a MagStacker backup bundle stream - * produced by {@link createEncryptStream}: it parses the leading - * {@link CryptoHeader} itself (the header is unencrypted preamble, so no key - * is needed to read it), then authenticates and decrypts each ciphertext - * chunk with `key`. - * - * Throws {@link InvalidHeaderError} for a malformed/truncated header and - * {@link DecryptionAuthError} for a wrong key, a tampered ciphertext byte - * (anywhere, including the final chunk), or a stream that ends before an - * authenticated final chunk — in every failure case, no unauthenticated - * plaintext is ever pushed downstream. - */ /** * Builds a `Transform` that decrypts a MagStacker backup bundle stream from a * `password` alone (U5's restore entry point). {@link createDecryptStream} @@ -494,6 +499,19 @@ export function createDecryptStreamFromPassword(password: string): Transform { return outer; } +/** + * Builds a `Transform` that decrypts a MagStacker backup bundle stream + * produced by {@link createEncryptStream}: it parses the leading + * {@link CryptoHeader} itself (the header is unencrypted preamble, so no key + * is needed to read it), then authenticates and decrypts each ciphertext + * chunk with `key`. + * + * Throws {@link InvalidHeaderError} for a malformed/truncated header and + * {@link DecryptionAuthError} for a wrong key, a tampered ciphertext byte + * (anywhere, including the final chunk), or a stream that ends before an + * authenticated final chunk — in every failure case, no unauthenticated + * plaintext is ever pushed downstream. + */ export function createDecryptStream(key: Buffer): Transform { if (key.byteLength !== SECRETSTREAM_KEY_BYTES) { throw new RangeError(`key must be ${SECRETSTREAM_KEY_BYTES} bytes`); From 6104cbca3e5f471bac602c015744d49410a8ea59 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 00:21:49 -0400 Subject: [PATCH 18/26] fix(backup): unify restore promote envelope + surface rollback failures + recovery/exhaustiveness hardening - Unify the empty-instance (F2) and force-replace (F3) restore promote paths through the same maintenance+snapshot+rollback envelope (forcePromote/emptyInstancePromote -> single `promote`). Closes a TOCTOU window where a non-force restore never raised the maintenance flag, so a concurrent ordinary write during staging/promote was neither blocked nor recoverable if the promote then failed. - `undoBlobSwap` now logs loudly and re-throws instead of swallowing rollback failures. When rollback itself fails inside `promote`, the error propagates as a generic thrown error (not a reassuring `rolled_back` outcome) and the maintenance flag is deliberately left active so writes stay blocked and recovery can retry. - `recoverInterruptedRestore` only clears the maintenance flag once recovery genuinely succeeded or wasn't needed; a partial `rollbackLiveFromSnapshot` failure now leaves the flag active and preserves the snapshot schema, logging a MANUAL INTERVENTION REQUIRED message. `sweepLeftoverSchemas` excludes any snapshot schema still referenced by an active flag. - `restore()`'s staging cleanup now logs drop/rm failures instead of swallowing them; `stageBundle`'s `BundleEvent` handling is now an exhaustive switch with a `never`-checked default; `_testFaultInjection` is now runtime-guarded to refuse outside `NODE_ENV=test`. Signed-off-by: UncleSp1d3r --- src/backup/__tests__/maintenance.test.ts | 65 ++++ src/backup/__tests__/restore-service.test.ts | 68 ++++ src/backup/maintenance.ts | 83 +++-- src/backup/restore-service.ts | 332 ++++++++++++------- 4 files changed, 401 insertions(+), 147 deletions(-) diff --git a/src/backup/__tests__/maintenance.test.ts b/src/backup/__tests__/maintenance.test.ts index c3dfdd3d..33a6f7c5 100644 --- a/src/backup/__tests__/maintenance.test.ts +++ b/src/backup/__tests__/maintenance.test.ts @@ -346,6 +346,71 @@ describe("maintenance envelope (KTD5 hardening)", () => { ).toBe(false); }); + test("clears the flag without touching data when maintenance was active but no snapshot was ever recorded (crash before the risky section began)", async () => { + await seedOwner(db, "Untouched Owner"); + const before = await snapshotTables(db); + + await enterMaintenance(db, "restore"); + // Deliberately no `recordMaintenanceSnapshotSchema` call — this is the + // "crash before the risky wipe+promote section ever began" state: + // nothing live has been touched, so recovery must be a pure flag clear. + expect(await isMaintenanceActive(db)).toBe(true); + + await recoverInterruptedRestore(db, uploadDir); + + expect(await snapshotTables(db)).toEqual(before); + expect(await isMaintenanceActive(db)).toBe(false); + }); + + test("leaves the maintenance flag ACTIVE and preserves the snapshot schema when the snapshot rollback fails partway (manual intervention required)", async () => { + await seedOwner(db, "Pre-Restore Owner"); + + const snapshotSchema = `${SNAPSHOT_SCHEMA_PREFIX}${randomUUID().replace(/-/g, "")}`; + // Deliberately incomplete snapshot: every table EXCEPT the last one in + // `EXPORT_TABLE_ORDER`'s insert order is copied, so + // `rollbackLiveFromSnapshot` successfully wipes+reinserts everything up + // to that point, then dies mid-loop on the missing table — simulating + // a crash partway through the non-transactional rollback. + await db.execute( + sql`DROP SCHEMA IF EXISTS ${sql.identifier(snapshotSchema)} CASCADE`, + ); + await db.execute(sql`CREATE SCHEMA ${sql.identifier(snapshotSchema)}`); + const tableNames = EXPORT_TABLE_ORDER.map((table) => getTableName(table)); + const skippedTable = tableNames[tableNames.length - 1]; + for (const table of EXPORT_TABLE_ORDER) { + const name = getTableName(table); + if (name === skippedTable) continue; + await db.execute(sql` + CREATE TABLE ${qualified(snapshotSchema, name)} + AS TABLE ${qualified("public", name)} + `); + } + + await wipeDatabase(db); + await seedOwner(db, "Half-Promoted New Owner"); + + await enterMaintenance(db, "restore"); + await recordMaintenanceSnapshotSchema(db, snapshotSchema); + + await recoverInterruptedRestore(db, uploadDir); + + // The flag must NOT have been cleared — a half-rolled-back DB must + // keep blocking ordinary writes until it's retried or resolved by + // hand, per the "MANUAL INTERVENTION REQUIRED" contract. + expect(await isMaintenanceActive(db)).toBe(true); + // The snapshot must be preserved, not swept away — dropping it would + // make the half-completed rollback unrecoverable. + expect(await schemaExists(db, snapshotSchema)).toBe(true); + + await expect(assertWritesAllowed(db)).rejects.toBeInstanceOf( + MaintenanceModeError, + ); + + // A subsequent sweep must also leave the still-referenced snapshot + // alone — it's still the only way back, not an orphan. + expect(await listRestoreSchemas(db)).toContain(snapshotSchema); + }); + test("is idempotent — calling it twice in a row after a rollback is a harmless no-op", async () => { await seedOwner(db, "Pre-Restore Owner"); const expectedSnapshot = await snapshotTables(db); diff --git a/src/backup/__tests__/restore-service.test.ts b/src/backup/__tests__/restore-service.test.ts index e82fd0e4..ea90c5f1 100644 --- a/src/backup/__tests__/restore-service.test.ts +++ b/src/backup/__tests__/restore-service.test.ts @@ -36,6 +36,7 @@ import { type BundleBlobEntry, writeBundle } from "../bundle"; import { createEncryptStream, deriveKey, generateSalt } from "../crypto"; import { exportDatabase } from "../db-export"; import { importDatabase, wipeDatabase } from "../db-import"; +import { isMaintenanceActive } from "../maintenance"; import { BACKUP_FORMAT_VERSION, type BackupManifest, @@ -759,4 +760,71 @@ describe("restore service (U5)", () => { expect(files).toEqual([winningBlobKey]); expect(files).not.toContain(losingBlobKey); }); + + test("the empty-instance (non-force) restore path also enters maintenance during promote, blocking concurrent ordinary writes (TOCTOU fix)", async () => { + // Empty-instance restores used to wipe+promote directly, with no + // maintenance flag raised at all — a concurrent ordinary write landing + // during that window was never blocked and had no snapshot to recover + // from if the promote then failed. `promote` is now unified across both + // paths, so this asserts the maintenance flag really does go active for + // a plain (non-force) restore too, not just `force: true` ones. + const bundle = await buildEncryptedBundle(db, {}); + + let observedActiveDuringPromote = false; + const outcomePromise = restore( + Readable.from([bundle]), + PASSWORD, + restoreOptions({ + _testFaultInjection: async (point) => { + if (point === "pre-commit") { + // Hold the promote transaction open long enough for the poll + // below to observe the maintenance flag while it's mid-flight. + await new Promise((resolve) => setTimeout(resolve, 200)); + } + }, + }), + ); + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if (await isMaintenanceActive(db)) { + observedActiveDuringPromote = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + const outcome = await outcomePromise; + + expect(observedActiveDuringPromote).toBe(true); + expect(outcome.kind).toBe("ok"); + // The flag must be cleared again once the (successful) restore finishes. + expect(await isMaintenanceActive(db)).toBe(false); + }); + + test("_testFaultInjection is refused outside a test environment (production footgun guard)", async () => { + // `NODE_ENV` is typed read-only (bun-types) — `Object.assign` on + // `process.env` sidesteps that at the type level while still mutating + // the real, live environment the running process reads from. + const originalNodeEnv = process.env.NODE_ENV; + Object.assign(process.env, { NODE_ENV: "production" }); + try { + let thrown: unknown; + try { + await restore( + Readable.from([]), + PASSWORD, + restoreOptions({ _testFaultInjection: () => {} }), + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).toMatch(/_testFaultInjection/); + expect(thrown).not.toBeInstanceOf(NotAuthorizedError); + } finally { + Object.assign(process.env, { NODE_ENV: originalNodeEnv }); + } + }); }); diff --git a/src/backup/maintenance.ts b/src/backup/maintenance.ts index c479aae8..0cc72bb4 100644 --- a/src/backup/maintenance.ts +++ b/src/backup/maintenance.ts @@ -1,17 +1,19 @@ /** - * Force-restore maintenance envelope (plan Unit U5, KTD5): a durable, + * Restore maintenance envelope (plan Unit U5, KTD5): a durable, * crash-recoverable "restore in progress" flag plus a pool-safe advisory * lock, both scoped OUTSIDE the `public` schema so they never show up as * live application tables (and so U3's `db-roundtrip.test.ts` regression * guard — which asserts `EXPORT_TABLE_ORDER` + `EPHEMERAL_TABLE_NAMES` cover * every `public`-schema table exactly — stays green without needing to know - * about restore's own bookkeeping). + * about restore's own bookkeeping). Entered for EVERY restore promote — + * empty-instance (F2) and force-replace (F3) alike, see + * `restore-service.ts`'s `promote` — not just force-restore. * * **Durable flag.** A single-row table (`restore_ops.maintenance_flag`) * rather than an in-memory flag: the flag must survive a process restart so - * a crash mid-force-restore is still visible afterward. The row also records - * which `restore_snapshot_*` schema (if any) belongs to the in-progress - * force-restore (`recordMaintenanceSnapshotSchema`) — this is what lets + * a crash mid-restore is still visible afterward. The row also records which + * `restore_snapshot_*` schema (if any) belongs to the in-progress restore + * (`recordMaintenanceSnapshotSchema`) — this is what lets * `recoverInterruptedRestore` tell "crashed before the risky section began, * nothing live touched" (`active = true`, no recorded snapshot) apart from * "crashed mid wipe+promote, live data may be in the new OR old state" @@ -432,13 +434,29 @@ async function restoreBlobsFromNewestPreRestoreDir( } } -/** Drops every leftover `restore_staging_*`/`restore_snapshot_*` schema found in the database — orphans from a restore that never reached its own cleanup. Uses a regex match (not `LIKE`) so the prefixes' literal underscores aren't treated as single-character wildcards. */ +/** + * Drops every leftover `restore_staging_*`/`restore_snapshot_*` schema found + * in the database — orphans from a restore that never reached its own + * cleanup. Uses a regex match (not `LIKE`) so the prefixes' literal + * underscores aren't treated as single-character wildcards. + * + * EXCLUDES whatever `restore_snapshot_*` schema is currently recorded by an + * ACTIVE maintenance flag: that schema is a still-live recovery artifact (a + * rollback that failed partway and deliberately left the flag active — see + * `recoverInterruptedRestore`'s doc comment), not an orphan. Sweeping it out + * from under a stuck recovery would make that half-completed rollback + * unrecoverable. + */ async function sweepLeftoverSchemas(db: DbOrTx): Promise { + const flag = await readMaintenanceFlag(db); + const protectedSchema = flag.active ? flag.snapshotSchema : null; + const result = await db.execute<{ nspname: string }>(sql` SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname ~ '^restore_(staging|snapshot)_' `); for (const row of result.rows) { + if (row.nspname === protectedSchema) continue; try { await dropSchemaIfExists(db, row.nspname); } catch (err) { @@ -474,12 +492,13 @@ async function sweepLeftoverStagingDirs(uploadDir: string): Promise { } /** - * Boot-time crash recovery for an interrupted force-restore (KTD5 fix): a - * process that dies mid force-restore (`restore-service.ts`'s - * `forcePromote`) can leave the durable maintenance flag stuck active, its - * `restore_snapshot_*` schema orphaned, the pre-restore blob directory never - * swapped back in, and staging schemas/directories leaked. Called once from - * `instrumentation.ts`'s `register()` on every server boot. + * Boot-time crash recovery for an interrupted restore (KTD5 fix): a process + * that dies mid-promote (`restore-service.ts`'s `promote`, which now runs + * for both the empty-instance and force-replace restore paths) can leave the + * durable maintenance flag stuck active, its `restore_snapshot_*` schema + * orphaned, the pre-restore blob directory never swapped back in, and + * staging schemas/directories leaked. Called once from `instrumentation.ts`'s + * `register()` on every server boot. * * Recovery contract: `active = true` together with a recorded * `snapshot_schema` that still exists in the database means the crash @@ -488,12 +507,25 @@ async function sweepLeftoverStagingDirs(uploadDir: string): Promise { * pre-restore directory, then the snapshot is dropped. `active = true` with * no recorded snapshot (or one that no longer exists) means the crash * happened before the risky section began — nothing live was touched, so no - * rollback is needed. Either way the flag is always cleared afterward, and a + * rollback is needed, and the flag is cleared immediately. Either way, once + * recovery genuinely succeeds (or wasn't needed), the flag is cleared and a * general sweep removes any other leftover restore staging/snapshot schema * or temp directory regardless of what the flag says (they can only be * orphans by the time this runs — nothing should be actively restoring at * boot). * + * **Partial-rollback failure.** `rollbackLiveFromSnapshot` is intentionally + * NOT transactional (see its own doc comment) — if it dies partway through + * its table-by-table loop, `public` may now be a genuine mix of wiped and + * restored tables. In that case the maintenance flag is deliberately left + * ACTIVE (not cleared) and the snapshot schema is deliberately preserved + * (not dropped, and excluded from the general schema sweep below — see + * {@link sweepLeftoverSchemas}) so `assertWritesAllowed` keeps blocking + * ordinary writes against the half-wiped DB and a future + * `recoverInterruptedRestore` call (or manual operator intervention) can + * still retry from that same snapshot. The failure is logged loudly as + * requiring manual intervention. + * * Idempotent (safe to call multiple times, e.g. across restarts) and * defensive: every step is individually caught and logged, so a failure in * one step never prevents the rest from running and this function never @@ -503,6 +535,11 @@ export async function recoverInterruptedRestore( db: DbOrTx, uploadDir: string, ): Promise { + // Set when `rollbackLiveFromSnapshot` fails partway — see the doc comment + // above. When true, the flag is deliberately left active below instead of + // cleared, so recovery can be retried (or resolved manually) later. + let rollbackFailed = false; + try { const flag = await readMaintenanceFlag(db); if (flag.active && flag.snapshotSchema) { @@ -518,8 +555,14 @@ export async function recoverInterruptedRestore( await dropSchemaIfExists(db, snapshotSchema); } } catch (err) { - logRecoveryFailure( - `roll back interrupted force-restore from snapshot "${snapshotSchema}"`, + rollbackFailed = true; + console.error( + `backup/maintenance: MANUAL INTERVENTION REQUIRED — rolling back an ` + + `interrupted restore from snapshot "${snapshotSchema}" failed partway; ` + + "the live database may now be a mix of wiped and restored tables. " + + "The maintenance flag is being left ACTIVE (blocking ordinary writes) " + + "and the snapshot schema is being preserved so a retry or manual " + + "recovery can still use it.", err, ); } @@ -527,10 +570,12 @@ export async function recoverInterruptedRestore( } catch (err) { logRecoveryFailure("read maintenance flag", err); } finally { - try { - await exitMaintenance(db); - } catch (err) { - logRecoveryFailure("clear maintenance flag", err); + if (!rollbackFailed) { + try { + await exitMaintenance(db); + } catch (err) { + logRecoveryFailure("clear maintenance flag", err); + } } } diff --git a/src/backup/restore-service.ts b/src/backup/restore-service.ts index 1793b54c..6ac76e95 100644 --- a/src/backup/restore-service.ts +++ b/src/backup/restore-service.ts @@ -22,25 +22,35 @@ * 4. Only once the whole bundle has authenticated does `restore()` check * instance emptiness (R6/AE1) and, if empty (or `force`), promote staging * to live. - * 5. A `force` restore additionally runs the KTD5 envelope (`maintenance.ts`): + * 5. EVERY promote — empty-instance (F2) or force-replace (F3) alike — runs + * through the same KTD5 envelope (`maintenance.ts` + `promote` below): * durable maintenance flag, a committed `restore_snapshot_` schema * of the pre-restore DB, and the pre-restore blob directory moved aside — * so a failure at ANY point in the wipe+promote step (including after the * DB side has already committed, but before the blob directory has been - * swapped in) rolls both stores back together. + * swapped in) rolls both stores back together. This used to be a + * force-only envelope, with the empty-instance path wiping+copying + * directly and unprotected; that left a TOCTOU window between the + * emptiness check (step 4) and promote where an ordinary concurrent write + * was never blocked and had no snapshot to recover from. Unifying both + * paths onto the maintenance envelope closes that window — the ONLY + * remaining difference between F2 and F3 is the refuse-unless-empty guard + * in step 4, which only `force` skips. * * **Concurrency (KTD5 hardening).** Every per-run schema (`restore_staging_*` - * and, for `force`, `restore_snapshot_*`) is named with a fresh random suffix - * per call, so two overlapping restores never collide on schema names. That - * alone isn't enough to prevent corruption, though: both restores still - * write to the shared `public` schema during promote, and interleaved - * wipe+promote transactions from two different restores could each commit - * different tables' worth of data, leaving `public` a genuine mix of both - * bundles. `restore()` therefore holds `withRestoreAdvisoryLock` for its - * ENTIRE body — staging through promote, both the empty-instance and force - * paths — so only one restore attempt is ever inside the risky section at a - * time; a second concurrent call blocks until the first fully finishes - * (commit or rollback) before it even begins staging. + * and `restore_snapshot_*`) is named with a fresh random suffix per call, so + * two overlapping restores never collide on schema names. That alone isn't + * enough to prevent corruption, though: both restores still write to the + * shared `public` schema during promote, and interleaved wipe+promote + * transactions from two different restores could each commit different + * tables' worth of data, leaving `public` a genuine mix of both bundles. + * `restore()` therefore holds `withRestoreAdvisoryLock` for its ENTIRE body — + * staging through promote, both the empty-instance and force paths — so only + * one restore attempt is ever inside the risky section at a time; a second + * concurrent call blocks until the first fully finishes (commit or rollback) + * before it even begins staging. On top of that, `promote`'s maintenance flag + * (active for both paths now) blocks ordinary, non-restore writes for the + * same window via `assertWritesAllowed`. */ import { randomUUID } from "node:crypto"; @@ -79,7 +89,7 @@ import { import { BACKUP_FORMAT_VERSION, type BackupManifest } from "./manifest"; import { EXPORT_TABLE_ORDER, WIPE_TABLE_ORDER } from "./table-order"; -/** Discriminated outcome of a restore attempt. Every branch carries an operator-facing `message`; none of them throw for expected restore-flow refusals — only a genuine programming/authorization error (see `restore`'s admin check) or an unclassified staging failure (see the module doc comment on error classification) throws. */ +/** Discriminated outcome of a restore attempt. Every branch carries an operator-facing `message`; none of them throw for expected restore-flow refusals — only a genuine programming/authorization error (see `restore`'s admin check) or an unclassified staging failure (see the error-classification `catch` block inside `restore()`) throws. */ export type RestoreOutcome = | { readonly kind: "ok"; readonly message: string } | { readonly kind: "refused_not_empty"; readonly message: string } @@ -127,6 +137,18 @@ function toError(err: unknown): Error { return err instanceof Error ? err : new Error(String(err)); } +/** Logs a cleanup-step failure loudly instead of swallowing it — mirrors `maintenance.ts`'s `logRecoveryFailure` pattern for its boot-time sweep. Cleanup failures here are leaked staging artifacts, not correctness bugs (the risky section has already committed or rolled back by the time this runs), so they're logged rather than thrown. */ +function logCleanupFailure(step: string, err: unknown): void { + console.error(`backup/restore-service: cleanup step "${step}" failed`, err); +} + +/** Exhaustiveness guard for `BundleEvent.kind`'s switch in `stageBundle`: if a new `BundleEvent` variant is ever added in `bundle.ts` without a corresponding `case` here, `event` fails to narrow to `never` and this file fails to typecheck — instead of the new event silently falling through a no-op default at runtime. */ +function assertNeverBundleEvent(event: never): never { + throw new Error( + `internal: unhandled BundleEvent kind: ${JSON.stringify(event)}`, + ); +} + async function pathExists(path: string): Promise { try { await stat(path); @@ -153,6 +175,17 @@ export async function restore( password: string, options: RestoreOptions = {}, ): Promise { + // Production footgun guard (KTD5 hardening): `_testFaultInjection` is an + // internal control-flow hook meant ONLY for this module's own test suite. + // `RestoreOptions` is a public exported type, so nothing at the type level + // stops a caller from wiring it up outside tests — this is the runtime + // backstop. `NODE_ENV=test` is set automatically by `bun test`. + if (options._testFaultInjection && process.env.NODE_ENV !== "test") { + throw new Error( + "restore(): options._testFaultInjection must never be supplied outside NODE_ENV=test", + ); + } + const checkIsAdmin = options.checkIsAdmin ?? isAdmin; if (!(await checkIsAdmin())) { // Not a modeled RestoreOutcome: an unauthorized caller is a programming/ @@ -172,9 +205,9 @@ export async function restore( // The ENTIRE staging+promote body runs under one advisory lock, held on a // single dedicated connection (`opsDb`) for the whole call — see the - // module doc comment. `forcePromote`/`emptyInstancePromote` therefore MUST - // NOT acquire this lock themselves (it isn't reentrant across connections; - // doing so would deadlock this very call). + // module doc comment. `promote` therefore MUST NOT acquire this lock + // itself (it isn't reentrant across connections; doing so would deadlock + // this very call). return await withRestoreAdvisoryLock(pool, async (client) => { const opsDb = drizzle(client, { schema }); @@ -233,23 +266,21 @@ export async function restore( } try { - if (force) { - await forcePromote({ - db: opsDb, - uploadDir, - stagingBlobDir, - stagingSchema, - snapshotSchema, - testFaultInjection: options._testFaultInjection, - }); - } else { - await emptyInstancePromote({ - db: opsDb, - uploadDir, - stagingBlobDir, - stagingSchema, - }); - } + // Both the empty-instance (F2) and force-replace (F3) paths now run + // through the SAME maintenance+snapshot+rollback envelope — see the + // module doc comment and `promote`'s own doc comment for why (KTD5 + // TOCTOU fix). The only remaining difference between the two is the + // refuse-unless-empty guard, which already ran inside `stageBundle` + // above. + await promote({ + db: opsDb, + uploadDir, + stagingBlobDir, + stagingSchema, + snapshotSchema, + reason: force ? "force-restore" : "empty-instance-restore", + testFaultInjection: options._testFaultInjection, + }); } catch (err) { if (err instanceof RestoreRolledBackError) { return { kind: "rolled_back", message: err.message }; @@ -263,9 +294,16 @@ export async function restore( .execute( sql`DROP SCHEMA IF EXISTS ${sql.identifier(stagingSchema)} CASCADE`, ) - .catch(() => {}); + .catch((err) => { + logCleanupFailure(`drop staging schema "${stagingSchema}"`, err); + }); await rm(stagingBlobDir, { recursive: true, force: true }).catch( - () => {}, + (err) => { + logCleanupFailure( + `remove staging blob directory "${stagingBlobDir}"`, + err, + ); + }, ); } }); @@ -321,26 +359,37 @@ async function stageBundle( let manifest: BackupManifest | undefined; for await (const event of generator) { - if (event.kind === "manifest") { - manifest = event.manifest; - if (manifest.backupFormatVersion !== BACKUP_FORMAT_VERSION) { - throw new VersionMismatchSignal( - `bundle backupFormatVersion ${manifest.backupFormatVersion} is incompatible with this instance's ${BACKUP_FORMAT_VERSION}`, - ); + switch (event.kind) { + case "manifest": { + manifest = event.manifest; + if (manifest.backupFormatVersion !== BACKUP_FORMAT_VERSION) { + throw new VersionMismatchSignal( + `bundle backupFormatVersion ${manifest.backupFormatVersion} is incompatible with this instance's ${BACKUP_FORMAT_VERSION}`, + ); + } + if ( + options.checkEmptiness && + (await options.instanceHasInventoryData()) + ) { + throw new NotEmptySignal( + "the instance already holds inventory data; use force-replace to overwrite it", + ); + } + break; } - if ( - options.checkEmptiness && - (await options.instanceHasInventoryData()) - ) { - throw new NotEmptySignal( - "the instance already holds inventory data; use force-replace to overwrite it", - ); + case "db": { + await importIntoStaging(pool, event.stream, stagingSchema); + break; + } + case "blob": { + // readBundle has already written + path-validated the file under + // stagingBlobDir (KTD11) — nothing further to do here. + break; + } + default: { + assertNeverBundleEvent(event); } - } else if (event.kind === "db") { - await importIntoStaging(pool, event.stream, stagingSchema); } - // "blob" events: readBundle has already written + path-validated the - // file under stagingBlobDir (KTD11) — nothing further to do here. } if (!manifest) { @@ -413,16 +462,37 @@ async function finishBlobSwap( await rename(stagingBlobDir, uploadDir); } -/** Undoes `beginBlobSwap` (and any partial `finishBlobSwap`): removes whatever now sits at `uploadDir` and restores the original contents. */ +/** + * Undoes `beginBlobSwap` (and any partial `finishBlobSwap`): removes whatever + * now sits at `uploadDir` and restores the original contents. + * + * Unlike most cleanup helpers in this module, a failure here is NOT + * swallowed: this runs only while unwinding an already-failed promote, so a + * failure means the live upload directory may now be missing or only + * partially restored — the operator must be told loudly (both `uploadDir` + * and the moved-aside directory are logged so recovery can be attempted by + * hand) and the caller MUST treat this as distinct from a clean rollback + * (see `promote`'s catch block). + */ async function undoBlobSwap( handle: BlobSwapHandle, uploadDir: string, ): Promise { - await rm(uploadDir, { recursive: true, force: true }).catch(() => {}); - if (handle.hadExistingDir) { - await rename(handle.movedAsideDir, uploadDir).catch(() => {}); - } else { - await mkdir(uploadDir, { recursive: true, mode: 0o700 }).catch(() => {}); + try { + await rm(uploadDir, { recursive: true, force: true }); + if (handle.hadExistingDir) { + await rename(handle.movedAsideDir, uploadDir); + } else { + await mkdir(uploadDir, { recursive: true, mode: 0o700 }); + } + } catch (err) { + console.error( + `backup/restore-service: CRITICAL — undoBlobSwap failed while rolling back a failed restore promote; ` + + `the live upload directory may be missing or corrupt. MANUAL RECOVERY REQUIRED. ` + + `uploadDir="${uploadDir}" movedAsideDir="${handle.movedAsideDir}" hadExistingDir=${handle.hadExistingDir}`, + err, + ); + throw err; } } @@ -456,65 +526,44 @@ async function wipeLive(tx: Transaction): Promise { } /** - * F2 (empty instance) promote: swap the staged blob directory in, then - * wipe-and-promote staging rows into live in one transaction. Either step - * failing undoes the other — no live change survives a partial failure. + * The unified promote envelope for BOTH F2 (empty instance) and F3 + * (force-replace) restores — the KTD5 envelope: durable maintenance flag, a + * committed pre-restore snapshot schema, wipe+promote in one transaction, and + * a blob-directory swap. A failure anywhere after the snapshot is committed + * rolls BOTH the DB (restored from the snapshot, if the wipe+promote + * transaction had already committed) and the blobs (restored from the + * moved-aside directory) back together. + * + * This used to be force-restore-only, with the empty-instance path wiping + * and copying directly — no maintenance flag, no snapshot. That left a TOCTOU + * window: emptiness is checked once, at the start of staging (`stageBundle`), + * but the maintenance flag (which is what `assertWritesAllowed` actually + * checks) was never raised for that path, so an ordinary write landing during + * staging or promote could interleave with the wipe+copy, and if THIS + * promote then failed, there was no snapshot to roll back to. Both paths now + * share this one envelope; "empty instance" only changes what `stageBundle` + * checked before promote was ever called. * * "Empty" here means no *inventory* data (`instanceHasInventoryData`); the - * instance can still hold bootstrap auth rows (the admin who is performing the - * restore always exists in `public.user`). Those must be replaced, not merged - * into — a plain `INSERT` of the backup's users would collide with the live - * admin on `user.email`'s UNIQUE constraint and roll the whole restore back. - * So this wipes live tables before copying, exactly like force-replace, and - * relies on the single wrapping transaction for atomic DB rollback (no - * separate snapshot schema is needed: a failed transaction reverts the wipe - * too, and the blob swap happened first and is undone on failure). + * instance can still hold bootstrap auth rows (the admin who is performing + * the restore always exists in `public.user`). Those must be replaced, not + * merged into — a plain `INSERT` of the backup's users would collide with the + * live admin on `user.email`'s UNIQUE constraint and roll the whole restore + * back. So this always wipes live tables before copying, for both paths. * - * `ctx.db` is bound to the SAME locked connection `restore()` holds for its - * entire body (see the module doc comment) — this doesn't need its own - * advisory lock. - */ -async function emptyInstancePromote(ctx: { - db: Database; - uploadDir: string; - stagingBlobDir: string; - stagingSchema: string; -}): Promise { - const swap = await beginBlobSwap(ctx.uploadDir); - try { - await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); - } catch (err) { - await undoBlobSwap(swap, ctx.uploadDir); - throw new RestoreRolledBackError( - `blob promotion failed: ${toError(err).message}`, - { cause: err }, - ); - } - - try { - await ctx.db.transaction(async (tx) => { - await wipeLive(tx); - await copySchemaToLive(tx, ctx.stagingSchema); - }); - } catch (err) { - await undoBlobSwap(swap, ctx.uploadDir); - throw new RestoreRolledBackError( - `database promotion failed: ${toError(err).message}`, - { cause: err }, - ); - } - - await commitBlobSwap(swap); -} - -/** - * F3 (force-replace) promote — the KTD5 envelope: durable maintenance flag, - * a committed pre-restore snapshot schema, wipe+promote in one transaction, - * and a blob-directory swap. A failure anywhere after the snapshot is - * committed rolls BOTH the DB (restored from the snapshot, if the - * wipe+promote transaction had already committed) and the blobs (restored - * from the moved-aside directory) back together, then always exits - * maintenance. + * **Rollback-failure handling.** If undoing a failed promote (restoring the + * DB from the snapshot, or `undoBlobSwap`) itself fails, this is NOT reported + * to the operator as a clean `RestoreRolledBackError` — that would tell them + * the instance is safely back to its pre-restore state when it may not be. + * Instead the original rollback failure is logged loudly (`undoBlobSwap` + * does its own logging; the DB-rollback failure is logged here) and + * re-thrown as a plain error, which `restore()` does NOT convert to a + * `'rolled_back'` outcome — it propagates as a generic thrown error instead + * (surfaced by the route as a generic 500/"error"). The maintenance flag is + * ALSO deliberately left active in that case (see the `finally` below) so + * `assertWritesAllowed` keeps blocking ordinary writes against a possibly + * half-wiped DB, and `recoverInterruptedRestore` can retry from the + * still-recorded snapshot on the next boot. * * `ctx.db` is bound to the SAME locked connection `restore()` holds for its * entire body (see the module doc comment) — this function does NOT acquire @@ -522,15 +571,21 @@ async function emptyInstancePromote(ctx: { * `pg_advisory_lock` call for the same key on a different connection blocks * until the first is released, and the first is this very call). */ -async function forcePromote(ctx: { +async function promote(ctx: { db: Database; uploadDir: string; stagingBlobDir: string; stagingSchema: string; snapshotSchema: string; + reason: string; testFaultInjection?: RestoreOptions["_testFaultInjection"]; }): Promise { - await enterMaintenance(ctx.db, "force-restore"); + await enterMaintenance(ctx.db, ctx.reason); + // Set when undoing a failed promote itself fails — see the doc comment + // above. When true, the `finally` below deliberately skips + // `exitMaintenance` so the flag stays active for `recoverInterruptedRestore` + // to pick up. + let rollbackFailed = false; try { await ctx.db.execute( sql`DROP SCHEMA IF EXISTS ${sql.identifier(ctx.snapshotSchema)} CASCADE`, @@ -564,22 +619,41 @@ async function forcePromote(ctx: { await ctx.testFaultInjection?.("post-commit-pre-blob-swap"); await finishBlobSwap(ctx.stagingBlobDir, ctx.uploadDir); } catch (err) { - if (dbCommitted) { - // The wipe+promote transaction already committed new data — the - // only way back is to explicitly restore from the snapshot. - await ctx.db.transaction(async (tx) => { - await wipeLive(tx); - await copySchemaToLive(tx, ctx.snapshotSchema); - }); + try { + if (dbCommitted) { + // The wipe+promote transaction already committed new data — the + // only way back is to explicitly restore from the snapshot. + await ctx.db.transaction(async (tx) => { + await wipeLive(tx); + await copySchemaToLive(tx, ctx.snapshotSchema); + }); + } + await undoBlobSwap(swap, ctx.uploadDir); + } catch (rollbackErr) { + // The rollback itself failed — do NOT report a clean 'rolled_back'. + // Leave the snapshot and the maintenance flag in place (see + // `rollbackFailed` above) and let this propagate as a generic error. + rollbackFailed = true; + console.error( + "backup/restore-service: CRITICAL — rollback of a failed restore promote itself failed; " + + "the instance may be left in a mixed/inconsistent state. MANUAL INTERVENTION REQUIRED. " + + `snapshotSchema="${ctx.snapshotSchema}" uploadDir="${ctx.uploadDir}"`, + { promoteError: err, rollbackError: rollbackErr }, + ); + throw rollbackErr; } - await undoBlobSwap(swap, ctx.uploadDir); await ctx.db .execute( sql`DROP SCHEMA IF EXISTS ${sql.identifier(ctx.snapshotSchema)} CASCADE`, ) - .catch(() => {}); + .catch((dropErr) => { + logCleanupFailure( + `drop snapshot schema "${ctx.snapshotSchema}" after a successful rollback`, + dropErr, + ); + }); throw new RestoreRolledBackError( - `force-restore promotion failed and was rolled back: ${toError(err).message}`, + `restore promotion failed and was rolled back: ${toError(err).message}`, { cause: err }, ); } @@ -589,6 +663,8 @@ async function forcePromote(ctx: { ); await commitBlobSwap(swap); } finally { - await exitMaintenance(ctx.db); + if (!rollbackFailed) { + await exitMaintenance(ctx.db); + } } } From 59ff896cbd90d5a5b484164388ecaf4b4ff1c80f Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 00:27:03 -0400 Subject: [PATCH 19/26] fix(backup): audit/stream error handling + CSRF fallback tests + render outcome message + instrumentation test Signed-off-by: UncleSp1d3r --- __tests__/instrumentation.test.ts | 141 ++++++++++++++++++++++++++ app/(admin)/backup/restore-panel.tsx | 11 ++ app/api/admin/backup/export/route.ts | 39 ++++++- app/api/admin/backup/restore/route.ts | 20 +++- src/backup/__tests__/routes.test.ts | 63 ++++++++++++ 5 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 __tests__/instrumentation.test.ts diff --git a/__tests__/instrumentation.test.ts b/__tests__/instrumentation.test.ts new file mode 100644 index 00000000..3c0cbd63 --- /dev/null +++ b/__tests__/instrumentation.test.ts @@ -0,0 +1,141 @@ +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + mock, + spyOn, + test, +} from "bun:test"; + +/** + * Focused unit tests for `register()` — `instrumentation.ts`'s Next.js + * server-startup hook. Covers the three real branches called out in that + * file's own doc comment: + * 1. `NEXT_RUNTIME !== "nodejs"` — early return, the recovery sweep never runs. + * 2. `DATABASE_URL` unset — early return, the recovery sweep never runs. + * 3. The try/catch around `recoverInterruptedRestore` — a recovery failure + * must be caught and logged, never rethrown (a boot-recovery failure + * must not crash the server). + * + * **Deliberately lives OUTSIDE `src/`** (not `src/__tests__/`) and is run as + * its own invocation (`bun test __tests__/instrumentation.test.ts`), never + * bundled into `bun run test`'s `bun test src` / `just ci-check`. Verified + * empirically against this repo's actual Bun version (1.3.14): `mock.module()` + * replaces a module specifier for the rest of the **process**, not just this + * file, and — critically — if any OTHER file anywhere in the same `bun test` + * invocation has a static `import` of that same module, the real module gets + * linked into the cache before this file's `mock.module()` call ever runs + * (regardless of file ordering), which either silently no-ops the mock here + * or (if this file's mock registers first) corrupts the real module for + * every other file that statically imports it — reproduced directly: a + * `mock.module("@/src/backup/maintenance", () => ({ POISONED: true }))` in a + * file that sorts before `src/backup/__tests__/maintenance.test.ts` made that + * file fail at load time with `SyntaxError: Export named 'isMaintenanceActive' + * not found`. `src/backup/__tests__/maintenance.test.ts` and + * `src/backup/__tests__/write-path-maintenance-guard.test.ts` both statically + * import the REAL `@/src/backup/maintenance`/`@/src/db/client`, so mocking + * those specifiers here would be unsafe inside `bun test src`. This file's + * own `afterAll` "restore" (re-registering `mock.module` with the real + * exports) does NOT fix this either — restoring a `mock.module()` override + * does not retroactively repair an already-linked static import in another + * file, the same constraint `src/backup/__tests__/routes.test.ts` documents + * for `@/src/db/client`. Living outside `src/` sidesteps the whole class of + * problem: this file never shares a `bun test` process with those tests. + */ + +const ORIGINAL_NEXT_RUNTIME = process.env.NEXT_RUNTIME; +const ORIGINAL_DATABASE_URL = process.env.DATABASE_URL; + +let recoverCalls = 0; +let recoverShouldThrow: unknown = null; + +mock.module("@/src/db/client", () => ({ + db: { fake: "db-handle" }, +})); +mock.module("@/src/storage", () => ({ + activeStorageRoot: () => "/fake/storage/root", +})); +mock.module("@/src/backup/maintenance", () => ({ + recoverInterruptedRestore: async () => { + recoverCalls += 1; + if (recoverShouldThrow) throw recoverShouldThrow; + }, +})); + +// instrumentation.ts has no top-level imports of its own — every dependency +// is dynamically imported inside `register()` at call time (see its doc +// comment) — so it's safe to statically import `register` here regardless +// of ordering relative to the `mock.module()` calls above. +const { register } = await import("../instrumentation"); + +function restoreEnv(): void { + if (ORIGINAL_NEXT_RUNTIME === undefined) { + delete process.env.NEXT_RUNTIME; + } else { + process.env.NEXT_RUNTIME = ORIGINAL_NEXT_RUNTIME; + } + if (ORIGINAL_DATABASE_URL === undefined) { + delete process.env.DATABASE_URL; + } else { + process.env.DATABASE_URL = ORIGINAL_DATABASE_URL; + } +} + +describe("instrumentation.register()", () => { + beforeEach(() => { + recoverCalls = 0; + recoverShouldThrow = null; + }); + + afterEach(() => { + restoreEnv(); + }); + + afterAll(() => { + restoreEnv(); + }); + + test('resolves without running the recovery sweep when NEXT_RUNTIME is not "nodejs"', async () => { + delete process.env.NEXT_RUNTIME; + process.env.DATABASE_URL = "postgres://ignored/ignored"; + + await expect(register()).resolves.toBeUndefined(); + expect(recoverCalls).toBe(0); + }); + + test("resolves without running the recovery sweep when DATABASE_URL is unset", async () => { + process.env.NEXT_RUNTIME = "nodejs"; + delete process.env.DATABASE_URL; + + await expect(register()).resolves.toBeUndefined(); + expect(recoverCalls).toBe(0); + }); + + test("runs the recovery sweep exactly once when both NEXT_RUNTIME and DATABASE_URL are set", async () => { + process.env.NEXT_RUNTIME = "nodejs"; + process.env.DATABASE_URL = "postgres://ignored/ignored"; + + await expect(register()).resolves.toBeUndefined(); + expect(recoverCalls).toBe(1); + }); + + test("swallows a recoverInterruptedRestore failure — register() never rethrows (a boot-recovery failure must not crash the server)", async () => { + process.env.NEXT_RUNTIME = "nodejs"; + process.env.DATABASE_URL = "postgres://ignored/ignored"; + recoverShouldThrow = new Error("simulated recovery failure"); + + const errorSpy = spyOn(console, "error").mockImplementation(() => {}); + try { + await expect(register()).resolves.toBeUndefined(); + expect(recoverCalls).toBe(1); + expect(errorSpy).toHaveBeenCalledTimes(1); + expect(errorSpy.mock.calls[0]?.[0]).toContain( + "crash-recovery sweep failed", + ); + } finally { + errorSpy.mockRestore(); + } + }); +}); diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx index ce600660..021512b3 100644 --- a/app/(admin)/backup/restore-panel.tsx +++ b/app/(admin)/backup/restore-panel.tsx @@ -215,6 +215,17 @@ export function RestorePanel() { ? RESTORE_UNEXPECTED_ERROR.detail : RESTORE_OUTCOME_COPY[outcome.kind].detail}

+ {outcome.kind !== "client_error" && outcome.message ? ( + // The server-computed detail for this specific outcome (e.g. + // the decrypt-error text, or version numbers on a + // `version_mismatch`) — additional to the fixed per-kind copy + // above, not a replacement for it. Deliberately excluded for + // `client_error`: that fallback's `message` is a raw + // network/parse error string, not server-vetted copy, and + // `RESTORE_UNEXPECTED_ERROR`'s generic detail is what's meant + // to be shown for it. +

{outcome.message}

+ ) : null} ) : null} diff --git a/app/api/admin/backup/export/route.ts b/app/api/admin/backup/export/route.ts index b00d92da..2a9e98c9 100644 --- a/app/api/admin/backup/export/route.ts +++ b/app/api/admin/backup/export/route.ts @@ -64,7 +64,12 @@ export async function POST(request: Request): Promise { actor: user.email, action: "export", outcome: `failure: ${errorMessage(error)}`, - }).catch(() => {}); + }).catch((auditError) => + console.error( + "backup export: failed to record a bad-password audit event", + auditError, + ), + ); return Response.json({ error: errorMessage(error) }, { status: 400 }); } @@ -76,17 +81,45 @@ export async function POST(request: Request): Promise { actor: user.email, action: "export", outcome: `failure: ${errorMessage(error)}`, - }).catch(() => {}); + }).catch((auditError) => + console.error( + "backup export: failed to record a build-failure audit event", + auditError, + ), + ); return Response.json({ error: "backup export failed" }, { status: 500 }); } + // Unlike the failure paths above (which already have a real result — a + // 400/500 — to return regardless of whether the audit write lands), a + // failure to record *this* row must not silently discard the bundle that + // was just built by letting the rejection propagate as an unhandled 500 + // (which the client couldn't distinguish from a real export failure). + // Log-and-still-deliver: the export already succeeded, so the bundle ships + // either way — the audit-write failure is only ever logged, never masked. await recordOperatorEvent({ actor: user.email, action: "export", outcome: "success", - }); + }).catch((auditError) => + console.error( + "backup export: failed to record the success audit event (bundle is being delivered anyway)", + auditError, + ), + ); const filename = `magstacker-backup-${timestampForFilename()}.magstacker-backup`; + // A mid-stream failure (e.g. a blob deleted between stat and read, an I/O + // error) would otherwise yield a truncated download with `operator_audit` + // permanently showing "success" above and zero server-side signal — attach + // an error listener before handing the stream off so at least the failure + // is logged. + bundle.on("error", (err) => + console.error( + "backup export: bundle stream failed after the success audit was already recorded", + err, + ), + ); return new Response(Readable.toWeb(bundle) as unknown as ReadableStream, { status: 200, headers: { diff --git a/app/api/admin/backup/restore/route.ts b/app/api/admin/backup/restore/route.ts index 6a0f0cf3..3a0adb8b 100644 --- a/app/api/admin/backup/restore/route.ts +++ b/app/api/admin/backup/restore/route.ts @@ -77,18 +77,34 @@ export async function POST(request: Request): Promise { actor: user.email, action: "restore", outcome: `failure: ${errorMessage(error)}`, - }).catch(() => {}); + }).catch((auditError) => + console.error( + "backup restore: failed to record a restore-failure audit event", + auditError, + ), + ); return Response.json( { outcome: "error", message: "restore failed unexpectedly" }, { status: 500 }, ); } + // Mirrors the export route's success-path handling: the restore already + // committed (or was refused) by the time this runs, so an audit-write + // failure here must be logged rather than left to propagate as an + // unhandled rejection — that would surface as a generic 500 to the client, + // indistinguishable from a real restore failure, even though the restore + // itself already has a real, decided `outcome`. await recordOperatorEvent({ actor: user.email, action: "restore", outcome: outcome.kind, - }); + }).catch((auditError) => + console.error( + `backup restore: failed to record the "${outcome.kind}" audit event`, + auditError, + ), + ); return Response.json( { outcome: outcome.kind, message: outcome.message }, diff --git a/src/backup/__tests__/routes.test.ts b/src/backup/__tests__/routes.test.ts index dfef70c0..afc6592c 100644 --- a/src/backup/__tests__/routes.test.ts +++ b/src/backup/__tests__/routes.test.ts @@ -420,6 +420,69 @@ describe("admin backup API routes (U6)", () => { expect(res.status).toBe(403); }); + // `same-origin.ts`'s third branch: neither `Origin` nor `Referer` present, + // decided from `Sec-Fetch-Site` alone. + test("an export POST with no Origin/Referer and Sec-Fetch-Site: cross-site is refused with 403", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportRequest(PASSWORD, { + Origin: null, + "Sec-Fetch-Site": "cross-site", + }), + ); + expect(res.status).toBe(403); + expect(await auditRowsFor("export")).toHaveLength(0); + }); + + test("an export POST with no Origin/Referer and Sec-Fetch-Site: same-origin is allowed", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportRequest(PASSWORD, { + Origin: null, + "Sec-Fetch-Site": "same-origin", + }), + ); + expect(res.status).toBe(200); + }); + + test("an export POST with no Origin/Referer and Sec-Fetch-Site: none is allowed", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportRequest(PASSWORD, { + Origin: null, + "Sec-Fetch-Site": "none", + }), + ); + expect(res.status).toBe(200); + }); + + // `same-origin.ts`'s fourth branch: none of the three signals present at + // all — default-deny rather than assumed same-origin. + test("an export POST with none of Origin, Referer, or Sec-Fetch-Site is refused with 403", async () => { + currentUser = ADMIN; + const res = await exportPost( + exportRequest(PASSWORD, { + Origin: null, + "Sec-Fetch-Site": null, + }), + ); + expect(res.status).toBe(403); + expect(await auditRowsFor("export")).toHaveLength(0); + }); + + test("a restore POST with none of Origin, Referer, or Sec-Fetch-Site is refused with 403", async () => { + currentUser = ADMIN; + const res = await restorePost( + restoreRequest({ + password: PASSWORD, + body: Readable.from([Buffer.alloc(0)]), + headerOverrides: { Origin: null, "Sec-Fetch-Site": null }, + }), + ); + expect(res.status).toBe(403); + expect(await auditRowsFor("restore")).toHaveLength(0); + }); + // --- Minimum export password length (hardening pass) ----------------------- test("an export with a too-short password returns 400 and records no backup", async () => { From 29b9a7ace87acda1c93caab02a3f1ca274170f7f Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 00:37:51 -0400 Subject: [PATCH 20/26] chore(backup): run root instrumentation test in CI + keep outcome message out of redundant UI - justfile test recipe also runs __tests__/instrumentation.test.ts in its own process (mock.module is process-global and would bleed into bun test src). - restore-panel no longer renders outcome.message: it paraphrases the curated per-kind copy, so showing both was redundant and tripped a Playwright strict-mode duplicate-text match. message stays in the JSON response + the client_error fallback. Signed-off-by: UncleSp1d3r --- app/(admin)/backup/restore-panel.tsx | 20 +++++++++----------- justfile | 5 +++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx index 021512b3..2588a6ae 100644 --- a/app/(admin)/backup/restore-panel.tsx +++ b/app/(admin)/backup/restore-panel.tsx @@ -215,17 +215,15 @@ export function RestorePanel() { ? RESTORE_UNEXPECTED_ERROR.detail : RESTORE_OUTCOME_COPY[outcome.kind].detail}

- {outcome.kind !== "client_error" && outcome.message ? ( - // The server-computed detail for this specific outcome (e.g. - // the decrypt-error text, or version numbers on a - // `version_mismatch`) — additional to the fixed per-kind copy - // above, not a replacement for it. Deliberately excluded for - // `client_error`: that fallback's `message` is a raw - // network/parse error string, not server-vetted copy, and - // `RESTORE_UNEXPECTED_ERROR`'s generic detail is what's meant - // to be shown for it. -

{outcome.message}

- ) : null} + {/* + `outcome.message` (the server-computed detail carried by every + RestoreOutcome) is intentionally NOT rendered here for known + kinds: it paraphrases the curated `RESTORE_OUTCOME_COPY` detail + above, so showing both is redundant and confusing. The curated + copy is the operator-facing source of truth; `message` remains + part of the route's JSON response (machine-readable / logged) and + drives the `client_error` fallback text above. + */} ) : null} diff --git a/justfile b/justfile index cba9263b..4fa88c28 100644 --- a/justfile +++ b/justfile @@ -137,6 +137,11 @@ alias t := test [group('test')] test: {{ mise_exec }} bun run test + # Root-level tests that must run in their own process (e.g. instrumentation + # tests use process-global `mock.module`, which would bleed into `src`). + # Target the exact file — a bare `__tests__` arg is a substring filter that + # would re-collect every `src/**/__tests__` file into one poisoned process. + {{ mise_exec }} bun test __tests__/instrumentation.test.ts # Install Playwright browsers (run once; idempotent) [group('test')] From 17d8cfefe8945eb5daeeb14e0d3700f9ce504e6c Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:40:19 -0400 Subject: [PATCH 21/26] fix(deploy): idempotent + owner-only secret handling, URL-safe DB password, doc clarity (PR review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit PR review findings on the deploy/secret-handling docs and scripts: - CONTRIBUTING.md: generate secrets/*.txt only when absent instead of overwriting them on every setup pass (an existing Postgres volume keeps its original password, so a silent overwrite locks devs out). Also stop instructing readers to embed unevaluated `$(cat ...)` shell substitution directly in .env.local — mise parses it as a literal dotenv file and never expands it; show the shell-evaluated file-generation commands instead. - docker-entrypoint.sh: enforce the documented hex-only password contract before interpolating POSTGRES_PASSWORD into DATABASE_URL, failing fast with a clear message instead of silently exporting a connection string a non-hex password (containing @ : / ? # %, etc.) would corrupt. - README.md: create the Docker secret files with owner-only permissions (umask 077) in both the quick-start and from-source flows; make the developer-loop postgres_password.txt generation idempotent for the same reason as CONTRIBUTING.md; clarify that `pg_dump` is database-only and does not capture uploaded document blobs (they live on the separate uploads volume), pointing readers at the in-app encrypted backup or a separate uploads-volume backup. - setup.sh: print owner-only (umask 077) secret-creation instructions, and have the preflight actively tighten existing secret files to 0600 (best-effort) rather than only checking they're non-empty. Verified with shellcheck (setup.sh, docker-entrypoint.sh) and pre-commit run --files, both clean. Signed-off-by: UncleSp1d3r --- CONTRIBUTING.md | 15 +++++++++++---- README.md | 20 ++++++++++++-------- docker-entrypoint.sh | 11 +++++++++++ setup.sh | 19 ++++++++++++++----- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 452589fc..4add6f98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,15 +33,22 @@ just env-setup # create .env.local from .env.example just install-hooks # install the pre-commit hooks (once) ``` -The database password and Better Auth signing secret are Docker secrets (R16), not `.env` values — create them once (see [`secrets/README.md`](secrets/README.md)): +The database password and Better Auth signing secret are Docker secrets (R16), not `.env` values — create them once, owner-only and only if they don't already exist (see [`secrets/README.md`](secrets/README.md)). Re-running the commands below is safe: an existing Postgres data volume keeps the password it was created with, so overwriting the file would just lock you out. ```bash mkdir -p secrets -openssl rand -hex 24 > secrets/postgres_password.txt -openssl rand -hex 32 > secrets/better_auth_secret.txt +[ -f secrets/postgres_password.txt ] || (umask 077 && openssl rand -hex 24 > secrets/postgres_password.txt) +[ -f secrets/better_auth_secret.txt ] || (umask 077 && openssl rand -hex 32 > secrets/better_auth_secret.txt) ``` -Then set `DATABASE_URL` and `BETTER_AUTH_SECRET` in `.env.local` so `mise` loads them into your shell — for the local Postgres below that's `postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker`. If you want a seeded admin, also fill in `ADMIN_EMAIL` / `ADMIN_PASSWORD`. Now bring up the database and start the app: +`mise` loads `DATABASE_URL` and `BETTER_AUTH_SECRET` from `.env.local` like any other variable, but it only parses `KEY=VALUE` lines — it doesn't run a shell, so a literal `$(cat ...)` typed into the file is never expanded and `DATABASE_URL` would end up containing that unevaluated text. Let your shell do the substitution once, when you write the file: + +```bash +echo "DATABASE_URL=postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker" >> .env.local +echo "BETTER_AUTH_SECRET=$(cat secrets/better_auth_secret.txt)" >> .env.local +``` + +If you want a seeded admin, also fill in `ADMIN_EMAIL` / `ADMIN_PASSWORD` in `.env.local`. Now bring up the database and start the app: ```bash docker compose up -d db # local Postgres on host port 5544 diff --git a/README.md b/README.md index f4502ca3..b4ed0050 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,11 @@ curl -o .env https://raw.githubusercontent.com/unclesp1d3r/mag_stacker/main/.env # set to the address you'll actually open it at. # 3. Create the two Docker secret files (R16) — the database password and the -# Better Auth signing secret are NOT set in .env: +# Better Auth signing secret are NOT set in .env. Restrict them to +# owner-only permissions as you create them: mkdir -p secrets -openssl rand -hex 24 > secrets/postgres_password.txt -openssl rand -hex 32 > secrets/better_auth_secret.txt +(umask 077 && openssl rand -hex 24 > secrets/postgres_password.txt) +(umask 077 && openssl rand -hex 32 > secrets/better_auth_secret.txt) # 4. Pull the published image and start the stack docker compose pull @@ -77,10 +78,11 @@ cp .env.example .env # to the address you'll actually open it at. # Create the two Docker secret files (R16) — the database password and the -# Better Auth signing secret are NOT set in .env: +# Better Auth signing secret are NOT set in .env. Restrict them to +# owner-only permissions as you create them: mkdir -p secrets -openssl rand -hex 24 > secrets/postgres_password.txt -openssl rand -hex 32 > secrets/better_auth_secret.txt +(umask 077 && openssl rand -hex 24 > secrets/postgres_password.txt) +(umask 077 && openssl rand -hex 32 > secrets/better_auth_secret.txt) docker compose up --build -d # migrates, seeds your first admin, starts the app ``` @@ -93,12 +95,14 @@ Open `http://:3000/login`, sign in, and add the rest of the account ### Backups -Everything lives in Postgres, so a normal `pg_dump` is your backup. Restoring it brings back every firearm, magazine, compatibility link, and share exactly as they were: +A `pg_dump` covers the database — every firearm, magazine, compatibility link, and share exactly as they were: ```bash docker compose exec db pg_dump -U "$POSTGRES_USER" -Fc -d "$POSTGRES_DB" > magstacker.dump ``` +**It does not include uploaded documents** (receipts, warranties, ATF forms) — those blobs live on the separate `magstacker-uploads` volume, not in Postgres, so a Postgres-only restore would come back missing every attachment. To back up both together, use the password-encrypted export on the **Admin → Backup** screen, or take a separate backup of the uploads volume alongside your `pg_dump`. + For running the Postgres and upload volumes on an encrypted host disk — and a rundown of which threats disk encryption covers versus which an encrypted in-app backup covers — see @@ -126,7 +130,7 @@ Stack: Next.js 16 (App Router), React 19, Bun, Drizzle ORM, Postgres, Better Aut ```bash mkdir -p secrets # once, if not already created -openssl rand -hex 24 > secrets/postgres_password.txt # (see secrets/README.md) +[ -f secrets/postgres_password.txt ] || (umask 077 && openssl rand -hex 24 > secrets/postgres_password.txt) # (see secrets/README.md) docker compose up -d db # local Postgres on host port 5544 export DATABASE_URL="postgres://magstacker:$(cat secrets/postgres_password.txt)@localhost:5544/magstacker" bun install diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 1019999b..165d8044 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -30,6 +30,17 @@ resolve_secret BETTER_AUTH_SECRET "${BETTER_AUTH_SECRET_FILE:-}" # that would require the plaintext password in the host's `.env`/shell # environment, defeating the point of the secret file. if [ -z "${DATABASE_URL:-}" ] && [ -n "${POSTGRES_PASSWORD:-}" ]; then + # secrets/README.md documents a hex-only password contract (openssl rand + # -hex): the password is interpolated unescaped below, so anything with + # `@ : / ? # %` etc. would be misparsed by the connection-string consumer + # and silently connect to the wrong host/db, or fail to connect at all. + # Enforce that contract instead of exporting a malformed URL. + case "${POSTGRES_PASSWORD}" in + *[!0-9A-Fa-f]*) + echo "docker-entrypoint.sh: POSTGRES_PASSWORD_FILE must contain a hex-only password (see secrets/README.md, 'openssl rand -hex'); refusing to build DATABASE_URL from a non-hex value." >&2 + exit 1 + ;; + esac export DATABASE_URL="postgres://${POSTGRES_USER:-magstacker}:${POSTGRES_PASSWORD}@${DB_HOST:-db}:${DB_PORT:-5432}/${POSTGRES_DB:-magstacker}" fi diff --git a/setup.sh b/setup.sh index f5bec1b6..c57a46b0 100755 --- a/setup.sh +++ b/setup.sh @@ -72,10 +72,11 @@ if [[ ! -f .env ]]; then echo " - BETTER_AUTH_URL (must match the origin you'll open the app at)" echo "" echo "Then create the two Docker secret files (R16) — the database password" - echo "and the Better Auth signing secret are NOT set in .env:" + echo "and the Better Auth signing secret are NOT set in .env. Restrict them to" + echo "owner-only permissions as you create them:" echo " mkdir -p secrets" - echo " openssl rand -hex 24 > secrets/postgres_password.txt" - echo " openssl rand -hex 32 > secrets/better_auth_secret.txt" + echo " (umask 077 && openssl rand -hex 24 > secrets/postgres_password.txt)" + echo " (umask 077 && openssl rand -hex 32 > secrets/better_auth_secret.txt)" echo "(hex, not base64 — the password lands unescaped in a connection URL.)" echo "" echo "Postgres only applies the password the first time its data volume is" @@ -95,16 +96,24 @@ auth_secret_file="secrets/better_auth_secret.txt" fail=0 +# Secret files inherit the caller's umask at creation time, which commonly +# leaves them 0644 (world-readable). Tighten any existing file to owner-only +# on every run rather than just checking non-emptiness — best-effort (a +# read-only bind mount or a file we don't own shouldn't block startup). if [[ ! -s "${postgres_password_file}" ]]; then echo "Error: ${postgres_password_file} is missing or empty." >&2 - echo " Create it: openssl rand -hex 24 > ${postgres_password_file}" >&2 + echo " Create it: (umask 077 && openssl rand -hex 24 > ${postgres_password_file})" >&2 fail=1 +else + chmod 600 "${postgres_password_file}" 2>/dev/null || true fi if [[ ! -s "${auth_secret_file}" ]]; then echo "Error: ${auth_secret_file} is missing or empty." >&2 - echo " Create it: openssl rand -hex 32 > ${auth_secret_file}" >&2 + echo " Create it: (umask 077 && openssl rand -hex 32 > ${auth_secret_file})" >&2 fail=1 +else + chmod 600 "${auth_secret_file}" 2>/dev/null || true fi # --- Read .env for presence/placeholder checks (values are never printed) -- From 52242da8c90c8ffc64034d7612e3d55a6ece5b2a Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:43:19 -0400 Subject: [PATCH 22/26] fix(backup): remove dead maintenance helper + correct export doc + test/migration nits (PR review) Resolves remaining PR-review findings on the backup backend/tests: - db-export.ts: correct the exportDatabase doc comment. selectAllRows buffers a whole table into memory via a single SELECT before yielding any of its rows, so it is table-by-table buffering, not row-level streaming from Postgres. Also note that export-service.ts's bufferDbExport still buffers the full concatenated NDJSON output for its own reasons (tar header byte length), which is a property of that caller, not of exportDatabase. - routes.test.ts: use `delete process.env.DATABASE_URL` instead of assigning `undefined`, which coerces to the string "undefined" and would leave a bogus DATABASE_URL for whichever test file runs next. isMaintenanceActive() (maintenance.ts) was NOT removed: it is exercised directly by maintenance.test.ts and restore-service.test.ts as a flag-state assertion helper, so it is not dead code. The underlying write-blocking concern the review raised is already handled independently by assertWritesAllowed, which is wired into authorize.ts, grants.ts, accessory-visibility.ts, restore-service.ts, and domain/accessories/service.ts, with dedicated integration coverage in write-path-maintenance-guard.test.ts. The migration/.sqlfluff finding (0018_spotty_blonde_phantom.sql) was declined: sqlfluff is not wired into this repo's pre-commit hooks or `just ci-check`, and every existing drizzle-generated migration has the same quoting/indent/line-length shape, so hand-editing this one generated file would not fix anything actually blocking CI. Signed-off-by: UncleSp1d3r --- src/backup/__tests__/routes.test.ts | 6 +++++- src/backup/db-export.ts | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/backup/__tests__/routes.test.ts b/src/backup/__tests__/routes.test.ts index afc6592c..c2cd4a06 100644 --- a/src/backup/__tests__/routes.test.ts +++ b/src/backup/__tests__/routes.test.ts @@ -223,7 +223,11 @@ describe("admin backup API routes (U6)", () => { // on its own first access) to the original target instead of this file's // now-stopped container. if (ORIGINAL_DATABASE_URL === undefined) { - process.env.DATABASE_URL = undefined; + // Assigning `undefined` would coerce to the string `"undefined"` + // (`process.env` values are always strings), leaving DATABASE_URL set + // to a bogus URL for whichever file runs next. Delete the key instead + // so its absence is genuine. + delete process.env.DATABASE_URL; } else { process.env.DATABASE_URL = ORIGINAL_DATABASE_URL; } diff --git a/src/backup/db-export.ts b/src/backup/db-export.ts index fa82dffd..b2331f8b 100644 --- a/src/backup/db-export.ts +++ b/src/backup/db-export.ts @@ -16,11 +16,19 @@ export interface ExportedRow { * (`EXPORT_TABLE_ORDER`). Ephemeral tables (session, rate-limit, idempotency) * are never touched — they simply aren't in that list. * - * Returned as a Node `Readable` so a caller can pipe it straight into an - * encryption/archive stage without buffering the whole export in memory: each - * table's rows are fetched and yielded table-by-table (not held alongside - * every other table's rows at once), and the generator only pulls the next - * table once the consumer has drained the current one. + * Returned as a Node `Readable` so a caller can pipe it into a downstream + * stage table-by-table rather than assembling the whole NDJSON payload up + * front itself: each table is fetched with a single `SELECT` — so that + * table's full row set is buffered in memory before any of its rows are + * yielded; this is whole-table buffering, NOT row-by-row/paginated streaming + * from Postgres — one table at a time, and the generator only issues the + * next table's `SELECT` once the consumer has drained the rows already + * yielded for the current one. So at most one table's rows are resident at + * once, never every table's at once. A caller can still choose to buffer the + * full concatenated output itself for its own reasons (e.g. + * `export-service.ts`'s `bufferDbExport`, which needs an exact byte length up + * front for a tar header) — that is a property of the caller, not of this + * generator. */ export function exportDatabase(db: DbOrTx): Readable { async function* generate(): AsyncGenerator { From 455c43f8d7cfede61724a10859469fb519015de9 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:46:21 -0400 Subject: [PATCH 23/26] fix(backup-ui): prevent duplicate concurrent export + restore-panel review fix (PR review) - export-panel: keep the export button disabled for the rest of the view's lifetime once submitted, not just while status === "pending". Previously the button re-enabled the moment the optimistic "started" state kicked in (ASSUME_STARTED_MS after submit), letting an admin fire a second full-instance encrypted export while the first was still streaming server-side. - restore-panel: reject restore passwords containing characters above U+00FF before ever calling fetch(), since Headers/fetch throw synchronously for header values outside the Latin-1 byte range (curly quotes, CJK, emoji, currency symbols like the euro sign). Surfaces an actionable field-level validation error instead of the previous opaque client_error fallback. Signed-off-by: UncleSp1d3r --- app/(admin)/backup/export-panel.tsx | 24 +++++++++++--- app/(admin)/backup/restore-panel.tsx | 49 ++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/app/(admin)/backup/export-panel.tsx b/app/(admin)/backup/export-panel.tsx index 18f615c0..e9be3ba9 100644 --- a/app/(admin)/backup/export-panel.tsx +++ b/app/(admin)/backup/export-panel.tsx @@ -47,9 +47,14 @@ export interface ExportGateState { * Whether the export trigger should be enabled (hardening pass, mirrors * `MIN_BACKUP_PASSWORD_LENGTH`/`readPassword` in the export route): the * password meets the minimum length, both password fields match, the - * no-recovery warning is acknowledged, and no export is already in flight. - * Exported as a pure function so the exact gating logic backing the - * rendered button's `disabled` state is unit-testable without a DOM. + * no-recovery warning is acknowledged, and no export has been submitted yet + * this session (`pending` covers the whole "pending" → "started" lifetime, + * not just the brief pre-navigation window — see `status` in `ExportPanel`). + * A genuine full-instance export streams for as long as the underlying data + * takes; there's no client-visible completion signal, so the only safe gate + * is "has this view already fired one," not a timer. Exported as a pure + * function so the exact gating logic backing the rendered button's + * `disabled` state is unit-testable without a DOM. */ export function canExportBackup(state: ExportGateState): boolean { const passwordLongEnough = @@ -73,18 +78,27 @@ export function ExportPanel() { const passwordLongEnough = password.length >= MIN_BACKUP_PASSWORD_LENGTH; const passwordsMatch = password.length > 0 && password === confirmPassword; + // Anything past "idle" means this view has already fired a submission — + // the "started" state (reached ASSUME_STARTED_MS after submit, purely to + // update the readout) must NOT re-enable the button, or an admin can fire + // a second full-instance encrypted export while the first is still + // streaming server-side (review finding: duplicate concurrent export). + const alreadySubmitted = status !== "idle"; const canExport = canExportBackup({ password, confirmPassword, acknowledged, - pending: status === "pending", + pending: alreadySubmitted, }); function onSubmit() { if (!canExport) return; setStatus("pending"); // The native form submission proceeds after this handler returns (no - // preventDefault) — this just drives the in-page status readout. + // preventDefault) — this just drives the in-page status readout. Status + // never returns to "idle" afterward, so `canExport` above stays false + // for the rest of this view's lifetime (a page reload is required to + // export again), which is the intended one-export-per-view guard. window.setTimeout(() => setStatus("started"), ASSUME_STARTED_MS); } diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx index 2588a6ae..df4bd54a 100644 --- a/app/(admin)/backup/restore-panel.tsx +++ b/app/(admin)/backup/restore-panel.tsx @@ -33,6 +33,37 @@ import { * progress, so this is a readable approximation, not a literal signal. */ const ASSUME_APPLYING_AFTER_MS = 4000; +/** Highest Unicode code point the Fetch spec allows in a header value (it + * restricts values to `ByteString` — code units U+0000 through U+00FF). */ +const MAX_HEADER_SAFE_CODE_POINT = 0xff; + +/** + * Whether `password` is safe to send as the `X-Backup-Password` request + * header (review finding: data integrity). `fetch()`/`Headers` throw + * synchronously while building the request for any header value containing + * a code point above `MAX_HEADER_SAFE_CODE_POINT` — e.g. a currency symbol + * like the euro sign, curly quotes, CJK, or emoji. `postRestore`'s + * try/catch already keeps that exception from crashing the page, but + * without this check it surfaced as an opaque browser error via the generic + * `client_error` fallback. Checked up front instead, so the operator gets + * an accurate, actionable message and the doomed request is never attempted. + * + * Exported as a pure function so the gate is unit-testable without a DOM, + * mirroring `canExportBackup` in the sibling export panel. Iterates code + * points (not UTF-16 code units) so a surrogate-pair character such as an + * emoji is correctly flagged as unsafe rather than accidentally passing on + * its individual halves. + */ +export function isRestorePasswordTransportSafe(password: string): boolean { + for (const character of password) { + const codePoint = character.codePointAt(0); + if (codePoint === undefined || codePoint > MAX_HEADER_SAFE_CODE_POINT) { + return false; + } + } + return true; +} + type Phase = "idle" | "verifying" | "applying"; /** `RestoreOutcome` plus a client-only kind for a request that never got a @@ -103,7 +134,9 @@ export function RestorePanel() { ); const restoring = phase !== "idle"; - const canSubmit = file !== null && password.length > 0 && !restoring; + const passwordTransportSafe = isRestorePasswordTransportSafe(password); + const canSubmit = + file !== null && password.length > 0 && passwordTransportSafe && !restoring; function onFileChange(event: ChangeEvent) { setFile(event.currentTarget.files?.[0] ?? null); @@ -148,10 +181,12 @@ export function RestorePanel() { function onSubmit(event: FormEvent) { event.preventDefault(); + if (!canSubmit) return; void runRestore(false); } function onConfirmForce() { + if (!passwordTransportSafe) return; void runRestore(true); } @@ -180,13 +215,23 @@ export function RestorePanel() { className="block w-full text-sm text-foreground file:mr-3 file:h-8 file:cursor-pointer file:rounded-md file:border-0 file:bg-primary file:px-3 file:text-sm file:font-medium file:text-primary-foreground file:transition-[filter] hover:file:brightness-105 disabled:cursor-not-allowed disabled:opacity-55" /> - + 0 && !passwordTransportSafe + ? "This password contains characters that can't be sent to the server (e.g. emoji, curly quotes, or non-Latin script). Use only Latin-1 characters." + : undefined + } + > 0 && !passwordTransportSafe} value={password} onChange={(event) => setPassword(event.target.value)} /> From 2dddd2b4d8491f059378abcf0bfbd34bcbb92bcc Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:55:19 -0400 Subject: [PATCH 24/26] feat(docker): add UPLOAD_DIR environment variable for uploads Signed-off-by: UncleSp1d3r --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8e6a867a..341e9e93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,7 @@ USER bun EXPOSE 3000 ENV PORT=3000 HOSTNAME=0.0.0.0 +ENV UPLOAD_DIR=/data/uploads ENTRYPOINT ["docker-entrypoint.sh"] CMD ["bun", "run", "start"] From 49ea0c7dd3f570900b1192decbf54cec2efb0566 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:56:57 -0400 Subject: [PATCH 25/26] feat(docker): make UPLOAD_DIR configurable and format healthcheck command Signed-off-by: UncleSp1d3r --- docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4d66c7ab..72c4f8a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,7 +76,11 @@ services: # with a host Postgres. The app reaches the db by service name internally. - "${POSTGRES_HOST_PORT:-5544}:5432" healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-magstacker} -d ${POSTGRES_DB:-magstacker}"] + test: + [ + "CMD-SHELL", + "pg_isready -U ${POSTGRES_USER:-magstacker} -d ${POSTGRES_DB:-magstacker}", + ] interval: 5s timeout: 5s retries: 10 @@ -171,7 +175,7 @@ services: # before `bun run start` runs. BETTER_AUTH_SECRET_FILE: /run/secrets/better_auth_secret BETTER_AUTH_URL: ${BETTER_AUTH_URL:-http://localhost:3000} - UPLOAD_DIR: /data/uploads + UPLOAD_DIR: ${UPLOAD_DIR:-/data/uploads} secrets: - postgres_password - better_auth_secret From a46580e421e89328896e1208185672a0fecc7f84 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Mon, 13 Jul 2026 19:56:12 -0400 Subject: [PATCH 26/26] fix(backup-ui): support non-Latin-1 restore passwords via percent-encoded header (PR review) Signed-off-by: UncleSp1d3r --- app/(admin)/backup/restore-panel.tsx | 63 +++++++-------------------- app/api/admin/backup/restore/route.ts | 23 ++++++++-- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/app/(admin)/backup/restore-panel.tsx b/app/(admin)/backup/restore-panel.tsx index df4bd54a..c423f2da 100644 --- a/app/(admin)/backup/restore-panel.tsx +++ b/app/(admin)/backup/restore-panel.tsx @@ -25,6 +25,16 @@ import { * force-replace intent carried in headers (`X-Backup-Password`, * `X-Backup-Force`) so the body stays exactly the encrypted bytes, matching * U6's route contract. + * + * Header values are restricted to `ByteString` (code points U+0000-U+00FF), + * so `fetch()`/`Headers` would throw synchronously for a password containing + * a currency symbol like the euro sign, curly quotes, CJK, or emoji. Rather + * than block those passwords client-side, the password is sent + * percent-encoded (`encodeURIComponent`, whose output is always pure ASCII + * and therefore always header-safe) and the route decodes it back + * (`decodeURIComponent`) before deriving the key — see `route.ts`. This + * keeps the header transport-safe while still supporting any password + * `restore()`/export can encrypt with. */ /** How long a restore runs before the progress label switches from @@ -33,37 +43,6 @@ import { * progress, so this is a readable approximation, not a literal signal. */ const ASSUME_APPLYING_AFTER_MS = 4000; -/** Highest Unicode code point the Fetch spec allows in a header value (it - * restricts values to `ByteString` — code units U+0000 through U+00FF). */ -const MAX_HEADER_SAFE_CODE_POINT = 0xff; - -/** - * Whether `password` is safe to send as the `X-Backup-Password` request - * header (review finding: data integrity). `fetch()`/`Headers` throw - * synchronously while building the request for any header value containing - * a code point above `MAX_HEADER_SAFE_CODE_POINT` — e.g. a currency symbol - * like the euro sign, curly quotes, CJK, or emoji. `postRestore`'s - * try/catch already keeps that exception from crashing the page, but - * without this check it surfaced as an opaque browser error via the generic - * `client_error` fallback. Checked up front instead, so the operator gets - * an accurate, actionable message and the doomed request is never attempted. - * - * Exported as a pure function so the gate is unit-testable without a DOM, - * mirroring `canExportBackup` in the sibling export panel. Iterates code - * points (not UTF-16 code units) so a surrogate-pair character such as an - * emoji is correctly flagged as unsafe rather than accidentally passing on - * its individual halves. - */ -export function isRestorePasswordTransportSafe(password: string): boolean { - for (const character of password) { - const codePoint = character.codePointAt(0); - if (codePoint === undefined || codePoint > MAX_HEADER_SAFE_CODE_POINT) { - return false; - } - } - return true; -} - type Phase = "idle" | "verifying" | "applying"; /** `RestoreOutcome` plus a client-only kind for a request that never got a @@ -82,7 +61,10 @@ async function postRestore( method: "POST", headers: { "Content-Type": "application/octet-stream", - "X-Backup-Password": password, + // Percent-encoded so the header stays within the `ByteString` + // (U+0000-U+00FF) range `fetch()` requires, even for a password + // containing non-Latin-1 characters — see the file doc comment. + "X-Backup-Password": encodeURIComponent(password), ...(force ? { "X-Backup-Force": "true" } : {}), }, body: file, @@ -134,9 +116,7 @@ export function RestorePanel() { ); const restoring = phase !== "idle"; - const passwordTransportSafe = isRestorePasswordTransportSafe(password); - const canSubmit = - file !== null && password.length > 0 && passwordTransportSafe && !restoring; + const canSubmit = file !== null && password.length > 0 && !restoring; function onFileChange(event: ChangeEvent) { setFile(event.currentTarget.files?.[0] ?? null); @@ -186,7 +166,6 @@ export function RestorePanel() { } function onConfirmForce() { - if (!passwordTransportSafe) return; void runRestore(true); } @@ -215,23 +194,13 @@ export function RestorePanel() { className="block w-full text-sm text-foreground file:mr-3 file:h-8 file:cursor-pointer file:rounded-md file:border-0 file:bg-primary file:px-3 file:text-sm file:font-medium file:text-primary-foreground file:transition-[filter] hover:file:brightness-105 disabled:cursor-not-allowed disabled:opacity-55" /> - 0 && !passwordTransportSafe - ? "This password contains characters that can't be sent to the server (e.g. emoji, curly quotes, or non-Latin script). Use only Latin-1 characters." - : undefined - } - > + 0 && !passwordTransportSafe} value={password} onChange={(event) => setPassword(event.target.value)} /> diff --git a/app/api/admin/backup/restore/route.ts b/app/api/admin/backup/restore/route.ts index 3a0adb8b..676bc57e 100644 --- a/app/api/admin/backup/restore/route.ts +++ b/app/api/admin/backup/restore/route.ts @@ -16,7 +16,13 @@ import { sameOriginError } from "@/src/backup/same-origin"; * * Request contract (documented, deliberately simple — not multipart, so no * streaming multipart parser is needed to keep the body unbuffered): - * - `X-Backup-Password` header — required, the bundle's password. + * - `X-Backup-Password` header — required, the bundle's password, + * percent-encoded with `encodeURIComponent` by the client (`restore-panel.tsx`) + * so a password containing non-Latin-1 characters (e.g. a currency symbol, + * curly quotes, CJK, or emoji) still fits the `ByteString` range `fetch()` + * requires for header values. Decoded back with `decodeURIComponent` below + * before being handed to `restore()`, recovering the exact raw password + * `export` encrypted with. * - `X-Backup-Force` header — optional, `"true"` to force-replace a * non-empty instance (R7); anything else (including absent) means the * safe refuse-unless-empty default (R6). @@ -53,13 +59,24 @@ export async function POST(request: Request): Promise { const originError = sameOriginError(request); if (originError) return originError; - const password = request.headers.get("x-backup-password"); - if (!password) { + const rawPasswordHeader = request.headers.get("x-backup-password"); + if (!rawPasswordHeader) { return Response.json( { outcome: "bad_request", message: "a password is required" }, { status: 400 }, ); } + let password: string; + try { + password = decodeURIComponent(rawPasswordHeader); + } catch { + // decodeURIComponent throws a URIError on malformed percent-encoding + // (e.g. a lone "%"); a crafted header must not crash the route. + return Response.json( + { outcome: "bad_request", message: "the password header is malformed" }, + { status: 400 }, + ); + } if (!request.body) { return Response.json( { outcome: "bad_request", message: "a backup bundle body is required" },