diff --git a/.gitignore b/.gitignore index fafd6e65dd..9413d7c901 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ node_modules/ tickets.db *.db +*.sqlite +*.sqlite-shm +*.sqlite-wal .db-key .env .db-key diff --git a/AGENTS.md b/AGENTS.md index 1f81ed7b7d..e8c7b56315 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -683,6 +683,7 @@ logging and table-scoped cache invalidation stay automatic. - `deno task lint:ci` - Strict, read-only lint (`check --error-on-warnings`, no `--write`). Fails on lint warnings (e.g. cognitive complexity) and on any code that *would* be reformatted, without touching the checkout. This is the lint `deno task precommit` runs in **every** environment, so a clean `precommit` locally means the lint step will pass in CI too. Run `deno task lint` to auto-fix before re-running. - `deno task build:edge` - Build for Bunny Edge deployment - `deno task backup` - Dump the database out-of-band to a `.zip`. Uploads to the configured storage zone by default (so it appears on the Backups page and lets the next migration skip its own inline backup); pass `--out ` to write a local file. Runs in a full Deno process, so unlike the in-edge backup it has no per-request subrequest budget and can dump arbitrarily large databases. +- `deno task snapshot --out ` - Sync the complete remote database to a standalone local SQLite file. The task prefers `DB_URL` and `DB_TOKEN` from `.env` over shell values. This developer-only task checkpoints and verifies the file, refuses to overwrite an existing path, and removes its temporary replica on success or failure. - `deno task precommit` - Run all checks (typecheck, lint, tests) - `deno task precommit:mutation` - The precommit mutation gate, runnable on its own: mutation-test every `src/` file this branch changed and demand a 100% kill rate. A source's mirror-located direct tests run first; changed tests under `test/integration/` or `test/e2e/` run only for direct-test survivors. Any other unmatched test is an error: move it to the path that mirrors its source while strengthening it to kill the mutants. The changed set is the branch's committed diff against the integration branch (`origin/main`, else a local `main`) via `base...HEAD` — three-dot/merge-base, so it's the branch's full diff vs main and stays bounded to the branch's own commits (precommit runs post-commit on a clean tree, so the index is empty). Skips cheaply when there is no base ref or no changed `src/` files (and likewise when src changed without any changed test). If a badly stale local `origin/main` balloons the changed set past `STALE_BASE_SOURCE_LIMIT`, it skips with a "run `git fetch origin main`" hint instead of mutating most of the tree. See [Mutation Testing](#mutation-testing). - `deno task mutation ` - Mutation-test your tests on demand in an isolated `.mutation-runs//work` copy: mutate operators in the source and check your tests catch it (see [Mutation Testing](#mutation-testing)) diff --git a/README.md b/README.md index d916fc20d0..5da3fae1ca 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,7 @@ deno task typecheck # Type check deno task build:edge # Build for Bunny Edge deno task deploy:edge # Build, upload, and publish to Bunny Edge using BUNNY_ACCESS_KEY from .env deno task backup # Dump the database out-of-band (uploads to storage; --out for a local .zip) +deno task snapshot --out # Copy the complete remote database to a local SQLite file deno task precommit # All checks (typecheck, lint, cpd, build:edge, test:coverage) ``` @@ -360,6 +361,15 @@ Optional: **Backups:** every table is dumped to a single `.zip`, with table reads keyset-paginated so no single response trips libsqld's "Response is too large" payload cap (the server limit behind Bunny's databases). Backups run **out-of-band**, not inside the migration: a full dump of a ~31-table schema can't fit alongside a migration within one edge request's [50-subrequest budget](https://docs.bunny.net/scripting/limits), so migrations just migrate, and a backup is taken by GitHub Actions (or `deno task backup`) beforehand. To enforce that, **`/admin/update` and the per-site update button refuse to deploy unless a backup of that database was taken in the last hour.** +**Local SQLite snapshots:** developers can copy a remote database into one complete SQLite file using the same `DB_URL` and `DB_TOKEN` as the app: + +```bash +mkdir -p .local-data +deno task snapshot --out .local-data/site.sqlite +``` + +The task reads `DB_URL` and `DB_TOKEN` from `.env`. Values in that file take priority over existing shell variables. It then uses libSQL page-level sync, moves all WAL data into the main file, and checks the finished database before publishing it. It refuses to replace an existing file. The temporary replica is created beside the output and removed on success or failure. Local `.sqlite` files and `.local-data/` are ignored by Git. Run this only on a development machine; it needs a local filesystem and the native libSQL client. The file contains the database exactly as stored, including encrypted personal data, but it does not include environment secrets or files held in Bunny Storage. + The deploy workflows back a site up (via `POST /instance/site-credentials`) before deploying to it (the staging push-to-`main` trigger is the one exception — see below): - **`.github/workflows/backup.yml`** (manual) — backs up the main instance's own database with `DB_URL` / `DB_TOKEN` / `STORAGE_ZONE_NAME` / `STORAGE_ZONE_KEY` repository secrets. diff --git a/deno.json b/deno.json index ffb74353b9..c80e5c108d 100644 --- a/deno.json +++ b/deno.json @@ -16,7 +16,7 @@ "test:raw": "deno task build:static && deno test --no-check --allow-net --allow-env --allow-read --allow-write --allow-run --allow-sys --allow-ffi test/", "lint": "deno run -A scripts/biome.ts check --write --error-on-warnings src test scripts cli e2e-payments", "lint:ci": "deno run -A scripts/biome.ts check --error-on-warnings src test scripts cli e2e-payments", - "typecheck": "deno check src/**/*.ts src/**/*.tsx test/**/*.ts cli/**/*.ts scripts/mutation.ts scripts/mutation/**/*.ts scripts/precommit.ts scripts/precommit/**/*.ts scripts/check-copy.ts scripts/check-copy/**/*.ts scripts/bench/bundle-composition/**/*.ts scripts/screenshots.ts scripts/screenshots/**/*.ts && deno check --config=e2e-payments/deno.json e2e-payments/src/**/*.ts", + "typecheck": "deno check src/**/*.ts src/**/*.tsx test/**/*.ts cli/**/*.ts scripts/database-snapshot.ts scripts/mutation.ts scripts/mutation/**/*.ts scripts/precommit.ts scripts/precommit/**/*.ts scripts/check-copy.ts scripts/check-copy/**/*.ts scripts/bench/bundle-composition/**/*.ts scripts/screenshots.ts scripts/screenshots/**/*.ts && deno check --config=e2e-payments/deno.json e2e-payments/src/**/*.ts", "cpd": "deno run -A scripts/cpd.ts --config .jscpd.json && deno run -A scripts/cpd.ts --config .jscpd.test.json && deno run -A scripts/cpd.ts --config .jscpd.css.json src/ui/static/style.scss", "check:copy": "deno run --allow-read=src/locales scripts/check-copy.ts", "precommit": "deno run -A scripts/precommit.ts", @@ -27,6 +27,7 @@ "unit-tests-report": "deno run --allow-read=src,test,deno.json scripts/unit-tests-report.ts", "upgrade": "deno run --allow-read --allow-write --allow-net scripts/safe-upgrade.ts", "backup": "deno run --env-file --allow-env --allow-read --allow-write --allow-net --allow-sys --allow-ffi scripts/backup.ts", + "snapshot": "deno run --allow-env --allow-read --allow-write --allow-net --allow-sys --allow-ffi scripts/database-snapshot.ts", "cli:tui": "deno run --allow-env --allow-read --allow-run cli/tui.ts", "cli:api": "deno run --allow-env --allow-read --allow-run cli/api.ts", "test:quality-audit": "deno run --allow-read scripts/test-quality-audit.ts", @@ -65,6 +66,7 @@ "fast-check": "npm:fast-check@^3.23.0", "intl-messageformat": "npm:intl-messageformat@^10", "@luca/esbuild-deno-loader": "jsr:@luca/esbuild-deno-loader@^0.11.1", + "@std/dotenv": "jsr:@std/dotenv@^0.225.7", "@std/http": "jsr:@std/http@^1.1.1", "@sentry/core": "npm:@sentry/core@^10.60.0", "@sentry/deno": "npm:@sentry/deno@^10.60.0", diff --git a/deno.lock b/deno.lock index 88d4292fce..10a7e6297e 100644 --- a/deno.lock +++ b/deno.lock @@ -12,6 +12,7 @@ "jsr:@std/collections@^1.1.3": "1.2.0", "jsr:@std/collections@^1.2.0": "1.2.0", "jsr:@std/data-structures@^1.0.10": "1.0.10", + "jsr:@std/dotenv@~0.225.7": "0.225.7", "jsr:@std/encoding@1": "1.0.10", "jsr:@std/encoding@^1.0.10": "1.0.10", "jsr:@std/encoding@^1.0.5": "1.0.10", @@ -22,7 +23,7 @@ "jsr:@std/fs@^1.0.24": "1.0.24", "jsr:@std/html@^1.0.7": "1.0.7", "jsr:@std/http@^1.1.1": "1.1.1", - "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/internal@^1.0.12": "1.0.14", "jsr:@std/internal@^1.0.14": "1.0.14", "jsr:@std/media-types@^1.1.0": "1.1.0", "jsr:@std/net@^1.0.6": "1.0.6", @@ -134,6 +135,9 @@ "@std/data-structures@1.0.10": { "integrity": "f574f86b0e07c69b9edc555fcc814b57d29258bad39fd5a34ba8a80ecf033cfe" }, + "@std/dotenv@0.225.7": { + "integrity": "11d8db03ca4ad5aba9eba809f2e8058b2a4f320b7b09fea4b360e162928329e3" + }, "@std/encoding@1.0.10": { "integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1" }, @@ -1905,6 +1909,7 @@ "jsr:@std/assert@^1.0.19", "jsr:@std/cli@^1.0.30", "jsr:@std/collections@^1.2.0", + "jsr:@std/dotenv@~0.225.7", "jsr:@std/expect@^1.0.18", "jsr:@std/http@^1.1.1", "jsr:@std/path@^1.1.4", diff --git a/scripts/database-snapshot-lib.ts b/scripts/database-snapshot-lib.ts new file mode 100644 index 0000000000..3bdba13f98 --- /dev/null +++ b/scripts/database-snapshot-lib.ts @@ -0,0 +1,319 @@ +import type { Config } from "@libsql/client"; +import { parseArgs } from "@std/cli/parse-args"; +import { load } from "@std/dotenv"; +import { basename, dirname, join, resolve, toFileUrl } from "@std/path"; +import * as v from "valibot"; +import { withCleanup } from "#scripts/cleanup.ts"; +import { getEnv } from "#shared/env.ts"; + +export const SNAPSHOT_USAGE = "Usage: deno task snapshot --out "; + +export interface SnapshotOptions { + outputPath: string; +} + +export interface SnapshotRequest extends SnapshotOptions { + dbToken: string; + dbUrl: string; +} + +export interface SnapshotQueryResult { + rows: readonly Readonly>[]; +} + +export interface SnapshotClient { + close(): void; + execute(sql: string): Promise; + sync(): Promise; +} + +export type SnapshotClientFactory = (config: Config) => SnapshotClient; +export type SnapshotEnvReader = (key: string) => string | undefined; +export type SnapshotEnvFileLoader = ( + path: string, +) => Promise>; + +export const SNAPSHOT_PROGRESS = { + checking: "[1/4] Checking destination", + publishing: "[4/4] Publishing standalone SQLite file", + syncing: "[2/4] Syncing remote database", + verifying: "[3/4] Checkpointing and checking integrity", +} as const; + +export type SnapshotProgress = + (typeof SNAPSHOT_PROGRESS)[keyof typeof SNAPSHOT_PROGRESS]; +export type SnapshotProgressWriter = (message: SnapshotProgress) => void; + +const ignoreSnapshotProgress: SnapshotProgressWriter = () => {}; + +const OutputValuesSchema = v.strictTuple([ + v.pipe( + v.string(), + v.check((path) => path.trim().length > 0), + ), +]); + +const EmptyStringsSchema = v.strictTuple([]); + +const snapshotArgsSchema = < + Help extends boolean, + OutSchema extends v.GenericSchema, +>( + help: Help, + out: OutSchema, +) => + v.strictObject({ + _: EmptyStringsSchema, + h: v.literal(help), + help: v.literal(help), + out, + }); + +const SnapshotArgsSchema = snapshotArgsSchema(false, OutputValuesSchema); +const SnapshotHelpArgsSchema = snapshotArgsSchema(true, EmptyStringsSchema); + +const CheckpointRowsSchema = v.pipe( + v.array( + v.strictObject({ + busy: v.literal(0), + checkpointed: v.literal(0), + log: v.literal(0), + }), + ), + v.length(1), +); + +const IntegrityRowsSchema = v.pipe( + v.array(v.strictObject({ integrity_check: v.literal("ok") })), + v.length(1), +); + +const SECURE_DATABASE_PROTOCOLS = new Set(["https:", "libsql:"]); +const HTTP_LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]); + +const invalidUsage = (): never => { + throw new Error(SNAPSHOT_USAGE); +}; + +export const parseSnapshotArgs = (args: string[]): SnapshotOptions | null => { + const parsed = parseArgs(args, { + alias: { h: "help" }, + boolean: ["help"], + collect: ["out"], + string: ["out"], + }); + if (parsed.help) { + if (!v.safeParse(SnapshotHelpArgsSchema, parsed).success) invalidUsage(); + return null; + } + const result = v.safeParse(SnapshotArgsSchema, parsed); + if (!result.success) return invalidUsage(); + return { outputPath: result.output.out[0] }; +}; + +const requiredEnv = (key: string, readEnv: SnapshotEnvReader): string => { + const value = readEnv(key); + if (!value || value.trim().length === 0) { + throw new Error(`${key} environment variable is required`); + } + return value; +}; + +const requireRemoteDatabaseUrl = (value: string): string => { + try { + const url = new URL(value); + const secure = SECURE_DATABASE_PROTOCOLS.has(url.protocol); + const loopback = HTTP_LOOPBACK_HOSTNAMES.has(url.hostname); + const plaintext = + url.protocol === "http:" || + (url.protocol === "libsql:" && + url.searchParams.getAll("tls").at(-1) === "0"); + if ( + url.hostname && + (secure || url.protocol === "http:") && + (!plaintext || loopback) + ) { + return value; + } + } catch { + // Invalid URLs share the same clear configuration error as local URLs. + } + throw new Error( + "DB_URL must use TLS. Plain connections are allowed only for loopback.", + ); +}; + +export const readSnapshotRequest = ( + options: SnapshotOptions, + readEnv: SnapshotEnvReader = getEnv, +): SnapshotRequest => ({ + dbToken: requiredEnv("DB_TOKEN", readEnv), + dbUrl: requireRemoteDatabaseUrl(requiredEnv("DB_URL", readEnv)), + outputPath: options.outputPath, +}); + +const loadSnapshotEnvFile: SnapshotEnvFileLoader = (path) => + load({ envPath: path }); + +export const readSnapshotRequestFromEnvFile = async ( + options: SnapshotOptions, + path = ".env", + readEnv: SnapshotEnvReader = getEnv, + loadFile: SnapshotEnvFileLoader = loadSnapshotEnvFile, +): Promise => { + const fileEnv = await loadFile(path); + return readSnapshotRequest(options, (key) => fileEnv[key] ?? readEnv(key)); +}; + +const outputAlreadyExists = (path: string): Error => + new Error(`Output already exists: ${path}`); + +const readOrNullIfMissing = async ( + read: () => Promise, +): Promise => { + try { + return await read(); + } catch (error) { + if (error instanceof Deno.errors.NotFound) return null; + throw error; + } +}; + +const fileInfoOrNull = + (getRead: () => (path: string) => Promise) => + (path: string): Promise => + readOrNullIfMissing(() => getRead()(path)); + +const statOrNull = fileInfoOrNull(() => Deno.stat); +const lstatOrNull = fileInfoOrNull(() => Deno.lstat); + +const requireOutputDirectory = async (path: string): Promise => { + const info = await statOrNull(path); + if (info === null) + throw new Error(`Output directory does not exist: ${path}`); + if (!info.isDirectory) + throw new Error(`Output directory is not a directory: ${path}`); +}; + +const outputFiles = (path: string): string[] => [ + path, + `${path}-wal`, + `${path}-shm`, +]; + +const requireMissingOutputFiles = async (path: string): Promise => { + const existingPath = ( + await Promise.all( + outputFiles(path).map(async (filePath) => + (await lstatOrNull(filePath)) === null ? null : filePath, + ), + ) + ).find((filePath) => filePath !== null); + if (existingPath !== undefined) throw outputAlreadyExists(existingPath); +}; + +const withSnapshotClient = ( + factory: SnapshotClientFactory, + config: Config, + run: (client: SnapshotClient) => Promise, +): Promise => { + const client = factory(config); + return withCleanup(() => run(client), [() => client.close()]); +}; + +const syncReplica = ( + path: string, + request: SnapshotRequest, + factory: SnapshotClientFactory, +): Promise => + withSnapshotClient( + factory, + { + authToken: request.dbToken, + syncUrl: request.dbUrl, + url: toFileUrl(path).href, + }, + (client) => client.sync(), + ); + +const verifyRows = ( + schema: v.GenericSchema, + result: SnapshotQueryResult, + message: string, +): void => { + if (!v.safeParse(schema, result.rows).success) throw new Error(message); +}; + +const checkpointAndVerify = ( + path: string, + factory: SnapshotClientFactory, +): Promise => + withSnapshotClient(factory, { url: toFileUrl(path).href }, async (client) => { + verifyRows( + CheckpointRowsSchema, + await client.execute("PRAGMA wal_checkpoint(TRUNCATE)"), + "Database snapshot checkpoint did not empty the WAL", + ); + verifyRows( + IntegrityRowsSchema, + await client.execute("PRAGMA integrity_check"), + "Database snapshot integrity check failed", + ); + }); + +const fileSizeOrNull = async (path: string): Promise => { + const info = await statOrNull(path); + // SQLite can either remove an empty WAL or leave a zero-byte file. + return info?.size ?? null; +}; + +const requireEmptyWal = async (path: string): Promise => { + const size = await fileSizeOrNull(`${path}-wal`); + if (size !== null && size !== 0) { + throw new Error("Database snapshot WAL is not empty"); + } +}; + +const publishSnapshot = async ( + temporaryPath: string, + outputPath: string, +): Promise => { + await Deno.chmod(temporaryPath, 0o600); + await requireMissingOutputFiles(outputPath); + try { + await Deno.link(temporaryPath, outputPath); + } catch (error) { + if (error instanceof Deno.errors.AlreadyExists) { + throw outputAlreadyExists(outputPath); + } + throw error; + } +}; + +export const createDatabaseSnapshot = async ( + request: SnapshotRequest, + factory: SnapshotClientFactory, + writeProgress: SnapshotProgressWriter = ignoreSnapshotProgress, +): Promise => { + const outputPath = resolve(request.outputPath); + const outputDirectory = dirname(outputPath); + writeProgress(SNAPSHOT_PROGRESS.checking); + await requireOutputDirectory(outputDirectory); + await requireMissingOutputFiles(outputPath); + const temporaryDirectory = await Deno.makeTempDir({ + dir: outputDirectory, + prefix: `.${basename(outputPath)}-snapshot-`, + }); + + return await withCleanup(async () => { + const temporaryPath = join(temporaryDirectory, "snapshot.sqlite"); + writeProgress(SNAPSHOT_PROGRESS.syncing); + await syncReplica(temporaryPath, request, factory); + writeProgress(SNAPSHOT_PROGRESS.verifying); + await checkpointAndVerify(temporaryPath, factory); + await requireEmptyWal(temporaryPath); + writeProgress(SNAPSHOT_PROGRESS.publishing); + await publishSnapshot(temporaryPath, outputPath); + return outputPath; + }, [() => Deno.remove(temporaryDirectory, { recursive: true })]); +}; diff --git a/scripts/database-snapshot-output.ts b/scripts/database-snapshot-output.ts new file mode 100644 index 0000000000..b1821e4fa2 --- /dev/null +++ b/scripts/database-snapshot-output.ts @@ -0,0 +1,78 @@ +import type { SnapshotProgress } from "#scripts/database-snapshot-lib.ts"; +import { formatMs } from "#shared/limits.ts"; + +const CLEAR_LINE = "\r\x1b[2K"; +const UPDATE_INTERVAL_MS = 1_000; + +export interface SnapshotOutputOptions { + terminal: boolean; + write: (text: string) => void; +} + +export interface SnapshotProgressOutput { + report(message: SnapshotProgress): void; + stop(): void; +} + +interface RenderedProgress { + message: SnapshotProgress; + renderedLine: string; + startedAt: number; +} + +interface ActiveProgress { + progress: RenderedProgress; + timer: number; +} + +export const createSnapshotProgressOutput = ({ + terminal, + write, +}: SnapshotOutputOptions): SnapshotProgressOutput => { + if (!terminal) { + return { + report: (message) => write(`${message}\n`), + stop: () => {}, + }; + } + + let active: ActiveProgress | null = null; + + const render = (progress: RenderedProgress): void => { + const elapsed = Date.now() - progress.startedAt; + const line = + elapsed < UPDATE_INTERVAL_MS + ? progress.message + : `${progress.message} (${formatMs(elapsed)})`; + if (line === progress.renderedLine) return; + progress.renderedLine = line; + write(`${CLEAR_LINE}${line}`); + }; + + const finish = (progress: ActiveProgress): void => { + clearInterval(progress.timer); + render(progress.progress); + write("\n"); + }; + + return { + report: (message) => { + if (active !== null) finish(active); + const progress = { + message, + renderedLine: "", + startedAt: Date.now(), + }; + render(progress); + active = { + progress, + timer: setInterval(() => render(progress), UPDATE_INTERVAL_MS), + }; + }, + stop: () => { + if (active === null) return; + finish(active); + active = null; + }, + }; +}; diff --git a/scripts/database-snapshot.ts b/scripts/database-snapshot.ts new file mode 100644 index 0000000000..638b994a62 --- /dev/null +++ b/scripts/database-snapshot.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env -S deno run --allow-env --allow-read --allow-write --allow-net --allow-sys --allow-ffi + +import { createClient } from "@libsql/client"; +import { resolve } from "@std/path"; +import { + createDatabaseSnapshot, + parseSnapshotArgs, + readSnapshotRequestFromEnvFile, + SNAPSHOT_USAGE, +} from "#scripts/database-snapshot-lib.ts"; +import { createSnapshotProgressOutput } from "#scripts/database-snapshot-output.ts"; +import { formatBytes, formatMs } from "#shared/limits.ts"; + +const encoder = new TextEncoder(); + +const options = parseSnapshotArgs(Deno.args); +if (options === null) { + console.log(SNAPSHOT_USAGE); +} else { + const startedAt = performance.now(); + const request = await readSnapshotRequestFromEnvFile(options); + console.log( + `Database snapshot\n Source: ${new URL(request.dbUrl).host}\n Output: ${resolve(request.outputPath)}\n`, + ); + const progress = createSnapshotProgressOutput({ + terminal: Deno.stdout.isTerminal(), + write: (text) => { + Deno.stdout.writeSync(encoder.encode(text)); + }, + }); + let outputPath: string; + try { + outputPath = await createDatabaseSnapshot( + request, + createClient, + progress.report, + ); + } finally { + progress.stop(); + } + const size = (await Deno.stat(outputPath)).size; + console.log( + `\nSnapshot ready\n File: ${outputPath}\n Size: ${formatBytes(size)}\n Time: ${formatMs(Math.round(performance.now() - startedAt))}`, + ); +} diff --git a/test/lib/test-utils/files.test.ts b/test/lib/test-utils/files.test.ts index 7c076b5277..f027017b09 100644 --- a/test/lib/test-utils/files.test.ts +++ b/test/lib/test-utils/files.test.ts @@ -1,6 +1,7 @@ import { expect } from "@std/expect"; import { describe, it as test } from "@std/testing/bdd"; import { + directoryNames, pathExists, tempDir, withTempDir, @@ -8,6 +9,14 @@ import { } from "#test-utils/files.ts"; describe("temporary paths", () => { + test("lists directory entry names in order", () => + withTempDir(async (path) => { + await Deno.writeTextFile(`${path}/second.txt`, "second"); + await Deno.mkdir(`${path}/first`); + + expect(await directoryNames(path)).toEqual(["first", "second.txt"]); + })); + test("removes a directory and everything inside it on disposal", async () => { const dir = tempDir({ prefix: "tickets-files-test-" }); const path = dir.path; diff --git a/test/scripts/database-snapshot-options.test.ts b/test/scripts/database-snapshot-options.test.ts new file mode 100644 index 0000000000..93db8f47aa --- /dev/null +++ b/test/scripts/database-snapshot-options.test.ts @@ -0,0 +1,160 @@ +import { expect } from "@std/expect"; +import { join } from "@std/path"; +import { describe, it as test } from "@std/testing/bdd"; +import { + parseSnapshotArgs, + readSnapshotRequest, + readSnapshotRequestFromEnvFile, + SNAPSHOT_USAGE, +} from "#scripts/database-snapshot-lib.ts"; +import { withEnv } from "#test-utils/env.ts"; +import { withTempDir } from "#test-utils/files.ts"; + +const envReader = + (values: Record) => + (key: string): string | undefined => + values[key]; + +describe("database snapshot options", () => { + test("accepts one output path", () => { + expect(parseSnapshotArgs(["--out", "backups/site.sqlite"])).toEqual({ + outputPath: "backups/site.sqlite", + }); + }); + + test("returns help without an output path", () => { + expect(parseSnapshotArgs(["--help"])).toBeNull(); + expect(parseSnapshotArgs(["-h"])).toBeNull(); + }); + + test("rejects missing, blank, duplicate, and extra output arguments", () => { + const invalidArguments = [ + [], + ["--out"], + ["--out", " "], + ["--out", "one.sqlite", "--out", "two.sqlite"], + ["--out", "site.sqlite", "extra"], + ["--out", "site.sqlite", "--unknown"], + ["--help", "--out", "site.sqlite"], + ]; + + for (const args of invalidArguments) { + expect(() => parseSnapshotArgs(args)).toThrow(SNAPSHOT_USAGE); + } + }); + + test("reads remote database credentials", () => { + using _env = withEnv({ + DB_TOKEN: "secret-token", + DB_URL: "libsql://site.example.com", + }); + expect(readSnapshotRequest({ outputPath: "site.sqlite" })).toEqual({ + dbToken: "secret-token", + dbUrl: "libsql://site.example.com", + outputPath: "site.sqlite", + }); + }); + + test("prefers database credentials from .env over shell values", () => + withTempDir(async (dir) => { + const envPath = join(dir, ".env"); + await Deno.writeTextFile( + envPath, + 'DB_URL="libsql://file.example.com"\nDB_TOKEN="file-token"\n', + ); + + await expect( + readSnapshotRequestFromEnvFile( + { outputPath: "site.sqlite" }, + envPath, + envReader({ DB_TOKEN: "shell-token", DB_URL: ":memory:" }), + ), + ).resolves.toEqual({ + dbToken: "file-token", + dbUrl: "libsql://file.example.com", + outputPath: "site.sqlite", + }); + })); + + test("uses shell credentials when .env is absent", () => + withTempDir( + async (dir) => + await expect( + readSnapshotRequestFromEnvFile( + { outputPath: "site.sqlite" }, + join(dir, ".env"), + envReader({ + DB_TOKEN: "shell-token", + DB_URL: "libsql://shell.example.com", + }), + ), + ).resolves.toEqual({ + dbToken: "shell-token", + dbUrl: "libsql://shell.example.com", + outputPath: "site.sqlite", + }), + )); + + test("rejects missing database credentials", () => { + expect(() => + readSnapshotRequest( + { outputPath: "site.sqlite" }, + envReader({ DB_TOKEN: "token" }), + ), + ).toThrow("DB_URL environment variable is required"); + expect(() => + readSnapshotRequest( + { outputPath: "site.sqlite" }, + envReader({ DB_URL: "libsql://site.example.com" }), + ), + ).toThrow("DB_TOKEN environment variable is required"); + expect(() => + readSnapshotRequest( + { outputPath: "site.sqlite" }, + envReader({ DB_TOKEN: " ", DB_URL: "libsql://site.example.com" }), + ), + ).toThrow("DB_TOKEN environment variable is required"); + }); + + test("rejects local, insecure, and malformed database URLs", () => { + for (const dbUrl of [ + ":memory:", + "file:site.sqlite", + "http://site.example.com", + "http://localhost.example.com", + "libsql://site.example.com:8080?tls=0", + "libsql://site.example.com:8080?tls=1&tls=0", + "not a URL", + ]) { + expect(() => + readSnapshotRequest( + { outputPath: "site.sqlite" }, + envReader({ DB_TOKEN: "token", DB_URL: dbUrl }), + ), + ).toThrow( + "DB_URL must use TLS. Plain connections are allowed only for loopback.", + ); + } + }); + + test("accepts each supported remote database protocol", () => { + for (const dbUrl of [ + "libsql://site.example.com", + "libsql://site.example.com:8080?tls=0&tls=1", + "https://site.example.com", + "http://localhost:8080", + "http://127.0.0.1:8080", + "http://[::1]:8080", + "libsql://localhost:8080?tls=0", + "libsql://127.0.0.1:8080?tls=0", + "libsql://[::1]:8080?tls=0", + ]) { + expect( + readSnapshotRequest( + { outputPath: "site.sqlite" }, + envReader({ DB_TOKEN: "token", DB_URL: dbUrl }), + ).dbUrl, + ).toBe(dbUrl); + } + }); +}); diff --git a/test/scripts/database-snapshot-output.test.ts b/test/scripts/database-snapshot-output.test.ts new file mode 100644 index 0000000000..2340965cbf --- /dev/null +++ b/test/scripts/database-snapshot-output.test.ts @@ -0,0 +1,57 @@ +import { expect } from "@std/expect"; +import { describe, it as test } from "@std/testing/bdd"; +import { FakeTime } from "@std/testing/time"; +import { SNAPSHOT_PROGRESS } from "#scripts/database-snapshot-lib.ts"; +import { createSnapshotProgressOutput } from "#scripts/database-snapshot-output.ts"; + +describe("database snapshot progress output", () => { + test("writes stable progress lines when output is redirected", () => { + const output: string[] = []; + const progress = createSnapshotProgressOutput({ + terminal: false, + write: (text) => output.push(text), + }); + + progress.report(SNAPSHOT_PROGRESS.checking); + progress.report(SNAPSHOT_PROGRESS.syncing); + progress.stop(); + + expect(output.join("")).toBe( + "[1/4] Checking destination\n[2/4] Syncing remote database\n", + ); + }); + + test("refreshes elapsed time on an active terminal line", async () => { + using time = new FakeTime(0); + const output: string[] = []; + const progress = createSnapshotProgressOutput({ + terminal: true, + write: (text) => output.push(text), + }); + + progress.report(SNAPSHOT_PROGRESS.checking); + await time.tickAsync(1_001); + progress.report(SNAPSHOT_PROGRESS.syncing); + await time.tickAsync(2_100); + progress.stop(); + + expect(output.join("")).toBe( + "\r\x1b[2K[1/4] Checking destination" + + "\r\x1b[2K[1/4] Checking destination (1s)\n" + + "\r\x1b[2K[2/4] Syncing remote database" + + "\r\x1b[2K[2/4] Syncing remote database (1s)" + + "\r\x1b[2K[2/4] Syncing remote database (2s)\n", + ); + }); + + test("can stop before progress starts", () => { + const progress = createSnapshotProgressOutput({ + terminal: true, + write: () => { + throw new Error("write should not be called"); + }, + }); + + progress.stop(); + }); +}); diff --git a/test/scripts/database-snapshot-safety.test.ts b/test/scripts/database-snapshot-safety.test.ts new file mode 100644 index 0000000000..7120bd8da1 --- /dev/null +++ b/test/scripts/database-snapshot-safety.test.ts @@ -0,0 +1,336 @@ +import type { Config } from "@libsql/client"; +import { expect } from "@std/expect"; +import { fromFileUrl, join } from "@std/path"; +import { describe, it as test } from "@std/testing/bdd"; +import { stub } from "@std/testing/mock"; +import { + createDatabaseSnapshot, + type SnapshotClient, + type SnapshotClientFactory, + type SnapshotRequest, +} from "#scripts/database-snapshot-lib.ts"; +import { directoryNames, pathExists, withTempDir } from "#test-utils/files.ts"; + +const request = (outputPath: string): SnapshotRequest => ({ + dbToken: "token", + dbUrl: "libsql://database.example.com", + outputPath, +}); + +const unusedExecute = (): Promise => + Promise.reject(new Error("execute should not be called")); + +const unusedSync = (): Promise => + Promise.reject(new Error("sync should not be called")); + +const replicaClient = ( + sync: () => Promise, + close: () => void = () => {}, +): SnapshotClient => ({ close, execute: unusedExecute, sync }); + +const localClient = ( + execute: SnapshotClient["execute"], + close: () => void = () => {}, +): SnapshotClient => ({ close, execute, sync: unusedSync }); + +const rows = (values: Record[]) => + Promise.resolve({ rows: values }); + +const checkpointOk = { busy: 0, checkpointed: 0, log: 0 }; +const integrityOk = { integrity_check: "ok" }; + +const fakeDatabaseFactory = ( + onSync: (path: string) => void | Promise, + localRows: Record[][], + onLocalClose: (path: string) => void = () => {}, +): SnapshotClientFactory => { + let localCall = 0; + return (config: Config): SnapshotClient => { + const path = fromFileUrl(config.url); + if (config.syncUrl) { + return replicaClient(async () => { + await onSync(path); + return {}; + }); + } + return localClient( + () => rows(localRows[localCall++] ?? []), + () => onLocalClose(path), + ); + }; +}; + +const expectExistingFilePreserved = async ( + outputPath: string, + existingPath: string, + content: string, + factory: SnapshotClientFactory, +): Promise => { + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toThrow(`Output already exists: ${existingPath}`); + expect(await Deno.readTextFile(existingPath)).toBe(content); +}; + +describe("database snapshot safety", () => { + test("refuses an existing output before syncing", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + await Deno.writeTextFile(outputPath, "keep me"); + let clientCreated = false; + + await expectExistingFilePreserved( + outputPath, + outputPath, + "keep me", + () => { + clientCreated = true; + return replicaClient(unusedSync); + }, + ); + + expect(clientCreated).toBe(false); + })); + + test("refuses existing SQLite sidecars before syncing", () => + withTempDir(async (dir) => { + for (const suffix of ["-wal", "-shm"]) { + const outputPath = join(dir, `snapshot${suffix}.sqlite`); + const sidecarPath = `${outputPath}${suffix}`; + await Deno.writeTextFile(sidecarPath, "keep me"); + let clientCreated = false; + + await expectExistingFilePreserved( + outputPath, + sidecarPath, + "keep me", + () => { + clientCreated = true; + return replicaClient(unusedSync); + }, + ); + + expect(clientCreated).toBe(false); + } + })); + + test("rejects a missing output directory before syncing", () => + withTempDir(async (dir) => { + const parentPath = join(dir, "missing"); + await expect( + createDatabaseSnapshot( + request(join(parentPath, "snapshot.sqlite")), + () => replicaClient(unusedSync), + ), + ).rejects.toThrow(`Output directory does not exist: ${parentPath}`); + })); + + test("rejects an output parent that is not a directory", () => + withTempDir(async (dir) => { + const parentPath = join(dir, "file"); + await Deno.writeTextFile(parentPath, "not a directory"); + await expect( + createDatabaseSnapshot( + request(join(parentPath, "snapshot.sqlite")), + () => replicaClient(unusedSync), + ), + ).rejects.toThrow(`Output directory is not a directory: ${parentPath}`); + })); + + test("surfaces output directory read errors", () => + withTempDir(async (dir) => { + const readError = new Deno.errors.PermissionDenied("blocked"); + using _stat = stub(Deno, "stat", () => Promise.reject(readError)); + + await expect( + createDatabaseSnapshot(request(join(dir, "snapshot.sqlite")), () => + replicaClient(unusedSync), + ), + ).rejects.toBe(readError); + })); + + test("does not overwrite an output created while syncing", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + async (path) => { + await Deno.writeTextFile(path, "new snapshot"); + await Deno.writeTextFile(outputPath, "racing output"); + }, + [[checkpointOk], [integrityOk]], + ); + + await expectExistingFilePreserved( + outputPath, + outputPath, + "racing output", + factory, + ); + expect(await directoryNames(dir)).toEqual(["snapshot.sqlite"]); + })); + + test("does not publish beside a sidecar created while syncing", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const sidecarPath = `${outputPath}-wal`; + const factory = fakeDatabaseFactory( + async (path) => { + await Deno.writeTextFile(path, "new snapshot"); + await Deno.writeTextFile(sidecarPath, "racing WAL"); + }, + [[checkpointOk], [integrityOk]], + ); + + await expectExistingFilePreserved( + outputPath, + sidecarPath, + "racing WAL", + factory, + ); + expect(await pathExists(outputPath)).toBe(false); + })); + + test("does not overwrite an output created during atomic publication", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "new snapshot"), + [[checkpointOk], [integrityOk]], + ); + const link = Deno.link; + using _link = stub(Deno, "link", async (oldPath, newPath) => { + await Deno.writeTextFile(newPath, "racing output"); + await link(oldPath, newPath); + }); + + await expectExistingFilePreserved( + outputPath, + outputPath, + "racing output", + factory, + ); + expect(await directoryNames(dir)).toEqual(["snapshot.sqlite"]); + })); + + test("closes the replica and cleans temporary files when sync fails", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const syncError = new Error("sync failed"); + let closed = false; + const factory: SnapshotClientFactory = (config) => + replicaClient( + async () => { + await Deno.writeTextFile(fromFileUrl(config.url), "partial"); + throw syncError; + }, + () => { + closed = true; + }, + ); + + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toBe(syncError); + + expect(closed).toBe(true); + expect(await directoryNames(dir)).toEqual([]); + })); + + test("rejects a busy WAL checkpoint and cleans temporary files", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "partial"), + [[{ busy: 1, checkpointed: 2, log: 3 }]], + ); + + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toThrow("Database snapshot checkpoint did not empty the WAL"); + expect(await directoryNames(dir)).toEqual([]); + })); + + test("rejects an invalid integrity result and cleans temporary files", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "partial"), + [[checkpointOk], [{ integrity_check: "page 4 is corrupt" }]], + ); + + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toThrow("Database snapshot integrity check failed"); + expect(await directoryNames(dir)).toEqual([]); + })); + + test("rejects WAL frames left after checkpoint and cleans temporary files", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "partial"), + [[checkpointOk], [integrityOk]], + (path) => Deno.writeTextFileSync(`${path}-wal`, "remaining frame"), + ); + + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toThrow("Database snapshot WAL is not empty"); + expect(await pathExists(outputPath)).toBe(false); + expect(await directoryNames(dir)).toEqual([]); + })); + + test("surfaces atomic publication errors and cleans temporary files", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const linkError = new Deno.errors.PermissionDenied("links blocked"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "complete"), + [[checkpointOk], [integrityOk]], + ); + using _link = stub(Deno, "link", () => Promise.reject(linkError)); + + await expect( + createDatabaseSnapshot(request(outputPath), factory), + ).rejects.toBe(linkError); + expect(await pathExists(outputPath)).toBe(false); + expect(await directoryNames(dir)).toEqual([]); + })); + + test("publishes the snapshot with owner-only permissions", () => + withTempDir(async (dir) => { + const outputPath = join(dir, "snapshot.sqlite"); + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "complete"), + [[checkpointOk], [integrityOk]], + ); + + await createDatabaseSnapshot(request(outputPath), factory); + + const mode = (await Deno.stat(outputPath)).mode; + if (mode === null) throw new Error("File mode is unavailable"); + expect(mode & 0o777).toBe(0o600); + })); + + test("reports each snapshot stage in order", () => + withTempDir(async (dir) => { + const progress: string[] = []; + const factory = fakeDatabaseFactory( + (path) => Deno.writeTextFile(path, "complete"), + [[checkpointOk], [integrityOk]], + ); + + await createDatabaseSnapshot( + request(join(dir, "snapshot.sqlite")), + factory, + (message: string) => progress.push(message), + ); + + expect(progress).toEqual([ + "[1/4] Checking destination", + "[2/4] Syncing remote database", + "[3/4] Checkpointing and checking integrity", + "[4/4] Publishing standalone SQLite file", + ]); + })); +}); diff --git a/test/scripts/database-snapshot.test.ts b/test/scripts/database-snapshot.test.ts new file mode 100644 index 0000000000..a9edb88896 --- /dev/null +++ b/test/scripts/database-snapshot.test.ts @@ -0,0 +1,121 @@ +import { createClient } from "@libsql/client"; +import { expect } from "@std/expect"; +import { join, toFileUrl } from "@std/path"; +import { describe, it as test } from "@std/testing/bdd"; +import { + createDatabaseSnapshot, + type SnapshotClientFactory, +} from "#scripts/database-snapshot-lib.ts"; +import { directoryNames, withTempDir } from "#test-utils/files.ts"; + +describe("database snapshot", () => { + test("publishes a complete SQLite database", () => + withTempDir(async (dir) => { + const sourceDirectory = join(dir, "source"); + await Deno.mkdir(sourceDirectory); + const sourcePath = join(sourceDirectory, "source.sqlite"); + const outputPath = join(dir, "snapshot.sqlite"); + const source = createClient({ + intMode: "bigint", + url: toFileUrl(sourcePath).href, + }); + await source.executeMultiple(` + PRAGMA journal_mode = WAL; + PRAGMA application_id = 1414090059; + PRAGMA user_version = 37; + CREATE TABLE outside_app ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + value INTEGER NOT NULL, + payload BLOB NOT NULL + ); + CREATE TABLE audit (outside_id INTEGER NOT NULL); + CREATE INDEX outside_value_idx ON outside_app (value); + CREATE VIEW outside_values AS SELECT id, value FROM outside_app; + CREATE TRIGGER outside_audit AFTER INSERT ON outside_app + BEGIN + INSERT INTO audit (outside_id) VALUES (NEW.id); + END; + `); + await source.execute({ + args: [9223372036854775806n, new Uint8Array([0, 127, 128, 255])], + sql: "INSERT INTO outside_app (value, payload) VALUES (?, ?)", + }); + await source.execute({ + args: [12n, new Uint8Array([1])], + sql: "INSERT INTO outside_app (value, payload) VALUES (?, ?)", + }); + await source.execute("DELETE FROM outside_app WHERE id = 2"); + await source.execute("PRAGMA wal_checkpoint(TRUNCATE)"); + source.close(); + + const factory: SnapshotClientFactory = (config) => { + if (!config.syncUrl) return createClient(config); + return { + close: () => {}, + execute: () => + Promise.reject(new Error("execute should not be called")), + sync: async () => { + await Deno.copyFile(sourcePath, new URL(config.url)); + return {}; + }, + }; + }; + + await createDatabaseSnapshot( + { + dbToken: "token", + dbUrl: "libsql://database.example.com", + outputPath, + }, + factory, + ); + + expect(await directoryNames(dir)).toEqual(["snapshot.sqlite", "source"]); + + const snapshot = createClient({ + intMode: "bigint", + url: toFileUrl(outputPath).href, + }); + try { + const objects = await snapshot.execute( + "SELECT type, name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", + ); + expect(objects.rows).toEqual([ + { name: "outside_value_idx", type: "index" }, + { name: "audit", type: "table" }, + { name: "outside_app", type: "table" }, + { name: "outside_audit", type: "trigger" }, + { name: "outside_values", type: "view" }, + ]); + const values = await snapshot.execute( + "SELECT id, value, payload FROM outside_app", + ); + const [stored] = values.rows; + if (!stored) throw new Error("Snapshot row is missing"); + expect(stored.id).toBe(1n); + expect(stored.value).toBe(9223372036854775806n); + expect(stored.payload).toBeInstanceOf(ArrayBuffer); + if (!(stored.payload instanceof ArrayBuffer)) { + throw new Error("Snapshot BLOB is not binary data"); + } + expect(Array.from(new Uint8Array(stored.payload))).toEqual([ + 0, 127, 128, 255, + ]); + expect((await snapshot.execute("SELECT * FROM audit")).rows).toEqual([ + { outside_id: 1n }, + { outside_id: 2n }, + ]); + expect( + (await snapshot.execute("SELECT seq FROM sqlite_sequence")).rows, + ).toEqual([{ seq: 2n }]); + expect((await snapshot.execute("PRAGMA application_id")).rows).toEqual([ + { application_id: 1414090059n }, + ]); + expect((await snapshot.execute("PRAGMA user_version")).rows).toEqual([ + { user_version: 37n }, + ]); + } finally { + snapshot.close(); + } + })); +}); diff --git a/test/test-utils/files.ts b/test/test-utils/files.ts index 89a5277a5d..8a05aed309 100644 --- a/test/test-utils/files.ts +++ b/test/test-utils/files.ts @@ -8,6 +8,9 @@ export const pathExists = async (path: string): Promise => { } }; +export const directoryNames = async (path: string): Promise => + (await Array.fromAsync(Deno.readDir(path))).map((entry) => entry.name).sort(); + export interface TempPath extends Disposable { dispose(): void; path: string;