From 7c688f68ccd06a6da03ce29b6499d08636b5da35 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 11:22:21 +0000 Subject: [PATCH 01/21] feat: add database reconciliation planner --- src/lib/v5/db-reconciliation.test.ts | 898 +++++++++++++ src/lib/v5/db-reconciliation.ts | 1754 ++++++++++++++++++++++++++ 2 files changed, 2652 insertions(+) create mode 100644 src/lib/v5/db-reconciliation.test.ts create mode 100644 src/lib/v5/db-reconciliation.ts diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts new file mode 100644 index 000000000..c66a09769 --- /dev/null +++ b/src/lib/v5/db-reconciliation.test.ts @@ -0,0 +1,898 @@ +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + IDENTICAL_HISTORY_ADDITION_LIMITATION, + ReconciliationError, + type ReconciliationRequest, + type TaskEventReconciliationValue, + dryRunDatabaseReconciliation, + planDatabaseReconciliation, +} from './db-reconciliation.js'; +import { openDb } from './genie-db.js'; + +let fixtureRoot: string; + +beforeEach(() => { + fixtureRoot = mkdtempSync(join(tmpdir(), 'genie-db-reconciliation-')); +}); + +afterEach(() => { + rmSync(fixtureRoot, { recursive: true, force: true }); +}); + +function pathFor(name: string): string { + return join(fixtureRoot, `${name}.db`); +} + +function currentDb(name: string, marker: string | null = '100'): string { + const path = pathFor(name); + openDb({ path }).close(); + const db = new Database(path); + if (marker === null) db.query("DELETE FROM meta WHERE key = 'stage_log_backfill_v1'").run(); + else { + db.query("UPDATE meta SET value = ? WHERE key = 'stage_log_backfill_v1'").run(marker); + } + db.close(); + return path; +} + +function mutate(path: string, callback: (db: Database) => void): void { + const db = new Database(path); + db.exec('PRAGMA foreign_keys = ON'); + callback(db); + db.close(); +} + +function insertBoard(db: Database, row: { id: string; name: string; createdAt?: number; lanes?: string | null }): void { + db.query('INSERT INTO boards (id, name, created_at, lanes) VALUES (?, ?, ?, ?)').run( + row.id, + row.name, + row.createdAt ?? 1, + row.lanes ?? null, + ); +} + +function insertTask( + db: Database, + row: { + id: string; + boardId?: string | null; + title?: string; + status?: string; + claimedBy?: string | null; + claimedAt?: number | null; + wish?: string | null; + groupName?: string | null; + createdAt?: number; + updatedAt?: number; + lane?: string | null; + agentKind?: string | null; + heartbeatAt?: number | null; + blockedBy?: string | null; + blockedReason?: string | null; + }, +): void { + db.query( + `INSERT INTO tasks ( + id, board_id, title, status, claimed_by, claimed_at, wish, group_name, + created_at, updated_at, lane, agent_kind, heartbeat_at, blocked_by, blocked_reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, + row.boardId ?? null, + row.title ?? row.id, + row.status ?? 'ready', + row.claimedBy ?? null, + row.claimedAt ?? null, + row.wish ?? null, + row.groupName ?? null, + row.createdAt ?? 1, + row.updatedAt ?? 1, + row.lane ?? null, + row.agentKind ?? null, + row.heartbeatAt ?? null, + row.blockedBy ?? null, + row.blockedReason ?? null, + ); +} + +function insertStage( + db: Database, + id: number, + row: { taskId: string; stage: string; note: string | null; createdAt: number }, +): void { + db.query('INSERT INTO stage_log (id, task_id, stage, note, created_at) VALUES (?, ?, ?, ?, ?)').run( + id, + row.taskId, + row.stage, + row.note, + row.createdAt, + ); +} + +function insertEvent( + db: Database, + id: number, + row: { + taskId: string; + kind: string; + note: string | null; + authorKind?: string | null; + author?: string | null; + createdAt: number; + }, +): void { + db.query( + 'INSERT INTO task_events (id, task_id, kind, note, author_kind, author, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)', + ).run(id, row.taskId, row.kind, row.note, row.authorKind ?? null, row.author ?? null, row.createdAt); +} + +function bidirectional(leftPath: string, rightPath: string): ReconciliationRequest { + return { mode: 'bidirectional', leftPath, rightPath }; +} + +function target(plan: ReturnType, role: 'left' | 'right' | 'destination') { + const found = plan.targets.find((candidate) => candidate.role === role); + if (found === undefined) throw new Error(`missing ${role} target`); + return found; +} + +function seedPreLanes(path: string): void { + const db = new Database(path); + db.exec(` + PRAGMA user_version = 1; + CREATE TABLE boards (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at INTEGER NOT NULL); + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + board_id TEXT REFERENCES boards(id) ON DELETE SET NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + claimed_by TEXT, + claimed_at INTEGER, + wish TEXT, + group_name TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE task_dependencies ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) + ); + CREATE TABLE stage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + stage TEXT NOT NULL, + note TEXT, + created_at INTEGER NOT NULL + ); + CREATE TABLE wish_groups ( + wish TEXT NOT NULL, + name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + depends_on TEXT NOT NULL DEFAULT '[]', + assignee TEXT, + started_at INTEGER, + completed_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (wish, name) + ); + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + `); + db.close(); +} + +function seedPreRuntime(path: string): void { + const db = new Database(path); + db.exec(` + PRAGMA user_version = 1; + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE boards ( + id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at INTEGER NOT NULL, lanes TEXT + ); + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + board_id TEXT REFERENCES boards(id) ON DELETE SET NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + claimed_by TEXT, claimed_at INTEGER, wish TEXT, group_name TEXT, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, lane TEXT + ); + CREATE TABLE task_dependencies ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) + ); + CREATE TABLE stage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + stage TEXT NOT NULL, note TEXT, created_at INTEGER NOT NULL + ); + CREATE TABLE task_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + kind TEXT NOT NULL, note TEXT, author_kind TEXT, author TEXT, created_at INTEGER NOT NULL + ); + CREATE TABLE wish_groups ( + wish TEXT NOT NULL, name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + depends_on TEXT NOT NULL DEFAULT '[]', assignee TEXT, started_at INTEGER, completed_at INTEGER, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (wish, name) + ); + CREATE TABLE hire_roster ( + wish TEXT NOT NULL, agent_adapter_id TEXT NOT NULL, profile TEXT, worktree TEXT NOT NULL, + hired_at INTEGER NOT NULL, state TEXT NOT NULL, PRIMARY KEY (wish, agent_adapter_id) + ); + CREATE INDEX idx_task_deps_dep ON task_dependencies(depends_on_id); + CREATE INDEX idx_tasks_status ON tasks(status); + CREATE INDEX idx_stage_log_task ON stage_log(task_id); + CREATE INDEX idx_task_events_task ON task_events(task_id); + `); + db.close(); +} + +function seedReorderedCurrent(path: string, taskStatuses = "'done', 'blocked', 'ready', 'in_progress'"): void { + const db = new Database(path); + db.exec(` + PRAGMA user_version = 1; + CREATE TABLE meta (value TEXT NOT NULL, key TEXT, PRIMARY KEY (key)); + CREATE TABLE boards ( + lanes TEXT, created_at INTEGER NOT NULL, name TEXT NOT NULL UNIQUE, id TEXT, PRIMARY KEY (id) + ); + CREATE TABLE tasks ( + blocked_reason TEXT, heartbeat_at INTEGER, agent_kind TEXT, lane TEXT, + updated_at INTEGER NOT NULL, created_at INTEGER NOT NULL, group_name TEXT, wish TEXT, + claimed_at INTEGER, claimed_by TEXT, + status TEXT NOT NULL CHECK(status IN (${taskStatuses})), + title TEXT NOT NULL, + board_id TEXT REFERENCES boards(id) ON DELETE SET NULL, + blocked_by TEXT, + id TEXT, + PRIMARY KEY (id) + ); + CREATE TABLE task_dependencies ( + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) + ); + CREATE TABLE stage_log ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + note TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at INTEGER NOT NULL, + stage TEXT NOT NULL + ); + CREATE TABLE task_events ( + author TEXT, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, + id INTEGER PRIMARY KEY AUTOINCREMENT, + note TEXT, + kind TEXT NOT NULL, + author_kind TEXT + ); + CREATE TABLE wish_groups ( + updated_at INTEGER NOT NULL, + assignee TEXT, + name TEXT NOT NULL, + depends_on TEXT NOT NULL DEFAULT '[]', + wish TEXT NOT NULL, + completed_at INTEGER, + status TEXT NOT NULL CHECK(status IN ('ready','in_progress','done','blocked')), + created_at INTEGER NOT NULL, + started_at INTEGER, + PRIMARY KEY (wish, name) + ); + CREATE TABLE hire_roster ( + state TEXT NOT NULL, + agent_adapter_id TEXT NOT NULL, + hired_at INTEGER NOT NULL, + wish TEXT NOT NULL, + worktree TEXT NOT NULL, + profile TEXT, + PRIMARY KEY (wish, agent_adapter_id) + ); + CREATE INDEX idx_task_events_task ON task_events(task_id); + CREATE INDEX idx_stage_log_task ON stage_log(task_id); + CREATE INDEX idx_tasks_status ON tasks(status); + CREATE INDEX idx_task_deps_dep ON task_dependencies(depends_on_id); + `); + db.close(); +} + +describe('closed normalized current schema', () => { + test('fresh, normalized additive-history, and reordered current schemas share one fingerprint', () => { + const fresh = currentDb('fresh', null); + const preLanes = pathFor('pre-lanes'); + const preRuntime = pathFor('pre-runtime'); + const reordered = pathFor('reordered'); + seedPreLanes(preLanes); + seedPreRuntime(preRuntime); + seedReorderedCurrent(reordered); + + // The normal current open path, not reconciliation, owns additive repair. + openDb({ path: preLanes }).close(); + openDb({ path: preRuntime }).close(); + openDb({ path: reordered }).close(); + + const fingerprints = [preLanes, preRuntime, reordered].map( + (candidate) => planDatabaseReconciliation(bidirectional(fresh, candidate)).schemaFingerprint, + ); + expect(new Set(fingerprints).size).toBe(1); + }); + + test('stale same-version additive shape is rejected read-only with normal-open guidance', () => { + const stale = pathFor('stale'); + const current = currentDb('current'); + seedPreLanes(stale); + const before = readFileSync(stale); + + const report = dryRunDatabaseReconciliation(bidirectional(stale, current)); + + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure).toEqual({ + code: 'stale-current-schema', + guidance: + 'Open the database once with Genie’s normal current open path (for example `genie board`) to normalize supported additive history, then retry reconciliation.', + }); + expect(readFileSync(stale)).toEqual(before); + const check = new Database(stale, { readonly: true }); + expect(check.query("SELECT 1 FROM sqlite_schema WHERE name = 'task_events'").get()).toBeNull(); + check.close(); + }); + + test.each([ + ['extra column', (db: Database) => db.exec('ALTER TABLE tasks ADD COLUMN guest_extra TEXT')], + ['extra table', (db: Database) => db.exec('CREATE TABLE guest_extra (id TEXT PRIMARY KEY)')], + ['extra index', (db: Database) => db.exec('CREATE INDEX guest_extra ON tasks(title)')], + ['view', (db: Database) => db.exec('CREATE VIEW guest_extra AS SELECT id FROM tasks')], + ['virtual table', (db: Database) => db.exec('CREATE VIRTUAL TABLE guest_extra USING fts5(content)')], + ])('%s is rejected before planning authority', (_name, alter) => { + const guest = currentDb('guest'); + const peer = currentDb('peer'); + mutate(guest, alter); + + const report = dryRunDatabaseReconciliation(bidirectional(guest, peer)); + + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure?.code).toBe('unsupported-schema'); + expect(report.targets).toEqual([]); + }); + + test('trigger is rejected and its guest-defined body never executes', () => { + const guest = currentDb('guest'); + const peer = currentDb('peer'); + mutate(guest, (db) => { + insertTask(db, { id: 't' }); + db.exec(` + CREATE TRIGGER guest_trigger AFTER UPDATE ON tasks + BEGIN + INSERT INTO meta (key, value) VALUES ('trigger_executed', 'yes'); + END + `); + }); + + const report = dryRunDatabaseReconciliation(bidirectional(guest, peer)); + + expect(report.operationalFailure?.code).toBe('unsupported-schema'); + const check = new Database(guest, { readonly: true }); + expect(check.query("SELECT value FROM meta WHERE key = 'trigger_executed'").get()).toBeNull(); + check.close(); + }); + + test('a non-schema-implied SQLite internal object is rejected', () => { + const guest = currentDb('guest'); + const peer = currentDb('peer'); + mutate(guest, (db) => { + // sqlite_stat1 is SQLite-owned, but unlike sqlite_sequence it is not + // implied by Genie's supported schema and therefore is still closed out. + db.exec('ANALYZE'); + }); + + const report = dryRunDatabaseReconciliation(bidirectional(guest, peer)); + + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure?.code).toBe('unsupported-schema'); + }); + + test('missing expected index is stale but altered constraints are unsupported', () => { + const peer = currentDb('peer'); + const missing = currentDb('missing'); + mutate(missing, (db) => db.exec('DROP INDEX idx_tasks_status')); + expect(dryRunDatabaseReconciliation(bidirectional(missing, peer)).operationalFailure?.code).toBe( + 'stale-current-schema', + ); + + const wrongCheck = pathFor('wrong-check'); + seedReorderedCurrent(wrongCheck, "'done', 'blocked', 'ready', 'paused'"); + expect(dryRunDatabaseReconciliation(bidirectional(wrongCheck, peer)).status).toBe('operational-failure'); + }); +}); + +describe('keyed, edge-set, and history-multiset planning', () => { + test('all tables, nullable columns, local-ID collisions, and meaningful duplicates reconcile deterministically', () => { + const left = currentDb('left', null); + const right = currentDb('right', null); + for (const [path, side] of [ + [left, 'left'], + [right, 'right'], + ] as const) { + mutate(path, (db) => { + insertBoard(db, { + id: `b-${side}`, + name: `board-${side}`, + lanes: side === 'left' ? null : '["ready"]', + }); + insertTask(db, { id: 'shared', title: 'same' }); + insertTask(db, { id: 'prerequisite', title: 'prerequisite' }); + insertTask( + db, + side === 'left' + ? { id: 'nullable', title: 'nullable' } + : { + id: 'populated', + boardId: 'b-right', + title: 'populated', + status: 'blocked', + claimedBy: 'worker', + claimedAt: 2, + wish: 'wish', + groupName: 'g', + createdAt: 2, + updatedAt: 3, + lane: 'blocked', + agentKind: 'codex', + heartbeatAt: 4, + blockedBy: 'prerequisite', + blockedReason: 'waiting', + }, + ); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + `wish-${side}`, + 'g', + 'ready', + '[]', + side === 'left' ? null : 'worker', + side === 'left' ? null : 2, + side === 'left' ? null : 3, + 1, + 1, + ); + db.query( + 'INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) VALUES (?, ?, ?, ?, ?, ?)', + ).run(`wish-${side}`, 'adapter', side === 'left' ? null : 'profile', `/wt/${side}`, 1, 'hired'); + db.query('INSERT INTO meta (key, value) VALUES (?, ?)').run(`meta-${side}`, side); + }); + } + mutate(left, (db) => { + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('shared', 'prerequisite'); + insertStage(db, 1, { taskId: 'shared', stage: 'planned', note: 'same', createdAt: 10 }); + insertStage(db, 2, { taskId: 'shared', stage: 'planned', note: 'same', createdAt: 10 }); + insertEvent(db, 1, { taskId: 'shared', kind: 'comment', note: 'same', createdAt: 20 }); + insertEvent(db, 2, { + taskId: 'shared', + kind: 'report', + note: 'left-only', + authorKind: 'agent', + author: 'a', + createdAt: 21, + }); + }); + mutate(right, (db) => { + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('prerequisite', 'shared'); + // IDs collide with left but are intentionally local and excluded from identity. + insertStage(db, 1, { taskId: 'shared', stage: 'planned', note: 'same', createdAt: 10 }); + insertStage(db, 2, { taskId: 'shared', stage: 'right-only', note: null, createdAt: 11 }); + insertEvent(db, 1, { taskId: 'shared', kind: 'comment', note: 'same', createdAt: 20 }); + insertEvent(db, 2, { taskId: 'shared', kind: 'comment', note: null, createdAt: 22 }); + }); + + const first = planDatabaseReconciliation(bidirectional(left, right)); + const second = planDatabaseReconciliation(bidirectional(left, right)); + + expect(first.status).toBe('changed'); + expect(first.report).toEqual(second.report); + expect(target(first, 'left').changes).toMatchObject({ + boards: [{ id: 'b-right' }], + tasks: [{ id: 'populated' }], + wishGroups: [{ wish: 'wish-right' }], + hireRoster: [{ wish: 'wish-right' }], + meta: [{ key: 'meta-right' }], + taskDependencies: [{ taskId: 'prerequisite', dependsOnId: 'shared' }], + stageLog: [{ count: 1, value: { stage: 'right-only' } }], + taskEvents: [{ count: 1, value: { note: null } }], + }); + expect(target(first, 'right').changes).toMatchObject({ + boards: [{ id: 'b-left' }], + tasks: [{ id: 'nullable' }], + wishGroups: [{ wish: 'wish-left' }], + hireRoster: [{ wish: 'wish-left', profile: null }], + meta: [{ key: 'meta-left' }], + taskDependencies: [{ taskId: 'shared', dependsOnId: 'prerequisite' }], + stageLog: [{ count: 1, value: { stage: 'planned', note: 'same' } }], + taskEvents: [{ count: 1, value: { kind: 'report', authorKind: 'agent', author: 'a' } }], + }); + expect(first.report.targets.every((item) => item.changes.deletions === 0)).toBe(true); + expect(Object.isFrozen(first)).toBe(true); + expect(Object.isFrozen(first.targets)).toBe(true); + expect(Object.isFrozen(first.targets[0].changes.stageLog)).toBe(true); + }); + + test('bidirectional same-key differences conflict without exposing hostile payloads', () => { + const left = currentDb('left'); + const right = currentDb('right'); + mutate(left, (db) => insertBoard(db, { id: 'same', name: '' })); + mutate(right, (db) => insertBoard(db, { id: 'same', name: ' right' })); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const reportJson = JSON.stringify(plan.report); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toHaveLength(1); + expect(plan.conflicts[0]).toMatchObject({ table: 'boards', reason: 'same-key-difference' }); + expect(plan.targets.every((item) => item.postimageDigest === null)).toBe(true); + expect(reportJson).not.toContain('hostile-title'); + expect(reportJson).not.toContain('left-title'); + }); + + test('directional source wins shared keyed rows and unknown meta while destination-only rows are preserved', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + for (const [path, suffix] of [ + [source, 'source'], + [destination, 'destination'], + ] as const) { + mutate(path, (db) => { + insertBoard(db, { id: 'shared-board', name: `board-${suffix}` }); + insertTask(db, { id: 'shared-task', boardId: 'shared-board', title: `task-${suffix}` }); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) + VALUES ('w', 'g', 'ready', '[]', ?, NULL, NULL, 1, 1)`, + ).run(suffix); + db.query( + `INSERT INTO hire_roster + (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('w', 'a', ?, ?, 1, 'hired')`, + ).run(suffix, `/wt/${suffix}`); + db.query("INSERT INTO meta (key, value) VALUES ('unknown', ?)").run(suffix); + }); + } + mutate(destination, (db) => insertBoard(db, { id: 'destination-only', name: 'keep-me' })); + + const plan = planDatabaseReconciliation({ + mode: 'directional', + sourcePath: source, + destinationPath: destination, + }); + const changes = target(plan, 'destination').changes; + + expect(changes.boards).toHaveLength(1); + expect(changes.boards[0].name).toBe('board-source'); + expect(changes.tasks[0].title).toBe('task-source'); + expect(changes.wishGroups[0].assignee).toBe('source'); + expect(changes.hireRoster[0].worktree).toBe('/wt/source'); + expect(changes.meta).toEqual([{ key: 'unknown', value: 'source' }]); + expect(changes.boards.some((row) => row.id === 'destination-only')).toBe(false); + expect(plan.report.targets[0].changes.deletions).toBe(0); + }); + + test('a cross-image uniqueness collision conflicts instead of planning an invalid target', () => { + const source = currentDb('source'); + const destination = currentDb('destination'); + mutate(source, (db) => insertBoard(db, { id: 'source-id', name: 'same-name' })); + mutate(destination, (db) => insertBoard(db, { id: 'destination-id', name: 'same-name' })); + + const plan = planDatabaseReconciliation({ + mode: 'directional', + sourcePath: source, + destinationPath: destination, + }); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ + table: 'boards', + reason: 'planned-integrity-failed', + side: 'destination', + }), + ); + expect(target(plan, 'destination').postimageDigest).toBeNull(); + }); + + test('exact event counts use max and identical independent additions remain indistinguishable', () => { + const left = currentDb('left'); + const right = currentDb('right'); + for (const [path, id] of [ + [left, 1], + [right, 999], + ] as const) { + mutate(path, (db) => { + insertTask(db, { id: 't' }); + insertEvent(db, id, { + taskId: 't', + kind: 'comment', + note: 'byte-identical', + authorKind: 'human', + author: 'same', + createdAt: 10, + }); + }); + } + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(plan.status).toBe('no-op'); + expect(plan.inputs[0].logicalDigest).toBe(plan.inputs[1].logicalDigest); + expect(plan.historyLimitation).toBe(IDENTICAL_HISTORY_ADDITION_LIMITATION); + expect(target(plan, 'left').changes.taskEvents).toEqual([]); + expect(target(plan, 'right').changes.taskEvents).toEqual([]); + }); + + test('directional history count is max(destination, source), never sum', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + for (const path of [source, destination]) { + mutate(path, (db) => insertTask(db, { id: 't' })); + } + mutate(source, (db) => { + for (let id = 1; id <= 3; id++) { + insertEvent(db, id, { taskId: 't', kind: 'comment', note: 'same', createdAt: 1 }); + } + }); + mutate(destination, (db) => { + insertEvent(db, 20, { taskId: 't', kind: 'comment', note: 'same', createdAt: 1 }); + }); + + const plan = planDatabaseReconciliation({ + mode: 'directional', + sourcePath: source, + destinationPath: destination, + }); + + expect(target(plan, 'destination').changes.taskEvents).toEqual([ + { + count: 2, + value: { + taskId: 't', + kind: 'comment', + note: 'same', + authorKind: null, + author: null, + createdAt: 1n, + }, + }, + ]); + }); +}); + +describe('stage_log_backfill_v1 marker semantics', () => { + test('all seven direct mappings plus both unknown mappings preserve exact task/time/null-author tuples', () => { + const left = currentDb('left', '200'); + const right = currentDb('right', null); + for (const path of [left, right]) mutate(path, (db) => insertTask(db, { id: 't' })); + const direct = ['comment', 'move', 'claim', 'release', 'block', 'unblock', 'report']; + const expectedEvents: TaskEventReconciliationValue[] = []; + mutate(left, (db) => { + let id = 1; + for (const kind of direct) { + const note = `note-${kind}`; + insertStage(db, id, { taskId: 't', stage: kind, note, createdAt: 100 + id }); + insertEvent(db, id, { taskId: 't', kind, note, createdAt: 100 + id }); + expectedEvents.push({ + taskId: 't', + kind, + note, + authorKind: null, + author: null, + createdAt: BigInt(100 + id), + }); + id++; + } + insertStage(db, id, { taskId: 't', stage: 'planned', note: 'kickoff', createdAt: 200 }); + insertEvent(db, id, { taskId: 't', kind: 'comment', note: 'planned: kickoff', createdAt: 200 }); + expectedEvents.push({ + taskId: 't', + kind: 'comment', + note: 'planned: kickoff', + authorKind: null, + author: null, + createdAt: 200n, + }); + id++; + insertStage(db, id, { taskId: 't', stage: 'implemented', note: null, createdAt: 201 }); + insertEvent(db, id, { taskId: 't', kind: 'comment', note: 'implemented', createdAt: 201 }); + expectedEvents.push({ + taskId: 't', + kind: 'comment', + note: 'implemented', + authorKind: null, + author: null, + createdAt: 201n, + }); + }); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const rightChanges = target(plan, 'right').changes; + + expect(plan.status).toBe('changed'); + expect(rightChanges.meta).toEqual([{ key: 'stage_log_backfill_v1', value: '200' }]); + expect(rightChanges.taskEvents.map((addition) => addition.value)).toEqual(expect.arrayContaining(expectedEvents)); + expect(rightChanges.taskEvents).toHaveLength(expectedEvents.length); + expect(rightChanges.taskEvents.every((addition) => addition.count === 1)).toBe(true); + expect( + rightChanges.taskEvents.every((addition) => addition.value.author === null && addition.value.authorKind === null), + ).toBe(true); + const stages = rightChanges.stageLog.map((addition) => addition.value); + expect(stages.every((stage) => stage.taskId === 't')).toBe(true); + expect(new Set(stages.map((stage) => stage.createdAt))).toEqual( + new Set(expectedEvents.map((event) => event.createdAt)), + ); + }); + + test('two valid decimal markers converge to the numerically smaller canonical timestamp', () => { + const left = currentDb('left', '000200'); + const right = currentDb('right', '100'); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(target(plan, 'left').changes.meta).toEqual([{ key: 'stage_log_backfill_v1', value: '100' }]); + expect(target(plan, 'right').changes.meta).toEqual([]); + expect(target(plan, 'left').postimageDigest).toBe(target(plan, 'right').postimageDigest); + }); + + test('invalid marker and invariant failure become deterministic conflicts', () => { + const invalid = currentDb('invalid', 'not-decimal'); + const peer = currentDb('peer', null); + const invalidPlan = planDatabaseReconciliation(bidirectional(invalid, peer)); + expect(invalidPlan.status).toBe('conflict'); + expect(invalidPlan.conflicts).toEqual([ + expect.objectContaining({ table: 'meta', reason: 'invalid-marker', side: 'left' }), + ]); + + const failing = currentDb('failing', '100'); + mutate(failing, (db) => { + insertTask(db, { id: 't' }); + insertStage(db, 1, { taskId: 't', stage: 'report', note: 'missing event', createdAt: 1 }); + }); + const first = planDatabaseReconciliation(bidirectional(failing, peer)); + const second = planDatabaseReconciliation(bidirectional(failing, peer)); + expect(first.status).toBe('conflict'); + expect(first.conflicts).toEqual(second.conflicts); + expect(first.conflicts).toContainEqual( + expect.objectContaining({ reason: 'marker-invariant-failed', side: 'left' }), + ); + }); + + test('mapped-count invariant aggregates distinct stage tuples that map to the same event tuple', () => { + const left = currentDb('left', '100'); + const right = currentDb('right', null); + mutate(left, (db) => { + insertTask(db, { id: 't' }); + insertStage(db, 1, { taskId: 't', stage: 'comment', note: 'x', createdAt: 1 }); + insertStage(db, 2, { taskId: 't', stage: 'x', note: null, createdAt: 1 }); + insertEvent(db, 1, { taskId: 't', kind: 'comment', note: 'x', createdAt: 1 }); + }); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toContainEqual(expect.objectContaining({ reason: 'marker-invariant-failed', side: 'left' })); + }); + + test('filling a missing marker checks the planned target event coverage', () => { + const left = currentDb('left', '100'); + const right = currentDb('right', null); + for (const path of [left, right]) mutate(path, (db) => insertTask(db, { id: 't' })); + mutate(left, (db) => { + insertStage(db, 1, { taskId: 't', stage: 'report', note: 'left', createdAt: 1 }); + insertEvent(db, 1, { taskId: 't', kind: 'report', note: 'left', createdAt: 1 }); + }); + mutate(right, (db) => { + insertStage(db, 1, { taskId: 't', stage: 'report', note: 'right', createdAt: 2 }); + }); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ reason: 'planned-marker-invariant-failed', side: 'left' }), + ); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ reason: 'planned-marker-invariant-failed', side: 'right' }), + ); + }); +}); + +describe('same-file, idempotency, and bounded failures', () => { + test('canonical aliases of the same physical database are a safe same-database no-op', () => { + const path = currentDb('one'); + const alias = join(fixtureRoot, 'alias.db'); + symlinkSync(path, alias); + + const plan = planDatabaseReconciliation(bidirectional(path, alias)); + + expect(plan.status).toBe('same-database'); + expect(plan.sameDatabase).toBe(true); + expect(plan.report.status).toBe('no-op'); + expect(plan.inputs[0].logicalDigest).toBe(plan.inputs[1].logicalDigest); + expect(plan.targets.every((item) => !Object.values(item.changes).some((rows) => rows.length > 0))).toBe(true); + }); + + test('equal logical images are idempotent despite local history IDs', () => { + const left = currentDb('left'); + const right = currentDb('right'); + for (const [path, id] of [ + [left, 1], + [right, 99], + ] as const) { + mutate(path, (db) => { + insertTask(db, { id: 't' }); + insertStage(db, id, { taskId: 't', stage: 'report', note: 'same', createdAt: 1 }); + insertEvent(db, id, { taskId: 't', kind: 'report', note: 'same', createdAt: 1 }); + }); + } + + const first = planDatabaseReconciliation(bidirectional(left, right)); + const second = planDatabaseReconciliation(bidirectional(left, right)); + + expect(first.status).toBe('no-op'); + expect(first.report).toEqual(second.report); + expect(first.inputs[0].logicalDigest).toBe(first.inputs[1].logicalDigest); + }); + + test('corrupt inputs and invalid row types fail with bounded deterministic reports', () => { + const peer = currentDb('peer'); + const corrupt = pathFor('corrupt'); + writeFileSync(corrupt, new Uint8Array([0, 1, 2, 3, 4, 5])); + + const first = dryRunDatabaseReconciliation(bidirectional(corrupt, peer)); + const second = dryRunDatabaseReconciliation(bidirectional(corrupt, peer)); + expect(first).toEqual(second); + expect(first).toMatchObject({ + status: 'operational-failure', + operationalFailure: { code: 'malformed-database' }, + targets: [], + }); + + const invalid = currentDb('invalid-row'); + mutate(invalid, (db) => { + insertTask(db, { id: 't' }); + db.exec("UPDATE tasks SET title = X'0102' WHERE id = 't'"); + }); + const invalidReport = dryRunDatabaseReconciliation(bidirectional(invalid, peer)); + expect(invalidReport.operationalFailure?.code).toBe('invalid-data'); + }); + + test('foreign-key-invalid content fails integrity validation before planning', () => { + const invalid = currentDb('invalid-fk'); + const peer = currentDb('peer'); + const db = new Database(invalid); + db.exec('PRAGMA foreign_keys = OFF'); + insertTask(db, { id: 'orphan', boardId: 'missing-board' }); + db.close(); + + const report = dryRunDatabaseReconciliation(bidirectional(invalid, peer)); + + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure?.code).toBe('integrity-failed'); + expect(report.targets).toEqual([]); + }); + + test('planner throws a typed bounded error while dry-run reduces it to a safe report', () => { + const missing = join(fixtureRoot, 'missing.db'); + const peer = currentDb('peer'); + expect(() => planDatabaseReconciliation(bidirectional(missing, peer))).toThrow(ReconciliationError); + const report = dryRunDatabaseReconciliation(bidirectional(missing, peer)); + expect(report.operationalFailure).toEqual({ code: 'input-unavailable' }); + expect(JSON.stringify(report)).not.toContain(missing); + }); +}); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts new file mode 100644 index 000000000..c0e651bad --- /dev/null +++ b/src/lib/v5/db-reconciliation.ts @@ -0,0 +1,1754 @@ +/** + * Read-only planning for reconciling two current Genie v5 databases. + * + * This module deliberately owns no locks, snapshots, or writes. It turns two + * transaction-consistent, structurally exact logical images into an immutable + * plan that a later apply boundary can revalidate and execute. + */ + +import { Database } from 'bun:sqlite'; +import { createHash } from 'node:crypto'; +import { realpathSync, statSync } from 'node:fs'; +import { normalize } from 'node:path'; +import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; + +const PLAN_VERSION = 1 as const; +const REPORT_VERSION = 1 as const; +const READ_BUSY_TIMEOUT_MS = 5_000; +const MAX_DATABASE_BYTES = 256 * 1024 * 1024; +const MAX_ROWS_PER_TABLE = 1_000_000; +const MAX_TOTAL_ROWS = 2_000_000; +const BACKFILL_MARKER = 'stage_log_backfill_v1'; +const DIRECT_BACKFILL_KINDS = new Set(['comment', 'move', 'claim', 'release', 'block', 'unblock', 'report']); + +/** + * Without ancestry, byte-identical events independently added on both forks + * are indistinguishable from one shared event and therefore converge to max, + * not sum. + */ +export const IDENTICAL_HISTORY_ADDITION_LIMITATION = + 'Independent byte-identical history additions are indistinguishable without ancestry; reconciliation keeps the maximum observed occurrence count.'; + +export type ReconciliationMode = 'bidirectional' | 'directional'; +export type ReconciliationPlanStatus = 'no-op' | 'changed' | 'conflict' | 'same-database'; +export type ReconciliationReportStatus = 'no-op' | 'changed' | 'conflict' | 'operational-failure'; +export type ReconciliationInputRole = 'left' | 'right' | 'source' | 'destination'; +export type ReconciliationTargetRole = 'left' | 'right' | 'destination'; +export type KeyedTableName = 'boards' | 'tasks' | 'wish_groups' | 'hire_roster' | 'meta'; +export type ReconciliationTableName = KeyedTableName | 'task_dependencies' | 'stage_log' | 'task_events'; + +export type ReconciliationRequest = + | { + readonly mode: 'bidirectional'; + readonly leftPath: string; + readonly rightPath: string; + } + | { + readonly mode: 'directional'; + readonly sourcePath: string; + readonly destinationPath: string; + }; + +export type ReconciliationErrorCode = + | 'input-unavailable' + | 'input-too-large' + | 'malformed-database' + | 'stale-current-schema' + | 'unsupported-schema' + | 'integrity-failed' + | 'invalid-data' + | 'input-changed'; + +export class ReconciliationError extends Error { + readonly code: ReconciliationErrorCode; + readonly guidance?: string; + + constructor(code: ReconciliationErrorCode, message: string, guidance?: string) { + super(message); + this.name = 'ReconciliationError'; + this.code = code; + this.guidance = guidance; + } +} + +export interface BoardReconciliationRow { + readonly id: string; + readonly name: string; + readonly createdAt: bigint; + readonly lanes: string | null; +} + +export interface TaskReconciliationRow { + readonly id: string; + readonly boardId: string | null; + readonly title: string; + readonly status: string; + readonly claimedBy: string | null; + readonly claimedAt: bigint | null; + readonly wish: string | null; + readonly groupName: string | null; + readonly createdAt: bigint; + readonly updatedAt: bigint; + readonly lane: string | null; + readonly agentKind: string | null; + readonly heartbeatAt: bigint | null; + readonly blockedBy: string | null; + readonly blockedReason: string | null; +} + +export interface WishGroupReconciliationRow { + readonly wish: string; + readonly name: string; + readonly status: string; + readonly dependsOn: string; + readonly assignee: string | null; + readonly startedAt: bigint | null; + readonly completedAt: bigint | null; + readonly createdAt: bigint; + readonly updatedAt: bigint; +} + +export interface HireRosterReconciliationRow { + readonly wish: string; + readonly agentAdapterId: string; + readonly profile: string | null; + readonly worktree: string; + readonly hiredAt: bigint; + readonly state: string; +} + +export interface MetaReconciliationRow { + readonly key: string; + readonly value: string; +} + +export interface TaskDependencyReconciliationRow { + readonly taskId: string; + readonly dependsOnId: string; +} + +export interface StageLogReconciliationValue { + readonly taskId: string; + readonly stage: string; + readonly note: string | null; + readonly createdAt: bigint; +} + +export interface TaskEventReconciliationValue { + readonly taskId: string; + readonly kind: string; + readonly note: string | null; + readonly authorKind: string | null; + readonly author: string | null; + readonly createdAt: bigint; +} + +export interface HistoryAddition { + readonly value: T; + readonly count: number; +} + +export interface ReconciliationTargetChanges { + readonly boards: readonly BoardReconciliationRow[]; + readonly tasks: readonly TaskReconciliationRow[]; + readonly wishGroups: readonly WishGroupReconciliationRow[]; + readonly hireRoster: readonly HireRosterReconciliationRow[]; + readonly meta: readonly MetaReconciliationRow[]; + readonly taskDependencies: readonly TaskDependencyReconciliationRow[]; + readonly stageLog: readonly HistoryAddition[]; + readonly taskEvents: readonly HistoryAddition[]; +} + +export interface ReconciliationConflict { + readonly table: ReconciliationTableName; + readonly reason: + | 'same-key-difference' + | 'invalid-marker' + | 'marker-invariant-failed' + | 'planned-marker-invariant-failed' + | 'planned-integrity-failed'; + readonly keyDigest: string; + readonly side?: ReconciliationInputRole | ReconciliationTargetRole; +} + +export interface ReconciliationPlanInput { + readonly role: ReconciliationInputRole; + readonly canonicalPath: string; + readonly logicalDigest: string; +} + +export interface ReconciliationTargetPlan { + readonly role: ReconciliationTargetRole; + readonly canonicalPath: string; + readonly preimageDigest: string; + readonly postimageDigest: string | null; + readonly changes: ReconciliationTargetChanges; +} + +export interface ReconciliationChangeCounts { + readonly boards: number; + readonly tasks: number; + readonly wishGroups: number; + readonly hireRoster: number; + readonly meta: number; + readonly taskDependencies: number; + readonly stageLog: number; + readonly taskEvents: number; + readonly deletions: 0; +} + +export interface ReconciliationTargetReport { + readonly role: ReconciliationTargetRole; + readonly preimageDigest: string; + readonly postimageDigest: string | null; + readonly changes: ReconciliationChangeCounts; +} + +export interface ReconciliationDryRunReport { + readonly reportVersion: typeof REPORT_VERSION; + readonly dryRun: true; + readonly mode: ReconciliationMode; + readonly status: ReconciliationReportStatus; + readonly sameDatabase: boolean; + readonly schemaFingerprint: string | null; + readonly targets: readonly ReconciliationTargetReport[]; + readonly conflicts: readonly ReconciliationConflict[]; + readonly operationalFailure: { + readonly code: ReconciliationErrorCode | 'unexpected-failure'; + readonly guidance?: string; + } | null; + readonly historyLimitation: typeof IDENTICAL_HISTORY_ADDITION_LIMITATION; +} + +export interface ReconciliationPlan { + readonly planVersion: typeof PLAN_VERSION; + readonly mode: ReconciliationMode; + readonly status: ReconciliationPlanStatus; + readonly sameDatabase: boolean; + readonly schemaFingerprint: string; + readonly inputs: readonly ReconciliationPlanInput[]; + readonly targets: readonly ReconciliationTargetPlan[]; + readonly conflicts: readonly ReconciliationConflict[]; + readonly report: ReconciliationDryRunReport; + readonly historyLimitation: typeof IDENTICAL_HISTORY_ADDITION_LIMITATION; +} + +type SqliteScalar = string | bigint | number | Uint8Array | null; +type CanonicalScalar = readonly ['null'] | readonly ['string', string] | readonly ['integer', string]; + +interface PhysicalInput { + readonly requestedPath: string; + readonly canonicalPath: string; + readonly device: string; + readonly inode: string; +} + +interface ColumnFingerprint { + readonly name: string; + readonly type: string; + readonly notNull: boolean; + readonly defaultValue: string | null; + readonly primaryKeyPosition: number; + readonly hidden: number; +} + +interface ForeignKeyFingerprint { + readonly table: string; + readonly from: string; + readonly to: string; + readonly onUpdate: string; + readonly onDelete: string; + readonly match: string; +} + +interface IndexFingerprint { + readonly name: string | null; + readonly origin: string; + readonly unique: boolean; + readonly partial: boolean; + readonly columns: readonly { + readonly name: string; + readonly descending: boolean; + readonly collation: string; + }[]; +} + +interface TableFingerprint { + readonly name: string; + readonly columns: readonly ColumnFingerprint[]; + readonly foreignKeys: readonly ForeignKeyFingerprint[]; + readonly indexes: readonly IndexFingerprint[]; + readonly checks: readonly string[]; + readonly autoIncrement: boolean; + readonly withoutRowid: boolean; + readonly strict: boolean; +} + +interface SchemaFingerprint { + readonly userVersion: number; + readonly tables: readonly TableFingerprint[]; +} + +interface CountedValue { + readonly value: T; + count: number; +} + +interface LogicalState { + readonly boards: Map; + readonly tasks: Map; + readonly wishGroups: Map; + readonly hireRoster: Map; + readonly meta: Map; + readonly taskDependencies: Map; + readonly stageLog: Map>; + readonly taskEvents: Map>; +} + +interface DatabaseImage { + readonly schemaFingerprint: string; + readonly logicalDigest: string; + readonly state: LogicalState; +} + +const NORMAL_OPEN_GUIDANCE = + 'Open the database once with Genie’s normal current open path (for example `genie board`) to normalize supported additive history, then retry reconciliation.'; + +const EXPECTED_COLUMNS: Readonly< + Record | 'task_dependencies', readonly ColumnFingerprint[]> +> = { + boards: [ + column('created_at', 'INTEGER', true), + column('id', 'TEXT', false, null, 1), + column('lanes', 'TEXT', false), + column('name', 'TEXT', true), + ], + hire_roster: [ + column('agent_adapter_id', 'TEXT', true, null, 2), + column('hired_at', 'INTEGER', true), + column('profile', 'TEXT', false), + column('state', 'TEXT', true), + column('wish', 'TEXT', true, null, 1), + column('worktree', 'TEXT', true), + ], + meta: [column('key', 'TEXT', false, null, 1), column('value', 'TEXT', true)], + stage_log: [ + column('created_at', 'INTEGER', true), + column('id', 'INTEGER', false, null, 1), + column('note', 'TEXT', false), + column('stage', 'TEXT', true), + column('task_id', 'TEXT', true), + ], + task_dependencies: [column('depends_on_id', 'TEXT', true, null, 2), column('task_id', 'TEXT', true, null, 1)], + task_events: [ + column('author', 'TEXT', false), + column('author_kind', 'TEXT', false), + column('created_at', 'INTEGER', true), + column('id', 'INTEGER', false, null, 1), + column('kind', 'TEXT', true), + column('note', 'TEXT', false), + column('task_id', 'TEXT', true), + ], + tasks: [ + column('agent_kind', 'TEXT', false), + column('blocked_by', 'TEXT', false), + column('blocked_reason', 'TEXT', false), + column('board_id', 'TEXT', false), + column('claimed_at', 'INTEGER', false), + column('claimed_by', 'TEXT', false), + column('created_at', 'INTEGER', true), + column('group_name', 'TEXT', false), + column('heartbeat_at', 'INTEGER', false), + column('id', 'TEXT', false, null, 1), + column('lane', 'TEXT', false), + column('status', 'TEXT', true), + column('title', 'TEXT', true), + column('updated_at', 'INTEGER', true), + column('wish', 'TEXT', false), + ], + wish_groups: [ + column('assignee', 'TEXT', false), + column('completed_at', 'INTEGER', false), + column('created_at', 'INTEGER', true), + column('depends_on', 'TEXT', true, "'[]'"), + column('name', 'TEXT', true, null, 2), + column('started_at', 'INTEGER', false), + column('status', 'TEXT', true), + column('updated_at', 'INTEGER', true), + column('wish', 'TEXT', true, null, 1), + ], +}; + +const EXPECTED_FOREIGN_KEYS: Readonly> = { + boards: [], + hire_roster: [], + meta: [], + stage_log: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], + task_dependencies: [ + foreignKey('tasks', 'depends_on_id', 'id', 'NO ACTION', 'CASCADE'), + foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE'), + ], + task_events: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], + tasks: [foreignKey('boards', 'board_id', 'id', 'NO ACTION', 'SET NULL')], + wish_groups: [], +}; + +const EXPECTED_INDEXES: Readonly> = { + boards: [index(null, 'pk', true, ['id']), index(null, 'u', true, ['name'])], + hire_roster: [index(null, 'pk', true, ['wish', 'agent_adapter_id'])], + meta: [index(null, 'pk', true, ['key'])], + stage_log: [index('idx_stage_log_task', 'c', false, ['task_id'])], + task_dependencies: [ + index('idx_task_deps_dep', 'c', false, ['depends_on_id']), + index(null, 'pk', true, ['task_id', 'depends_on_id']), + ], + task_events: [index('idx_task_events_task', 'c', false, ['task_id'])], + tasks: [index('idx_tasks_status', 'c', false, ['status']), index(null, 'pk', true, ['id'])], + wish_groups: [index(null, 'pk', true, ['wish', 'name'])], +}; + +const EXPECTED_CHECKS: Readonly> = { + boards: [], + hire_roster: [], + meta: [], + stage_log: [], + task_dependencies: [], + task_events: [], + tasks: ['status-in:blocked,done,in_progress,ready'], + wish_groups: ['status-in:blocked,done,in_progress,ready'], +}; + +const TABLE_NAMES = [ + 'boards', + 'hire_roster', + 'meta', + 'stage_log', + 'task_dependencies', + 'task_events', + 'tasks', + 'wish_groups', +] as const satisfies readonly ReconciliationTableName[]; + +const EXPLICIT_INDEX_TO_TABLE = new Map([ + ['idx_stage_log_task', 'stage_log'], + ['idx_task_deps_dep', 'task_dependencies'], + ['idx_task_events_task', 'task_events'], + ['idx_tasks_status', 'tasks'], +]); + +function column( + name: string, + type: string, + notNull: boolean, + defaultValue: string | null = null, + primaryKeyPosition = 0, +): ColumnFingerprint { + return { name, type, notNull, defaultValue, primaryKeyPosition, hidden: 0 }; +} + +function foreignKey( + table: string, + from: string, + to: string, + onUpdate: string, + onDelete: string, +): ForeignKeyFingerprint { + return { table, from, to, onUpdate, onDelete, match: 'NONE' }; +} + +function index(name: string | null, origin: string, unique: boolean, columns: readonly string[]): IndexFingerprint { + return { + name, + origin, + unique, + partial: false, + columns: columns.map((columnName) => ({ name: columnName, descending: false, collation: 'BINARY' })), + }; +} + +function error(code: ReconciliationErrorCode, message: string, guidance?: string): ReconciliationError { + return new ReconciliationError(code, message, guidance); +} + +function staleSchema(): never { + throw error('stale-current-schema', 'The database has a stale same-version additive schema.', NORMAL_OPEN_GUIDANCE); +} + +function unsupportedSchema(): never { + throw error('unsupported-schema', 'The database does not have the exact supported current Genie schema.'); +} + +function canonicalScalar(value: string | bigint | null): CanonicalScalar { + if (value === null) return ['null']; + if (typeof value === 'string') return ['string', value]; + return ['integer', value.toString()]; +} + +function canonicalTuple(values: readonly (string | bigint | null)[]): string { + return JSON.stringify(values.map(canonicalScalar)); +} + +function digestCanonical(value: unknown): string { + return createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +function compareCanonical(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function schemaInteger(value: unknown): number { + const numeric = typeof value === 'bigint' ? Number(value) : value; + if (typeof numeric !== 'number' || !Number.isSafeInteger(numeric)) unsupportedSchema(); + return numeric; +} + +function rowString(row: Record, name: string): string { + const value = row[name]; + if (typeof value !== 'string') throw error('invalid-data', 'A database row contains an invalid value type.'); + return value; +} + +function rowNullableString(row: Record, name: string): string | null { + const value = row[name]; + if (value === null) return null; + if (typeof value !== 'string') throw error('invalid-data', 'A database row contains an invalid value type.'); + return value; +} + +function rowInteger(row: Record, name: string): bigint { + const value = row[name]; + if (typeof value !== 'bigint') throw error('invalid-data', 'A database row contains an invalid value type.'); + return value; +} + +function rowNullableInteger(row: Record, name: string): bigint | null { + const value = row[name]; + if (value === null) return null; + if (typeof value !== 'bigint') throw error('invalid-data', 'A database row contains an invalid value type.'); + return value; +} + +function resolvePhysicalInput(path: string): PhysicalInput { + if (path === ':memory:') { + throw error('input-unavailable', 'Reconciliation requires a physical database file.'); + } + try { + const canonicalPath = normalize(realpathSync(path)); + const stats = statSync(canonicalPath); + if (!stats.isFile()) throw new Error('not a regular file'); + if (stats.size > MAX_DATABASE_BYTES) { + throw error('input-too-large', 'The database exceeds the bounded reconciliation input size.'); + } + return { + requestedPath: path, + canonicalPath, + device: String(stats.dev), + inode: String(stats.ino), + }; + } catch (caught) { + if (caught instanceof ReconciliationError) throw caught; + throw error('input-unavailable', 'A reconciliation input is unavailable or is not a regular file.'); + } +} + +function revalidatePhysicalInput(input: PhysicalInput): void { + let current: PhysicalInput; + try { + current = resolvePhysicalInput(input.requestedPath); + } catch { + throw error('input-changed', 'A reconciliation input changed identity while it was being read.'); + } + if ( + current.canonicalPath !== input.canonicalPath || + current.device !== input.device || + current.inode !== input.inode + ) { + throw error('input-changed', 'A reconciliation input changed identity while it was being read.'); + } +} + +function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { + return left.canonicalPath === right.canonicalPath || (left.device === right.device && left.inode === right.inode); +} + +function readInventory(db: Database): Array<{ type: string; name: string; tableName: string; sql: string | null }> { + const rows = db + .query('SELECT type, name, tbl_name AS tableName, sql FROM sqlite_schema ORDER BY type, name') + .all() as Array>; + return rows.map((row) => { + const type = rowString(row, 'type'); + const name = rowString(row, 'name'); + const tableName = rowString(row, 'tableName'); + const sqlValue = row.sql; + if (sqlValue !== null && typeof sqlValue !== 'string') unsupportedSchema(); + return { type, name, tableName, sql: sqlValue }; + }); +} + +function collectInventoryObject( + object: { type: string; name: string; tableName: string; sql: string | null }, + expectedTables: ReadonlySet, + actualTables: Set, + actualExplicitIndexes: Set, + tableSql: Map, +): void { + if (object.type === 'table') { + if (object.name === 'sqlite_sequence' && object.tableName === 'sqlite_sequence') return; + if (!expectedTables.has(object.name) || object.tableName !== object.name || object.sql === null) { + unsupportedSchema(); + } + actualTables.add(object.name); + tableSql.set(object.name, object.sql); + return; + } + if (object.type === 'index') { + if (object.sql === null) { + if (!object.name.startsWith('sqlite_autoindex_') || !expectedTables.has(object.tableName)) { + unsupportedSchema(); + } + return; + } + const expectedTable = EXPLICIT_INDEX_TO_TABLE.get(object.name); + if (expectedTable === undefined || expectedTable !== object.tableName) unsupportedSchema(); + actualExplicitIndexes.add(object.name); + return; + } + // Views, triggers, and every other non-internal schema object are rejected + // before integrity checks or logical row reads can grant planning authority. + unsupportedSchema(); +} + +function validateInventory( + inventory: readonly { type: string; name: string; tableName: string; sql: string | null }[], +): Map { + const expectedTables = new Set(TABLE_NAMES); + const actualTables = new Set(); + const actualExplicitIndexes = new Set(); + const tableSql = new Map(); + + for (const object of inventory) { + collectInventoryObject(object, expectedTables, actualTables, actualExplicitIndexes, tableSql); + } + + for (const table of TABLE_NAMES) { + if (!actualTables.has(table)) staleSchema(); + } + for (const indexName of EXPLICIT_INDEX_TO_TABLE.keys()) { + if (!actualExplicitIndexes.has(indexName)) staleSchema(); + } + if (!inventory.some((object) => object.type === 'table' && object.name === 'sqlite_sequence')) { + unsupportedSchema(); + } + return tableSql; +} + +function tableColumns(db: Database, table: ReconciliationTableName): readonly ColumnFingerprint[] { + const rows = db.query(`PRAGMA table_xinfo(${table})`).all() as Array>; + const observed = rows + .map((row) => ({ + name: rowString(row, 'name'), + type: rowString(row, 'type').toUpperCase(), + notNull: schemaInteger(row.notnull) === 1, + defaultValue: row.dflt_value === null ? null : rowString(row, 'dflt_value'), + primaryKeyPosition: schemaInteger(row.pk), + hidden: schemaInteger(row.hidden), + })) + .sort((left, right) => compareCanonical(left.name, right.name)); + + const expected = EXPECTED_COLUMNS[table]; + const observedNames = new Set(observed.map((item) => item.name)); + if (observed.some((item) => !expected.some((candidate) => candidate.name === item.name))) unsupportedSchema(); + if (expected.some((item) => !observedNames.has(item.name))) staleSchema(); + if (JSON.stringify(observed) !== JSON.stringify(expected)) unsupportedSchema(); + return observed; +} + +function tableForeignKeys(db: Database, table: ReconciliationTableName): readonly ForeignKeyFingerprint[] { + const rows = db.query(`PRAGMA foreign_key_list(${table})`).all() as Array>; + const observed = rows + .map((row) => ({ + table: rowString(row, 'table'), + from: rowString(row, 'from'), + to: rowString(row, 'to'), + onUpdate: rowString(row, 'on_update'), + onDelete: rowString(row, 'on_delete'), + match: rowString(row, 'match'), + })) + .sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); + if (JSON.stringify(observed) !== JSON.stringify(EXPECTED_FOREIGN_KEYS[table])) unsupportedSchema(); + return observed; +} + +function indexColumns(db: Database, name: string): IndexFingerprint['columns'] { + const rows = db + .query('SELECT seqno, cid, name, desc, coll, key FROM pragma_index_xinfo(?) ORDER BY seqno') + .all(name) as Array>; + const keyRows = rows.filter((row) => schemaInteger(row.key) === 1); + return keyRows.map((row) => { + if (schemaInteger(row.cid) < 0) unsupportedSchema(); + return { + name: rowString(row, 'name'), + descending: schemaInteger(row.desc) === 1, + collation: rowString(row, 'coll'), + }; + }); +} + +function tableIndexes(db: Database, table: ReconciliationTableName): readonly IndexFingerprint[] { + const rows = db.query(`PRAGMA index_list(${table})`).all() as Array>; + const observed = rows + .map((row) => { + const name = rowString(row, 'name'); + const origin = rowString(row, 'origin'); + return { + name: origin === 'c' ? name : null, + origin, + unique: schemaInteger(row.unique) === 1, + partial: schemaInteger(row.partial) === 1, + columns: indexColumns(db, name), + }; + }) + .sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); + const expected = [...EXPECTED_INDEXES[table]].sort((left, right) => + compareCanonical(JSON.stringify(left), JSON.stringify(right)), + ); + if (JSON.stringify(observed) !== JSON.stringify(expected)) unsupportedSchema(); + return observed; +} + +interface SqlToken { + readonly kind: 'word' | 'string' | 'punctuation'; + readonly value: string; +} + +function skipSqlTrivia(sql: string, start: number): number { + let index = start; + for (;;) { + while (index < sql.length && /\s/.test(sql[index])) index++; + if (sql[index] === '-' && sql[index + 1] === '-') { + index += 2; + while (index < sql.length && sql[index] !== '\n') index++; + continue; + } + if (sql[index] === '/' && sql[index + 1] === '*') { + const end = sql.indexOf('*/', index + 2); + if (end < 0) unsupportedSchema(); + index = end + 2; + continue; + } + return index; + } +} + +function readQuotedSqlToken( + sql: string, + start: number, + closing: string, + kind: SqlToken['kind'], +): { token: SqlToken; nextIndex: number } { + let value = ''; + let index = start + 1; + while (index < sql.length) { + if (sql[index] !== closing) { + value += sql[index]; + index++; + continue; + } + if (closing !== ']' && sql[index + 1] === closing) { + value += closing; + index += 2; + continue; + } + return { + token: { kind, value: kind === 'word' ? value.toLowerCase() : value }, + nextIndex: index + 1, + }; + } + unsupportedSchema(); +} + +function readSqlToken(sql: string, index: number): { token: SqlToken; nextIndex: number } { + const char = sql[index]; + if (char === "'") return readQuotedSqlToken(sql, index, "'", 'string'); + if (char === '"') return readQuotedSqlToken(sql, index, '"', 'word'); + if (char === '`') return readQuotedSqlToken(sql, index, '`', 'word'); + if (char === '[') return readQuotedSqlToken(sql, index, ']', 'word'); + if (/[A-Za-z_]/.test(char)) { + const start = index; + let cursor = index + 1; + while (cursor < sql.length && /[A-Za-z0-9_$]/.test(sql[cursor])) cursor++; + return { token: { kind: 'word', value: sql.slice(start, cursor).toLowerCase() }, nextIndex: cursor }; + } + if ('(),'.includes(char)) { + return { token: { kind: 'punctuation', value: char }, nextIndex: index + 1 }; + } + return { token: { kind: 'word', value: char.toLowerCase() }, nextIndex: index + 1 }; +} + +function sqlTokens(sql: string): readonly SqlToken[] { + const tokens: SqlToken[] = []; + let index = 0; + while (index < sql.length) { + index = skipSqlTrivia(sql, index); + if (index >= sql.length) break; + const parsed = readSqlToken(sql, index); + tokens.push(parsed.token); + index = parsed.nextIndex; + } + return tokens; +} + +function readCheckExpression( + tokens: readonly SqlToken[], + openingIndex: number, +): { expression: readonly SqlToken[]; closingIndex: number } { + if (tokens[openingIndex]?.value !== '(') unsupportedSchema(); + let depth = 1; + const expression: SqlToken[] = []; + let index = openingIndex + 1; + while (index < tokens.length && depth > 0) { + const current = tokens[index]; + if (current.value === '(') depth++; + if (current.value === ')') depth--; + if (depth > 0) expression.push(current); + index++; + } + if (depth !== 0) unsupportedSchema(); + return { expression, closingIndex: index - 1 }; +} + +function normalizeStatusCheck(expression: readonly SqlToken[]): string { + if ( + expression.length !== 11 || + expression[0].kind !== 'word' || + expression[0].value !== 'status' || + expression[1].kind !== 'word' || + expression[1].value !== 'in' || + expression[2].value !== '(' || + expression[10].value !== ')' + ) { + unsupportedSchema(); + } + const values: string[] = []; + for (let valueIndex = 3; valueIndex < 10; valueIndex += 2) { + const value = expression[valueIndex]; + if (value.kind !== 'string') unsupportedSchema(); + values.push(value.value); + if (valueIndex < 9 && expression[valueIndex + 1]?.value !== ',') unsupportedSchema(); + } + return `status-in:${values.sort(compareCanonical).join(',')}`; +} + +function normalizeChecks(sql: string): readonly string[] { + const tokens = sqlTokens(sql); + const checks: string[] = []; + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + if (token.kind !== 'word' || token.value !== 'check') continue; + const parsed = readCheckExpression(tokens, index + 1); + checks.push(normalizeStatusCheck(parsed.expression)); + index = parsed.closingIndex; + } + return checks.sort(compareCanonical); +} + +/* + * Keep keyword recognition on the same comment/string-aware token stream as + * CHECK extraction so hostile DDL text cannot smuggle a false match. + */ +function hasKeyword(sql: string, keyword: string): boolean { + return sqlTokens(sql).some((token) => token.kind === 'word' && token.value === keyword); +} + +function readSchemaFingerprint(db: Database): SchemaFingerprint { + const versionRow = db.query('PRAGMA user_version').get() as Record | null; + if (versionRow === null || schemaInteger(versionRow.user_version) !== CURRENT_SCHEMA_VERSION) unsupportedSchema(); + + const tableSql = validateInventory(readInventory(db)); + const tableList = db.query('PRAGMA table_list').all() as Array>; + const mainTables = new Map>(); + for (const row of tableList) { + if (rowString(row, 'schema') !== 'main') continue; + mainTables.set(rowString(row, 'name'), row); + } + + const tables = TABLE_NAMES.map((name): TableFingerprint => { + const listRow = mainTables.get(name); + if (listRow === undefined) staleSchema(); + if ( + rowString(listRow, 'type') !== 'table' || + schemaInteger(listRow.wr) !== 0 || + schemaInteger(listRow.strict) !== 0 + ) { + unsupportedSchema(); + } + const sql = tableSql.get(name); + if (sql === undefined) staleSchema(); + if (hasKeyword(sql, 'deferrable') || hasKeyword(sql, 'initially')) unsupportedSchema(); + const checks = normalizeChecks(sql); + if (JSON.stringify(checks) !== JSON.stringify(EXPECTED_CHECKS[name])) unsupportedSchema(); + const autoIncrement = hasKeyword(sql, 'autoincrement'); + if (autoIncrement !== (name === 'stage_log' || name === 'task_events')) unsupportedSchema(); + return { + name, + columns: tableColumns(db, name), + foreignKeys: tableForeignKeys(db, name), + indexes: tableIndexes(db, name), + checks, + autoIncrement, + withoutRowid: false, + strict: false, + }; + }); + return { userVersion: CURRENT_SCHEMA_VERSION, tables }; +} + +function checkIntegrity(db: Database): void { + const integrity = db.query('PRAGMA integrity_check(1)').get() as Record | null; + const result = integrity === null ? null : Object.values(integrity)[0]; + if (result !== 'ok') throw error('integrity-failed', 'A reconciliation input failed SQLite integrity validation.'); + const foreignKeyFailure = db.query('PRAGMA foreign_key_check').get(); + if (foreignKeyFailure !== null) { + throw error('integrity-failed', 'A reconciliation input failed SQLite foreign-key validation.'); + } +} + +function boundedTableRows( + db: Database, + table: ReconciliationTableName, + sql: string, +): Array> { + const countRow = db.query(`SELECT count(*) AS rowCount FROM ${table}`).get() as Record; + const count = rowInteger(countRow, 'rowCount'); + if (count > BigInt(MAX_ROWS_PER_TABLE)) { + throw error('invalid-data', 'A reconciliation input exceeds the bounded row-count limit.'); + } + return db.query(sql).all() as Array>; +} + +function countValues(values: readonly T[], key: (value: T) => string): Map> { + const counts = new Map>(); + for (const value of values) { + const identity = key(value); + const existing = counts.get(identity); + if (existing === undefined) counts.set(identity, { value, count: 1 }); + else existing.count++; + } + return counts; +} + +function keyedValues(values: readonly T[], key: (value: T) => string): Map { + const result = new Map(); + for (const value of values) { + const identity = key(value); + if (result.has(identity)) throw error('invalid-data', 'A database table contains a duplicate logical key.'); + result.set(identity, value); + } + return result; +} + +function readLogicalState(db: Database): LogicalState { + const boards = boundedTableRows(db, 'boards', 'SELECT id, name, created_at, lanes FROM boards').map( + (row): BoardReconciliationRow => ({ + id: rowString(row, 'id'), + name: rowString(row, 'name'), + createdAt: rowInteger(row, 'created_at'), + lanes: rowNullableString(row, 'lanes'), + }), + ); + const tasks = boundedTableRows( + db, + 'tasks', + `SELECT id, board_id, title, status, claimed_by, claimed_at, wish, group_name, + created_at, updated_at, lane, agent_kind, heartbeat_at, blocked_by, blocked_reason + FROM tasks`, + ).map( + (row): TaskReconciliationRow => ({ + id: rowString(row, 'id'), + boardId: rowNullableString(row, 'board_id'), + title: rowString(row, 'title'), + status: rowString(row, 'status'), + claimedBy: rowNullableString(row, 'claimed_by'), + claimedAt: rowNullableInteger(row, 'claimed_at'), + wish: rowNullableString(row, 'wish'), + groupName: rowNullableString(row, 'group_name'), + createdAt: rowInteger(row, 'created_at'), + updatedAt: rowInteger(row, 'updated_at'), + lane: rowNullableString(row, 'lane'), + agentKind: rowNullableString(row, 'agent_kind'), + heartbeatAt: rowNullableInteger(row, 'heartbeat_at'), + blockedBy: rowNullableString(row, 'blocked_by'), + blockedReason: rowNullableString(row, 'blocked_reason'), + }), + ); + const wishGroups = boundedTableRows( + db, + 'wish_groups', + `SELECT wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at + FROM wish_groups`, + ).map( + (row): WishGroupReconciliationRow => ({ + wish: rowString(row, 'wish'), + name: rowString(row, 'name'), + status: rowString(row, 'status'), + dependsOn: rowString(row, 'depends_on'), + assignee: rowNullableString(row, 'assignee'), + startedAt: rowNullableInteger(row, 'started_at'), + completedAt: rowNullableInteger(row, 'completed_at'), + createdAt: rowInteger(row, 'created_at'), + updatedAt: rowInteger(row, 'updated_at'), + }), + ); + const hireRoster = boundedTableRows( + db, + 'hire_roster', + 'SELECT wish, agent_adapter_id, profile, worktree, hired_at, state FROM hire_roster', + ).map( + (row): HireRosterReconciliationRow => ({ + wish: rowString(row, 'wish'), + agentAdapterId: rowString(row, 'agent_adapter_id'), + profile: rowNullableString(row, 'profile'), + worktree: rowString(row, 'worktree'), + hiredAt: rowInteger(row, 'hired_at'), + state: rowString(row, 'state'), + }), + ); + const meta = boundedTableRows(db, 'meta', 'SELECT key, value FROM meta').map( + (row): MetaReconciliationRow => ({ + key: rowString(row, 'key'), + value: rowString(row, 'value'), + }), + ); + const dependencies = boundedTableRows( + db, + 'task_dependencies', + 'SELECT task_id, depends_on_id FROM task_dependencies', + ).map( + (row): TaskDependencyReconciliationRow => ({ + taskId: rowString(row, 'task_id'), + dependsOnId: rowString(row, 'depends_on_id'), + }), + ); + const stages = boundedTableRows(db, 'stage_log', 'SELECT id, task_id, stage, note, created_at FROM stage_log').map( + (row): StageLogReconciliationValue => { + rowInteger(row, 'id'); + return { + taskId: rowString(row, 'task_id'), + stage: rowString(row, 'stage'), + note: rowNullableString(row, 'note'), + createdAt: rowInteger(row, 'created_at'), + }; + }, + ); + const events = boundedTableRows( + db, + 'task_events', + 'SELECT id, task_id, kind, note, author_kind, author, created_at FROM task_events', + ).map((row): TaskEventReconciliationValue => { + rowInteger(row, 'id'); + return { + taskId: rowString(row, 'task_id'), + kind: rowString(row, 'kind'), + note: rowNullableString(row, 'note'), + authorKind: rowNullableString(row, 'author_kind'), + author: rowNullableString(row, 'author'), + createdAt: rowInteger(row, 'created_at'), + }; + }); + + const totalRows = + boards.length + + tasks.length + + wishGroups.length + + hireRoster.length + + meta.length + + dependencies.length + + stages.length + + events.length; + if (totalRows > MAX_TOTAL_ROWS) { + throw error('invalid-data', 'A reconciliation input exceeds the bounded total row-count limit.'); + } + + return { + boards: keyedValues(boards, boardKey), + tasks: keyedValues(tasks, taskKey), + wishGroups: keyedValues(wishGroups, wishGroupKey), + hireRoster: keyedValues(hireRoster, hireRosterKey), + meta: keyedValues(meta, metaKey), + taskDependencies: keyedValues(dependencies, dependencyKey), + stageLog: countValues(stages, stageLogKey), + taskEvents: countValues(events, taskEventKey), + }; +} + +function loadDatabaseImage(input: PhysicalInput): DatabaseImage { + let db: Database; + try { + db = new Database(input.canonicalPath, { readonly: true, strict: true, safeIntegers: true }); + } catch { + throw error('malformed-database', 'A reconciliation input is not a readable SQLite database.'); + } + let transactionOpen = false; + try { + db.exec(`PRAGMA busy_timeout = ${READ_BUSY_TIMEOUT_MS}`); + db.exec('PRAGMA query_only = ON'); + db.exec('PRAGMA trusted_schema = OFF'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('BEGIN'); + transactionOpen = true; + const schema = readSchemaFingerprint(db); + // Integrity is intentionally after the closed inventory: a guest trigger or + // virtual object is rejected before any logical data receives authority. + checkIntegrity(db); + const state = readLogicalState(db); + const schemaFingerprint = digestCanonical(['genie-schema-v1', schema]); + const logicalDigest = logicalStateDigest(schemaFingerprint, state); + db.exec('COMMIT'); + transactionOpen = false; + return { schemaFingerprint, logicalDigest, state }; + } catch (caught) { + if (transactionOpen) { + try { + db.exec('ROLLBACK'); + } catch { + // The original bounded failure is authoritative. + } + } + if (caught instanceof ReconciliationError) throw caught; + throw error('malformed-database', 'A reconciliation input could not be validated as SQLite.'); + } finally { + db.close(); + } +} + +function boardKey(row: BoardReconciliationRow): string { + return canonicalTuple([row.id]); +} + +function taskKey(row: TaskReconciliationRow): string { + return canonicalTuple([row.id]); +} + +function wishGroupKey(row: WishGroupReconciliationRow): string { + return canonicalTuple([row.wish, row.name]); +} + +function hireRosterKey(row: HireRosterReconciliationRow): string { + return canonicalTuple([row.wish, row.agentAdapterId]); +} + +function metaKey(row: MetaReconciliationRow): string { + return canonicalTuple([row.key]); +} + +function dependencyKey(row: TaskDependencyReconciliationRow): string { + return canonicalTuple([row.taskId, row.dependsOnId]); +} + +function stageLogKey(row: StageLogReconciliationValue): string { + return canonicalTuple([row.taskId, row.stage, row.note, row.createdAt]); +} + +function taskEventKey(row: TaskEventReconciliationValue): string { + return canonicalTuple([row.taskId, row.kind, row.note, row.authorKind, row.author, row.createdAt]); +} + +function boardValues(row: BoardReconciliationRow): readonly (string | bigint | null)[] { + return [row.id, row.name, row.createdAt, row.lanes]; +} + +function taskValues(row: TaskReconciliationRow): readonly (string | bigint | null)[] { + return [ + row.id, + row.boardId, + row.title, + row.status, + row.claimedBy, + row.claimedAt, + row.wish, + row.groupName, + row.createdAt, + row.updatedAt, + row.lane, + row.agentKind, + row.heartbeatAt, + row.blockedBy, + row.blockedReason, + ]; +} + +function wishGroupValues(row: WishGroupReconciliationRow): readonly (string | bigint | null)[] { + return [ + row.wish, + row.name, + row.status, + row.dependsOn, + row.assignee, + row.startedAt, + row.completedAt, + row.createdAt, + row.updatedAt, + ]; +} + +function hireRosterValues(row: HireRosterReconciliationRow): readonly (string | bigint | null)[] { + return [row.wish, row.agentAdapterId, row.profile, row.worktree, row.hiredAt, row.state]; +} + +function metaValues(row: MetaReconciliationRow): readonly (string | bigint | null)[] { + return [row.key, row.value]; +} + +function dependencyValues(row: TaskDependencyReconciliationRow): readonly (string | bigint | null)[] { + return [row.taskId, row.dependsOnId]; +} + +function stageValues(row: StageLogReconciliationValue): readonly (string | bigint | null)[] { + return [row.taskId, row.stage, row.note, row.createdAt]; +} + +function eventValues(row: TaskEventReconciliationValue): readonly (string | bigint | null)[] { + return [row.taskId, row.kind, row.note, row.authorKind, row.author, row.createdAt]; +} + +function mapDigestRows(map: ReadonlyMap, values: (row: T) => readonly (string | bigint | null)[]) { + return [...map.values()] + .map((row) => values(row).map(canonicalScalar)) + .sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); +} + +function mapDigestCounts( + map: ReadonlyMap>, + values: (row: T) => readonly (string | bigint | null)[], +) { + return [...map.values()] + .map((entry) => [values(entry.value).map(canonicalScalar), entry.count] as const) + .sort((left, right) => compareCanonical(JSON.stringify(left[0]), JSON.stringify(right[0]))); +} + +function logicalStateDigest(schemaFingerprint: string, state: LogicalState): string { + return digestCanonical([ + 'genie-logical-image-v1', + schemaFingerprint, + { + boards: mapDigestRows(state.boards, boardValues), + hireRoster: mapDigestRows(state.hireRoster, hireRosterValues), + meta: mapDigestRows(state.meta, metaValues), + stageLog: mapDigestCounts(state.stageLog, stageValues), + taskDependencies: mapDigestRows(state.taskDependencies, dependencyValues), + taskEvents: mapDigestCounts(state.taskEvents, eventValues), + tasks: mapDigestRows(state.tasks, taskValues), + wishGroups: mapDigestRows(state.wishGroups, wishGroupValues), + }, + ]); +} + +function cloneCountMap(source: ReadonlyMap>): Map> { + return new Map([...source].map(([key, entry]) => [key, { value: entry.value, count: entry.count }])); +} + +function cloneState(source: LogicalState): LogicalState { + return { + boards: new Map(source.boards), + tasks: new Map(source.tasks), + wishGroups: new Map(source.wishGroups), + hireRoster: new Map(source.hireRoster), + meta: new Map(source.meta), + taskDependencies: new Map(source.taskDependencies), + stageLog: cloneCountMap(source.stageLog), + taskEvents: cloneCountMap(source.taskEvents), + }; +} + +function conflictKeyDigest(table: ReconciliationTableName, key: string): string { + return digestCanonical(['genie-reconciliation-conflict-key-v1', table, key]); +} + +function addConflict( + conflicts: ReconciliationConflict[], + table: ReconciliationTableName, + reason: ReconciliationConflict['reason'], + key: string, + side?: ReconciliationConflict['side'], +): void { + conflicts.push({ table, reason, keyDigest: conflictKeyDigest(table, key), ...(side === undefined ? {} : { side }) }); +} + +function rowEqual(left: T, right: T, values: (row: T) => readonly (string | bigint | null)[]): boolean { + return canonicalTuple(values(left)) === canonicalTuple(values(right)); +} + +function reconcileBidirectionalKeyed( + table: KeyedTableName, + left: Map, + right: Map, + values: (row: T) => readonly (string | bigint | null)[], + conflicts: ReconciliationConflict[], + excludedKey?: string, +): void { + const keys = new Set([...left.keys(), ...right.keys()]); + for (const key of [...keys].sort(compareCanonical)) { + if (key === excludedKey) continue; + const leftRow = left.get(key); + const rightRow = right.get(key); + if (leftRow === undefined && rightRow !== undefined) left.set(key, rightRow); + else if (rightRow === undefined && leftRow !== undefined) right.set(key, leftRow); + else if (leftRow !== undefined && rightRow !== undefined && !rowEqual(leftRow, rightRow, values)) { + addConflict(conflicts, table, 'same-key-difference', key); + } + } +} + +function reconcileDirectionalKeyed( + source: ReadonlyMap, + destination: Map, + values: (row: T) => readonly (string | bigint | null)[], + excludedKey?: string, +): void { + for (const key of [...source.keys()].sort(compareCanonical)) { + if (key === excludedKey) continue; + const sourceRow = source.get(key); + if (sourceRow === undefined) continue; + const destinationRow = destination.get(key); + if (destinationRow === undefined || !rowEqual(sourceRow, destinationRow, values)) { + destination.set(key, sourceRow); + } + } +} + +function reconcileBidirectionalSet(left: Map, right: Map): void { + for (const [key, value] of left) if (!right.has(key)) right.set(key, value); + for (const [key, value] of right) if (!left.has(key)) left.set(key, value); +} + +function reconcileDirectionalSet(source: ReadonlyMap, destination: Map): void { + for (const [key, value] of source) if (!destination.has(key)) destination.set(key, value); +} + +function reconcileBidirectionalCounts( + left: Map>, + right: Map>, +): void { + const keys = new Set([...left.keys(), ...right.keys()]); + for (const key of keys) { + const leftEntry = left.get(key); + const rightEntry = right.get(key); + const count = Math.max(leftEntry?.count ?? 0, rightEntry?.count ?? 0); + const value = leftEntry?.value ?? rightEntry?.value; + if (value === undefined) continue; + left.set(key, { value, count }); + right.set(key, { value, count }); + } +} + +function reconcileDirectionalCounts( + source: ReadonlyMap>, + destination: Map>, +): void { + for (const [key, sourceEntry] of source) { + const destinationEntry = destination.get(key); + const count = Math.max(sourceEntry.count, destinationEntry?.count ?? 0); + destination.set(key, { value: destinationEntry?.value ?? sourceEntry.value, count }); + } +} + +function mappedStageEvent(stage: StageLogReconciliationValue): TaskEventReconciliationValue { + const direct = DIRECT_BACKFILL_KINDS.has(stage.stage); + return { + taskId: stage.taskId, + kind: direct ? stage.stage : 'comment', + note: direct ? stage.note : stage.note === null ? stage.stage : `${stage.stage}: ${stage.note}`, + authorKind: null, + author: null, + createdAt: stage.createdAt, + }; +} + +function markerInvariant(state: LogicalState): boolean { + const required = new Map(); + for (const entry of state.stageLog.values()) { + const mapped = mappedStageEvent(entry.value); + const key = taskEventKey(mapped); + required.set(key, (required.get(key) ?? 0) + entry.count); + } + for (const [key, count] of required) { + if ((state.taskEvents.get(key)?.count ?? 0) < count) return false; + } + return true; +} + +function parseMarker(value: string): bigint | null { + if (value.length > 128 || !/^[0-9]+$/.test(value)) return null; + try { + return BigInt(value); + } catch { + return null; + } +} + +function markerRow(state: LogicalState): MetaReconciliationRow | undefined { + return state.meta.get(canonicalTuple([BACKFILL_MARKER])); +} + +function validateExistingMarker( + state: LogicalState, + side: ReconciliationInputRole, + conflicts: ReconciliationConflict[], +): bigint | null | undefined { + const marker = markerRow(state); + if (marker === undefined) return undefined; + const parsed = parseMarker(marker.value); + if (parsed === null) { + addConflict(conflicts, 'meta', 'invalid-marker', canonicalTuple([BACKFILL_MARKER]), side); + return null; + } + if (!markerInvariant(state)) { + addConflict(conflicts, 'meta', 'marker-invariant-failed', canonicalTuple([BACKFILL_MARKER]), side); + } + return parsed; +} + +function reconcileMarker( + mode: ReconciliationMode, + leftInput: LogicalState, + rightInput: LogicalState, + leftTarget: LogicalState, + rightTarget: LogicalState, + leftRole: ReconciliationInputRole, + rightRole: ReconciliationInputRole, + conflicts: ReconciliationConflict[], +): void { + const leftValue = validateExistingMarker(leftInput, leftRole, conflicts); + const rightValue = validateExistingMarker(rightInput, rightRole, conflicts); + if (leftValue === null || rightValue === null) return; + + const values = [leftValue, rightValue].filter((value): value is bigint => value !== undefined); + if (values.length === 0) return; + const selected = values.reduce((smallest, value) => (value < smallest ? value : smallest)).toString(); + const key = canonicalTuple([BACKFILL_MARKER]); + const row = { key: BACKFILL_MARKER, value: selected }; + if (mode === 'bidirectional') { + leftTarget.meta.set(key, row); + rightTarget.meta.set(key, row); + } else { + rightTarget.meta.set(key, row); + } + + const targets: Array<[LogicalState, ReconciliationTargetRole]> = + mode === 'bidirectional' + ? [ + [leftTarget, 'left'], + [rightTarget, 'right'], + ] + : [[rightTarget, 'destination']]; + for (const [target, role] of targets) { + if (!markerInvariant(target)) { + addConflict(conflicts, 'meta', 'planned-marker-invariant-failed', key, role); + } + } +} + +function validatePlannedTargetIntegrity( + state: LogicalState, + side: ReconciliationTargetRole, + conflicts: ReconciliationConflict[], +): void { + const boardNames = new Set(); + for (const board of state.boards.values()) { + const nameKey = canonicalTuple([board.name]); + if (boardNames.has(nameKey)) { + addConflict(conflicts, 'boards', 'planned-integrity-failed', nameKey, side); + } + boardNames.add(nameKey); + } + for (const [key, task] of state.tasks) { + if (task.boardId !== null && !state.boards.has(canonicalTuple([task.boardId]))) { + addConflict(conflicts, 'tasks', 'planned-integrity-failed', key, side); + } + } + for (const [key, dependency] of state.taskDependencies) { + if ( + !state.tasks.has(canonicalTuple([dependency.taskId])) || + !state.tasks.has(canonicalTuple([dependency.dependsOnId])) + ) { + addConflict(conflicts, 'task_dependencies', 'planned-integrity-failed', key, side); + } + } + for (const [key, stage] of state.stageLog) { + if (!state.tasks.has(canonicalTuple([stage.value.taskId]))) { + addConflict(conflicts, 'stage_log', 'planned-integrity-failed', key, side); + } + } + for (const [key, event] of state.taskEvents) { + if (!state.tasks.has(canonicalTuple([event.value.taskId]))) { + addConflict(conflicts, 'task_events', 'planned-integrity-failed', key, side); + } + } +} + +function reconcileStates( + mode: ReconciliationMode, + leftInput: LogicalState, + rightInput: LogicalState, +): { left: LogicalState; right: LogicalState; conflicts: ReconciliationConflict[] } { + const left = cloneState(leftInput); + const right = cloneState(rightInput); + const conflicts: ReconciliationConflict[] = []; + const markerKey = canonicalTuple([BACKFILL_MARKER]); + + if (mode === 'bidirectional') { + reconcileBidirectionalKeyed('boards', left.boards, right.boards, boardValues, conflicts); + reconcileBidirectionalKeyed('tasks', left.tasks, right.tasks, taskValues, conflicts); + reconcileBidirectionalKeyed('wish_groups', left.wishGroups, right.wishGroups, wishGroupValues, conflicts); + reconcileBidirectionalKeyed('hire_roster', left.hireRoster, right.hireRoster, hireRosterValues, conflicts); + reconcileBidirectionalKeyed('meta', left.meta, right.meta, metaValues, conflicts, markerKey); + reconcileBidirectionalSet(left.taskDependencies, right.taskDependencies); + reconcileBidirectionalCounts(left.stageLog, right.stageLog); + reconcileBidirectionalCounts(left.taskEvents, right.taskEvents); + reconcileMarker('bidirectional', leftInput, rightInput, left, right, 'left', 'right', conflicts); + } else { + reconcileDirectionalKeyed(left.boards, right.boards, boardValues); + reconcileDirectionalKeyed(left.tasks, right.tasks, taskValues); + reconcileDirectionalKeyed(left.wishGroups, right.wishGroups, wishGroupValues); + reconcileDirectionalKeyed(left.hireRoster, right.hireRoster, hireRosterValues); + reconcileDirectionalKeyed(left.meta, right.meta, metaValues, markerKey); + reconcileDirectionalSet(left.taskDependencies, right.taskDependencies); + reconcileDirectionalCounts(left.stageLog, right.stageLog); + reconcileDirectionalCounts(left.taskEvents, right.taskEvents); + reconcileMarker('directional', leftInput, rightInput, left, right, 'source', 'destination', conflicts); + } + + if (conflicts.length === 0) { + if (mode === 'bidirectional') validatePlannedTargetIntegrity(left, 'left', conflicts); + validatePlannedTargetIntegrity(right, mode === 'bidirectional' ? 'right' : 'destination', conflicts); + } + conflicts.sort((a, b) => compareCanonical(JSON.stringify(a), JSON.stringify(b))); + return { left, right, conflicts }; +} + +function changedRows( + current: ReadonlyMap, + target: ReadonlyMap, + values: (row: T) => readonly (string | bigint | null)[], +): readonly T[] { + return [...target.entries()] + .filter(([key, row]) => { + const existing = current.get(key); + return existing === undefined || !rowEqual(existing, row, values); + }) + .sort(([left], [right]) => compareCanonical(left, right)) + .map(([, row]) => row); +} + +function addedSetRows(current: ReadonlyMap, target: ReadonlyMap): readonly T[] { + return [...target.entries()] + .filter(([key]) => !current.has(key)) + .sort(([left], [right]) => compareCanonical(left, right)) + .map(([, row]) => row); +} + +function historyAdditions( + current: ReadonlyMap>, + target: ReadonlyMap>, +): readonly HistoryAddition[] { + return [...target.entries()] + .map(([key, entry]) => ({ key, value: entry.value, count: entry.count - (current.get(key)?.count ?? 0) })) + .filter((entry) => entry.count > 0) + .sort((left, right) => compareCanonical(left.key, right.key)) + .map(({ value, count }) => ({ value, count })); +} + +function targetChanges(current: LogicalState, target: LogicalState): ReconciliationTargetChanges { + return { + boards: changedRows(current.boards, target.boards, boardValues), + tasks: changedRows(current.tasks, target.tasks, taskValues), + wishGroups: changedRows(current.wishGroups, target.wishGroups, wishGroupValues), + hireRoster: changedRows(current.hireRoster, target.hireRoster, hireRosterValues), + meta: changedRows(current.meta, target.meta, metaValues), + taskDependencies: addedSetRows(current.taskDependencies, target.taskDependencies), + stageLog: historyAdditions(current.stageLog, target.stageLog), + taskEvents: historyAdditions(current.taskEvents, target.taskEvents), + }; +} + +function emptyChanges(): ReconciliationTargetChanges { + return { + boards: [], + tasks: [], + wishGroups: [], + hireRoster: [], + meta: [], + taskDependencies: [], + stageLog: [], + taskEvents: [], + }; +} + +function changeCounts(changes: ReconciliationTargetChanges): ReconciliationChangeCounts { + return { + boards: changes.boards.length, + tasks: changes.tasks.length, + wishGroups: changes.wishGroups.length, + hireRoster: changes.hireRoster.length, + meta: changes.meta.length, + taskDependencies: changes.taskDependencies.length, + stageLog: changes.stageLog.reduce((total, addition) => total + addition.count, 0), + taskEvents: changes.taskEvents.reduce((total, addition) => total + addition.count, 0), + deletions: 0, + }; +} + +function hasChanges(changes: ReconciliationTargetChanges): boolean { + return Object.entries(changeCounts(changes)).some(([name, count]) => name !== 'deletions' && count > 0); +} + +function deepFreeze(value: T): T { + if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + } + return value; +} + +function makeReport( + mode: ReconciliationMode, + status: ReconciliationReportStatus, + sameDatabase: boolean, + schemaFingerprint: string | null, + targets: readonly ReconciliationTargetPlan[], + conflicts: readonly ReconciliationConflict[], + operationalFailure: ReconciliationDryRunReport['operationalFailure'] = null, +): ReconciliationDryRunReport { + return { + reportVersion: REPORT_VERSION, + dryRun: true, + mode, + status, + sameDatabase, + schemaFingerprint, + targets: targets.map((target) => ({ + role: target.role, + preimageDigest: target.preimageDigest, + postimageDigest: target.postimageDigest, + changes: changeCounts(target.changes), + })), + conflicts, + operationalFailure, + historyLimitation: IDENTICAL_HISTORY_ADDITION_LIMITATION, + }; +} + +function makeTargetPlan( + role: ReconciliationTargetRole, + input: PhysicalInput, + image: DatabaseImage, + target: LogicalState, + conflict: boolean, +): ReconciliationTargetPlan { + return { + role, + canonicalPath: input.canonicalPath, + preimageDigest: image.logicalDigest, + postimageDigest: conflict ? null : logicalStateDigest(image.schemaFingerprint, target), + changes: conflict ? emptyChanges() : targetChanges(image.state, target), + }; +} + +function buildTargetPlans( + mode: ReconciliationMode, + first: PhysicalInput, + second: PhysicalInput, + firstImage: DatabaseImage, + secondImage: DatabaseImage, + reconciled: { left: LogicalState; right: LogicalState; conflicts: readonly ReconciliationConflict[] }, +): readonly ReconciliationTargetPlan[] { + const conflict = reconciled.conflicts.length > 0; + if (mode === 'directional') { + return [makeTargetPlan('destination', second, secondImage, reconciled.right, conflict)]; + } + return [ + makeTargetPlan('left', first, firstImage, reconciled.left, conflict), + makeTargetPlan('right', second, secondImage, reconciled.right, conflict), + ]; +} + +function planStatus(conflict: boolean, sameDatabase: boolean, anyChanges: boolean): ReconciliationPlanStatus { + if (conflict) return 'conflict'; + if (sameDatabase) return 'same-database'; + return anyChanges ? 'changed' : 'no-op'; +} + +/** + * Load two exact current schemas read-only and produce a deeply frozen logical + * reconciliation plan. No schema normalization, snapshot, lock, or write is + * attempted here. + */ +export function planDatabaseReconciliation(request: ReconciliationRequest): ReconciliationPlan { + const mode = request.mode; + const firstRole: ReconciliationInputRole = mode === 'bidirectional' ? 'left' : 'source'; + const secondRole: ReconciliationInputRole = mode === 'bidirectional' ? 'right' : 'destination'; + const first = resolvePhysicalInput(mode === 'bidirectional' ? request.leftPath : request.sourcePath); + const second = resolvePhysicalInput(mode === 'bidirectional' ? request.rightPath : request.destinationPath); + const sameDatabase = samePhysicalInput(first, second); + + const firstImage = loadDatabaseImage(first); + const secondImage = sameDatabase ? firstImage : loadDatabaseImage(second); + revalidatePhysicalInput(first); + revalidatePhysicalInput(second); + if (firstImage.schemaFingerprint !== secondImage.schemaFingerprint) unsupportedSchema(); + + const reconciled = reconcileStates(mode, firstImage.state, secondImage.state); + const inputs: ReconciliationPlanInput[] = [ + { + role: firstRole, + canonicalPath: first.canonicalPath, + logicalDigest: firstImage.logicalDigest, + }, + { + role: secondRole, + canonicalPath: second.canonicalPath, + logicalDigest: secondImage.logicalDigest, + }, + ]; + const conflict = reconciled.conflicts.length > 0; + const targets = buildTargetPlans(mode, first, second, firstImage, secondImage, reconciled); + const anyChanges = targets.some((target) => hasChanges(target.changes)); + const status = planStatus(conflict, sameDatabase, anyChanges); + const reportStatus: ReconciliationReportStatus = conflict ? 'conflict' : anyChanges ? 'changed' : 'no-op'; + const report = makeReport( + mode, + reportStatus, + sameDatabase, + firstImage.schemaFingerprint, + targets, + reconciled.conflicts, + ); + return deepFreeze({ + planVersion: PLAN_VERSION, + mode, + status, + sameDatabase, + schemaFingerprint: firstImage.schemaFingerprint, + inputs, + targets, + conflicts: reconciled.conflicts, + report, + historyLimitation: IDENTICAL_HISTORY_ADDITION_LIMITATION, + }); +} + +/** + * Safe reporting wrapper for dry-run surfaces. Operational failures are + * reduced to bounded codes/guidance and never include guest titles or notes. + */ +export function dryRunDatabaseReconciliation(request: ReconciliationRequest): ReconciliationDryRunReport { + try { + return planDatabaseReconciliation(request).report; + } catch (caught) { + const failure = + caught instanceof ReconciliationError + ? { code: caught.code, ...(caught.guidance === undefined ? {} : { guidance: caught.guidance }) } + : { code: 'unexpected-failure' as const }; + return deepFreeze(makeReport(request.mode, 'operational-failure', false, null, [], [], failure)); + } +} From 777bb19422b6d431034875363c7ef9ffff8a0a20 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 11:32:59 +0000 Subject: [PATCH 02/21] fix(reconciliation): enforce marker and dependency invariants --- src/lib/v5/db-reconciliation.test.ts | 76 +++++++++++++++++++++++++++- src/lib/v5/db-reconciliation.ts | 52 +++++++++++++++++-- 2 files changed, 121 insertions(+), 7 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index c66a09769..cd531f740 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -487,7 +487,7 @@ describe('keyed, edge-set, and history-multiset planning', () => { }); }); mutate(right, (db) => { - db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('prerequisite', 'shared'); + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('populated', 'shared'); // IDs collide with left but are intentionally local and excluded from identity. insertStage(db, 1, { taskId: 'shared', stage: 'planned', note: 'same', createdAt: 10 }); insertStage(db, 2, { taskId: 'shared', stage: 'right-only', note: null, createdAt: 11 }); @@ -506,7 +506,7 @@ describe('keyed, edge-set, and history-multiset planning', () => { wishGroups: [{ wish: 'wish-right' }], hireRoster: [{ wish: 'wish-right' }], meta: [{ key: 'meta-right' }], - taskDependencies: [{ taskId: 'prerequisite', dependsOnId: 'shared' }], + taskDependencies: [{ taskId: 'populated', dependsOnId: 'shared' }], stageLog: [{ count: 1, value: { stage: 'right-only' } }], taskEvents: [{ count: 1, value: { note: null } }], }); @@ -526,6 +526,54 @@ describe('keyed, edge-set, and history-multiset planning', () => { expect(Object.isFrozen(first.targets[0].changes.stageLog)).toBe(true); }); + test('pre-existing self-dependencies and cycles are rejected as invalid input graphs', () => { + const peer = currentDb('peer'); + for (const kind of ['self', 'cycle'] as const) { + const invalid = currentDb(kind); + mutate(invalid, (db) => { + insertTask(db, { id: 'x' }); + if (kind === 'self') { + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('x', 'x'); + return; + } + insertTask(db, { id: 'y' }); + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('x', 'y'); + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('y', 'x'); + }); + + expect(() => planDatabaseReconciliation(bidirectional(invalid, peer))).toThrow( + expect.objectContaining({ code: 'invalid-data' }), + ); + } + }); + + test('a cycle formed only by the cross-endpoint dependency union conflicts before planning changes', () => { + const left = currentDb('left'); + const right = currentDb('right'); + for (const path of [left, right]) { + mutate(path, (db) => { + insertTask(db, { id: 'x' }); + insertTask(db, { id: 'y' }); + }); + } + mutate(left, (db) => { + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('x', 'y'); + }); + mutate(right, (db) => { + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run('y', 'x'); + }); + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toEqual([ + expect.objectContaining({ table: 'task_dependencies', reason: 'planned-integrity-failed', side: 'left' }), + expect.objectContaining({ table: 'task_dependencies', reason: 'planned-integrity-failed', side: 'right' }), + ]); + expect(target(plan, 'left').changes.taskDependencies).toEqual([]); + expect(target(plan, 'right').changes.taskDependencies).toEqual([]); + }); + test('bidirectional same-key differences conflict without exposing hostile payloads', () => { const left = currentDb('left'); const right = currentDb('right'); @@ -749,6 +797,30 @@ describe('stage_log_backfill_v1 marker semantics', () => { expect(target(plan, 'left').postimageDigest).toBe(target(plan, 'right').postimageDigest); }); + test('directional reconciliation uses the valid source marker whenever it is present', () => { + const fixtures = [ + { name: 'greater', source: '200', destination: '100', expected: '200' }, + { name: 'smaller', source: '100', destination: '200', expected: '100' }, + { name: 'destination-missing', source: '200', destination: null, expected: '200' }, + { name: 'source-missing', source: null, destination: '100', expected: null }, + ] as const; + + for (const fixture of fixtures) { + const source = currentDb(`${fixture.name}-source`, fixture.source); + const destination = currentDb(`${fixture.name}-destination`, fixture.destination); + + const plan = planDatabaseReconciliation({ + mode: 'directional', + sourcePath: source, + destinationPath: destination, + }); + + expect(target(plan, 'destination').changes.meta).toEqual( + fixture.expected === null ? [] : [{ key: 'stage_log_backfill_v1', value: fixture.expected }], + ); + } + }); + test('invalid marker and invariant failure become deterministic conflicts', () => { const invalid = currentDb('invalid', 'not-decimal'); const peer = currentDb('peer', null); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index c0e651bad..fe0c7501e 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -947,6 +947,32 @@ function keyedValues(values: readonly T[], key: (value: T) => string): Map): boolean { + const outgoing = new Map(); + const incomingCount = new Map(); + for (const dependency of dependencies.values()) { + if (dependency.taskId === dependency.dependsOnId) return false; + const outgoingEdges = outgoing.get(dependency.taskId); + if (outgoingEdges === undefined) outgoing.set(dependency.taskId, [dependency.dependsOnId]); + else outgoingEdges.push(dependency.dependsOnId); + incomingCount.set(dependency.taskId, incomingCount.get(dependency.taskId) ?? 0); + incomingCount.set(dependency.dependsOnId, (incomingCount.get(dependency.dependsOnId) ?? 0) + 1); + } + + const ready = [...incomingCount].filter(([, count]) => count === 0).map(([taskId]) => taskId); + let visited = 0; + while (ready.length > 0) { + const taskId = ready.pop() as string; + visited++; + for (const dependsOnId of outgoing.get(taskId) ?? []) { + const nextCount = (incomingCount.get(dependsOnId) as number) - 1; + incomingCount.set(dependsOnId, nextCount); + if (nextCount === 0) ready.push(dependsOnId); + } + } + return visited === incomingCount.size; +} + function readLogicalState(db: Database): LogicalState { const boards = boundedTableRows(db, 'boards', 'SELECT id, name, created_at, lanes FROM boards').map( (row): BoardReconciliationRow => ({ @@ -1069,7 +1095,7 @@ function readLogicalState(db: Database): LogicalState { throw error('invalid-data', 'A reconciliation input exceeds the bounded total row-count limit.'); } - return { + const state = { boards: keyedValues(boards, boardKey), tasks: keyedValues(tasks, taskKey), wishGroups: keyedValues(wishGroups, wishGroupKey), @@ -1079,6 +1105,10 @@ function readLogicalState(db: Database): LogicalState { stageLog: countValues(stages, stageLogKey), taskEvents: countValues(events, taskEventKey), }; + if (!dependencyGraphIsAcyclic(state.taskDependencies)) { + throw error('invalid-data', 'A reconciliation input contains an invalid task dependency graph.'); + } + return state; } function loadDatabaseImage(input: PhysicalInput): DatabaseImage { @@ -1404,6 +1434,7 @@ function validateExistingMarker( } if (!markerInvariant(state)) { addConflict(conflicts, 'meta', 'marker-invariant-failed', canonicalTuple([BACKFILL_MARKER]), side); + return null; } return parsed; } @@ -1422,11 +1453,19 @@ function reconcileMarker( const rightValue = validateExistingMarker(rightInput, rightRole, conflicts); if (leftValue === null || rightValue === null) return; - const values = [leftValue, rightValue].filter((value): value is bigint => value !== undefined); - if (values.length === 0) return; - const selected = values.reduce((smallest, value) => (value < smallest ? value : smallest)).toString(); + let selected: bigint | undefined; + if (mode === 'directional') { + selected = leftValue ?? rightValue; + } else { + const values = [leftValue, rightValue].filter((value): value is bigint => value !== undefined); + selected = values.reduce( + (smallest, value) => (smallest === undefined || value < smallest ? value : smallest), + undefined, + ); + } + if (selected === undefined) return; const key = canonicalTuple([BACKFILL_MARKER]); - const row = { key: BACKFILL_MARKER, value: selected }; + const row = { key: BACKFILL_MARKER, value: selected.toString() }; if (mode === 'bidirectional') { leftTarget.meta.set(key, row); rightTarget.meta.set(key, row); @@ -1474,6 +1513,9 @@ function validatePlannedTargetIntegrity( addConflict(conflicts, 'task_dependencies', 'planned-integrity-failed', key, side); } } + if (!dependencyGraphIsAcyclic(state.taskDependencies)) { + addConflict(conflicts, 'task_dependencies', 'planned-integrity-failed', canonicalTuple(['dependency-cycle']), side); + } for (const [key, stage] of state.stageLog) { if (!state.tasks.has(canonicalTuple([stage.value.taskId]))) { addConflict(conflicts, 'stage_log', 'planned-integrity-failed', key, side); From 49fb7a38f9da1a7cd2eff9a713701ba85c4b6337 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 11:51:58 +0000 Subject: [PATCH 03/21] fix(reconciliation): close validation and resource gaps --- src/lib/v5/db-reconciliation.test.ts | 178 ++++++++++++++++++++-- src/lib/v5/db-reconciliation.ts | 215 +++++++++++++++++++++------ 2 files changed, 334 insertions(+), 59 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index cd531f740..f2e26cbd3 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -1,6 +1,6 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { linkSync, mkdtempSync, readFileSync, rmSync, symlinkSync, truncateSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { @@ -235,9 +235,14 @@ function seedPreRuntime(path: string): void { db.close(); } -function seedReorderedCurrent(path: string, taskStatuses = "'done', 'blocked', 'ready', 'in_progress'"): void { +function seedReorderedCurrent( + path: string, + taskStatuses = "'done', 'blocked', 'ready', 'in_progress'", + transform: (sql: string) => string = (sql) => sql, +): void { const db = new Database(path); - db.exec(` + db.exec( + transform(` PRAGMA user_version = 1; CREATE TABLE meta (value TEXT NOT NULL, key TEXT, PRIMARY KEY (key)); CREATE TABLE boards ( @@ -300,7 +305,8 @@ function seedReorderedCurrent(path: string, taskStatuses = "'done', 'blocked', ' CREATE INDEX idx_stage_log_task ON stage_log(task_id); CREATE INDEX idx_tasks_status ON tasks(status); CREATE INDEX idx_task_deps_dep ON task_dependencies(depends_on_id); - `); + `), + ); db.close(); } @@ -411,6 +417,33 @@ describe('closed normalized current schema', () => { seedReorderedCurrent(wrongCheck, "'done', 'blocked', 'ready', 'paused'"); expect(dryRunDatabaseReconciliation(bidirectional(wrongCheck, peer)).status).toBe('operational-failure'); }); + + test.each([ + ['declared collation', (sql: string) => sql.replace('title TEXT NOT NULL', 'title TEXT COLLATE NOCASE NOT NULL')], + [ + 'constraint conflict policy', + (sql: string) => sql.replace('name TEXT NOT NULL UNIQUE', 'name TEXT NOT NULL UNIQUE ON CONFLICT IGNORE'), + ], + [ + 'regrouped foreign keys', + (sql: string) => + sql.replace( + /depends_on_id TEXT NOT NULL REFERENCES tasks\(id\) ON DELETE CASCADE,\s*task_id TEXT NOT NULL REFERENCES tasks\(id\) ON DELETE CASCADE,/, + `task_id TEXT NOT NULL, + depends_on_id TEXT NOT NULL, + FOREIGN KEY (task_id, depends_on_id) REFERENCES tasks(id, id) ON DELETE CASCADE,`, + ), + ], + ])('%s is rejected by the closed schema fingerprint', (_name, transform) => { + const altered = pathFor('altered'); + const peer = currentDb('peer'); + seedReorderedCurrent(altered, "'done', 'blocked', 'ready', 'in_progress'", transform); + + const report = dryRunDatabaseReconciliation(bidirectional(altered, peer)); + + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure?.code).toBe('unsupported-schema'); + }); }); describe('keyed, edge-set, and history-multiset planning', () => { @@ -574,6 +607,55 @@ describe('keyed, edge-set, and history-multiset planning', () => { expect(target(plan, 'right').changes.taskDependencies).toEqual([]); }); + test('invalid loaded wish-group dependency graphs are rejected while valid per-wish graphs reconcile', () => { + const invalidFixtures = [ + { name: 'malformed', rows: [['w', 'a', '{']] }, + { name: 'non-array', rows: [['w', 'a', '{}']] }, + { name: 'non-string', rows: [['w', 'a', '[1]']] }, + { name: 'dangling', rows: [['w', 'a', '["missing"]']] }, + { name: 'self', rows: [['w', 'a', '["a"]']] }, + { + name: 'cycle', + rows: [ + ['w', 'a', '["b"]'], + ['w', 'b', '["a"]'], + ], + }, + ] as const; + const peer = currentDb('peer'); + for (const fixture of invalidFixtures) { + const invalid = currentDb(fixture.name); + mutate(invalid, (db) => { + for (const [wish, name, dependsOn] of fixture.rows) { + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, created_at, updated_at) + VALUES (?, ?, 'ready', ?, 1, 1)`, + ).run(wish, name, dependsOn); + } + }); + expect(() => planDatabaseReconciliation(bidirectional(invalid, peer))).toThrow( + expect.objectContaining({ code: 'invalid-data' }), + ); + } + + const valid = currentDb('valid'); + mutate(valid, (db) => { + for (const [wish, name, dependsOn] of [ + ['w1', 'a', '[]'], + ['w1', 'b', '["a"]'], + ['w2', 'a', '[]'], + ]) { + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, created_at, updated_at) + VALUES (?, ?, 'ready', ?, 1, 1)`, + ).run(wish, name, dependsOn); + } + }); + expect(planDatabaseReconciliation(bidirectional(valid, peer)).status).toBe('changed'); + }); + test('bidirectional same-key differences conflict without exposing hostile payloads', () => { const left = currentDb('left'); const right = currentDb('right'); @@ -885,18 +967,24 @@ describe('stage_log_backfill_v1 marker semantics', () => { }); describe('same-file, idempotency, and bounded failures', () => { - test('canonical aliases of the same physical database are a safe same-database no-op', () => { - const path = currentDb('one'); - const alias = join(fixtureRoot, 'alias.db'); - symlinkSync(path, alias); - - const plan = planDatabaseReconciliation(bidirectional(path, alias)); - - expect(plan.status).toBe('same-database'); - expect(plan.sameDatabase).toBe(true); - expect(plan.report.status).toBe('no-op'); - expect(plan.inputs[0].logicalDigest).toBe(plan.inputs[1].logicalDigest); - expect(plan.targets.every((item) => !Object.values(item.changes).some((rows) => rows.length > 0))).toBe(true); + test('symlink and hardlink aliases are true same-database no-ops in both modes', () => { + const path = currentDb('one', '000100'); + for (const aliasKind of ['symlink', 'hardlink'] as const) { + const alias = join(fixtureRoot, `${aliasKind}.db`); + if (aliasKind === 'symlink') symlinkSync(path, alias); + else linkSync(path, alias); + for (const request of [ + bidirectional(path, alias), + { mode: 'directional', sourcePath: alias, destinationPath: path } as const, + ]) { + const plan = planDatabaseReconciliation(request); + expect(plan.status).toBe('same-database'); + expect(plan.sameDatabase).toBe(true); + expect(plan.report.status).toBe('no-op'); + expect(plan.inputs[0].logicalDigest).toBe(plan.inputs[1].logicalDigest); + expect(plan.targets.every((item) => !Object.values(item.changes).some((rows) => rows.length > 0))).toBe(true); + } + } }); test('equal logical images are idempotent despite local history IDs', () => { @@ -967,4 +1055,62 @@ describe('same-file, idempotency, and bounded failures', () => { expect(report.operationalFailure).toEqual({ code: 'input-unavailable' }); expect(JSON.stringify(report)).not.toContain(missing); }); + + test('WAL-backed bytes and row counts are bounded before logical row materialization', () => { + const peer = currentDb('peer'); + const walHeavy = currentDb('wal-heavy'); + writeFileSync(`${walHeavy}-wal`, ''); + truncateSync(`${walHeavy}-wal`, 256 * 1024 * 1024 + 1); + expect(dryRunDatabaseReconciliation(bidirectional(walHeavy, peer)).operationalFailure?.code).toBe( + 'input-too-large', + ); + + const rowHeavy = currentDb('row-heavy'); + mutate(rowHeavy, (db) => { + db.exec("INSERT INTO boards (id, name, created_at) VALUES ('bad', X'01', 1)"); + db.exec(` + WITH RECURSIVE seq(value) AS ( + VALUES(0) + UNION ALL + SELECT value + 1 FROM seq WHERE value < 1000000 + ) + INSERT INTO meta (key, value) + SELECT printf('key-%07d', value), 'v' FROM seq + `); + }); + expect(() => planDatabaseReconciliation(bidirectional(rowHeavy, peer))).toThrow( + expect.objectContaining({ + code: 'invalid-data', + message: expect.stringContaining('row-count'), + }), + ); + }, 30_000); + + test('conflict diagnostics have a bounded cardinality', () => { + const left = currentDb('many-left'); + const right = currentDb('many-right'); + for (const [path, prefix] of [ + [left, 'left'], + [right, 'right'], + ] as const) { + mutate(path, (db) => { + db.exec(` + WITH RECURSIVE seq(value) AS ( + VALUES(0) + UNION ALL + SELECT value + 1 FROM seq WHERE value < 10000 + ) + INSERT INTO boards (id, name, created_at) + SELECT printf('id-%05d', value), '${prefix}-' || value, 1 FROM seq + `); + }); + } + + expect(() => planDatabaseReconciliation(bidirectional(left, right))).toThrow( + expect.objectContaining({ + code: 'invalid-data', + message: expect.stringContaining('diagnostic'), + }), + ); + }); }); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index fe0c7501e..db7a31bdf 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -8,7 +8,7 @@ import { Database } from 'bun:sqlite'; import { createHash } from 'node:crypto'; -import { realpathSync, statSync } from 'node:fs'; +import { existsSync, realpathSync, statSync } from 'node:fs'; import { normalize } from 'node:path'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; @@ -18,6 +18,8 @@ const READ_BUSY_TIMEOUT_MS = 5_000; const MAX_DATABASE_BYTES = 256 * 1024 * 1024; const MAX_ROWS_PER_TABLE = 1_000_000; const MAX_TOTAL_ROWS = 2_000_000; +const MAX_CONFLICTS = 10_000; +const MAX_SCHEMA_OBJECTS = 64; const BACKFILL_MARKER = 'stage_log_backfill_v1'; const DIRECT_BACKFILL_KINDS = new Set(['comment', 'move', 'claim', 'release', 'block', 'unblock', 'report']); @@ -276,7 +278,7 @@ interface IndexFingerprint { interface TableFingerprint { readonly name: string; readonly columns: readonly ColumnFingerprint[]; - readonly foreignKeys: readonly ForeignKeyFingerprint[]; + readonly foreignKeys: readonly (readonly ForeignKeyFingerprint[])[]; readonly indexes: readonly IndexFingerprint[]; readonly checks: readonly string[]; readonly autoIncrement: boolean; @@ -379,19 +381,20 @@ const EXPECTED_COLUMNS: Readonly< ], }; -const EXPECTED_FOREIGN_KEYS: Readonly> = { - boards: [], - hire_roster: [], - meta: [], - stage_log: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], - task_dependencies: [ - foreignKey('tasks', 'depends_on_id', 'id', 'NO ACTION', 'CASCADE'), - foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE'), - ], - task_events: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], - tasks: [foreignKey('boards', 'board_id', 'id', 'NO ACTION', 'SET NULL')], - wish_groups: [], -}; +const EXPECTED_FOREIGN_KEYS: Readonly> = + { + boards: [], + hire_roster: [], + meta: [], + stage_log: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], + task_dependencies: [ + foreignKey('tasks', 'depends_on_id', 'id', 'NO ACTION', 'CASCADE'), + foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE'), + ], + task_events: [foreignKey('tasks', 'task_id', 'id', 'NO ACTION', 'CASCADE')], + tasks: [foreignKey('boards', 'board_id', 'id', 'NO ACTION', 'SET NULL')], + wish_groups: [], + }; const EXPECTED_INDEXES: Readonly> = { boards: [index(null, 'pk', true, ['id']), index(null, 'u', true, ['name'])], @@ -452,8 +455,8 @@ function foreignKey( to: string, onUpdate: string, onDelete: string, -): ForeignKeyFingerprint { - return { table, from, to, onUpdate, onDelete, match: 'NONE' }; +): readonly ForeignKeyFingerprint[] { + return [{ table, from, to, onUpdate, onDelete, match: 'NONE' }]; } function index(name: string | null, origin: string, unique: boolean, columns: readonly string[]): IndexFingerprint { @@ -536,7 +539,15 @@ function resolvePhysicalInput(path: string): PhysicalInput { const canonicalPath = normalize(realpathSync(path)); const stats = statSync(canonicalPath); if (!stats.isFile()) throw new Error('not a regular file'); - if (stats.size > MAX_DATABASE_BYTES) { + const walPath = `${canonicalPath}-wal`; + let walBytes = 0; + if (existsSync(walPath)) { + const walStats = statSync(walPath); + if (!walStats.isFile()) throw new Error('WAL is not a regular file'); + walBytes = walStats.size; + } + const logicalBytes = stats.size + walBytes; + if (logicalBytes > MAX_DATABASE_BYTES) { throw error('input-too-large', 'The database exceeds the bounded reconciliation input size.'); } return { @@ -573,8 +584,9 @@ function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { function readInventory(db: Database): Array<{ type: string; name: string; tableName: string; sql: string | null }> { const rows = db - .query('SELECT type, name, tbl_name AS tableName, sql FROM sqlite_schema ORDER BY type, name') - .all() as Array>; + .query('SELECT type, name, tbl_name AS tableName, sql FROM sqlite_schema ORDER BY type, name LIMIT ?') + .all(MAX_SCHEMA_OBJECTS + 1) as Array>; + if (rows.length > MAX_SCHEMA_OBJECTS) unsupportedSchema(); return rows.map((row) => { const type = rowString(row, 'type'); const name = rowString(row, 'name'); @@ -663,18 +675,34 @@ function tableColumns(db: Database, table: ReconciliationTableName): readonly Co return observed; } -function tableForeignKeys(db: Database, table: ReconciliationTableName): readonly ForeignKeyFingerprint[] { +function tableForeignKeys(db: Database, table: ReconciliationTableName): readonly (readonly ForeignKeyFingerprint[])[] { const rows = db.query(`PRAGMA foreign_key_list(${table})`).all() as Array>; - const observed = rows - .map((row) => ({ - table: rowString(row, 'table'), - from: rowString(row, 'from'), - to: rowString(row, 'to'), - onUpdate: rowString(row, 'on_update'), - onDelete: rowString(row, 'on_delete'), - match: rowString(row, 'match'), - })) - .sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); + const rawGroups = new Map>(); + for (const row of rows) { + const id = schemaInteger(row.id); + const group = rawGroups.get(id) ?? []; + group.push({ + sequence: schemaInteger(row.seq), + fingerprint: { + table: rowString(row, 'table'), + from: rowString(row, 'from'), + to: rowString(row, 'to'), + onUpdate: rowString(row, 'on_update'), + onDelete: rowString(row, 'on_delete'), + match: rowString(row, 'match'), + }, + }); + rawGroups.set(id, group); + } + const observed = [...rawGroups.values()] + .map((group) => { + group.sort((left, right) => left.sequence - right.sequence); + if (group.some((item, index) => item.sequence !== index)) unsupportedSchema(); + return group.map((item) => item.fingerprint); + }) + .map((group) => ({ group, key: JSON.stringify(group) })) + .sort((left, right) => compareCanonical(left.key, right.key)) + .map((item) => item.group); if (JSON.stringify(observed) !== JSON.stringify(EXPECTED_FOREIGN_KEYS[table])) unsupportedSchema(); return observed; } @@ -884,7 +912,13 @@ function readSchemaFingerprint(db: Database): SchemaFingerprint { } const sql = tableSql.get(name); if (sql === undefined) staleSchema(); - if (hasKeyword(sql, 'deferrable') || hasKeyword(sql, 'initially')) unsupportedSchema(); + if ( + ['collate', 'conflict', 'deferrable', 'generated', 'initially', 'match'].some((keyword) => + hasKeyword(sql, keyword), + ) + ) { + unsupportedSchema(); + } const checks = normalizeChecks(sql); if (JSON.stringify(checks) !== JSON.stringify(EXPECTED_CHECKS[name])) unsupportedSchema(); const autoIncrement = hasKeyword(sql, 'autoincrement'); @@ -915,17 +949,27 @@ function checkIntegrity(db: Database): void { function boundedTableRows( db: Database, - table: ReconciliationTableName, + _table: ReconciliationTableName, sql: string, ): Array> { - const countRow = db.query(`SELECT count(*) AS rowCount FROM ${table}`).get() as Record; - const count = rowInteger(countRow, 'rowCount'); - if (count > BigInt(MAX_ROWS_PER_TABLE)) { - throw error('invalid-data', 'A reconciliation input exceeds the bounded row-count limit.'); - } return db.query(sql).all() as Array>; } +function preflightRowCounts(db: Database): void { + let total = 0n; + for (const table of TABLE_NAMES) { + const countRow = db.query(`SELECT count(*) AS rowCount FROM ${table}`).get() as Record; + const count = rowInteger(countRow, 'rowCount'); + if (count > BigInt(MAX_ROWS_PER_TABLE)) { + throw error('invalid-data', 'A reconciliation input exceeds the bounded row-count limit.'); + } + total += count; + if (total > BigInt(MAX_TOTAL_ROWS)) { + throw error('invalid-data', 'A reconciliation input exceeds the bounded total row-count limit.'); + } + } +} + function countValues(values: readonly T[], key: (value: T) => string): Map> { const counts = new Map>(); for (const value of values) { @@ -973,6 +1017,72 @@ function dependencyGraphIsAcyclic(dependencies: ReadonlyMap typeof dependency === 'string') + ? (parsed as string[]) + : null; + } catch { + return null; + } +} + +function wishGroupGraphIsValid(groups: readonly ParsedWishGroup[]): boolean { + const names = new Set(groups.map((group) => group.name)); + const incomingCount = new Map(groups.map((group) => [group.name, 0])); + const dependents = new Map(); + for (const group of groups) { + if ( + group.dependencies === null || + group.dependencies.includes(group.name) || + group.dependencies.some((dependency) => !names.has(dependency)) + ) { + return false; + } + incomingCount.set(group.name, group.dependencies.length); + for (const dependency of group.dependencies) { + const rows = dependents.get(dependency) ?? []; + rows.push(group.name); + dependents.set(dependency, rows); + } + } + + const ready = [...incomingCount].filter(([, count]) => count === 0).map(([name]) => name); + let visited = 0; + while (ready.length > 0) { + const name = ready.pop() as string; + visited++; + for (const dependent of dependents.get(name) ?? []) { + const count = (incomingCount.get(dependent) as number) - 1; + incomingCount.set(dependent, count); + if (count === 0) ready.push(dependent); + } + } + return visited === groups.length; +} + +function invalidWishGroupGraphs(groups: ReadonlyMap): Set { + const byWish = new Map(); + for (const group of groups.values()) { + const wishKey = canonicalTuple([group.wish]); + const wishGroups = byWish.get(wishKey) ?? []; + wishGroups.push({ name: group.name, dependencies: parseWishGroupDependencies(group.dependsOn) }); + byWish.set(wishKey, wishGroups); + } + + const invalid = new Set(); + for (const [wishKey, wishGroups] of byWish) { + if (!wishGroupGraphIsValid(wishGroups)) invalid.add(wishKey); + } + return invalid; +} + function readLogicalState(db: Database): LogicalState { const boards = boundedTableRows(db, 'boards', 'SELECT id, name, created_at, lanes FROM boards').map( (row): BoardReconciliationRow => ({ @@ -1108,6 +1218,9 @@ function readLogicalState(db: Database): LogicalState { if (!dependencyGraphIsAcyclic(state.taskDependencies)) { throw error('invalid-data', 'A reconciliation input contains an invalid task dependency graph.'); } + if (invalidWishGroupGraphs(state.wishGroups).size > 0) { + throw error('invalid-data', 'A reconciliation input contains an invalid wish-group dependency graph.'); + } return state; } @@ -1130,6 +1243,7 @@ function loadDatabaseImage(input: PhysicalInput): DatabaseImage { // Integrity is intentionally after the closed inventory: a guest trigger or // virtual object is rejected before any logical data receives authority. checkIntegrity(db); + preflightRowCounts(db); const state = readLogicalState(db); const schemaFingerprint = digestCanonical(['genie-schema-v1', schema]); const logicalDigest = logicalStateDigest(schemaFingerprint, state); @@ -1244,7 +1358,9 @@ function eventValues(row: TaskEventReconciliationValue): readonly (string | bigi function mapDigestRows(map: ReadonlyMap, values: (row: T) => readonly (string | bigint | null)[]) { return [...map.values()] .map((row) => values(row).map(canonicalScalar)) - .sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); + .map((row) => ({ row, key: JSON.stringify(row) })) + .sort((left, right) => compareCanonical(left.key, right.key)) + .map((item) => item.row); } function mapDigestCounts( @@ -1253,7 +1369,9 @@ function mapDigestCounts( ) { return [...map.values()] .map((entry) => [values(entry.value).map(canonicalScalar), entry.count] as const) - .sort((left, right) => compareCanonical(JSON.stringify(left[0]), JSON.stringify(right[0]))); + .map((row) => ({ row, key: JSON.stringify(row[0]) })) + .sort((left, right) => compareCanonical(left.key, right.key)) + .map((item) => item.row); } function logicalStateDigest(schemaFingerprint: string, state: LogicalState): string { @@ -1301,6 +1419,9 @@ function addConflict( key: string, side?: ReconciliationConflict['side'], ): void { + if (conflicts.length >= MAX_CONFLICTS) { + throw error('invalid-data', 'Reconciliation exceeds the bounded conflict diagnostic limit.'); + } conflicts.push({ table, reason, keyDigest: conflictKeyDigest(table, key), ...(side === undefined ? {} : { side }) }); } @@ -1516,6 +1637,9 @@ function validatePlannedTargetIntegrity( if (!dependencyGraphIsAcyclic(state.taskDependencies)) { addConflict(conflicts, 'task_dependencies', 'planned-integrity-failed', canonicalTuple(['dependency-cycle']), side); } + for (const wishKey of invalidWishGroupGraphs(state.wishGroups)) { + addConflict(conflicts, 'wish_groups', 'planned-integrity-failed', wishKey, side); + } for (const [key, stage] of state.stageLog) { if (!state.tasks.has(canonicalTuple([stage.value.taskId]))) { addConflict(conflicts, 'stage_log', 'planned-integrity-failed', key, side); @@ -1564,8 +1688,11 @@ function reconcileStates( if (mode === 'bidirectional') validatePlannedTargetIntegrity(left, 'left', conflicts); validatePlannedTargetIntegrity(right, mode === 'bidirectional' ? 'right' : 'destination', conflicts); } - conflicts.sort((a, b) => compareCanonical(JSON.stringify(a), JSON.stringify(b))); - return { left, right, conflicts }; + const sortedConflicts = conflicts + .map((conflict) => ({ conflict, key: JSON.stringify(conflict) })) + .sort((leftConflict, rightConflict) => compareCanonical(leftConflict.key, rightConflict.key)) + .map((item) => item.conflict); + return { left, right, conflicts: sortedConflicts }; } function changedRows( @@ -1739,7 +1866,9 @@ export function planDatabaseReconciliation(request: ReconciliationRequest): Reco revalidatePhysicalInput(second); if (firstImage.schemaFingerprint !== secondImage.schemaFingerprint) unsupportedSchema(); - const reconciled = reconcileStates(mode, firstImage.state, secondImage.state); + const reconciled = sameDatabase + ? { left: cloneState(firstImage.state), right: cloneState(firstImage.state), conflicts: [] } + : reconcileStates(mode, firstImage.state, secondImage.state); const inputs: ReconciliationPlanInput[] = [ { role: firstRole, From 6bb372fece9d601424c679cdd4beeafa248b17c8 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 12:00:04 +0000 Subject: [PATCH 04/21] fix(reconciliation): validate physical aliases safely --- src/lib/v5/db-reconciliation.test.ts | 99 +++++++++++++++++++++++++--- src/lib/v5/db-reconciliation.ts | 48 +++++++++++--- 2 files changed, 130 insertions(+), 17 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index f2e26cbd3..3d8450134 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -1,6 +1,15 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import { linkSync, mkdtempSync, readFileSync, rmSync, symlinkSync, truncateSync, writeFileSync } from 'node:fs'; +import { + existsSync, + linkSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { @@ -968,15 +977,16 @@ describe('stage_log_backfill_v1 marker semantics', () => { describe('same-file, idempotency, and bounded failures', () => { test('symlink and hardlink aliases are true same-database no-ops in both modes', () => { - const path = currentDb('one', '000100'); for (const aliasKind of ['symlink', 'hardlink'] as const) { - const alias = join(fixtureRoot, `${aliasKind}.db`); - if (aliasKind === 'symlink') symlinkSync(path, alias); - else linkSync(path, alias); - for (const request of [ - bidirectional(path, alias), - { mode: 'directional', sourcePath: alias, destinationPath: path } as const, - ]) { + for (const mode of ['bidirectional', 'directional'] as const) { + const path = currentDb(`${aliasKind}-${mode}`, '000100'); + const alias = join(fixtureRoot, `${aliasKind}-${mode}-alias.db`); + if (aliasKind === 'symlink') symlinkSync(path, alias); + else linkSync(path, alias); + const request = + mode === 'bidirectional' + ? bidirectional(path, alias) + : ({ mode, sourcePath: alias, destinationPath: path } as const); const plan = planDatabaseReconciliation(request); expect(plan.status).toBe('same-database'); expect(plan.sameDatabase).toBe(true); @@ -987,6 +997,77 @@ describe('same-file, idempotency, and bounded failures', () => { } }); + test.each( + (['invalid-marker', 'marker-invariant-failed'] as const).flatMap((failure) => + (['symlink', 'hardlink'] as const).flatMap((aliasKind) => + (['bidirectional', 'directional'] as const).map((mode) => [failure, aliasKind, mode] as const), + ), + ), + )('same-physical %s %s aliases validate markers in %s mode', (failure, aliasKind, mode) => { + const path = currentDb(`${failure}-${aliasKind}-${mode}`, failure === 'invalid-marker' ? 'not-decimal' : '100'); + if (failure === 'marker-invariant-failed') { + mutate(path, (db) => { + insertTask(db, { id: 't' }); + insertStage(db, 1, { taskId: 't', stage: 'report', note: 'missing event', createdAt: 1 }); + }); + } + const alias = join(fixtureRoot, `${failure}-${aliasKind}-${mode}-alias.db`); + if (aliasKind === 'symlink') symlinkSync(path, alias); + else linkSync(path, alias); + const request = + mode === 'bidirectional' + ? bidirectional(path, alias) + : ({ mode, sourcePath: alias, destinationPath: path } as const); + + const plan = planDatabaseReconciliation(request); + + expect(plan.status).toBe('conflict'); + expect(plan.sameDatabase).toBe(true); + expect(plan.report.status).toBe('conflict'); + expect(plan.conflicts).toHaveLength(2); + expect(plan.conflicts.every((conflict) => conflict.reason === failure)).toBe(true); + expect(plan.targets.every((item) => !Object.values(item.changes).some((rows) => rows.length > 0))).toBe(true); + }); + + test('hardlink aliases with a committed path-specific WAL are rejected independent of order and mode', () => { + const path = currentDb('wal-alias'); + const alias = join(fixtureRoot, 'wal-hardlink.db'); + linkSync(path, alias); + const writer = new Database(path); + try { + writer.exec('PRAGMA journal_mode = WAL'); + writer.exec('PRAGMA wal_autocheckpoint = 0'); + writer.exec('BEGIN IMMEDIATE'); + insertBoard(writer, { id: 'wal-only', name: 'committed in WAL' }); + writer.exec('COMMIT'); + expect(existsSync(`${path}-wal`)).toBe(true); + expect(existsSync(`${path}-shm`)).toBe(true); + + const bidirectionalForward = dryRunDatabaseReconciliation(bidirectional(path, alias)); + const bidirectionalReverse = dryRunDatabaseReconciliation(bidirectional(alias, path)); + const directionalForward = dryRunDatabaseReconciliation({ + mode: 'directional', + sourcePath: path, + destinationPath: alias, + }); + const directionalReverse = dryRunDatabaseReconciliation({ + mode: 'directional', + sourcePath: alias, + destinationPath: path, + }); + + expect(bidirectionalForward).toEqual(bidirectionalReverse); + expect(directionalForward).toEqual(directionalReverse); + for (const report of [bidirectionalForward, directionalForward]) { + expect(report.status).toBe('operational-failure'); + expect(report.operationalFailure?.code).toBe('invalid-data'); + expect(report.targets).toEqual([]); + } + } finally { + writer.close(); + } + }); + test('equal logical images are idempotent despite local history IDs', () => { const left = currentDb('left'); const right = currentDb('right'); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index db7a31bdf..3dd02c1f2 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -243,6 +243,7 @@ interface PhysicalInput { readonly canonicalPath: string; readonly device: string; readonly inode: string; + readonly hasSidecars: boolean; } interface ColumnFingerprint { @@ -540,12 +541,18 @@ function resolvePhysicalInput(path: string): PhysicalInput { const stats = statSync(canonicalPath); if (!stats.isFile()) throw new Error('not a regular file'); const walPath = `${canonicalPath}-wal`; + const hasWal = existsSync(walPath); let walBytes = 0; - if (existsSync(walPath)) { + if (hasWal) { const walStats = statSync(walPath); if (!walStats.isFile()) throw new Error('WAL is not a regular file'); walBytes = walStats.size; } + const shmPath = `${canonicalPath}-shm`; + const hasShm = existsSync(shmPath); + if (hasShm && !statSync(shmPath).isFile()) { + throw new Error('SHM is not a regular file'); + } const logicalBytes = stats.size + walBytes; if (logicalBytes > MAX_DATABASE_BYTES) { throw error('input-too-large', 'The database exceeds the bounded reconciliation input size.'); @@ -555,6 +562,7 @@ function resolvePhysicalInput(path: string): PhysicalInput { canonicalPath, device: String(stats.dev), inode: String(stats.ino), + hasSidecars: hasWal || hasShm, }; } catch (caught) { if (caught instanceof ReconciliationError) throw caught; @@ -579,7 +587,12 @@ function revalidatePhysicalInput(input: PhysicalInput): void { } function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { - return left.canonicalPath === right.canonicalPath || (left.device === right.device && left.inode === right.inode); + if (left.canonicalPath === right.canonicalPath) return true; + if (left.device !== right.device || left.inode !== right.inode) return false; + if (left.hasSidecars || right.hasSidecars) { + throw error('invalid-data', 'Hardlink aliases with path-specific SQLite sidecars are ambiguous.'); + } + return true; } function readInventory(db: Database): Array<{ type: string; name: string; tableName: string; sql: string | null }> { @@ -1652,6 +1665,29 @@ function validatePlannedTargetIntegrity( } } +function sortConflicts(conflicts: readonly ReconciliationConflict[]): ReconciliationConflict[] { + return conflicts + .map((conflict) => ({ conflict, key: JSON.stringify(conflict) })) + .sort((leftConflict, rightConflict) => compareCanonical(leftConflict.key, rightConflict.key)) + .map((item) => item.conflict); +} + +function validateSameDatabase( + mode: ReconciliationMode, + state: LogicalState, +): { left: LogicalState; right: LogicalState; conflicts: ReconciliationConflict[] } { + const conflicts: ReconciliationConflict[] = []; + const firstRole: ReconciliationInputRole = mode === 'bidirectional' ? 'left' : 'source'; + const secondRole: ReconciliationInputRole = mode === 'bidirectional' ? 'right' : 'destination'; + validateExistingMarker(state, firstRole, conflicts); + validateExistingMarker(state, secondRole, conflicts); + return { + left: cloneState(state), + right: cloneState(state), + conflicts: sortConflicts(conflicts), + }; +} + function reconcileStates( mode: ReconciliationMode, leftInput: LogicalState, @@ -1688,11 +1724,7 @@ function reconcileStates( if (mode === 'bidirectional') validatePlannedTargetIntegrity(left, 'left', conflicts); validatePlannedTargetIntegrity(right, mode === 'bidirectional' ? 'right' : 'destination', conflicts); } - const sortedConflicts = conflicts - .map((conflict) => ({ conflict, key: JSON.stringify(conflict) })) - .sort((leftConflict, rightConflict) => compareCanonical(leftConflict.key, rightConflict.key)) - .map((item) => item.conflict); - return { left, right, conflicts: sortedConflicts }; + return { left, right, conflicts: sortConflicts(conflicts) }; } function changedRows( @@ -1867,7 +1899,7 @@ export function planDatabaseReconciliation(request: ReconciliationRequest): Reco if (firstImage.schemaFingerprint !== secondImage.schemaFingerprint) unsupportedSchema(); const reconciled = sameDatabase - ? { left: cloneState(firstImage.state), right: cloneState(firstImage.state), conflicts: [] } + ? validateSameDatabase(mode, firstImage.state) : reconcileStates(mode, firstImage.state, secondImage.state); const inputs: ReconciliationPlanInput[] = [ { From 12dce921e7ff58e103768fe967df35927548daf8 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 12:30:34 +0000 Subject: [PATCH 05/21] feat: add transactional database reconciliation apply --- src/lib/v5/db-reconciliation.test.ts | 451 +++++++++++ src/lib/v5/db-reconciliation.ts | 1109 +++++++++++++++++++++++++- 2 files changed, 1541 insertions(+), 19 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index 3d8450134..23d3a7e6c 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -6,17 +6,21 @@ import { mkdtempSync, readFileSync, rmSync, + statSync, symlinkSync, truncateSync, + unlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { IDENTICAL_HISTORY_ADDITION_LIMITATION, + type ReconciliationApplyEvent, ReconciliationError, type ReconciliationRequest, type TaskEventReconciliationValue, + applyDatabaseReconciliation, dryRunDatabaseReconciliation, planDatabaseReconciliation, } from './db-reconciliation.js'; @@ -143,12 +147,78 @@ function bidirectional(leftPath: string, rightPath: string): ReconciliationReque return { mode: 'bidirectional', leftPath, rightPath }; } +function directional(sourcePath: string, destinationPath: string): ReconciliationRequest { + return { mode: 'directional', sourcePath, destinationPath }; +} + function target(plan: ReturnType, role: 'left' | 'right' | 'destination') { const found = plan.targets.find((candidate) => candidate.role === role); if (found === undefined) throw new Error(`missing ${role} target`); return found; } +function scalarCount(path: string, table: string): number { + const db = new Database(path, { readonly: true }); + try { + const row = db.query(`SELECT count(*) AS count FROM ${table}`).get() as { count: number }; + return row.count; + } finally { + db.close(); + } +} + +function metaValue(path: string, key: string): string | null { + const db = new Database(path, { readonly: true }); + try { + const row = db.query('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | null; + return row?.value ?? null; + } finally { + db.close(); + } +} + +function taskTitle(path: string, id: string): string | null { + const db = new Database(path, { readonly: true }); + try { + const row = db.query('SELECT title FROM tasks WHERE id = ?').get(id) as { title: string } | null; + return row?.title ?? null; + } finally { + db.close(); + } +} + +function seedBidirectionalApplyPair(): { left: string; right: string } { + const left = currentDb('apply-left', null); + const right = currentDb('apply-right', null); + for (const [path, side] of [ + [left, 'left'], + [right, 'right'], + ] as const) { + mutate(path, (db) => { + insertBoard(db, { id: `board-${side}`, name: `Board ${side}` }); + insertTask(db, { id: `task-${side}`, boardId: `board-${side}` }); + insertTask(db, { id: `dependency-${side}` }); + db.query('INSERT INTO task_dependencies (task_id, depends_on_id) VALUES (?, ?)').run( + `task-${side}`, + `dependency-${side}`, + ); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) + VALUES (?, ?, 'ready', '[]', NULL, NULL, NULL, 1, 1)`, + ).run(`wish-${side}`, 'group'); + db.query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES (?, 'adapter', NULL, ?, 1, 'hired')`, + ).run(`wish-${side}`, `/worktree/${side}`); + db.query('INSERT INTO meta (key, value) VALUES (?, ?)').run(`meta-${side}`, side); + insertStage(db, 1, { taskId: `task-${side}`, stage: 'report', note: side, createdAt: 1 }); + insertEvent(db, 1, { taskId: `task-${side}`, kind: 'report', note: side, createdAt: 1 }); + }); + } + return { left, right }; +} + function seedPreLanes(path: string): void { const db = new Database(path); db.exec(` @@ -1195,3 +1265,384 @@ describe('same-file, idempotency, and bounded failures', () => { ); }); }); + +describe('canonical locking and transactional apply', () => { + test('applies every logical table through live handles and repeats as a no-op', () => { + const { left, right } = seedBidirectionalApplyPair(); + const beforeInodes = [statSync(left).ino, statSync(right).ino]; + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + const report = applyDatabaseReconciliation(plan); + + expect(report).toMatchObject({ status: 'changed', converged: true, failure: null }); + expect(report.targets.every((item) => item.observation === 'expected-postimage')).toBe(true); + for (const path of [left, right]) { + expect(scalarCount(path, 'boards')).toBe(2); + expect(scalarCount(path, 'tasks')).toBe(4); + expect(scalarCount(path, 'wish_groups')).toBe(2); + expect(scalarCount(path, 'hire_roster')).toBe(2); + expect(scalarCount(path, 'meta')).toBe(2); + expect(scalarCount(path, 'task_dependencies')).toBe(2); + expect(scalarCount(path, 'stage_log')).toBe(2); + expect(scalarCount(path, 'task_events')).toBe(2); + } + expect([statSync(left).ino, statSync(right).ino]).toEqual(beforeInodes); + + const repeatedPlan = planDatabaseReconciliation(bidirectional(right, left)); + const repeated = applyDatabaseReconciliation(repeatedPlan); + expect(repeated).toMatchObject({ status: 'no-op', converged: true, failure: null }); + }); + + test('directional apply keeps the locked source unchanged and preserves destination-only rows', () => { + const source = currentDb('directional-source', null); + const destination = currentDb('directional-destination', null); + mutate(source, (db) => { + insertTask(db, { id: 'shared', title: 'source authority' }); + insertTask(db, { id: 'source-only' }); + }); + mutate(destination, (db) => { + insertTask(db, { id: 'shared', title: 'destination old' }); + insertTask(db, { id: 'destination-only' }); + }); + const plan = planDatabaseReconciliation(directional(source, destination)); + const sourcePreimage = plan.inputs.find((input) => input.role === 'source')?.logicalDigest; + + const report = applyDatabaseReconciliation(plan); + const after = planDatabaseReconciliation(directional(source, destination)); + + expect(report).toMatchObject({ status: 'changed', converged: true, failure: null }); + expect(after.inputs.find((input) => input.role === 'source')?.logicalDigest).toBe(sourcePreimage); + expect(taskTitle(source, 'shared')).toBe('source authority'); + expect(taskTitle(source, 'destination-only')).toBeNull(); + expect(taskTitle(destination, 'shared')).toBe('source authority'); + expect(taskTitle(destination, 'source-only')).toBe('source-only'); + expect(taskTitle(destination, 'destination-only')).toBe('destination-only'); + }); + + test('an older write after planning is preserved and aborts locked preimage revalidation', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + mutate(source, (db) => db.query("INSERT INTO meta (key, value) VALUES ('planned', 'source')").run()); + const plan = planDatabaseReconciliation(directional(source, destination)); + mutate(destination, (db) => db.query("INSERT INTO meta (key, value) VALUES ('older', 'writer')").run()); + + const report = applyDatabaseReconciliation(plan); + + expect(report).toMatchObject({ + status: 'preimage-changed', + converged: false, + failure: { code: 'input-changed', phase: 'revalidation' }, + }); + expect(metaValue(destination, 'older')).toBe('writer'); + expect(metaValue(destination, 'planned')).toBeNull(); + }); + + test('a schema change after planning is rejected under locks before a guest trigger can execute', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + mutate(source, (db) => db.query("INSERT INTO meta (key, value) VALUES ('planned', 'source')").run()); + const plan = planDatabaseReconciliation(directional(source, destination)); + mutate(destination, (db) => { + db.exec(` + CREATE TRIGGER intervening_trigger AFTER INSERT ON meta + BEGIN + INSERT INTO meta (key, value) VALUES ('trigger-fired', 'yes'); + END + `); + }); + + const report = applyDatabaseReconciliation(plan); + + expect(report).toMatchObject({ + status: 'preimage-changed', + converged: false, + failure: { code: 'unsupported-schema', phase: 'revalidation', role: 'destination' }, + }); + expect(metaValue(destination, 'planned')).toBeNull(); + expect(metaValue(destination, 'trigger-fired')).toBeNull(); + }); + + test('SQLite contention is bounded and rolls back earlier canonical locks without mutation', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(right, left)); + const heldPath = [left, right].sort()[0]; + const holder = new Database(heldPath); + holder.exec('BEGIN IMMEDIATE'); + const startedAt = Date.now(); + try { + const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 25 }); + expect(report).toMatchObject({ + status: 'lock-timeout', + converged: false, + failure: { code: 'sqlite-lock-timeout', phase: 'sqlite-lock' }, + }); + expect(Date.now() - startedAt).toBeLessThan(1_000); + expect(scalarCount(left, 'boards')).toBe(1); + expect(scalarCount(right, 'boards')).toBe(1); + } finally { + holder.exec('ROLLBACK'); + holder.close(); + } + }); + + test('reversed arguments stop on the first canonical advisory lock without taking the second', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(right, left)); + const [first, second] = [left, right].sort(); + const firstLock = `${first}.genie-reconciliation.lock`; + const secondLock = `${second}.genie-reconciliation.lock`; + writeFileSync(firstLock, JSON.stringify({ pid: process.pid, token: 'test-holder' })); + try { + const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); + expect(report).toMatchObject({ + status: 'lock-timeout', + failure: { code: 'advisory-lock-timeout', phase: 'advisory-lock' }, + }); + expect(existsSync(secondLock)).toBe(false); + } finally { + unlinkSync(firstLock); + } + }); + + test('opposite-order lock-aware reconcilers serialize without deadlock or overwrite', async () => { + const { left, right } = seedBidirectionalApplyPair(); + const moduleUrl = new URL('./db-reconciliation.ts', import.meta.url).href; + const go = join(fixtureRoot, 'concurrent-go'); + const readyA = join(fixtureRoot, 'concurrent-ready-a'); + const readyB = join(fixtureRoot, 'concurrent-ready-b'); + const childCode = ` + import { existsSync, writeFileSync } from 'node:fs'; + const api = await import(Bun.argv[1]); + const request = Bun.argv[6] === 'reverse' + ? { mode: 'bidirectional', leftPath: Bun.argv[3], rightPath: Bun.argv[2] } + : { mode: 'bidirectional', leftPath: Bun.argv[2], rightPath: Bun.argv[3] }; + const plan = api.planDatabaseReconciliation(request); + writeFileSync(Bun.argv[4], 'ready'); + while (!existsSync(Bun.argv[5])) await Bun.sleep(5); + const report = api.applyDatabaseReconciliation(plan, { busyTimeoutMs: 1_000 }); + process.stdout.write(JSON.stringify({ status: report.status, converged: report.converged })); + `; + const spawn = (ready: string, order: 'forward' | 'reverse') => + Bun.spawn({ + cmd: [process.execPath, '-e', childCode, moduleUrl, left, right, ready, go, order], + stderr: 'pipe', + stdout: 'pipe', + }); + const childA = spawn(readyA, 'forward'); + const childB = spawn(readyB, 'reverse'); + for (let attempt = 0; attempt < 400 && (!existsSync(readyA) || !existsSync(readyB)); attempt++) { + await Bun.sleep(5); + } + expect(existsSync(readyA)).toBe(true); + expect(existsSync(readyB)).toBe(true); + writeFileSync(go, 'go'); + + const [exitA, exitB, stdoutA, stdoutB] = await Promise.all([ + childA.exited, + childB.exited, + new Response(childA.stdout).text(), + new Response(childB.stdout).text(), + ]); + expect([exitA, exitB]).toEqual([0, 0]); + const statuses = [JSON.parse(stdoutA).status, JSON.parse(stdoutB).status].sort(); + expect(statuses).toEqual(['changed', 'preimage-changed']); + expect(planDatabaseReconciliation(bidirectional(left, right)).status).toBe('no-op'); + }, 5_000); + + test('an older writer racing held SQLite locks receives bounded busy and cannot be overwritten', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + mutate(source, (db) => db.query("INSERT INTO meta (key, value) VALUES ('planned', 'source')").run()); + const plan = planDatabaseReconciliation(directional(source, destination)); + const olderExitCodes: number[] = []; + + const report = applyDatabaseReconciliation(plan, { + busyTimeoutMs: 100, + onLocked: () => { + for (const path of [source, destination]) { + const child = Bun.spawnSync({ + cmd: [ + process.execPath, + '-e', + `import { Database } from 'bun:sqlite'; + const db = new Database(Bun.argv[1], { readwrite: true, create: false }); + db.exec('PRAGMA busy_timeout = 25'); + try { + db.exec('BEGIN IMMEDIATE'); + db.query("INSERT INTO meta (key, value) VALUES ('older', 'writer')").run(); + db.exec('COMMIT'); + process.exit(0); + } catch { + process.exit(7); + } finally { + db.close(); + }`, + path, + ], + stderr: 'pipe', + stdout: 'pipe', + }); + olderExitCodes.push(child.exitCode); + } + }, + }); + + expect(olderExitCodes).toEqual([7, 7]); + expect(report).toMatchObject({ status: 'changed', converged: true }); + expect(metaValue(destination, 'planned')).toBe('source'); + expect(metaValue(destination, 'older')).toBeNull(); + }); + + test('foreign-key, integrity, and logical postimage checks all finish before commit', () => { + const source = currentDb('source', null); + const destination = currentDb('destination', null); + mutate(source, (db) => { + insertTask(db, { id: 'source-task' }); + insertEvent(db, 1, { taskId: 'source-task', kind: 'report', note: 'ready', createdAt: 1 }); + }); + const events: string[] = []; + + const report = applyDatabaseReconciliation(planDatabaseReconciliation(directional(source, destination)), { + onEvent: (event: ReconciliationApplyEvent) => { + if ('role' in event && event.role === 'destination') { + events.push(`${event.phase}:${event.state}`); + } + }, + }); + + expect(report.status).toBe('changed'); + expect(events).toEqual([ + 'mutation:before', + 'mutation:after', + 'foreign-key-check:before', + 'foreign-key-check:after', + 'integrity-check:before', + 'integrity-check:after', + 'logical-postimage-check:before', + 'logical-postimage-check:after', + 'commit:before', + 'commit:after', + ]); + }); + + test('a precommit failure rolls both live transactions back to their planned preimages', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'before') throw new Error('injected before commit'); + }, + }); + + expect(report).toMatchObject({ status: 'rolled-back', converged: false }); + expect(report.targets.every((item) => item.observation === 'expected-preimage')).toBe(true); + expect(report.targets.every((item) => item.committed === false)).toBe(true); + expect(scalarCount(left, 'boards')).toBe(1); + expect(scalarCount(right, 'boards')).toBe(1); + }); + + test('a failure after the first destination commit reports a known partial commit', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + let completedCommits = 0; + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'after' && ++completedCommits === 1) { + throw new Error('injected after first commit'); + } + }, + }); + + expect(report).toMatchObject({ status: 'partial-commit', converged: false }); + expect(report.targets.filter((item) => item.observation === 'expected-postimage')).toHaveLength(1); + expect(report.targets.filter((item) => item.observation === 'expected-preimage')).toHaveLength(1); + expect(report.targets.filter((item) => item.committed)).toHaveLength(1); + }); + + test('an error after all commits reports expected postimages without claiming convergence', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + let completedCommits = 0; + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'after' && ++completedCommits === 2) { + throw new Error('injected after second commit'); + } + }, + }); + + expect(report).toMatchObject({ status: 'expected-postimage', converged: false }); + expect(report.targets.every((item) => item.observation === 'expected-postimage')).toBe(true); + expect(planDatabaseReconciliation(bidirectional(left, right)).status).toBe('no-op'); + }); + + test('an older write after one commit is reported as unexpected and is never overwritten', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const paths = { left, right }; + let injected = false; + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase !== 'commit' || event.state !== 'after' || injected) return; + injected = true; + mutate(paths[event.role as 'left' | 'right'], (db) => { + db.query("INSERT INTO meta (key, value) VALUES ('intervening', 'older-writer')").run(); + }); + throw new Error('stop after intervening write'); + }, + }); + + expect(report).toMatchObject({ status: 'unexpected-intervening-write', converged: false }); + const unexpected = report.targets.find((item) => item.observation === 'unexpected'); + expect(unexpected).toBeDefined(); + expect(metaValue(paths[unexpected?.role as 'left' | 'right'], 'intervening')).toBe('older-writer'); + }); + + test('an unclassifiable postcommit schema intervention is reported as uncertain', () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const paths = { left, right }; + let injected = false; + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase !== 'commit' || event.state !== 'after' || injected) return; + injected = true; + mutate(paths[event.role as 'left' | 'right'], (db) => { + db.exec('CREATE TABLE intervening_schema (id TEXT PRIMARY KEY)'); + }); + throw new Error('stop after unclassifiable intervention'); + }, + }); + + expect(report).toMatchObject({ status: 'uncertain', converged: false }); + expect(report.targets.some((item) => item.observation === 'not-observed')).toBe(true); + }); + + test('directional board-name swaps use a transaction-local parking order', () => { + const source = currentDb('swap-source', null); + const destination = currentDb('swap-destination', null); + mutate(source, (db) => { + insertBoard(db, { id: 'a', name: 'A' }); + insertBoard(db, { id: 'b', name: 'B' }); + }); + mutate(destination, (db) => { + insertBoard(db, { id: 'a', name: 'B' }); + insertBoard(db, { id: 'b', name: 'A' }); + }); + + const report = applyDatabaseReconciliation(planDatabaseReconciliation(directional(source, destination))); + + expect(report).toMatchObject({ status: 'changed', converged: true }); + const db = new Database(destination, { readonly: true }); + expect(db.query('SELECT id, name FROM boards ORDER BY id').all()).toEqual([ + { id: 'a', name: 'A' }, + { id: 'b', name: 'B' }, + ]); + db.close(); + }); +}); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 3dd02c1f2..49ead7f3b 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -1,20 +1,37 @@ /** - * Read-only planning for reconciling two current Genie v5 databases. + * Planning and locked transactional apply for reconciling two current Genie + * v5 databases. * - * This module deliberately owns no locks, snapshots, or writes. It turns two - * transaction-consistent, structurally exact logical images into an immutable - * plan that a later apply boundary can revalidate and execute. + * Read-only planning turns two transaction-consistent, structurally exact + * logical images into an immutable plan. Apply then takes stable advisory and + * SQLite write locks in canonical order, revalidates that exact plan, and + * writes only through the live handles. Snapshot publication remains outside + * this module. */ import { Database } from 'bun:sqlite'; -import { createHash } from 'node:crypto'; -import { existsSync, realpathSync, statSync } from 'node:fs'; -import { normalize } from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import { + closeSync, + existsSync, + linkSync, + openSync, + readFileSync, + realpathSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, join, normalize } from 'node:path'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; +import { BUSY_TIMEOUT_MS, isBusyError } from './sqlite-open.js'; const PLAN_VERSION = 1 as const; const REPORT_VERSION = 1 as const; const READ_BUSY_TIMEOUT_MS = 5_000; +const MAX_BUSY_TIMEOUT_MS = 2_147_483_647; +const ADVISORY_RETRY_MS = 10; const MAX_DATABASE_BYTES = 256 * 1024 * 1024; const MAX_ROWS_PER_TABLE = 1_000_000; const MAX_TOTAL_ROWS = 2_000_000; @@ -34,6 +51,47 @@ export const IDENTICAL_HISTORY_ADDITION_LIMITATION = export type ReconciliationMode = 'bidirectional' | 'directional'; export type ReconciliationPlanStatus = 'no-op' | 'changed' | 'conflict' | 'same-database'; export type ReconciliationReportStatus = 'no-op' | 'changed' | 'conflict' | 'operational-failure'; +export type ReconciliationApplyStatus = + | 'no-op' + | 'changed' + | 'conflict' + | 'same-database' + | 'preimage-changed' + | 'lock-timeout' + | 'rolled-back' + | 'expected-postimage' + | 'partial-commit' + | 'unexpected-intervening-write' + | 'uncertain'; +export type ReconciliationApplyPhase = + | 'plan-validation' + | 'advisory-lock' + | 'sqlite-lock' + | 'revalidation' + | 'locked' + | 'mutation' + | 'foreign-key-check' + | 'integrity-check' + | 'logical-postimage-check' + | 'commit' + | 'rollback' + | 'observation'; +export type ReconciliationApplyFailureCode = + | ReconciliationErrorCode + | 'invalid-plan' + | 'advisory-lock-timeout' + | 'sqlite-lock-timeout' + | 'apply-failed' + | 'postimage-mismatch' + | 'commit-failed' + | 'rollback-failed' + | 'observation-failed' + | 'unexpected-failure'; +export type ReconciliationTargetObservation = + | 'not-observed' + | 'expected-preimage' + | 'expected-postimage' + | 'unexpected'; export type ReconciliationInputRole = 'left' | 'right' | 'source' | 'destination'; export type ReconciliationTargetRole = 'left' | 'right' | 'destination'; export type KeyedTableName = 'boards' | 'tasks' | 'wish_groups' | 'hire_roster' | 'meta'; @@ -235,6 +293,58 @@ export interface ReconciliationPlan { readonly historyLimitation: typeof IDENTICAL_HISTORY_ADDITION_LIMITATION; } +export interface ReconciliationApplyTargetReport { + readonly role: ReconciliationTargetRole; + readonly preimageDigest: string; + readonly postimageDigest: string | null; + readonly observedDigest: string | null; + readonly observation: ReconciliationTargetObservation; + readonly committed: boolean; +} + +export interface ReconciliationApplyFailure { + readonly code: ReconciliationApplyFailureCode; + readonly phase: ReconciliationApplyPhase; + readonly role?: ReconciliationInputRole | ReconciliationTargetRole; +} + +export interface ReconciliationApplyReport { + readonly reportVersion: typeof REPORT_VERSION; + readonly dryRun: false; + readonly mode: ReconciliationMode; + readonly status: ReconciliationApplyStatus; + readonly converged: boolean; + readonly targets: readonly ReconciliationApplyTargetReport[]; + readonly failure: ReconciliationApplyFailure | null; +} + +export type ReconciliationApplyEvent = + | { readonly phase: 'locked' } + | { + readonly phase: 'mutation' | 'foreign-key-check' | 'integrity-check' | 'logical-postimage-check' | 'commit'; + readonly role: ReconciliationTargetRole; + readonly state: 'before' | 'after'; + }; + +export interface ReconciliationLockedInput { + readonly role: ReconciliationInputRole; + readonly canonicalPath: string; + readonly target: boolean; + serialize(): Uint8Array; +} + +export interface ReconciliationApplyOptions { + /** Shared total wait bound for all advisory and SQLite lock acquisition. */ + readonly busyTimeoutMs?: number; + /** + * Runs after every input has been revalidated while all SQLite write locks + * remain held. Group 3 uses the read-only serializers to publish snapshots. + */ + readonly onLocked?: (inputs: readonly ReconciliationLockedInput[]) => void; + /** Bounded lifecycle observation and deterministic failure injection for tests. */ + readonly onEvent?: (event: ReconciliationApplyEvent) => void; +} + type SqliteScalar = string | bigint | number | Uint8Array | null; type CanonicalScalar = readonly ['null'] | readonly ['string', string] | readonly ['integer', string]; @@ -314,6 +424,34 @@ interface DatabaseImage { readonly state: LogicalState; } +interface AdvisoryLock { + readonly path: string; + readonly token: string; +} + +interface LockedDatabase { + readonly input: PhysicalInput; + readonly roles: readonly ReconciliationInputRole[]; + readonly db: Database; + transactionOpen: boolean; +} + +interface ApplyFailureContext { + readonly code: ReconciliationApplyFailureCode; + readonly phase: ReconciliationApplyPhase; + readonly role?: ReconciliationInputRole | ReconciliationTargetRole; +} + +class ApplyBoundaryError extends Error { + readonly context: ApplyFailureContext; + + constructor(context: ApplyFailureContext, cause?: unknown) { + super(cause instanceof Error ? cause.message : context.code); + this.name = 'ApplyBoundaryError'; + this.context = context; + } +} + const NORMAL_OPEN_GUIDANCE = 'Open the database once with Genie’s normal current open path (for example `genie board`) to normalize supported additive history, then retry reconciliation.'; @@ -595,6 +733,126 @@ function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { return true; } +const SLEEP_SIGNAL = new Int32Array(new SharedArrayBuffer(4)); + +function sleepMs(ms: number): void { + Atomics.wait(SLEEP_SIGNAL, 0, 0, ms); +} + +function errnoCode(caught: unknown): string | undefined { + if (!(caught instanceof Error)) return undefined; + const code = (caught as NodeJS.ErrnoException).code; + return typeof code === 'string' ? code : undefined; +} + +function advisoryLockPath(canonicalPath: string): string { + return `${canonicalPath}.genie-reconciliation.lock`; +} + +function advisoryOwner(pid: number, token: string): string { + return JSON.stringify({ pid, token }); +} + +function parseAdvisoryOwner(path: string): { pid: number; token: string } | null { + try { + const value: unknown = JSON.parse(readFileSync(path, 'utf8')); + if ( + typeof value !== 'object' || + value === null || + !('pid' in value) || + !('token' in value) || + !Number.isSafeInteger(value.pid) || + (value.pid as number) <= 0 || + typeof value.token !== 'string' + ) { + return null; + } + return { pid: value.pid as number, token: value.token }; + } catch { + return null; + } +} + +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (caught) { + return errnoCode(caught) !== 'ESRCH'; + } +} + +function removeStaleAdvisoryLock(path: string): boolean { + const owner = parseAdvisoryOwner(path); + if (owner === null || processIsAlive(owner.pid)) return false; + const stalePath = `${path}.${randomUUID()}.stale`; + try { + renameSync(path, stalePath); + } catch (caught) { + if (errnoCode(caught) === 'ENOENT') return true; + return false; + } + try { + unlinkSync(stalePath); + } catch { + // The stable name is already free. A stranded uniquely named stale inode + // does not grant authority and cannot block another reconciler. + } + return true; +} + +function acquireAdvisoryLock(canonicalPath: string, timeoutMs: number): AdvisoryLock { + const path = advisoryLockPath(canonicalPath); + const token = randomUUID(); + const candidatePath = join(dirname(path), `.${basename(path)}.${process.pid}.${token}.candidate`); + const descriptor = openSync(candidatePath, 'wx', 0o600); + try { + writeFileSync(descriptor, advisoryOwner(process.pid, token)); + } finally { + closeSync(descriptor); + } + + const deadline = Date.now() + timeoutMs; + try { + for (;;) { + try { + linkSync(candidatePath, path); + return { path, token }; + } catch (caught) { + if (errnoCode(caught) !== 'EEXIST') { + throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }, caught); + } + if (removeStaleAdvisoryLock(path)) continue; + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new ApplyBoundaryError({ code: 'advisory-lock-timeout', phase: 'advisory-lock' }, caught); + } + sleepMs(Math.min(ADVISORY_RETRY_MS, remaining)); + } + } + } finally { + try { + unlinkSync(candidatePath); + } catch { + // The stable hard link, not this unique candidate name, owns exclusion. + } + } +} + +function releaseAdvisoryLock(lock: AdvisoryLock): void { + const owner = parseAdvisoryOwner(lock.path); + if (owner === null) { + if (!existsSync(lock.path)) return; + throw new Error('Advisory lock ownership became unreadable.'); + } + if (owner.pid !== process.pid || owner.token !== lock.token) { + throw new Error('Advisory lock ownership changed before release.'); + } + const releasePath = `${lock.path}.${lock.token}.release`; + renameSync(lock.path, releasePath); + unlinkSync(releasePath); +} + function readInventory(db: Database): Array<{ type: string; name: string; tableName: string; sql: string | null }> { const rows = db .query('SELECT type, name, tbl_name AS tableName, sql FROM sqlite_schema ORDER BY type, name LIMIT ?') @@ -950,16 +1208,24 @@ function readSchemaFingerprint(db: Database): SchemaFingerprint { return { userVersion: CURRENT_SCHEMA_VERSION, tables }; } -function checkIntegrity(db: Database): void { +function checkSqliteIntegrity(db: Database): void { const integrity = db.query('PRAGMA integrity_check(1)').get() as Record | null; const result = integrity === null ? null : Object.values(integrity)[0]; if (result !== 'ok') throw error('integrity-failed', 'A reconciliation input failed SQLite integrity validation.'); +} + +function checkForeignKeys(db: Database): void { const foreignKeyFailure = db.query('PRAGMA foreign_key_check').get(); if (foreignKeyFailure !== null) { throw error('integrity-failed', 'A reconciliation input failed SQLite foreign-key validation.'); } } +function checkIntegrity(db: Database): void { + checkSqliteIntegrity(db); + checkForeignKeys(db); +} + function boundedTableRows( db: Database, _table: ReconciliationTableName, @@ -1237,7 +1503,19 @@ function readLogicalState(db: Database): LogicalState { return state; } -function loadDatabaseImage(input: PhysicalInput): DatabaseImage { +function readDatabaseImage(db: Database): DatabaseImage { + const schema = readSchemaFingerprint(db); + // Integrity is intentionally after the closed inventory: a guest trigger or + // virtual object is rejected before any logical data receives authority. + checkIntegrity(db); + preflightRowCounts(db); + const state = readLogicalState(db); + const schemaFingerprint = digestCanonical(['genie-schema-v1', schema]); + const logicalDigest = logicalStateDigest(schemaFingerprint, state); + return { schemaFingerprint, logicalDigest, state }; +} + +function loadDatabaseImage(input: PhysicalInput, busyTimeoutMs = READ_BUSY_TIMEOUT_MS): DatabaseImage { let db: Database; try { db = new Database(input.canonicalPath, { readonly: true, strict: true, safeIntegers: true }); @@ -1246,23 +1524,16 @@ function loadDatabaseImage(input: PhysicalInput): DatabaseImage { } let transactionOpen = false; try { - db.exec(`PRAGMA busy_timeout = ${READ_BUSY_TIMEOUT_MS}`); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); db.exec('PRAGMA query_only = ON'); db.exec('PRAGMA trusted_schema = OFF'); db.exec('PRAGMA foreign_keys = ON'); db.exec('BEGIN'); transactionOpen = true; - const schema = readSchemaFingerprint(db); - // Integrity is intentionally after the closed inventory: a guest trigger or - // virtual object is rejected before any logical data receives authority. - checkIntegrity(db); - preflightRowCounts(db); - const state = readLogicalState(db); - const schemaFingerprint = digestCanonical(['genie-schema-v1', schema]); - const logicalDigest = logicalStateDigest(schemaFingerprint, state); + const image = readDatabaseImage(db); db.exec('COMMIT'); transactionOpen = false; - return { schemaFingerprint, logicalDigest, state }; + return image; } catch (caught) { if (transactionOpen) { try { @@ -1955,3 +2226,803 @@ export function dryRunDatabaseReconciliation(request: ReconciliationRequest): Re return deepFreeze(makeReport(request.mode, 'operational-failure', false, null, [], [], failure)); } } + +function validateBusyTimeout(value: number | undefined): number { + const timeout = value ?? BUSY_TIMEOUT_MS; + if (!Number.isSafeInteger(timeout) || timeout < 0 || timeout > MAX_BUSY_TIMEOUT_MS) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } + return timeout; +} + +function inputForTarget(plan: ReconciliationPlan, role: ReconciliationTargetRole): ReconciliationPlanInput | undefined { + const inputRole: ReconciliationInputRole = role === 'destination' ? 'destination' : role; + return plan.inputs.find((input) => input.role === inputRole); +} + +function expectedRoles(mode: ReconciliationMode): { + readonly inputs: readonly ReconciliationInputRole[]; + readonly targets: readonly ReconciliationTargetRole[]; +} { + return mode === 'bidirectional' + ? { inputs: ['left', 'right'], targets: ['left', 'right'] } + : { inputs: ['source', 'destination'], targets: ['destination'] }; +} + +function validateApplicablePlan(plan: ReconciliationPlan): void { + if (plan.planVersion !== PLAN_VERSION || (plan.mode !== 'bidirectional' && plan.mode !== 'directional')) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } + const expected = expectedRoles(plan.mode); + if ( + plan.inputs.length !== expected.inputs.length || + plan.targets.length !== expected.targets.length || + plan.inputs.some((input, index) => input.role !== expected.inputs[index]) || + plan.targets.some((target, index) => target.role !== expected.targets[index]) + ) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } + for (const target of plan.targets) { + const input = inputForTarget(plan, target.role); + if ( + input === undefined || + input.canonicalPath !== target.canonicalPath || + input.logicalDigest !== target.preimageDigest || + (plan.status === 'conflict') !== (target.postimageDigest === null) + ) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation', role: target.role }); + } + } + const conflict = plan.conflicts.length > 0; + const invalidSameDatabaseStatus = + plan.status === 'conflict' ? false : plan.sameDatabase !== (plan.status === 'same-database'); + if (conflict !== (plan.status === 'conflict') || invalidSameDatabaseStatus) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } +} + +function initialApplyTargets(plan: ReconciliationPlan): ReconciliationApplyTargetReport[] { + return plan.targets.map((target) => ({ + role: target.role, + preimageDigest: target.preimageDigest, + postimageDigest: target.postimageDigest, + observedDigest: null, + observation: 'not-observed', + committed: false, + })); +} + +function makeApplyReport( + plan: ReconciliationPlan, + status: ReconciliationApplyStatus, + converged: boolean, + targets: readonly ReconciliationApplyTargetReport[], + failure: ReconciliationApplyFailure | null, +): ReconciliationApplyReport { + return deepFreeze({ + reportVersion: REPORT_VERSION, + dryRun: false, + mode: plan.mode, + status, + converged, + targets, + failure, + }); +} + +function failureFrom(caught: unknown, fallback: ApplyFailureContext): ReconciliationApplyFailure { + if (caught instanceof ApplyBoundaryError) return caught.context; + if (caught instanceof ReconciliationError) { + return { + code: caught.code, + phase: fallback.phase, + ...(fallback.role === undefined ? {} : { role: fallback.role }), + }; + } + return { + ...fallback, + code: fallback.code === 'unexpected-failure' ? fallback.code : 'unexpected-failure', + }; +} + +function emitApplyEvent(options: ReconciliationApplyOptions, event: ReconciliationApplyEvent): void { + try { + options.onEvent?.(event); + } catch (caught) { + const role = 'role' in event ? event.role : undefined; + throw new ApplyBoundaryError( + { + code: event.phase === 'commit' ? 'commit-failed' : 'apply-failed', + phase: event.phase, + ...(role === undefined ? {} : { role }), + }, + caught, + ); + } +} + +function resolvePlannedInputs(plan: ReconciliationPlan): Array<{ + readonly planned: ReconciliationPlanInput; + readonly physical: PhysicalInput; +}> { + const resolved = plan.inputs.map((planned) => { + let physical: PhysicalInput; + try { + physical = resolvePhysicalInput(planned.canonicalPath); + } catch (caught) { + throw new ApplyBoundaryError( + { + code: caught instanceof ReconciliationError ? caught.code : 'input-changed', + phase: 'revalidation', + role: planned.role, + }, + caught, + ); + } + if (physical.canonicalPath !== planned.canonicalPath) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation', role: planned.role }); + } + return { planned, physical }; + }); + let sameNow: boolean; + try { + sameNow = samePhysicalInput(resolved[0].physical, resolved[1].physical); + } catch (caught) { + throw new ApplyBoundaryError( + { + code: caught instanceof ReconciliationError ? caught.code : 'input-changed', + phase: 'revalidation', + }, + caught, + ); + } + if (sameNow !== plan.sameDatabase) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation' }); + } + return resolved; +} + +function acquirePlannedAdvisoryLocks( + inputs: readonly { readonly physical: PhysicalInput }[], + deadline: number, +): AdvisoryLock[] { + const locks: AdvisoryLock[] = []; + const paths = [...new Set(inputs.map((input) => input.physical.canonicalPath))].sort(compareCanonical); + try { + for (const path of paths) locks.push(acquireAdvisoryLock(path, remainingLockWait(deadline))); + return locks; + } catch (caught) { + for (const lock of locks.reverse()) { + try { + releaseAdvisoryLock(lock); + } catch { + // The acquisition failure remains authoritative. + } + } + throw caught; + } +} + +function remainingLockWait(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +function physicalIdentity(input: PhysicalInput): string { + return `${input.device}:${input.inode}`; +} + +function revalidateApplyInput(input: PhysicalInput, role: ReconciliationInputRole): void { + try { + revalidatePhysicalInput(input); + } catch (caught) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation', role }, caught); + } +} + +function openLockedDatabases( + inputs: readonly { readonly planned: ReconciliationPlanInput; readonly physical: PhysicalInput }[], + deadline: number, +): LockedDatabase[] { + for (const input of inputs) revalidateApplyInput(input.physical, input.planned.role); + const grouped = new Map(); + for (const { planned, physical } of inputs) { + const key = physicalIdentity(physical); + const existing = grouped.get(key); + if (existing === undefined) { + grouped.set(key, { input: physical, roles: [planned.role], paths: [physical] }); + } else { + existing.roles.push(planned.role); + existing.paths.push(physical); + if (compareCanonical(physical.canonicalPath, existing.input.canonicalPath) < 0) { + existing.input = physical; + } + } + } + + const locked: LockedDatabase[] = []; + const ordered = [...grouped.values()].sort((left, right) => + compareCanonical(left.input.canonicalPath, right.input.canonicalPath), + ); + try { + for (const group of ordered) { + let db: Database | null = null; + try { + db = new Database(group.input.canonicalPath, { + readwrite: true, + create: false, + strict: true, + safeIntegers: true, + }); + db.exec(`PRAGMA busy_timeout = ${remainingLockWait(deadline)}`); + db.exec('PRAGMA trusted_schema = OFF'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('BEGIN IMMEDIATE'); + } catch (caught) { + db?.close(); + throw new ApplyBoundaryError( + { + code: isBusyError(caught) ? 'sqlite-lock-timeout' : 'input-changed', + phase: isBusyError(caught) ? 'sqlite-lock' : 'revalidation', + role: group.roles[0], + }, + caught, + ); + } + locked.push({ input: group.input, roles: group.roles, db, transactionOpen: true }); + for (const path of group.paths) revalidateApplyInput(path, group.roles[0]); + } + return locked; + } catch (caught) { + rollbackOpenTransactions(locked); + closeLockedDatabases(locked); + throw caught; + } +} + +function lockedDatabaseForRole( + locked: readonly LockedDatabase[], + role: ReconciliationInputRole | ReconciliationTargetRole, +): LockedDatabase { + const inputRole: ReconciliationInputRole = role === 'destination' ? 'destination' : role; + const found = locked.find((item) => item.roles.includes(inputRole)); + if (found === undefined) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation', role }); + } + return found; +} + +function rollbackOpenTransactions(locked: readonly LockedDatabase[]): boolean { + let complete = true; + for (const item of [...locked].reverse()) { + if (!item.transactionOpen) continue; + try { + item.db.exec('ROLLBACK'); + item.transactionOpen = false; + } catch { + complete = false; + } + } + return complete; +} + +function closeLockedDatabases(locked: readonly LockedDatabase[]): void { + for (const item of [...locked].reverse()) { + try { + item.db.close(); + } catch { + // Observation classifies the durable logical state after close failure. + } + } +} + +function releaseAdvisoryLocks(locks: readonly AdvisoryLock[]): boolean { + let complete = true; + for (const lock of [...locks].reverse()) { + try { + releaseAdvisoryLock(lock); + } catch { + complete = false; + } + } + return complete; +} + +function revalidateLockedPlan( + plan: ReconciliationPlan, + locked: readonly LockedDatabase[], +): ReadonlyMap { + const images = new Map(); + for (const item of locked) { + let image: DatabaseImage; + try { + image = readDatabaseImage(item.db); + } catch (caught) { + throw new ApplyBoundaryError( + { + code: caught instanceof ReconciliationError ? caught.code : 'input-changed', + phase: 'revalidation', + role: item.roles[0], + }, + caught, + ); + } + if (image.schemaFingerprint !== plan.schemaFingerprint) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation', role: item.roles[0] }); + } + for (const role of item.roles) { + const expected = plan.inputs.find((input) => input.role === role); + if (expected === undefined || image.logicalDigest !== expected.logicalDigest) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation', role }); + } + images.set(role, image); + } + } + return images; +} + +function lockedInputsForHook( + plan: ReconciliationPlan, + locked: readonly LockedDatabase[], +): readonly ReconciliationLockedInput[] { + return plan.inputs.map((input) => { + const item = lockedDatabaseForRole(locked, input.role); + const targetRole: ReconciliationTargetRole = + input.role === 'destination' + ? 'destination' + : input.role === 'left' || input.role === 'right' + ? input.role + : 'destination'; + const target = plan.targets.some((candidate) => candidate.role === targetRole && input.role !== 'source'); + return Object.freeze({ + role: input.role, + canonicalPath: input.canonicalPath, + target, + serialize: () => new Uint8Array(item.db.serialize()), + }); + }); +} + +function runLockedHook( + plan: ReconciliationPlan, + locked: readonly LockedDatabase[], + options: ReconciliationApplyOptions, +): void { + emitApplyEvent(options, { phase: 'locked' }); + try { + options.onLocked?.(lockedInputsForHook(plan, locked)); + } catch (caught) { + throw new ApplyBoundaryError({ code: 'apply-failed', phase: 'locked' }, caught); + } +} + +function temporaryBoardName(db: Database, nonce: string, index: number): string { + let attempt = 0; + for (;;) { + const value = `__genie_reconciliation_${nonce}_${index}_${attempt}`; + const found = db.query('SELECT 1 AS present FROM boards WHERE name = ?').get(value); + if (found === null) return value; + attempt++; + } +} + +function applyBoards(db: Database, rows: readonly BoardReconciliationRow[]): void { + const changing: BoardReconciliationRow[] = []; + for (const row of rows) { + const current = db.query('SELECT name FROM boards WHERE id = ?').get(row.id) as Record | null; + if (current !== null && rowString(current, 'name') !== row.name) changing.push(row); + } + const nonce = randomUUID().replaceAll('-', ''); + const park = db.query('UPDATE boards SET name = ? WHERE id = ?'); + for (const [index, row] of changing.entries()) { + park.run(temporaryBoardName(db, nonce, index), row.id); + } + const upsert = db.query( + `INSERT INTO boards (id, name, created_at, lanes) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + name = excluded.name, + created_at = excluded.created_at, + lanes = excluded.lanes`, + ); + for (const row of rows) upsert.run(...boardValues(row)); +} + +function applyTasks(db: Database, rows: readonly TaskReconciliationRow[]): void { + const upsert = db.query( + `INSERT INTO tasks ( + id, board_id, title, status, claimed_by, claimed_at, wish, group_name, + created_at, updated_at, lane, agent_kind, heartbeat_at, blocked_by, blocked_reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + board_id = excluded.board_id, + title = excluded.title, + status = excluded.status, + claimed_by = excluded.claimed_by, + claimed_at = excluded.claimed_at, + wish = excluded.wish, + group_name = excluded.group_name, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + lane = excluded.lane, + agent_kind = excluded.agent_kind, + heartbeat_at = excluded.heartbeat_at, + blocked_by = excluded.blocked_by, + blocked_reason = excluded.blocked_reason`, + ); + for (const row of rows) upsert.run(...taskValues(row)); +} + +function applyWishGroups(db: Database, rows: readonly WishGroupReconciliationRow[]): void { + const upsert = db.query( + `INSERT INTO wish_groups ( + wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(wish, name) DO UPDATE SET + status = excluded.status, + depends_on = excluded.depends_on, + assignee = excluded.assignee, + started_at = excluded.started_at, + completed_at = excluded.completed_at, + created_at = excluded.created_at, + updated_at = excluded.updated_at`, + ); + for (const row of rows) upsert.run(...wishGroupValues(row)); +} + +function applyHireRoster(db: Database, rows: readonly HireRosterReconciliationRow[]): void { + const upsert = db.query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(wish, agent_adapter_id) DO UPDATE SET + profile = excluded.profile, + worktree = excluded.worktree, + hired_at = excluded.hired_at, + state = excluded.state`, + ); + for (const row of rows) upsert.run(...hireRosterValues(row)); +} + +function applyTaskDependencies(db: Database, rows: readonly TaskDependencyReconciliationRow[]): void { + const insert = db.query( + `INSERT INTO task_dependencies (task_id, depends_on_id) + VALUES (?, ?) + ON CONFLICT(task_id, depends_on_id) DO NOTHING`, + ); + for (const row of rows) insert.run(...dependencyValues(row)); +} + +function applyStageLog(db: Database, additions: readonly HistoryAddition[]): void { + const insert = db.query('INSERT INTO stage_log (task_id, stage, note, created_at) VALUES (?, ?, ?, ?)'); + for (const addition of additions) { + for (let count = 0; count < addition.count; count++) insert.run(...stageValues(addition.value)); + } +} + +function applyTaskEvents(db: Database, additions: readonly HistoryAddition[]): void { + const insert = db.query( + 'INSERT INTO task_events (task_id, kind, note, author_kind, author, created_at) VALUES (?, ?, ?, ?, ?, ?)', + ); + for (const addition of additions) { + for (let count = 0; count < addition.count; count++) insert.run(...eventValues(addition.value)); + } +} + +function applyMeta(db: Database, rows: readonly MetaReconciliationRow[]): void { + const upsert = db.query( + `INSERT INTO meta (key, value) + VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + ); + for (const row of rows) upsert.run(...metaValues(row)); +} + +function applyTargetChanges(db: Database, changes: ReconciliationTargetChanges): void { + // Parent rows precede dependents. The backfill marker is written last so its + // logical coverage invariant is never transiently asserted ahead of history. + applyBoards(db, changes.boards); + applyTasks(db, changes.tasks); + applyWishGroups(db, changes.wishGroups); + applyHireRoster(db, changes.hireRoster); + applyTaskDependencies(db, changes.taskDependencies); + applyStageLog(db, changes.stageLog); + applyTaskEvents(db, changes.taskEvents); + applyMeta(db, changes.meta); +} + +function runPostimageCheck( + phase: 'foreign-key-check' | 'integrity-check' | 'logical-postimage-check', + role: ReconciliationTargetRole, + check: () => void, +): void { + try { + check(); + } catch (caught) { + if (caught instanceof ApplyBoundaryError) throw caught; + throw new ApplyBoundaryError( + { + code: caught instanceof ReconciliationError ? caught.code : 'postimage-mismatch', + phase, + role, + }, + caught, + ); + } +} + +function validateTargetPostimage( + plan: ReconciliationPlan, + target: ReconciliationTargetPlan, + db: Database, + options: ReconciliationApplyOptions, +): void { + let schemaFingerprint = ''; + runPostimageCheck('logical-postimage-check', target.role, () => { + schemaFingerprint = digestCanonical(['genie-schema-v1', readSchemaFingerprint(db)]); + if (schemaFingerprint !== plan.schemaFingerprint) { + throw new ApplyBoundaryError({ + code: 'postimage-mismatch', + phase: 'logical-postimage-check', + role: target.role, + }); + } + }); + + emitApplyEvent(options, { phase: 'foreign-key-check', role: target.role, state: 'before' }); + runPostimageCheck('foreign-key-check', target.role, () => checkForeignKeys(db)); + emitApplyEvent(options, { phase: 'foreign-key-check', role: target.role, state: 'after' }); + + emitApplyEvent(options, { phase: 'integrity-check', role: target.role, state: 'before' }); + runPostimageCheck('integrity-check', target.role, () => checkSqliteIntegrity(db)); + emitApplyEvent(options, { phase: 'integrity-check', role: target.role, state: 'after' }); + + emitApplyEvent(options, { phase: 'logical-postimage-check', role: target.role, state: 'before' }); + runPostimageCheck('logical-postimage-check', target.role, () => { + preflightRowCounts(db); + const state = readLogicalState(db); + const logicalDigest = logicalStateDigest(schemaFingerprint, state); + if (target.postimageDigest === null || logicalDigest !== target.postimageDigest) { + throw new ApplyBoundaryError({ + code: 'postimage-mismatch', + phase: 'logical-postimage-check', + role: target.role, + }); + } + }); + emitApplyEvent(options, { phase: 'logical-postimage-check', role: target.role, state: 'after' }); +} + +function ensureDirectionalSourceUnchanged(plan: ReconciliationPlan, locked: readonly LockedDatabase[]): void { + if (plan.mode !== 'directional') return; + const source = plan.inputs.find((input) => input.role === 'source'); + if (source === undefined) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation', role: 'source' }); + } + let image: DatabaseImage; + try { + image = readDatabaseImage(lockedDatabaseForRole(locked, 'source').db); + } catch (caught) { + throw new ApplyBoundaryError( + { + code: caught instanceof ReconciliationError ? caught.code : 'input-changed', + phase: 'revalidation', + role: 'source', + }, + caught, + ); + } + if (image.schemaFingerprint !== plan.schemaFingerprint || image.logicalDigest !== source.logicalDigest) { + throw new ApplyBoundaryError({ code: 'input-changed', phase: 'revalidation', role: 'source' }); + } +} + +function orderedTargets(plan: ReconciliationPlan): ReconciliationTargetPlan[] { + return [...plan.targets].sort((left, right) => { + const pathOrder = compareCanonical(left.canonicalPath, right.canonicalPath); + return pathOrder === 0 ? compareCanonical(left.role, right.role) : pathOrder; + }); +} + +function commitTarget( + target: ReconciliationTargetPlan, + item: LockedDatabase, + options: ReconciliationApplyOptions, + committed: Set, +): void { + emitApplyEvent(options, { phase: 'commit', role: target.role, state: 'before' }); + try { + item.db.exec('COMMIT'); + item.transactionOpen = false; + committed.add(target.role); + } catch (caught) { + throw new ApplyBoundaryError({ code: 'commit-failed', phase: 'commit', role: target.role }, caught); + } + emitApplyEvent(options, { phase: 'commit', role: target.role, state: 'after' }); +} + +function commitNonTargetTransactions( + plan: ReconciliationPlan, + locked: readonly LockedDatabase[], + committed: Set, +): void { + for (const item of locked) { + if (!item.transactionOpen) continue; + try { + item.db.exec('COMMIT'); + item.transactionOpen = false; + } catch (caught) { + throw new ApplyBoundaryError({ code: 'commit-failed', phase: 'commit', role: item.roles[0] }, caught); + } + for (const target of plan.targets) { + if (item.roles.includes(target.role === 'destination' ? 'destination' : target.role)) { + committed.add(target.role); + } + } + } +} + +function mutateValidateAndCommit( + plan: ReconciliationPlan, + locked: readonly LockedDatabase[], + options: ReconciliationApplyOptions, + committed: Set, +): void { + if (plan.status === 'same-database') { + commitNonTargetTransactions(plan, locked, committed); + return; + } + + const targets = orderedTargets(plan); + for (const target of targets) { + const item = lockedDatabaseForRole(locked, target.role); + emitApplyEvent(options, { phase: 'mutation', role: target.role, state: 'before' }); + applyTargetChanges(item.db, target.changes); + emitApplyEvent(options, { phase: 'mutation', role: target.role, state: 'after' }); + } + for (const target of targets) { + validateTargetPostimage(plan, target, lockedDatabaseForRole(locked, target.role).db, options); + } + ensureDirectionalSourceUnchanged(plan, locked); + + for (const target of targets) { + commitTarget(target, lockedDatabaseForRole(locked, target.role), options, committed); + } + commitNonTargetTransactions(plan, locked, committed); +} + +function observeTarget( + target: ReconciliationTargetPlan, + committed: ReadonlySet, + timeoutMs: number, +): ReconciliationApplyTargetReport { + let observedDigest: string | null = null; + try { + observedDigest = loadDatabaseImage(resolvePhysicalInput(target.canonicalPath), timeoutMs).logicalDigest; + } catch { + return { + role: target.role, + preimageDigest: target.preimageDigest, + postimageDigest: target.postimageDigest, + observedDigest: null, + observation: 'not-observed', + committed: committed.has(target.role), + }; + } + const observation: ReconciliationTargetObservation = + target.postimageDigest !== null && observedDigest === target.postimageDigest + ? 'expected-postimage' + : observedDigest === target.preimageDigest + ? 'expected-preimage' + : 'unexpected'; + return { + role: target.role, + preimageDigest: target.preimageDigest, + postimageDigest: target.postimageDigest, + observedDigest, + observation, + committed: committed.has(target.role), + }; +} + +function observeTargets( + plan: ReconciliationPlan, + committed: ReadonlySet, + timeoutMs: number, +): ReconciliationApplyTargetReport[] { + return plan.targets.map((target) => observeTarget(target, committed, timeoutMs)); +} + +function classifyApplyStatus( + plan: ReconciliationPlan, + targets: readonly ReconciliationApplyTargetReport[], + failure: ReconciliationApplyFailure | null, +): { status: ReconciliationApplyStatus; converged: boolean } { + if (failure?.code === 'advisory-lock-timeout' || failure?.code === 'sqlite-lock-timeout') { + return { status: 'lock-timeout', converged: false }; + } + if (failure?.phase === 'revalidation' || failure?.code === 'input-changed') { + return { status: 'preimage-changed', converged: false }; + } + if (targets.some((target) => target.observation === 'unexpected')) { + return { status: 'unexpected-intervening-write', converged: false }; + } + if (targets.some((target) => target.observation === 'not-observed')) { + return { status: 'uncertain', converged: false }; + } + if (failure === null) { + if (plan.status === 'same-database') return { status: 'same-database', converged: true }; + if (plan.status === 'no-op') return { status: 'no-op', converged: true }; + if (targets.every((target) => target.observation === 'expected-postimage')) { + return { status: 'changed', converged: true }; + } + return { status: 'uncertain', converged: false }; + } + if (targets.every((target) => target.observation === 'expected-postimage')) { + return { status: 'expected-postimage', converged: false }; + } + if ( + targets.some((target) => target.observation === 'expected-postimage') && + targets.every((target) => target.observation === 'expected-postimage' || target.observation === 'expected-preimage') + ) { + return { status: 'partial-commit', converged: false }; + } + if (targets.every((target) => target.observation === 'expected-preimage')) { + return { status: 'rolled-back', converged: false }; + } + return { status: 'uncertain', converged: false }; +} + +/** + * Apply an immutable plan through live SQLite handles only. + * + * Advisory locks coordinate reconciliation-aware writers. `BEGIN IMMEDIATE` + * on every input remains authoritative for older Genie writers that do not + * know those locks. All acquisition, rollback, commit, and post-failure + * observation paths return a bounded state report rather than claiming pair + * convergence after an ambiguous boundary. + */ +export function applyDatabaseReconciliation( + plan: ReconciliationPlan, + options: ReconciliationApplyOptions = {}, +): ReconciliationApplyReport { + let timeoutMs = BUSY_TIMEOUT_MS; + let failure: ReconciliationApplyFailure | null = null; + let advisoryLocks: AdvisoryLock[] = []; + let locked: LockedDatabase[] = []; + const committed = new Set(); + + try { + timeoutMs = validateBusyTimeout(options.busyTimeoutMs); + validateApplicablePlan(plan); + } catch (caught) { + failure = failureFrom(caught, { code: 'invalid-plan', phase: 'plan-validation' }); + return makeApplyReport(plan, 'uncertain', false, initialApplyTargets(plan), failure); + } + + if (plan.status === 'conflict') { + return makeApplyReport(plan, 'conflict', false, initialApplyTargets(plan), null); + } + + try { + const inputs = resolvePlannedInputs(plan); + const lockDeadline = Date.now() + timeoutMs; + advisoryLocks = acquirePlannedAdvisoryLocks(inputs, lockDeadline); + locked = openLockedDatabases(inputs, lockDeadline); + revalidateLockedPlan(plan, locked); + runLockedHook(plan, locked, options); + mutateValidateAndCommit(plan, locked, options, committed); + } catch (caught) { + failure = failureFrom(caught, { code: 'apply-failed', phase: 'mutation' }); + if (!rollbackOpenTransactions(locked)) { + failure = { code: 'rollback-failed', phase: 'rollback' }; + } + } finally { + closeLockedDatabases(locked); + } + + const targets = advisoryLocks.length > 0 ? observeTargets(plan, committed, timeoutMs) : initialApplyTargets(plan); + if (!releaseAdvisoryLocks(advisoryLocks)) { + failure = failure ?? { code: 'unexpected-failure', phase: 'advisory-lock' }; + } + const classified = classifyApplyStatus(plan, targets, failure); + return makeApplyReport(plan, classified.status, classified.converged, targets, failure); +} From f3c7455cd479b0ef55c6b722e65d77692bb1d44b Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 12:50:25 +0000 Subject: [PATCH 06/21] fix(reconciliation): harden apply locking boundaries --- src/lib/v5/db-reconciliation.test.ts | 293 ++++++++++++++++++++++++- src/lib/v5/db-reconciliation.ts | 307 ++++++++++++++++----------- 2 files changed, 463 insertions(+), 137 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index 23d3a7e6c..2c2dd8d8e 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -3,13 +3,14 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { existsSync, linkSync, + mkdirSync, mkdtempSync, readFileSync, + renameSync, rmSync, statSync, symlinkSync, truncateSync, - unlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -52,6 +53,30 @@ function currentDb(name: string, marker: string | null = '100'): string { return path; } +async function spawnFlockHolder(lockPath: string, holdMs: number): Promise> { + const readyPath = `${lockPath}.ready`; + writeFileSync(lockPath, ''); + const code = ` + import { dlopen, FFIType } from 'bun:ffi'; + import { openSync } from 'node:fs'; + const libc = dlopen('libc.so.6', { + flock: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, + }); + const fd = openSync(Bun.argv[1], 'r+'); + if (libc.symbols.flock(fd, 2) !== 0) process.exit(9); + await Bun.write(Bun.argv[2], 'ready'); + await Bun.sleep(Number(Bun.argv[3])); + `; + const child = Bun.spawn({ + cmd: [process.execPath, '-e', code, lockPath, readyPath, String(holdMs)], + stderr: 'pipe', + stdout: 'pipe', + }); + for (let attempt = 0; attempt < 400 && !existsSync(readyPath); attempt++) await Bun.sleep(5); + expect(existsSync(readyPath)).toBe(true); + return child; +} + function mutate(path: string, callback: (db: Database) => void): void { const db = new Database(path); db.exec('PRAGMA foreign_keys = ON'); @@ -1362,6 +1387,65 @@ describe('canonical locking and transactional apply', () => { expect(metaValue(destination, 'trigger-fired')).toBeNull(); }); + test('an opened SQLite handle fails closed across an A-to-B-open/B-to-A pathname restore', () => { + const source = currentDb('aba-source', null); + const destination = currentDb('aba-destination', null); + mutate(source, (db) => db.query("INSERT INTO meta (key, value) VALUES ('planned', 'source')").run()); + const plan = planDatabaseReconciliation(directional(source, destination)); + let injected = false; + + const report = applyDatabaseReconciliation(plan, { + openDatabase: (path, options) => { + if (injected) return new Database(path, options); + injected = true; + const other = path === source ? destination : source; + const saved = `${path}.saved`; + renameSync(path, saved); + renameSync(other, path); + const opened = new Database(path, options); + renameSync(path, other); + renameSync(saved, path); + return opened; + }, + }); + + expect(report).toMatchObject({ + status: 'preimage-changed', + converged: false, + failure: { code: 'input-changed', phase: 'revalidation' }, + }); + expect(metaValue(destination, 'planned')).toBeNull(); + }); + + test('locked serialization rechecks the exact opened handle before reading bytes', () => { + const source = currentDb('serialize-source', null); + const destination = currentDb('serialize-destination', null); + mutate(source, (db) => insertTask(db, { id: 'planned' })); + const plan = planDatabaseReconciliation(directional(source, destination)); + + const report = applyDatabaseReconciliation(plan, { + onLocked: (inputs) => { + const destinationInput = inputs.find((input) => input.role === 'destination'); + if (destinationInput === undefined) throw new Error('missing destination'); + const saved = `${destination}.saved`; + renameSync(destination, saved); + renameSync(source, destination); + try { + destinationInput.serialize(); + } finally { + renameSync(destination, source); + renameSync(saved, destination); + } + }, + }); + + expect(report).toMatchObject({ + status: 'preimage-changed', + failure: { code: 'input-changed', phase: 'revalidation', role: 'destination' }, + }); + expect(taskTitle(destination, 'planned')).toBeNull(); + }); + test('SQLite contention is bounded and rolls back earlier canonical locks without mutation', () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(right, left)); @@ -1385,25 +1469,127 @@ describe('canonical locking and transactional apply', () => { } }); - test('reversed arguments stop on the first canonical advisory lock without taking the second', () => { + test('reversed arguments stop on the first canonical advisory descriptor lock without taking the second', async () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(right, left)); const [first, second] = [left, right].sort(); const firstLock = `${first}.genie-reconciliation.lock`; const secondLock = `${second}.genie-reconciliation.lock`; - writeFileSync(firstLock, JSON.stringify({ pid: process.pid, token: 'test-holder' })); - try { + const holder = await spawnFlockHolder(firstLock, 250); + const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); + expect(report).toMatchObject({ + status: 'lock-timeout', + failure: { code: 'advisory-lock-timeout', phase: 'advisory-lock' }, + }); + expect(existsSync(secondLock)).toBe(false); + expect(await holder.exited).toBe(0); + }); + + test('second advisory-lock contention releases the first lock and a retry succeeds', async () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(right, left)); + const [, second] = [left, right].sort(); + const holder = await spawnFlockHolder(`${second}.genie-reconciliation.lock`, 250); + + const blocked = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); + expect(blocked).toMatchObject({ + status: 'lock-timeout', + failure: { code: 'advisory-lock-timeout', phase: 'advisory-lock' }, + cleanupFailures: [], + }); + expect(await holder.exited).toBe(0); + + const retried = applyDatabaseReconciliation(plan, { busyTimeoutMs: 500 }); + expect(retried).toMatchObject({ status: 'changed', converged: true, failure: null, cleanupFailures: [] }); + }); + + test('an advisory writer that finishes during the shared wait bound is followed safely', async () => { + const { left, right } = seedBidirectionalApplyPair(); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const [first] = [left, right].sort(); + const holder = await spawnFlockHolder(`${first}.genie-reconciliation.lock`, 75); + + const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 1_000 }); + + expect(await holder.exited).toBe(0); + expect(report).toMatchObject({ status: 'changed', converged: true, failure: null, cleanupFailures: [] }); + }); + + test('advisory inputs are no-follow, regular, empty, and bounded without parsing owner data', () => { + const cases: Array<{ name: string; prepare(path: string): void }> = [ + { + name: 'symlink', + prepare(path) { + const target = `${path}.target`; + writeFileSync(target, ''); + symlinkSync(target, path); + }, + }, + { + name: 'directory', + prepare(path) { + mkdirSync(path); + }, + }, + { + name: 'fifo', + prepare(path) { + const made = Bun.spawnSync({ cmd: ['mkfifo', path], stderr: 'pipe', stdout: 'pipe' }); + expect(made.exitCode).toBe(0); + }, + }, + { + name: 'malformed', + prepare(path) { + writeFileSync(path, '{not-json'); + }, + }, + { + name: 'oversized', + prepare(path) { + writeFileSync(path, new Uint8Array(1024 * 1024)); + }, + }, + ]; + + for (const fixture of cases) { + const left = currentDb(`${fixture.name}-left`, null); + const right = currentDb(`${fixture.name}-right`, null); + mutate(left, (db) => insertTask(db, { id: fixture.name })); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + const [first] = [left, right].sort(); + const lockPath = `${first}.genie-reconciliation.lock`; + fixture.prepare(lockPath); + const started = Date.now(); + const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); + expect(report).toMatchObject({ - status: 'lock-timeout', - failure: { code: 'advisory-lock-timeout', phase: 'advisory-lock' }, + status: 'uncertain', + failure: { code: 'unexpected-failure', phase: 'advisory-lock' }, }); - expect(existsSync(secondLock)).toBe(false); - } finally { - unlinkSync(firstLock); + expect(Date.now() - started).toBeLessThan(500); + rmSync(lockPath, { recursive: true, force: true }); } }); + test('release never removes a replacement advisory pathname', () => { + const { left, right } = seedBidirectionalApplyPair(); + const replacements: string[] = []; + const report = applyDatabaseReconciliation(planDatabaseReconciliation(bidirectional(left, right)), { + onLocked: (inputs) => { + for (const path of new Set(inputs.map((input) => `${input.canonicalPath}.genie-reconciliation.lock`))) { + renameSync(path, `${path}.held-inode`); + writeFileSync(path, ''); + replacements.push(path); + } + }, + }); + + expect(report).toMatchObject({ status: 'changed', converged: true, cleanupFailures: [] }); + expect(replacements.every((path) => existsSync(path))).toBe(true); + }); + test('opposite-order lock-aware reconcilers serialize without deadlock or overwrite', async () => { const { left, right } = seedBidirectionalApplyPair(); const moduleUrl = new URL('./db-reconciliation.ts', import.meta.url).href; @@ -1542,6 +1728,50 @@ describe('canonical locking and transactional apply', () => { expect(scalarCount(right, 'boards')).toBe(1); }); + test('a precommit failure with one no-op target is rolled back with zero commits', () => { + const left = currentDb('a-no-op-target', null); + const right = currentDb('z-changing-target', null); + mutate(left, (db) => insertTask(db, { id: 'left-only' })); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + expect(plan.targets.filter((target) => target.preimageDigest === target.postimageDigest)).toHaveLength(1); + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'before') throw new Error('before first commit'); + }, + }); + + expect(report).toMatchObject({ status: 'rolled-back', converged: false }); + expect(report.targets.every((target) => !target.committed)).toBe(true); + expect(report.targets.map((target) => target.observation).sort()).toEqual([ + 'expected-pre-and-postimage', + 'expected-preimage', + ]); + expect(taskTitle(left, 'left-only')).toBe('left-only'); + expect(taskTitle(right, 'left-only')).toBeNull(); + }); + + test('the first-commit boundary records a committed no-op target without claiming rollback', () => { + const left = currentDb('a-no-op-target', null); + const right = currentDb('z-changing-target', null); + mutate(left, (db) => insertTask(db, { id: 'left-only' })); + const plan = planDatabaseReconciliation(bidirectional(left, right)); + let commits = 0; + + const report = applyDatabaseReconciliation(plan, { + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'after' && ++commits === 1) { + throw new Error('after first commit'); + } + }, + }); + + expect(report).toMatchObject({ status: 'partial-commit', converged: false }); + expect(report.targets.filter((target) => target.committed)).toHaveLength(1); + expect(report.targets.find((target) => target.committed)?.observation).toBe('expected-pre-and-postimage'); + expect(taskTitle(right, 'left-only')).toBeNull(); + }); + test('a failure after the first destination commit reports a known partial commit', () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(left, right)); @@ -1623,6 +1853,51 @@ describe('canonical locking and transactional apply', () => { expect(report.targets.some((item) => item.observation === 'not-observed')).toBe(true); }); + test('a primary apply failure is preserved alongside rollback, close, and advisory cleanup failures', () => { + const source = currentDb('cleanup-source', null); + const destination = currentDb('cleanup-destination', null); + mutate(source, (db) => insertTask(db, { id: 'planned' })); + + const report = applyDatabaseReconciliation(planDatabaseReconciliation(directional(source, destination)), { + openDatabase: (path, options) => { + const db = new Database(path, options); + return new Proxy(db, { + get(target, property) { + if (property === 'exec') { + return (sql: string) => { + if (sql === 'ROLLBACK') throw new Error('injected rollback cleanup failure'); + return target.exec(sql); + }; + } + if (property === 'close') { + return () => { + target.close(); + throw new Error('injected close cleanup failure'); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }, + advisoryUnlock: () => -1, + onEvent: (event) => { + if (event.phase === 'commit' && event.state === 'before') throw new Error('primary commit failure'); + }, + }); + + expect(report).toMatchObject({ + status: 'rolled-back', + failure: { code: 'commit-failed', phase: 'commit' }, + cleanupFailures: [ + { code: 'rollback-failed', phase: 'rollback' }, + { code: 'close-failed', phase: 'cleanup' }, + { code: 'advisory-lock-release-failed', phase: 'cleanup' }, + ], + }); + expect(taskTitle(destination, 'planned')).toBeNull(); + }); + test('directional board-name swaps use a transaction-local parking order', () => { const source = currentDb('swap-source', null); const destination = currentDb('swap-destination', null); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 49ead7f3b..2f54b262c 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -9,21 +9,11 @@ * this module. */ -import { Database } from 'bun:sqlite'; +import { FFIType, dlopen } from 'bun:ffi'; +import { Database, constants as sqliteConstants } from 'bun:sqlite'; import { createHash, randomUUID } from 'node:crypto'; -import { - closeSync, - existsSync, - linkSync, - openSync, - readFileSync, - realpathSync, - renameSync, - statSync, - unlinkSync, - writeFileSync, -} from 'node:fs'; -import { basename, dirname, join, normalize } from 'node:path'; +import { closeSync, existsSync, constants as fsConstants, fstatSync, openSync, realpathSync, statSync } from 'node:fs'; +import { normalize } from 'node:path'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; import { BUSY_TIMEOUT_MS, isBusyError } from './sqlite-open.js'; @@ -32,6 +22,8 @@ const REPORT_VERSION = 1 as const; const READ_BUSY_TIMEOUT_MS = 5_000; const MAX_BUSY_TIMEOUT_MS = 2_147_483_647; const ADVISORY_RETRY_MS = 10; +const LOCK_EXCLUSIVE_NONBLOCKING = 2 | 4; +const LOCK_UNLOCK = 8; const MAX_DATABASE_BYTES = 256 * 1024 * 1024; const MAX_ROWS_PER_TABLE = 1_000_000; const MAX_TOTAL_ROWS = 2_000_000; @@ -75,6 +67,7 @@ export type ReconciliationApplyPhase = | 'logical-postimage-check' | 'commit' | 'rollback' + | 'cleanup' | 'observation'; export type ReconciliationApplyFailureCode = | ReconciliationErrorCode @@ -85,12 +78,15 @@ export type ReconciliationApplyFailureCode = | 'postimage-mismatch' | 'commit-failed' | 'rollback-failed' + | 'close-failed' + | 'advisory-lock-release-failed' | 'observation-failed' | 'unexpected-failure'; export type ReconciliationTargetObservation = | 'not-observed' | 'expected-preimage' | 'expected-postimage' + | 'expected-pre-and-postimage' | 'unexpected'; export type ReconciliationInputRole = 'left' | 'right' | 'source' | 'destination'; export type ReconciliationTargetRole = 'left' | 'right' | 'destination'; @@ -316,6 +312,7 @@ export interface ReconciliationApplyReport { readonly converged: boolean; readonly targets: readonly ReconciliationApplyTargetReport[]; readonly failure: ReconciliationApplyFailure | null; + readonly cleanupFailures: readonly ReconciliationApplyFailure[]; } export type ReconciliationApplyEvent = @@ -343,6 +340,10 @@ export interface ReconciliationApplyOptions { readonly onLocked?: (inputs: readonly ReconciliationLockedInput[]) => void; /** Bounded lifecycle observation and deterministic failure injection for tests. */ readonly onEvent?: (event: ReconciliationApplyEvent) => void; + /** Deterministic constructor seam for physical-identity regression tests. */ + readonly openDatabase?: (path: string, options: ConstructorParameters[1]) => Database; + /** Deterministic lifecycle seam; production always calls descriptor-bound flock. */ + readonly advisoryUnlock?: (descriptor: number) => number; } type SqliteScalar = string | bigint | number | Uint8Array | null; @@ -426,7 +427,7 @@ interface DatabaseImage { interface AdvisoryLock { readonly path: string; - readonly token: string; + readonly descriptor: number; } interface LockedDatabase { @@ -444,11 +445,17 @@ interface ApplyFailureContext { class ApplyBoundaryError extends Error { readonly context: ApplyFailureContext; + readonly cleanupFailures: ReconciliationApplyFailure[]; - constructor(context: ApplyFailureContext, cause?: unknown) { + constructor( + context: ApplyFailureContext, + cause?: unknown, + cleanupFailures: readonly ReconciliationApplyFailure[] = [], + ) { super(cause instanceof Error ? cause.message : context.code); this.name = 'ApplyBoundaryError'; this.context = context; + this.cleanupFailures = [...cleanupFailures]; } } @@ -734,123 +741,91 @@ function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { } const SLEEP_SIGNAL = new Int32Array(new SharedArrayBuffer(4)); +const FLOCK_SYMBOL = { + flock: { + args: [FFIType.i32, FFIType.i32], + returns: FFIType.i32, + }, +} as const; +const LIBC = + process.platform === 'linux' + ? dlopen('libc.so.6', FLOCK_SYMBOL) + : process.platform === 'darwin' + ? dlopen('/usr/lib/libSystem.B.dylib', FLOCK_SYMBOL) + : null; function sleepMs(ms: number): void { Atomics.wait(SLEEP_SIGNAL, 0, 0, ms); } -function errnoCode(caught: unknown): string | undefined { - if (!(caught instanceof Error)) return undefined; - const code = (caught as NodeJS.ErrnoException).code; - return typeof code === 'string' ? code : undefined; -} - function advisoryLockPath(canonicalPath: string): string { return `${canonicalPath}.genie-reconciliation.lock`; } -function advisoryOwner(pid: number, token: string): string { - return JSON.stringify({ pid, token }); -} - -function parseAdvisoryOwner(path: string): { pid: number; token: string } | null { +function databaseHandleMatchesPath(db: Pick): boolean { + const moved = new Int32Array(1); try { - const value: unknown = JSON.parse(readFileSync(path, 'utf8')); - if ( - typeof value !== 'object' || - value === null || - !('pid' in value) || - !('token' in value) || - !Number.isSafeInteger(value.pid) || - (value.pid as number) <= 0 || - typeof value.token !== 'string' - ) { - return null; - } - return { pid: value.pid as number, token: value.token }; + return db.fileControl(sqliteConstants.SQLITE_FCNTL_HAS_MOVED, moved) === 0 && moved[0] === 0; } catch { - return null; - } -} - -function processIsAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (caught) { - return errnoCode(caught) !== 'ESRCH'; + return false; } } -function removeStaleAdvisoryLock(path: string): boolean { - const owner = parseAdvisoryOwner(path); - if (owner === null || processIsAlive(owner.pid)) return false; - const stalePath = `${path}.${randomUUID()}.stale`; - try { - renameSync(path, stalePath); - } catch (caught) { - if (errnoCode(caught) === 'ENOENT') return true; - return false; - } - try { - unlinkSync(stalePath); - } catch { - // The stable name is already free. A stranded uniquely named stale inode - // does not grant authority and cannot block another reconciler. +function requireDatabaseHandleMatchesPath( + db: Pick, + role?: ReconciliationInputRole | ReconciliationTargetRole, +): void { + if (!databaseHandleMatchesPath(db)) { + throw new ApplyBoundaryError({ + code: 'input-changed', + phase: 'revalidation', + ...(role === undefined ? {} : { role }), + }); } - return true; } function acquireAdvisoryLock(canonicalPath: string, timeoutMs: number): AdvisoryLock { const path = advisoryLockPath(canonicalPath); - const token = randomUUID(); - const candidatePath = join(dirname(path), `.${basename(path)}.${process.pid}.${token}.candidate`); - const descriptor = openSync(candidatePath, 'wx', 0o600); + if (LIBC === null) { + throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }); + } + let descriptor: number; try { - writeFileSync(descriptor, advisoryOwner(process.pid, token)); - } finally { - closeSync(descriptor); + descriptor = openSync( + path, + fsConstants.O_RDWR | fsConstants.O_CREAT | fsConstants.O_NOFOLLOW | fsConstants.O_NONBLOCK, + 0o600, + ); + const stats = fstatSync(descriptor); + if (!stats.isFile() || stats.size !== 0) { + throw new Error('Advisory lock path must be an empty regular file.'); + } + } catch (caught) { + throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }, caught); } const deadline = Date.now() + timeoutMs; try { for (;;) { - try { - linkSync(candidatePath, path); - return { path, token }; - } catch (caught) { - if (errnoCode(caught) !== 'EEXIST') { - throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }, caught); - } - if (removeStaleAdvisoryLock(path)) continue; - const remaining = deadline - Date.now(); - if (remaining <= 0) { - throw new ApplyBoundaryError({ code: 'advisory-lock-timeout', phase: 'advisory-lock' }, caught); - } - sleepMs(Math.min(ADVISORY_RETRY_MS, remaining)); + if (LIBC.symbols.flock(descriptor, LOCK_EXCLUSIVE_NONBLOCKING) === 0) { + return { path, descriptor }; } + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new ApplyBoundaryError({ code: 'advisory-lock-timeout', phase: 'advisory-lock' }); + } + sleepMs(Math.min(ADVISORY_RETRY_MS, remaining)); } - } finally { - try { - unlinkSync(candidatePath); - } catch { - // The stable hard link, not this unique candidate name, owns exclusion. - } + } catch (caught) { + closeSync(descriptor); + throw caught; } } -function releaseAdvisoryLock(lock: AdvisoryLock): void { - const owner = parseAdvisoryOwner(lock.path); - if (owner === null) { - if (!existsSync(lock.path)) return; - throw new Error('Advisory lock ownership became unreadable.'); - } - if (owner.pid !== process.pid || owner.token !== lock.token) { - throw new Error('Advisory lock ownership changed before release.'); - } - const releasePath = `${lock.path}.${lock.token}.release`; - renameSync(lock.path, releasePath); - unlinkSync(releasePath); +function releaseAdvisoryLock(lock: AdvisoryLock, options?: ReconciliationApplyOptions): void { + const result = options?.advisoryUnlock?.(lock.descriptor) ?? LIBC?.symbols.flock(lock.descriptor, LOCK_UNLOCK) ?? -1; + closeSync(lock.descriptor); + if (result !== 0) throw new Error('Advisory descriptor unlock failed.'); } function readInventory(db: Database): Array<{ type: string; name: string; tableName: string; sql: string | null }> { @@ -1524,12 +1499,22 @@ function loadDatabaseImage(input: PhysicalInput, busyTimeoutMs = READ_BUSY_TIMEO } let transactionOpen = false; try { + if (!databaseHandleMatchesPath(db)) { + throw error('input-changed', 'A reconciliation input changed identity while it was being opened.'); + } + revalidatePhysicalInput(input); + if (!databaseHandleMatchesPath(db)) { + throw error('input-changed', 'A reconciliation input changed identity while it was being opened.'); + } db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); db.exec('PRAGMA query_only = ON'); db.exec('PRAGMA trusted_schema = OFF'); db.exec('PRAGMA foreign_keys = ON'); db.exec('BEGIN'); transactionOpen = true; + if (!databaseHandleMatchesPath(db)) { + throw error('input-changed', 'A reconciliation input changed identity while it was being read.'); + } const image = readDatabaseImage(db); db.exec('COMMIT'); transactionOpen = false; @@ -2298,6 +2283,7 @@ function makeApplyReport( converged: boolean, targets: readonly ReconciliationApplyTargetReport[], failure: ReconciliationApplyFailure | null, + cleanupFailures: readonly ReconciliationApplyFailure[] = [], ): ReconciliationApplyReport { return deepFreeze({ reportVersion: REPORT_VERSION, @@ -2307,6 +2293,7 @@ function makeApplyReport( converged, targets, failure, + cleanupFailures, }); } @@ -2392,11 +2379,18 @@ function acquirePlannedAdvisoryLocks( for (const path of paths) locks.push(acquireAdvisoryLock(path, remainingLockWait(deadline))); return locks; } catch (caught) { + const cleanupFailures: ReconciliationApplyFailure[] = []; for (const lock of locks.reverse()) { try { releaseAdvisoryLock(lock); } catch { - // The acquisition failure remains authoritative. + cleanupFailures.push({ code: 'advisory-lock-release-failed', phase: 'cleanup' }); + } + } + if (cleanupFailures.length > 0) { + if (caught instanceof ApplyBoundaryError) caught.cleanupFailures.push(...cleanupFailures); + else { + throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }, caught, cleanupFailures); } } throw caught; @@ -2422,6 +2416,7 @@ function revalidateApplyInput(input: PhysicalInput, role: ReconciliationInputRol function openLockedDatabases( inputs: readonly { readonly planned: ReconciliationPlanInput; readonly physical: PhysicalInput }[], deadline: number, + options: ReconciliationApplyOptions, ): LockedDatabase[] { for (const input of inputs) revalidateApplyInput(input.physical, input.planned.role); const grouped = new Map(); @@ -2447,16 +2442,23 @@ function openLockedDatabases( for (const group of ordered) { let db: Database | null = null; try { - db = new Database(group.input.canonicalPath, { + const databaseOptions = { readwrite: true, create: false, strict: true, safeIntegers: true, - }); + } as const; + db = + options.openDatabase?.(group.input.canonicalPath, databaseOptions) ?? + new Database(group.input.canonicalPath, databaseOptions); + requireDatabaseHandleMatchesPath(db, group.roles[0]); + for (const path of group.paths) revalidateApplyInput(path, group.roles[0]); + requireDatabaseHandleMatchesPath(db, group.roles[0]); db.exec(`PRAGMA busy_timeout = ${remainingLockWait(deadline)}`); db.exec('PRAGMA trusted_schema = OFF'); db.exec('PRAGMA foreign_keys = ON'); db.exec('BEGIN IMMEDIATE'); + requireDatabaseHandleMatchesPath(db, group.roles[0]); } catch (caught) { db?.close(); throw new ApplyBoundaryError( @@ -2469,7 +2471,6 @@ function openLockedDatabases( ); } locked.push({ input: group.input, roles: group.roles, db, transactionOpen: true }); - for (const path of group.paths) revalidateApplyInput(path, group.roles[0]); } return locked; } catch (caught) { @@ -2505,21 +2506,23 @@ function rollbackOpenTransactions(locked: readonly LockedDatabase[]): boolean { return complete; } -function closeLockedDatabases(locked: readonly LockedDatabase[]): void { +function closeLockedDatabases(locked: readonly LockedDatabase[]): boolean { + let complete = true; for (const item of [...locked].reverse()) { try { item.db.close(); } catch { - // Observation classifies the durable logical state after close failure. + complete = false; } } + return complete; } -function releaseAdvisoryLocks(locks: readonly AdvisoryLock[]): boolean { +function releaseAdvisoryLocks(locks: readonly AdvisoryLock[], options?: ReconciliationApplyOptions): boolean { let complete = true; for (const lock of [...locks].reverse()) { try { - releaseAdvisoryLock(lock); + releaseAdvisoryLock(lock, options); } catch { complete = false; } @@ -2535,6 +2538,9 @@ function revalidateLockedPlan( for (const item of locked) { let image: DatabaseImage; try { + requireDatabaseHandleMatchesPath(item.db, item.roles[0]); + revalidateApplyInput(item.input, item.roles[0]); + requireDatabaseHandleMatchesPath(item.db, item.roles[0]); image = readDatabaseImage(item.db); } catch (caught) { throw new ApplyBoundaryError( @@ -2577,7 +2583,12 @@ function lockedInputsForHook( role: input.role, canonicalPath: input.canonicalPath, target, - serialize: () => new Uint8Array(item.db.serialize()), + serialize: () => { + requireDatabaseHandleMatchesPath(item.db, input.role); + revalidateApplyInput(item.input, input.role); + requireDatabaseHandleMatchesPath(item.db, input.role); + return new Uint8Array(item.db.serialize()); + }, }); }); } @@ -2591,6 +2602,7 @@ function runLockedHook( try { options.onLocked?.(lockedInputsForHook(plan, locked)); } catch (caught) { + if (caught instanceof ApplyBoundaryError) throw caught; throw new ApplyBoundaryError({ code: 'apply-failed', phase: 'locked' }, caught); } } @@ -2829,6 +2841,9 @@ function commitTarget( committed: Set, ): void { emitApplyEvent(options, { phase: 'commit', role: target.role, state: 'before' }); + requireDatabaseHandleMatchesPath(item.db, target.role); + revalidateApplyInput(item.input, target.role); + requireDatabaseHandleMatchesPath(item.db, target.role); try { item.db.exec('COMMIT'); item.transactionOpen = false; @@ -2847,6 +2862,9 @@ function commitNonTargetTransactions( for (const item of locked) { if (!item.transactionOpen) continue; try { + requireDatabaseHandleMatchesPath(item.db, item.roles[0]); + revalidateApplyInput(item.input, item.roles[0]); + requireDatabaseHandleMatchesPath(item.db, item.roles[0]); item.db.exec('COMMIT'); item.transactionOpen = false; } catch (caught) { @@ -2875,6 +2893,9 @@ function mutateValidateAndCommit( for (const target of targets) { const item = lockedDatabaseForRole(locked, target.role); emitApplyEvent(options, { phase: 'mutation', role: target.role, state: 'before' }); + requireDatabaseHandleMatchesPath(item.db, target.role); + revalidateApplyInput(item.input, target.role); + requireDatabaseHandleMatchesPath(item.db, target.role); applyTargetChanges(item.db, target.changes); emitApplyEvent(options, { phase: 'mutation', role: target.role, state: 'after' }); } @@ -2907,12 +2928,16 @@ function observeTarget( committed: committed.has(target.role), }; } + const equalsPreimage = observedDigest === target.preimageDigest; + const equalsPostimage = target.postimageDigest !== null && observedDigest === target.postimageDigest; const observation: ReconciliationTargetObservation = - target.postimageDigest !== null && observedDigest === target.postimageDigest - ? 'expected-postimage' - : observedDigest === target.preimageDigest - ? 'expected-preimage' - : 'unexpected'; + equalsPreimage && equalsPostimage + ? 'expected-pre-and-postimage' + : equalsPostimage + ? 'expected-postimage' + : equalsPreimage + ? 'expected-preimage' + : 'unexpected'; return { role: target.role, preimageDigest: target.preimageDigest, @@ -2951,21 +2976,43 @@ function classifyApplyStatus( if (failure === null) { if (plan.status === 'same-database') return { status: 'same-database', converged: true }; if (plan.status === 'no-op') return { status: 'no-op', converged: true }; - if (targets.every((target) => target.observation === 'expected-postimage')) { + if ( + targets.every( + (target) => target.observation === 'expected-postimage' || target.observation === 'expected-pre-and-postimage', + ) + ) { return { status: 'changed', converged: true }; } return { status: 'uncertain', converged: false }; } - if (targets.every((target) => target.observation === 'expected-postimage')) { + const hasPostCommitEvidence = targets.some( + (target) => target.committed || target.observation === 'expected-postimage', + ); + if ( + hasPostCommitEvidence && + targets.every( + (target) => target.observation === 'expected-postimage' || target.observation === 'expected-pre-and-postimage', + ) + ) { return { status: 'expected-postimage', converged: false }; } if ( - targets.some((target) => target.observation === 'expected-postimage') && - targets.every((target) => target.observation === 'expected-postimage' || target.observation === 'expected-preimage') + hasPostCommitEvidence && + targets.every( + (target) => + target.observation === 'expected-postimage' || + target.observation === 'expected-preimage' || + target.observation === 'expected-pre-and-postimage', + ) ) { return { status: 'partial-commit', converged: false }; } - if (targets.every((target) => target.observation === 'expected-preimage')) { + if ( + targets.every((target) => !target.committed) && + targets.every( + (target) => target.observation === 'expected-preimage' || target.observation === 'expected-pre-and-postimage', + ) + ) { return { status: 'rolled-back', converged: false }; } return { status: 'uncertain', converged: false }; @@ -2989,6 +3036,7 @@ export function applyDatabaseReconciliation( let advisoryLocks: AdvisoryLock[] = []; let locked: LockedDatabase[] = []; const committed = new Set(); + const cleanupFailures: ReconciliationApplyFailure[] = []; try { timeoutMs = validateBusyTimeout(options.busyTimeoutMs); @@ -3006,23 +3054,26 @@ export function applyDatabaseReconciliation( const inputs = resolvePlannedInputs(plan); const lockDeadline = Date.now() + timeoutMs; advisoryLocks = acquirePlannedAdvisoryLocks(inputs, lockDeadline); - locked = openLockedDatabases(inputs, lockDeadline); + locked = openLockedDatabases(inputs, lockDeadline, options); revalidateLockedPlan(plan, locked); runLockedHook(plan, locked, options); mutateValidateAndCommit(plan, locked, options, committed); } catch (caught) { + if (caught instanceof ApplyBoundaryError) cleanupFailures.push(...caught.cleanupFailures); failure = failureFrom(caught, { code: 'apply-failed', phase: 'mutation' }); if (!rollbackOpenTransactions(locked)) { - failure = { code: 'rollback-failed', phase: 'rollback' }; + cleanupFailures.push({ code: 'rollback-failed', phase: 'rollback' }); } } finally { - closeLockedDatabases(locked); + if (!closeLockedDatabases(locked)) { + cleanupFailures.push({ code: 'close-failed', phase: 'cleanup' }); + } } const targets = advisoryLocks.length > 0 ? observeTargets(plan, committed, timeoutMs) : initialApplyTargets(plan); - if (!releaseAdvisoryLocks(advisoryLocks)) { - failure = failure ?? { code: 'unexpected-failure', phase: 'advisory-lock' }; + if (!releaseAdvisoryLocks(advisoryLocks, options)) { + cleanupFailures.push({ code: 'advisory-lock-release-failed', phase: 'cleanup' }); } const classified = classifyApplyStatus(plan, targets, failure); - return makeApplyReport(plan, classified.status, classified.converged, targets, failure); + return makeApplyReport(plan, classified.status, classified.converged, targets, failure, cleanupFailures); } From 972a02621c6a83d6f266a25e893779ee9aa7cf09 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 13:05:31 +0000 Subject: [PATCH 07/21] fix(reconciliation): close locking portability gaps --- src/lib/v5/db-reconciliation.test.ts | 218 +++++++++++++++++++++++++-- src/lib/v5/db-reconciliation.ts | 181 ++++++++++++++++++---- 2 files changed, 359 insertions(+), 40 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index 2c2dd8d8e..b8bfcd0df 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -2,10 +2,12 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { existsSync, + fstatSync, linkSync, mkdirSync, mkdtempSync, readFileSync, + readdirSync, renameSync, rmSync, statSync, @@ -15,6 +17,7 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { linuxLibcCandidates } from '../install-transaction.js'; import { IDENTICAL_HISTORY_ADDITION_LIMITATION, type ReconciliationApplyEvent, @@ -24,6 +27,7 @@ import { applyDatabaseReconciliation, dryRunDatabaseReconciliation, planDatabaseReconciliation, + reconciliationAdvisoryLockPath, } from './db-reconciliation.js'; import { openDb } from './genie-db.js'; @@ -59,16 +63,31 @@ async function spawnFlockHolder(lockPath: string, holdMs: number): Promise { expect(taskTitle(destination, 'planned')).toBeNull(); }); - test('SQLite contention is bounded and rolls back earlier canonical locks without mutation', () => { + test('locked serialization rechecks handle/path/handle identity after the real serialize call', () => { + const source = currentDb('serialize-after-source', null); + const destination = currentDb('serialize-after-destination', null); + mutate(source, (db) => insertTask(db, { id: 'planned' })); + const plan = planDatabaseReconciliation(directional(source, destination)); + let swapped = false; + + const report = applyDatabaseReconciliation(plan, { + openDatabase: (path, options) => { + const db = new Database(path, options); + if (path !== destination) return db; + return new Proxy(db, { + get(target, property) { + if (property === 'serialize') { + return () => { + const bytes = target.serialize(); + renameSync(destination, `${destination}.saved`); + renameSync(source, destination); + swapped = true; + return bytes; + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }, + onLocked: (inputs) => { + inputs.find((input) => input.role === 'destination')?.serialize(); + }, + }); + if (swapped) { + renameSync(destination, source); + renameSync(`${destination}.saved`, destination); + } + + expect(report).toMatchObject({ + status: 'preimage-changed', + failure: { code: 'input-changed', phase: 'revalidation', role: 'destination' }, + }); + expect(taskTitle(destination, 'planned')).toBeNull(); + }); + + test('SQLite contention on the second canonical database cleans the earlier transaction and reports failures', () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(right, left)); - const heldPath = [left, right].sort()[0]; + const [firstPath, heldPath] = [left, right].sort(); const holder = new Database(heldPath); holder.exec('BEGIN IMMEDIATE'); const startedAt = Date.now(); try { - const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 25 }); + const report = applyDatabaseReconciliation(plan, { + busyTimeoutMs: 25, + openDatabase: (path, options) => { + const db = new Database(path, options); + if (path !== firstPath) return db; + return new Proxy(db, { + get(target, property) { + if (property === 'exec') { + return (sql: string) => { + if (sql === 'ROLLBACK') { + target.exec(sql); + throw new Error('injected earlier rollback cleanup failure'); + } + return target.exec(sql); + }; + } + if (property === 'close') { + return () => { + target.close(); + throw new Error('injected earlier close cleanup failure'); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }, + }); expect(report).toMatchObject({ status: 'lock-timeout', converged: false, failure: { code: 'sqlite-lock-timeout', phase: 'sqlite-lock' }, + cleanupFailures: [ + { code: 'rollback-failed', phase: 'rollback' }, + { code: 'close-failed', phase: 'cleanup' }, + ], }); expect(Date.now() - startedAt).toBeLessThan(1_000); expect(scalarCount(left, 'boards')).toBe(1); @@ -1467,14 +1560,19 @@ describe('canonical locking and transactional apply', () => { holder.exec('ROLLBACK'); holder.close(); } + expect(applyDatabaseReconciliation(plan, { busyTimeoutMs: 500 })).toMatchObject({ + status: 'changed', + converged: true, + cleanupFailures: [], + }); }); test('reversed arguments stop on the first canonical advisory descriptor lock without taking the second', async () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(right, left)); const [first, second] = [left, right].sort(); - const firstLock = `${first}.genie-reconciliation.lock`; - const secondLock = `${second}.genie-reconciliation.lock`; + const firstLock = reconciliationAdvisoryLockPath(first); + const secondLock = reconciliationAdvisoryLockPath(second); const holder = await spawnFlockHolder(firstLock, 250); const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); expect(report).toMatchObject({ @@ -1489,7 +1587,7 @@ describe('canonical locking and transactional apply', () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(right, left)); const [, second] = [left, right].sort(); - const holder = await spawnFlockHolder(`${second}.genie-reconciliation.lock`, 250); + const holder = await spawnFlockHolder(reconciliationAdvisoryLockPath(second), 250); const blocked = applyDatabaseReconciliation(plan, { busyTimeoutMs: 20 }); expect(blocked).toMatchObject({ @@ -1507,7 +1605,7 @@ describe('canonical locking and transactional apply', () => { const { left, right } = seedBidirectionalApplyPair(); const plan = planDatabaseReconciliation(bidirectional(left, right)); const [first] = [left, right].sort(); - const holder = await spawnFlockHolder(`${first}.genie-reconciliation.lock`, 75); + const holder = await spawnFlockHolder(reconciliationAdvisoryLockPath(first), 75); const report = applyDatabaseReconciliation(plan, { busyTimeoutMs: 1_000 }); @@ -1558,7 +1656,7 @@ describe('canonical locking and transactional apply', () => { mutate(left, (db) => insertTask(db, { id: fixture.name })); const plan = planDatabaseReconciliation(bidirectional(left, right)); const [first] = [left, right].sort(); - const lockPath = `${first}.genie-reconciliation.lock`; + const lockPath = reconciliationAdvisoryLockPath(first); fixture.prepare(lockPath); const started = Date.now(); @@ -1578,7 +1676,7 @@ describe('canonical locking and transactional apply', () => { const replacements: string[] = []; const report = applyDatabaseReconciliation(planDatabaseReconciliation(bidirectional(left, right)), { onLocked: (inputs) => { - for (const path of new Set(inputs.map((input) => `${input.canonicalPath}.genie-reconciliation.lock`))) { + for (const path of new Set(inputs.map((input) => reconciliationAdvisoryLockPath(input.canonicalPath)))) { renameSync(path, `${path}.held-inode`); writeFileSync(path, ''); replacements.push(path); @@ -1590,6 +1688,100 @@ describe('canonical locking and transactional apply', () => { expect(replacements.every((path) => existsSync(path))).toBe(true); }); + test('advisory flock resolution is lazy and falls through glibc, musl, and unavailable candidates', () => { + const source = currentDb('flock-source', null); + const destination = currentDb('flock-destination', null); + mutate(source, (db) => insertTask(db, { id: 'planned' })); + const request = directional(source, destination); + const cases = [ + { available: 'libc.so.6', expected: ['libc.so.6'] }, + { available: '/lib/ld-musl-x86_64.so.1', expected: ['libc.so.6', '/lib/ld-musl-x86_64.so.1'] }, + ]; + for (const fixture of cases) { + const attempted: string[] = []; + const report = applyDatabaseReconciliation(planDatabaseReconciliation(request), { + advisoryFlock: { + platform: 'linux', + architecture: 'x64', + linuxCandidates: ['libc.so.6', '/lib/ld-musl-x86_64.so.1'], + linuxOpener: (candidate) => { + attempted.push(candidate); + return candidate === fixture.available ? () => 0 : null; + }, + }, + }); + expect(report.failure).toBeNull(); + expect(attempted).toEqual([...fixture.expected, ...fixture.expected]); + } + + const attempted: string[] = []; + const unavailable = applyDatabaseReconciliation(planDatabaseReconciliation(request), { + advisoryFlock: { + platform: 'linux', + architecture: 'x64', + linuxCandidates: ['missing-glibc', 'missing-musl'], + linuxOpener: (candidate) => { + attempted.push(candidate); + return null; + }, + }, + }); + expect(attempted).toEqual(['missing-glibc', 'missing-musl']); + expect(unavailable).toMatchObject({ + status: 'uncertain', + failure: { code: 'unexpected-failure', phase: 'advisory-lock' }, + }); + }); + + test('post-open advisory validation failure closes the owned descriptor', () => { + const { left, right } = seedBidirectionalApplyPair(); + const [first] = [left, right].sort(); + writeFileSync(reconciliationAdvisoryLockPath(first), 'invalid'); + let descriptor: number | null = null; + + const report = applyDatabaseReconciliation(planDatabaseReconciliation(bidirectional(left, right)), { + onAdvisoryDescriptorOpened: (opened) => { + descriptor ??= opened; + }, + }); + + expect(report).toMatchObject({ + status: 'uncertain', + failure: { code: 'unexpected-failure', phase: 'advisory-lock' }, + }); + expect(descriptor).not.toBeNull(); + expect(() => fstatSync(descriptor as number)).toThrow(); + }); + + test('throwing advisory unlock still closes the descriptor and permits reacquisition', () => { + const { left, right } = seedBidirectionalApplyPair(); + const first = applyDatabaseReconciliation(planDatabaseReconciliation(bidirectional(left, right)), { + advisoryUnlock: () => { + throw new Error('injected unlock failure'); + }, + }); + expect(first.cleanupFailures).toContainEqual({ code: 'advisory-lock-release-failed', phase: 'cleanup' }); + + const retried = applyDatabaseReconciliation(planDatabaseReconciliation(bidirectional(left, right)), { + busyTimeoutMs: 100, + }); + expect(retried).toMatchObject({ status: 'no-op', converged: true, failure: null, cleanupFailures: [] }); + }); + + test('reconciliation advisory locks never dirty database directories', () => { + const source = currentDb('clean-source', null); + const destination = currentDb('clean-destination', null); + mutate(source, (db) => insertTask(db, { id: 'planned' })); + expect(applyDatabaseReconciliation(planDatabaseReconciliation(directional(source, destination)))).toMatchObject({ + status: 'changed', + converged: true, + }); + + expect(readdirSync(fixtureRoot).filter((name) => name.includes('genie-reconciliation.lock'))).toEqual([]); + expect(reconciliationAdvisoryLockPath(source).startsWith(fixtureRoot)).toBe(false); + expect(reconciliationAdvisoryLockPath(destination).startsWith(fixtureRoot)).toBe(false); + }); + test('opposite-order lock-aware reconcilers serialize without deadlock or overwrite', async () => { const { left, right } = seedBidirectionalApplyPair(); const moduleUrl = new URL('./db-reconciliation.ts', import.meta.url).href; diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 2f54b262c..61138117b 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -12,8 +12,20 @@ import { FFIType, dlopen } from 'bun:ffi'; import { Database, constants as sqliteConstants } from 'bun:sqlite'; import { createHash, randomUUID } from 'node:crypto'; -import { closeSync, existsSync, constants as fsConstants, fstatSync, openSync, realpathSync, statSync } from 'node:fs'; -import { normalize } from 'node:path'; +import { + closeSync, + existsSync, + constants as fsConstants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + realpathSync, + statSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, normalize } from 'node:path'; +import { linuxLibcCandidates } from '../install-transaction.js'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; import { BUSY_TIMEOUT_MS, isBusyError } from './sqlite-open.js'; @@ -342,10 +354,24 @@ export interface ReconciliationApplyOptions { readonly onEvent?: (event: ReconciliationApplyEvent) => void; /** Deterministic constructor seam for physical-identity regression tests. */ readonly openDatabase?: (path: string, options: ConstructorParameters[1]) => Database; + /** Lazy native advisory-lock resolution seam for libc portability tests. */ + readonly advisoryFlock?: ReconciliationAdvisoryFlockDependencies; + /** Descriptor ownership observation after open and before object validation. */ + readonly onAdvisoryDescriptorOpened?: (descriptor: number) => void; /** Deterministic lifecycle seam; production always calls descriptor-bound flock. */ readonly advisoryUnlock?: (descriptor: number) => number; } +export type ReconciliationAdvisoryFlock = (descriptor: number, operation: number) => number; + +export interface ReconciliationAdvisoryFlockDependencies { + readonly platform?: NodeJS.Platform; + readonly architecture?: NodeJS.Architecture; + readonly linuxCandidates?: readonly string[]; + readonly linuxOpener?: (candidate: string) => ReconciliationAdvisoryFlock | null; + readonly darwinOpener?: () => ReconciliationAdvisoryFlock | null; +} + type SqliteScalar = string | bigint | number | Uint8Array | null; type CanonicalScalar = readonly ['null'] | readonly ['string', string] | readonly ['integer', string]; @@ -428,6 +454,7 @@ interface DatabaseImage { interface AdvisoryLock { readonly path: string; readonly descriptor: number; + readonly flock: ReconciliationAdvisoryFlock; } interface LockedDatabase { @@ -747,19 +774,83 @@ const FLOCK_SYMBOL = { returns: FFIType.i32, }, } as const; -const LIBC = - process.platform === 'linux' - ? dlopen('libc.so.6', FLOCK_SYMBOL) - : process.platform === 'darwin' - ? dlopen('/usr/lib/libSystem.B.dylib', FLOCK_SYMBOL) - : null; +const ADVISORY_LOCK_ROOT_NAME = `genie-reconciliation-locks-${process.getuid?.() ?? 'unknown'}-v1`; +let defaultAdvisoryFlock: ReconciliationAdvisoryFlock | null | undefined; function sleepMs(ms: number): void { Atomics.wait(SLEEP_SIGNAL, 0, 0, ms); } -function advisoryLockPath(canonicalPath: string): string { - return `${canonicalPath}.genie-reconciliation.lock`; +function openFlock(candidate: string): ReconciliationAdvisoryFlock | null { + try { + const library = dlopen(candidate, FLOCK_SYMBOL); + return (descriptor, operation) => library.symbols.flock(descriptor, operation); + } catch { + return null; + } +} + +function openDarwinFlock(): ReconciliationAdvisoryFlock | null { + return openFlock('/usr/lib/libSystem.B.dylib'); +} + +function resolveAdvisoryFlock( + dependencies?: ReconciliationAdvisoryFlockDependencies, +): ReconciliationAdvisoryFlock | null { + if (dependencies === undefined && defaultAdvisoryFlock !== undefined) return defaultAdvisoryFlock; + const platform = dependencies?.platform ?? process.platform; + const architecture = dependencies?.architecture ?? process.arch; + let resolved: ReconciliationAdvisoryFlock | null = null; + if (platform === 'linux') { + const opener = dependencies?.linuxOpener ?? openFlock; + const candidates = dependencies?.linuxCandidates ?? linuxLibcCandidates(architecture); + for (const candidate of candidates) { + try { + resolved = opener(candidate); + } catch { + resolved = null; + } + if (resolved !== null) break; + } + } else if (platform === 'darwin') { + try { + resolved = (dependencies?.darwinOpener ?? openDarwinFlock)(); + } catch { + resolved = null; + } + } + if (dependencies === undefined) defaultAdvisoryFlock = resolved; + return resolved; +} + +function advisoryLockRoot(): string { + const requestedRoot = join(tmpdir(), ADVISORY_LOCK_ROOT_NAME); + try { + mkdirSync(requestedRoot, { mode: 0o700 }); + } catch (caught) { + if (!(caught instanceof Error) || !('code' in caught) || (caught as NodeJS.ErrnoException).code !== 'EEXIST') { + throw caught; + } + } + const stats = lstatSync(requestedRoot); + const expectedUid = process.getuid?.(); + if ( + !stats.isDirectory() || + stats.isSymbolicLink() || + (stats.mode & 0o077) !== 0 || + (expectedUid !== undefined && stats.uid !== expectedUid) + ) { + throw new Error('Advisory lock root must be a private owned directory.'); + } + return realpathSync(requestedRoot); +} + +export function reconciliationAdvisoryLockPath(canonicalPath: string): string { + const digest = createHash('sha256') + .update('genie-reconciliation-advisory-lock-v1\0') + .update(normalize(canonicalPath)) + .digest('hex'); + return join(advisoryLockRoot(), `${digest}.lock`); } function databaseHandleMatchesPath(db: Pick): boolean { @@ -784,31 +875,46 @@ function requireDatabaseHandleMatchesPath( } } -function acquireAdvisoryLock(canonicalPath: string, timeoutMs: number): AdvisoryLock { - const path = advisoryLockPath(canonicalPath); - if (LIBC === null) { +function acquireAdvisoryLock( + canonicalPath: string, + timeoutMs: number, + options: ReconciliationApplyOptions, +): AdvisoryLock { + const path = reconciliationAdvisoryLockPath(canonicalPath); + const flock = resolveAdvisoryFlock(options.advisoryFlock); + if (flock === null) { throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }); } - let descriptor: number; + let descriptor: number | null = null; try { descriptor = openSync( path, fsConstants.O_RDWR | fsConstants.O_CREAT | fsConstants.O_NOFOLLOW | fsConstants.O_NONBLOCK, 0o600, ); + options.onAdvisoryDescriptorOpened?.(descriptor); const stats = fstatSync(descriptor); if (!stats.isFile() || stats.size !== 0) { throw new Error('Advisory lock path must be an empty regular file.'); } } catch (caught) { + if (descriptor !== null) { + try { + closeSync(descriptor); + } catch { + // The validation failure remains primary; descriptor close was still attempted. + } + } throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'advisory-lock' }, caught); } const deadline = Date.now() + timeoutMs; + let ownershipTransferred = false; try { for (;;) { - if (LIBC.symbols.flock(descriptor, LOCK_EXCLUSIVE_NONBLOCKING) === 0) { - return { path, descriptor }; + if (flock(descriptor, LOCK_EXCLUSIVE_NONBLOCKING) === 0) { + ownershipTransferred = true; + return { path, descriptor, flock }; } const remaining = deadline - Date.now(); if (remaining <= 0) { @@ -816,15 +922,18 @@ function acquireAdvisoryLock(canonicalPath: string, timeoutMs: number): Advisory } sleepMs(Math.min(ADVISORY_RETRY_MS, remaining)); } - } catch (caught) { - closeSync(descriptor); - throw caught; + } finally { + if (!ownershipTransferred) closeSync(descriptor); } } function releaseAdvisoryLock(lock: AdvisoryLock, options?: ReconciliationApplyOptions): void { - const result = options?.advisoryUnlock?.(lock.descriptor) ?? LIBC?.symbols.flock(lock.descriptor, LOCK_UNLOCK) ?? -1; - closeSync(lock.descriptor); + let result = -1; + try { + result = options?.advisoryUnlock?.(lock.descriptor) ?? lock.flock(lock.descriptor, LOCK_UNLOCK); + } finally { + closeSync(lock.descriptor); + } if (result !== 0) throw new Error('Advisory descriptor unlock failed.'); } @@ -2372,11 +2481,12 @@ function resolvePlannedInputs(plan: ReconciliationPlan): Array<{ function acquirePlannedAdvisoryLocks( inputs: readonly { readonly physical: PhysicalInput }[], deadline: number, + options: ReconciliationApplyOptions, ): AdvisoryLock[] { const locks: AdvisoryLock[] = []; const paths = [...new Set(inputs.map((input) => input.physical.canonicalPath))].sort(compareCanonical); try { - for (const path of paths) locks.push(acquireAdvisoryLock(path, remainingLockWait(deadline))); + for (const path of paths) locks.push(acquireAdvisoryLock(path, remainingLockWait(deadline), options)); return locks; } catch (caught) { const cleanupFailures: ReconciliationApplyFailure[] = []; @@ -2474,9 +2584,7 @@ function openLockedDatabases( } return locked; } catch (caught) { - rollbackOpenTransactions(locked); - closeLockedDatabases(locked); - throw caught; + throwWithLockedCleanup(caught, locked); } } @@ -2518,6 +2626,21 @@ function closeLockedDatabases(locked: readonly LockedDatabase[]): boolean { return complete; } +function throwWithLockedCleanup(caught: unknown, locked: readonly LockedDatabase[]): never { + const cleanupFailures: ReconciliationApplyFailure[] = []; + if (!rollbackOpenTransactions(locked)) { + cleanupFailures.push({ code: 'rollback-failed', phase: 'rollback' }); + } + if (!closeLockedDatabases(locked)) { + cleanupFailures.push({ code: 'close-failed', phase: 'cleanup' }); + } + if (caught instanceof ApplyBoundaryError) { + caught.cleanupFailures.push(...cleanupFailures); + throw caught; + } + throw new ApplyBoundaryError({ code: 'unexpected-failure', phase: 'sqlite-lock' }, caught, cleanupFailures); +} + function releaseAdvisoryLocks(locks: readonly AdvisoryLock[], options?: ReconciliationApplyOptions): boolean { let complete = true; for (const lock of [...locks].reverse()) { @@ -2587,7 +2710,11 @@ function lockedInputsForHook( requireDatabaseHandleMatchesPath(item.db, input.role); revalidateApplyInput(item.input, input.role); requireDatabaseHandleMatchesPath(item.db, input.role); - return new Uint8Array(item.db.serialize()); + const bytes = new Uint8Array(item.db.serialize()); + requireDatabaseHandleMatchesPath(item.db, input.role); + revalidateApplyInput(item.input, input.role); + requireDatabaseHandleMatchesPath(item.db, input.role); + return bytes; }, }); }); @@ -3053,7 +3180,7 @@ export function applyDatabaseReconciliation( try { const inputs = resolvePlannedInputs(plan); const lockDeadline = Date.now() + timeoutMs; - advisoryLocks = acquirePlannedAdvisoryLocks(inputs, lockDeadline); + advisoryLocks = acquirePlannedAdvisoryLocks(inputs, lockDeadline, options); locked = openLockedDatabases(inputs, lockDeadline, options); revalidateLockedPlan(plan, locked); runLockedHook(plan, locked, options); From 063064cc36306c1589357b938ebd37089847ef67 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 13:45:15 +0000 Subject: [PATCH 08/21] feat(db-sync): add durable snapshot recovery --- src/lib/v5/db-reconciliation.ts | 221 ++++ src/lib/v5/db-sync-snapshots.test.ts | 912 +++++++++++++++++ src/lib/v5/db-sync-snapshots.ts | 1406 ++++++++++++++++++++++++++ 3 files changed, 2539 insertions(+) create mode 100644 src/lib/v5/db-sync-snapshots.test.ts create mode 100644 src/lib/v5/db-sync-snapshots.ts diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 61138117b..aae982205 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -342,6 +342,57 @@ export interface ReconciliationLockedInput { serialize(): Uint8Array; } +export interface ReconciliationDatabaseObservation { + readonly schemaFingerprint: string; + readonly logicalDigest: string; +} + +export interface ReconciliationLockedDatabaseInput { + readonly role: ReconciliationInputRole; + readonly canonicalPath: string; + observe(): ReconciliationDatabaseObservation; + serialize(): Uint8Array; + restoreFrom(source: Database): ReconciliationDatabaseObservation; +} + +export type ReconciliationLockedOperationEvent = { + readonly phase: 'commit'; + readonly role: ReconciliationInputRole; + readonly state: 'before' | 'after'; +}; + +export interface ReconciliationLockedOperationOptions { + readonly busyTimeoutMs?: number; + readonly onEvent?: (event: ReconciliationLockedOperationEvent) => void; + readonly openDatabase?: ReconciliationApplyOptions['openDatabase']; + readonly advisoryFlock?: ReconciliationAdvisoryFlockDependencies; + readonly onAdvisoryDescriptorOpened?: (descriptor: number) => void; + readonly advisoryUnlock?: (descriptor: number) => number; +} + +export interface ReconciliationLockedOperationResult { + readonly value: T; + readonly afterCommit?: () => void; +} + +export class ReconciliationLockedOperationError extends Error { + readonly failure: ReconciliationApplyFailure; + readonly cleanupFailures: readonly ReconciliationApplyFailure[]; + readonly operationCause: unknown; + + constructor( + failure: ReconciliationApplyFailure, + cause?: unknown, + cleanupFailures: readonly ReconciliationApplyFailure[] = [], + ) { + super(cause instanceof Error ? cause.message : failure.code); + this.name = 'ReconciliationLockedOperationError'; + this.failure = failure; + this.cleanupFailures = cleanupFailures; + this.operationCause = cause; + } +} + export interface ReconciliationApplyOptions { /** Shared total wait bound for all advisory and SQLite lock acquisition. */ readonly busyTimeoutMs?: number; @@ -1599,6 +1650,12 @@ function readDatabaseImage(db: Database): DatabaseImage { return { schemaFingerprint, logicalDigest, state }; } +/** Validate and identify an already-open exact-current reconciliation image. */ +export function inspectReconciliationDatabase(db: Database): ReconciliationDatabaseObservation { + const image = readDatabaseImage(db); + return { schemaFingerprint: image.schemaFingerprint, logicalDigest: image.logicalDigest }; +} + function loadDatabaseImage(input: PhysicalInput, busyTimeoutMs = READ_BUSY_TIMEOUT_MS): DatabaseImage { let db: Database; try { @@ -2868,6 +2925,41 @@ function applyTargetChanges(db: Database, changes: ReconciliationTargetChanges): applyMeta(db, changes.meta); } +function completeStateChanges(state: LogicalState): ReconciliationTargetChanges { + return { + boards: [...state.boards.values()], + tasks: [...state.tasks.values()], + wishGroups: [...state.wishGroups.values()], + hireRoster: [...state.hireRoster.values()], + meta: [...state.meta.values()], + taskDependencies: [...state.taskDependencies.values()], + stageLog: [...state.stageLog.values()].map(({ value, count }) => ({ value, count })), + taskEvents: [...state.taskEvents.values()].map(({ value, count }) => ({ value, count })), + }; +} + +function replaceLogicalState(db: Database, source: DatabaseImage): DatabaseImage { + db.exec(` + DELETE FROM task_dependencies; + DELETE FROM stage_log; + DELETE FROM task_events; + DELETE FROM hire_roster; + DELETE FROM wish_groups; + DELETE FROM tasks; + DELETE FROM boards; + DELETE FROM meta; + `); + applyTargetChanges(db, completeStateChanges(source.state)); + const restored = readDatabaseImage(db); + if (restored.schemaFingerprint !== source.schemaFingerprint || restored.logicalDigest !== source.logicalDigest) { + throw new ReconciliationError( + 'integrity-failed', + 'A logical snapshot restore did not reproduce its validated image.', + ); + } + return restored; +} + function runPostimageCheck( phase: 'foreign-key-check' | 'integrity-check' | 'logical-postimage-check', role: ReconciliationTargetRole, @@ -3204,3 +3296,132 @@ export function applyDatabaseReconciliation( const classified = classifyApplyStatus(plan, targets, failure); return makeApplyReport(plan, classified.status, classified.converged, targets, failure, cleanupFailures); } + +function lockedOperationInputs( + requested: readonly { readonly planned: ReconciliationPlanInput }[], + locked: readonly LockedDatabase[], +): readonly ReconciliationLockedDatabaseInput[] { + return requested.map(({ planned }) => { + const item = lockedDatabaseForRole(locked, planned.role); + const guarded = (operation: () => T): T => { + requireDatabaseHandleMatchesPath(item.db, planned.role); + revalidateApplyInput(item.input, planned.role); + requireDatabaseHandleMatchesPath(item.db, planned.role); + const value = operation(); + requireDatabaseHandleMatchesPath(item.db, planned.role); + revalidateApplyInput(item.input, planned.role); + requireDatabaseHandleMatchesPath(item.db, planned.role); + return value; + }; + return Object.freeze({ + role: planned.role, + canonicalPath: planned.canonicalPath, + observe: () => + guarded(() => { + const image = readDatabaseImage(item.db); + return { schemaFingerprint: image.schemaFingerprint, logicalDigest: image.logicalDigest }; + }), + serialize: () => guarded(() => new Uint8Array(item.db.serialize())), + restoreFrom: (source: Database) => + guarded(() => { + const restored = replaceLogicalState(item.db, readDatabaseImage(source)); + return { schemaFingerprint: restored.schemaFingerprint, logicalDigest: restored.logicalDigest }; + }), + }); + }); +} + +function emitLockedOperationEvent( + options: ReconciliationLockedOperationOptions, + event: ReconciliationLockedOperationEvent, +): void { + try { + options.onEvent?.(event); + } catch (caught) { + throw new ApplyBoundaryError({ code: 'commit-failed', phase: 'commit', role: event.role }, caught); + } +} + +/** + * Run a bounded operation while both reconciliation inputs are held by the + * same canonical advisory and SQLite write-lock contract as normal apply. + * + * The callback may restore logical contents only through the provided live + * handles. Its optional afterCommit callback runs while advisory locks remain + * held, after every SQLite transaction committed, which is the safe boundary + * for durable snapshot-state transitions. + */ +export function withLockedReconciliationDatabases( + request: ReconciliationRequest, + operation: (inputs: readonly ReconciliationLockedDatabaseInput[]) => ReconciliationLockedOperationResult, + options: ReconciliationLockedOperationOptions = {}, +): T { + let advisoryLocks: AdvisoryLock[] = []; + let locked: LockedDatabase[] = []; + const cleanupFailures: ReconciliationApplyFailure[] = []; + const lockOptions: ReconciliationApplyOptions = { + busyTimeoutMs: options.busyTimeoutMs, + openDatabase: options.openDatabase, + advisoryFlock: options.advisoryFlock, + onAdvisoryDescriptorOpened: options.onAdvisoryDescriptorOpened, + advisoryUnlock: options.advisoryUnlock, + }; + try { + const timeoutMs = validateBusyTimeout(options.busyTimeoutMs); + const roles: readonly [ReconciliationInputRole, ReconciliationInputRole] = + request.mode === 'bidirectional' ? ['left', 'right'] : ['source', 'destination']; + const paths: readonly [string, string] = + request.mode === 'bidirectional' + ? [request.leftPath, request.rightPath] + : [request.sourcePath, request.destinationPath]; + const requested = paths.map((path, index) => { + const physical = resolvePhysicalInput(path); + return { + planned: { + role: roles[index], + canonicalPath: physical.canonicalPath, + logicalDigest: '', + }, + physical, + }; + }); + if (samePhysicalInput(requested[0].physical, requested[1].physical)) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } + const deadline = Date.now() + timeoutMs; + advisoryLocks = acquirePlannedAdvisoryLocks(requested, deadline, lockOptions); + locked = openLockedDatabases(requested, deadline, lockOptions); + const inputs = lockedOperationInputs(requested, locked); + const initial = inputs.map((input) => input.observe()); + if (initial[0].schemaFingerprint !== initial[1].schemaFingerprint) unsupportedSchema(); + const result = operation(inputs); + const final = inputs.map((input) => input.observe()); + if (final.some((image) => image.schemaFingerprint !== initial[0].schemaFingerprint)) unsupportedSchema(); + for (const item of locked) { + const role = item.roles[0]; + emitLockedOperationEvent(options, { phase: 'commit', role, state: 'before' }); + requireDatabaseHandleMatchesPath(item.db, role); + revalidateApplyInput(item.input, role); + requireDatabaseHandleMatchesPath(item.db, role); + item.db.exec('COMMIT'); + item.transactionOpen = false; + emitLockedOperationEvent(options, { phase: 'commit', role, state: 'after' }); + } + result.afterCommit?.(); + return result.value; + } catch (caught) { + if (caught instanceof ApplyBoundaryError) cleanupFailures.push(...caught.cleanupFailures); + if (!rollbackOpenTransactions(locked)) { + cleanupFailures.push({ code: 'rollback-failed', phase: 'rollback' }); + } + const failure = failureFrom(caught, { code: 'apply-failed', phase: 'mutation' }); + throw new ReconciliationLockedOperationError(failure, caught, cleanupFailures); + } finally { + if (!closeLockedDatabases(locked)) { + cleanupFailures.push({ code: 'close-failed', phase: 'cleanup' }); + } + if (!releaseAdvisoryLocks(advisoryLocks, lockOptions)) { + cleanupFailures.push({ code: 'advisory-lock-release-failed', phase: 'cleanup' }); + } + } +} diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts new file mode 100644 index 000000000..87213f409 --- /dev/null +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -0,0 +1,912 @@ +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { applyDatabaseReconciliation, planDatabaseReconciliation } from './db-reconciliation.js'; +import { + SnapshotError, + applyDatabaseReconciliationWithSnapshots, + databaseSyncSnapshotIdentity, + deserializeSnapshotBytes, + normalizeSerializedSqliteForDeserialize, + recoverDatabaseReconciliation, + rollbackDatabaseReconciliation, +} from './db-sync-snapshots.js'; +import { openDb } from './genie-db.js'; + +let fixtureRoot: string; + +beforeEach(() => { + fixtureRoot = mkdtempSync(join(tmpdir(), 'genie-db-sync-snapshots-')); +}); + +afterEach(() => { + rmSync(fixtureRoot, { recursive: true, force: true }); +}); + +function currentDb(name: string): string { + const path = join(fixtureRoot, `${name}.db`); + openDb({ path }).close(); + const db = new Database(path); + db.query("UPDATE meta SET value = '100' WHERE key = 'stage_log_backfill_v1'").run(); + db.close(); + return path; +} + +function logicalRows(db: Database): unknown { + return { + boards: db.query('SELECT id, name, created_at, lanes FROM boards ORDER BY id').all(), + hireRoster: db + .query( + `SELECT wish, agent_adapter_id, profile, worktree, hired_at, state + FROM hire_roster ORDER BY wish, agent_adapter_id`, + ) + .all(), + meta: db.query('SELECT key, value FROM meta ORDER BY key').all(), + stageLog: db.query('SELECT id, task_id, stage, note, created_at FROM stage_log ORDER BY id').all(), + taskDependencies: db + .query('SELECT task_id, depends_on_id FROM task_dependencies ORDER BY task_id, depends_on_id') + .all(), + taskEvents: db + .query( + `SELECT id, task_id, kind, note, author_kind, author, created_at + FROM task_events ORDER BY id`, + ) + .all(), + tasks: db + .query( + `SELECT id, board_id, title, status, claimed_by, claimed_at, wish, group_name, + created_at, updated_at, lane, agent_kind, heartbeat_at, blocked_by, blocked_reason + FROM tasks ORDER BY id`, + ) + .all(), + wishGroups: db + .query( + `SELECT wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at + FROM wish_groups ORDER BY wish, name`, + ) + .all(), + }; +} + +function insertBoard(path: string, id: string, name = id): void { + const db = new Database(path); + db.query('INSERT INTO boards (id, name, created_at, lanes) VALUES (?, ?, 1, NULL)').run(id, name); + db.close(); +} + +function boardNames(path: string): string[] { + const db = new Database(path); + const values = (db.query('SELECT name FROM boards ORDER BY name').all() as Array<{ name: string }>).map( + (row) => row.name, + ); + db.close(); + return values; +} + +function generationDirectories(root: string): string[] { + if (!existsSync(root)) return []; + return readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.staging-')) + .map((entry) => join(root, entry.name)) + .sort(); +} + +function leaveCompleteGeneration(left: string, right: string): { root: string; directory: string } { + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let injected = false; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!injected && event.phase === 'recovery-classify' && event.state === 'before') { + injected = true; + throw new Error('leave complete'); + } + }, + }); + expect(report.status).toBe('operational-failure'); + const root = databaseSyncSnapshotIdentity(request).root; + const directories = generationDirectories(root); + expect(directories).toHaveLength(1); + return { root, directory: directories[0] }; +} + +describe('database sync snapshots', () => { + test('serialize under the intended held write locks includes committed WAL logical content', () => { + const left = currentDb('left'); + const right = currentDb('right'); + const writer = new Database(right, { safeIntegers: true }); + writer.exec('PRAGMA foreign_keys = ON'); + writer.exec('PRAGMA journal_mode = WAL'); + writer.exec('PRAGMA wal_autocheckpoint = 0'); + writer.exec('BEGIN IMMEDIATE'); + writer.query("INSERT INTO boards (id, name, created_at, lanes) VALUES ('board', 'Board', 1, NULL)").run(); + writer + .query( + `INSERT INTO tasks ( + id, board_id, title, status, claimed_by, claimed_at, wish, group_name, + created_at, updated_at, lane, agent_kind, heartbeat_at, blocked_by, blocked_reason + ) VALUES ('first', 'board', 'First', 'ready', NULL, NULL, 'wish', 'group', 2, 2, NULL, NULL, NULL, NULL, NULL), + ('second', 'board', 'Second', 'ready', NULL, NULL, 'wish', 'group', 3, 3, NULL, NULL, NULL, NULL, NULL)`, + ) + .run(); + writer.query("INSERT INTO task_dependencies (task_id, depends_on_id) VALUES ('second', 'first')").run(); + writer + .query("INSERT INTO stage_log (task_id, stage, note, created_at) VALUES ('first', 'comment', 'legacy', 4)") + .run(); + writer + .query( + `INSERT INTO task_events (task_id, kind, note, author_kind, author, created_at) + VALUES ('first', 'comment', 'legacy', NULL, NULL, 4), + ('first', 'comment', 'event', 'worker', 'gate', 5)`, + ) + .run(); + writer + .query( + `INSERT INTO wish_groups ( + wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at + ) VALUES ('wish', 'group', 'ready', '[]', NULL, NULL, NULL, 6, 6)`, + ) + .run(); + writer + .query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('wish', 'agent', NULL, '/tmp/worktree', 7, 'active')`, + ) + .run(); + writer.exec('COMMIT'); + + expect(existsSync(`${right}-wal`)).toBe(true); + expect(statSync(`${right}-wal`).size).toBeGreaterThan(0); + const before = logicalRows(writer); + let captured: Uint8Array | undefined; + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + const report = applyDatabaseReconciliation(plan, { + onLocked: (inputs) => { + captured = inputs.find((input) => input.canonicalPath === right)?.serialize(); + }, + }); + writer.close(); + + expect(report.status).toBe('changed'); + expect(captured).toBeDefined(); + const original = captured ?? new Uint8Array(); + const originalHeaderMode = original.slice(18, 20); + const normalized = normalizeSerializedSqliteForDeserialize(original); + expect([...original.slice(18, 20)]).toEqual([...originalHeaderMode]); + expect([...originalHeaderMode]).toEqual([2, 2]); + expect([...normalized.slice(18, 20)]).toEqual([1, 1]); + const restored = deserializeSnapshotBytes(original); + expect(logicalRows(restored)).toEqual(before); + expect(restored.query('PRAGMA integrity_check').get()).toEqual({ integrity_check: 'ok' }); + restored.close(); + }); + + test('serialized snapshot normalization rejects short, non-SQLite, and inconsistent format headers', () => { + expect(() => normalizeSerializedSqliteForDeserialize(new Uint8Array(99))).toThrow(SnapshotError); + + const db = new Database(':memory:'); + db.exec('CREATE TABLE value (id INTEGER PRIMARY KEY)'); + const badMagic = db.serialize(); + badMagic[0] = 0; + expect(() => normalizeSerializedSqliteForDeserialize(badMagic)).toThrow( + expect.objectContaining({ code: 'snapshot-invalid-header' }), + ); + + const badMode = db.serialize(); + badMode[18] = 2; + badMode[19] = 1; + expect(() => normalizeSerializedSqliteForDeserialize(badMode)).toThrow( + expect.objectContaining({ code: 'snapshot-unsupported-header-mode' }), + ); + db.close(); + }); + + test('bidirectional identity is argument-order independent while directional authority is role-sensitive', () => { + const left = currentDb('identity-left'); + const right = currentDb('identity-right'); + const forward = databaseSyncSnapshotIdentity({ mode: 'bidirectional', leftPath: left, rightPath: right }); + const reversed = databaseSyncSnapshotIdentity({ mode: 'bidirectional', leftPath: right, rightPath: left }); + expect(reversed).toEqual(forward); + + const directional = databaseSyncSnapshotIdentity({ + mode: 'directional', + sourcePath: left, + destinationPath: right, + }); + const authorityReversed = databaseSyncSnapshotIdentity({ + mode: 'directional', + sourcePath: right, + destinationPath: left, + }); + expect(authorityReversed.operationId).not.toBe(directional.operationId); + expect(directional.root).toBe(join(fixtureRoot, 'sync-snapshots')); + }); + + test('directional snapshots stay destination-adjacent, preserve source authority, and reverse independently', () => { + const source = currentDb('direction-source'); + const destination = currentDb('direction-destination'); + insertBoard(source, 'source-only'); + const request = { mode: 'directional' as const, sourcePath: source, destinationPath: destination }; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request)); + expect(report.apply?.status).toBe('changed'); + expect(boardNames(source)).toEqual(['source-only']); + expect(boardNames(destination)).toEqual(['source-only']); + const identity = databaseSyncSnapshotIdentity(request); + const manifest = JSON.parse(readFileSync(join(generationDirectories(identity.root)[0], 'manifest.json'), 'utf8')); + expect(manifest.targets).toEqual([expect.objectContaining({ role: 'destination', path: destination })]); + expect( + recoverDatabaseReconciliation({ + mode: 'directional', + sourcePath: destination, + destinationPath: source, + }), + ).toMatchObject({ status: 'none', generationId: null }); + }); + + test('an explicit snapshot root overrides the default without changing manifest roles or pair identity', () => { + const left = currentDb('override-left'); + const right = currentDb('override-right'); + insertBoard(right, 'override'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const override = join(fixtureRoot, 'operator-snapshots'); + const expected = databaseSyncSnapshotIdentity(request, override); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + snapshotRoot: override, + }); + expect(report.status).toBe('changed'); + expect(generationDirectories(override)).toHaveLength(1); + const manifest = JSON.parse(readFileSync(join(generationDirectories(override)[0], 'manifest.json'), 'utf8')); + expect(manifest).toMatchObject({ operation_id: expected.operationId, state: 'converged' }); + expect(manifest.targets.map((target: { role: string }) => target.role).sort()).toEqual(['left', 'right']); + expect(existsSync(databaseSyncSnapshotIdentity(request).root)).toBe(false); + }); + + test('publishes normalized complete payloads in the durable order and finalizes both-post as converged', () => { + const left = currentDb('publish-left'); + const right = currentDb('publish-right'); + insertBoard(right, 'right-only'); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + const events: string[] = []; + const report = applyDatabaseReconciliationWithSnapshots(plan, { + onEvent: (event) => + events.push(`${event.phase}:${event.state}${event.role === undefined ? '' : `:${event.role}`}`), + now: () => new Date('2026-07-28T12:00:00.000Z'), + randomId: () => '00000000-0000-4000-8000-000000000001', + }); + + expect(report.status).toBe('changed'); + expect(report.apply?.status).toBe('changed'); + expect(report.recovery.status).toBe('converged'); + expect(boardNames(left)).toEqual(['right-only']); + const identity = databaseSyncSnapshotIdentity({ mode: 'bidirectional', leftPath: left, rightPath: right }); + const generations = generationDirectories(identity.root); + expect(generations).toHaveLength(1); + const manifest = JSON.parse(readFileSync(join(generations[0], 'manifest.json'), 'utf8')); + expect(manifest).toMatchObject({ + format_version: 1, + operation_version: 1, + operation_id: identity.operationId, + mode: 'bidirectional', + state: 'converged', + }); + expect(manifest.targets).toHaveLength(2); + for (const target of manifest.targets) { + const bytes = new Uint8Array(readFileSync(join(generations[0], target.snapshot_file))); + expect([...bytes.slice(18, 20)]).toEqual([1, 1]); + expect(target.snapshot_sha256).toMatch(/^[a-f0-9]{64}$/); + } + expect(events.slice(0, 18)).toEqual([ + 'payload-write:before:left', + 'payload-write:after:left', + 'payload-write:before:right', + 'payload-write:after:right', + 'provisional-manifest-write:before', + 'provisional-manifest-write:after', + 'payload-fsync:before:left', + 'payload-fsync:after:left', + 'payload-fsync:before:right', + 'payload-fsync:after:right', + 'complete-manifest-write:before', + 'complete-manifest-write:after', + 'complete-manifest-fsync:before', + 'complete-manifest-fsync:after', + 'staging-fsync:before', + 'staging-fsync:after', + 'generation-rename:before', + 'generation-rename:after', + ]); + expect(events).toContain('root-fsync:after'); + expect(events).toContain('state-rewrite-rename:after'); + expect(events).toContain('generation-fsync:after'); + }); + + test('a first-commit cut is recovered immediately by restoring the post side to its preimage', () => { + const left = currentDb('mixed-left'); + const right = currentDb('mixed-right'); + insertBoard(left, 'left-only'); + insertBoard(right, 'right-only'); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + let committed = 0; + const report = applyDatabaseReconciliationWithSnapshots(plan, { + onApplyEvent: (event) => { + if (event.phase === 'commit' && event.state === 'after' && ++committed === 1) { + throw new Error('simulated crash after first commit'); + } + }, + }); + + expect(report.status).toBe('recovered'); + expect(report.apply?.status).toBe('partial-commit'); + expect(report.recovery).toMatchObject({ status: 'recovered' }); + expect(report.recovery.restoredPaths).toHaveLength(1); + expect(boardNames(left)).toEqual(['left-only']); + expect(boardNames(right)).toEqual(['right-only']); + }); + + test('a pre-commit cut classifies both-pre as recovered without database writes', () => { + const left = currentDb('pre-left'); + const right = currentDb('pre-right'); + insertBoard(right, 'right-only'); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + const report = applyDatabaseReconciliationWithSnapshots(plan, { + onApplyEvent: (event) => { + if (event.phase === 'mutation' && event.state === 'before') throw new Error('simulated pre-commit crash'); + }, + }); + + expect(report.status).toBe('recovered'); + expect(report.recovery).toMatchObject({ status: 'recovered', restoredPaths: [] }); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['right-only']); + }); + + test('unexpected current digests persist uncertain and later recovery never overwrites them', () => { + const left = currentDb('uncertain-left'); + const right = currentDb('uncertain-right'); + insertBoard(right, 'planned'); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + let injected = false; + const interrupted = applyDatabaseReconciliationWithSnapshots(plan, { + onEvent: (event) => { + if (!injected && event.phase === 'recovery-classify' && event.state === 'before') { + injected = true; + throw new Error('simulated recovery crash'); + } + }, + }); + expect(interrupted.status).toBe('operational-failure'); + insertBoard(left, 'unexpected'); + + const request = { mode: 'bidirectional' as const, leftPath: right, rightPath: left }; + const recovery = recoverDatabaseReconciliation(request); + expect(recovery).toMatchObject({ status: 'uncertain', restoredPaths: [] }); + expect(boardNames(left)).toEqual(['planned', 'unexpected']); + expect(boardNames(right)).toEqual(['planned']); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ status: 'uncertain', restoredPaths: [] }); + }); + + test('hash, schema, digest, and integrity-invalid recovery inputs are refused without overwrite', () => { + const left = currentDb('validation-left'); + const right = currentDb('validation-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + const manifestPath = join(directory, 'manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + const target = manifest.targets[0]; + const payloadPath = join(directory, target.snapshot_file); + const original = new Uint8Array(readFileSync(payloadPath)); + const originalHash = target.snapshot_sha256; + + const hashCorrupt = original.slice(); + hashCorrupt[hashCorrupt.length - 1] ^= 0xff; + writeFileSync(payloadPath, hashCorrupt); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'snapshot-hash-mismatch', + }); + expect(boardNames(left)).toEqual(['planned']); + expect(boardNames(right)).toEqual(['planned']); + + const digestDb = Database.deserialize(original, { strict: true }); + digestDb.query("INSERT INTO meta (key, value) VALUES ('tampered', 'yes')").run(); + const digestBytes = normalizeSerializedSqliteForDeserialize(digestDb.serialize()); + digestDb.close(); + writeFileSync(payloadPath, digestBytes); + manifest.targets[0].snapshot_sha256 = createHash('sha256').update(digestBytes).digest('hex'); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'snapshot-image-mismatch', + }); + + const schemaDb = Database.deserialize(original, { strict: true }); + schemaDb.exec('CREATE TABLE unexpected_schema (value TEXT)'); + const schemaBytes = normalizeSerializedSqliteForDeserialize(schemaDb.serialize()); + schemaDb.close(); + writeFileSync(payloadPath, schemaBytes); + manifest.targets[0].snapshot_sha256 = createHash('sha256').update(schemaBytes).digest('hex'); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'snapshot-image-mismatch', + }); + + const structurallyCorrupt = original.slice(); + structurallyCorrupt.fill(0xff, 100, Math.min(structurallyCorrupt.length, 512)); + writeFileSync(payloadPath, structurallyCorrupt); + manifest.targets[0].snapshot_sha256 = createHash('sha256').update(structurallyCorrupt).digest('hex'); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'snapshot-image-mismatch', + }); + expect(originalHash).toMatch(/^[a-f0-9]{64}$/); + expect(boardNames(left)).toEqual(['planned']); + expect(boardNames(right)).toEqual(['planned']); + }); + + test('manifest version, shape, digest, identity, and state validation refuse recovery authority', () => { + const mutations: Array<(manifest: Record) => void> = [ + (manifest) => { + manifest.format_version = 2; + }, + (manifest) => { + manifest.unknown = true; + }, + (manifest) => { + manifest.targets[0].preimage_digest = 'not-a-digest'; + }, + (manifest) => { + manifest.targets[0].identity = '0'.repeat(64); + }, + (manifest) => { + manifest.state = 'invented'; + }, + ]; + for (const [index, mutateManifest] of mutations.entries()) { + const left = currentDb(`manifest-${index}-left`); + const right = currentDb(`manifest-${index}-right`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + const manifestPath = join(directory, 'manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + mutateManifest(manifest); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'manifest-invalid', + }); + expect(boardNames(left)).toEqual(['planned']); + expect(boardNames(right)).toEqual(['planned']); + } + }); + + test('every durable publication cut exposes only staging or a recoverable complete generation', () => { + const phases = [ + 'payload-write', + 'provisional-manifest-write', + 'payload-fsync', + 'complete-manifest-write', + 'complete-manifest-fsync', + 'staging-fsync', + 'generation-rename', + 'root-fsync', + ] as const; + for (const [index, phase] of phases.entries()) { + const left = currentDb(`cut-${index}-left`); + const right = currentDb(`cut-${index}-right`); + insertBoard(right, `planned-${index}`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let injected = false; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!injected && event.phase === phase && event.state === 'after') { + injected = true; + throw new Error(`cut after ${phase}`); + } + }, + }); + expect(report.apply?.converged ?? false).toBe(false); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual([`planned-${index}`]); + const recovery = recoverDatabaseReconciliation(request); + expect(['none', 'recovered']).toContain(recovery.status); + expect(recovery.status).not.toBe('converged'); + } + }); + + test('state-rewrite cuts remain safely reclassifiable and never claim a false database restore', () => { + const phases = ['state-rewrite-write', 'state-rewrite-fsync', 'state-rewrite-rename', 'generation-fsync'] as const; + for (const [index, phase] of phases.entries()) { + const left = currentDb(`state-cut-${index}-left`); + const right = currentDb(`state-cut-${index}-right`); + insertBoard(right, `planned-${index}`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let injected = false; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!injected && event.phase === phase && event.state === 'after') { + injected = true; + throw new Error(`cut after ${phase}`); + } + }, + }); + expect(report.status).toBe('operational-failure'); + expect(boardNames(left)).toEqual([`planned-${index}`]); + expect(boardNames(right)).toEqual([`planned-${index}`]); + const recovery = recoverDatabaseReconciliation(request); + expect(['none', 'converged']).toContain(recovery.status); + expect(recovery.status).not.toBe('recovered'); + } + }); + + test('a recovery-restore cut rolls back the live transaction and the next invocation restores safely', () => { + const left = currentDb('recovery-cut-left'); + const right = currentDb('recovery-cut-right'); + insertBoard(left, 'left-only'); + insertBoard(right, 'right-only'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let leaveComplete = false; + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!leaveComplete && event.phase === 'recovery-classify' && event.state === 'before') { + leaveComplete = true; + throw new Error('leave complete'); + } + }, + }); + const writer = new Database(left); + writer.query("DELETE FROM boards WHERE id = 'right-only'").run(); + writer.close(); + let cut = false; + const interrupted = recoverDatabaseReconciliation(request, { + onEvent: (event) => { + if (!cut && event.phase === 'recovery-restore' && event.state === 'after') { + cut = true; + throw new Error('recovery restore cut'); + } + }, + }); + expect(interrupted.status).toBe('operational-failure'); + expect(boardNames(left)).toEqual(['left-only']); + expect(boardNames(right)).toEqual(['left-only', 'right-only']); + expect(recoverDatabaseReconciliation(request).status).toBe('recovered'); + expect(boardNames(left)).toEqual(['left-only']); + expect(boardNames(right)).toEqual(['right-only']); + }); + + test('a recovery commit cut leaves the complete generation authoritative for the next invocation', () => { + const left = currentDb('recovery-commit-left'); + const right = currentDb('recovery-commit-right'); + insertBoard(left, 'left-only'); + insertBoard(right, 'right-only'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let leaveComplete = false; + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!leaveComplete && event.phase === 'recovery-classify' && event.state === 'before') { + leaveComplete = true; + throw new Error('leave complete'); + } + }, + }); + const writer = new Database(left); + writer.query("DELETE FROM boards WHERE id = 'right-only'").run(); + writer.close(); + let committed = false; + const interrupted = recoverDatabaseReconciliation(request, { + onLockedOperationEvent: (event) => { + if (!committed && event.state === 'after') { + committed = true; + throw new Error('recovery commit cut'); + } + }, + }); + expect(interrupted.status).toBe('operational-failure'); + expect(recoverDatabaseReconciliation(request).status).toBe('recovered'); + expect(boardNames(left)).toEqual(['left-only']); + expect(boardNames(right)).toEqual(['right-only']); + }); + + test('explicit rollback snapshots arbitrary current state before restoring the selected preimages', () => { + const left = currentDb('rollback-left'); + const right = currentDb('rollback-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const applied = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request)); + expect(applied.recovery.status).toBe('converged'); + const selected = applied.generationId; + expect(selected).not.toBeNull(); + insertBoard(left, 'arbitrary-left'); + insertBoard(right, 'arbitrary-right'); + + const rolledBack = rollbackDatabaseReconciliation( + { mode: 'bidirectional', leftPath: right, rightPath: left }, + selected ?? '', + ); + expect(rolledBack).toMatchObject({ status: 'rolled-back', selectedGenerationId: selected }); + expect(rolledBack.safetyGenerationId).not.toBeNull(); + expect(rolledBack.safetyGenerationId).not.toBe(selected); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + }); + + test('older writers are bounded by SQLite locks during recovery restore and explicit rollback', () => { + const left = currentDb('writer-left'); + const right = currentDb('writer-right'); + insertBoard(left, 'left-only'); + insertBoard(right, 'right-only'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let interrupted = false; + const applied = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!interrupted && event.phase === 'recovery-classify' && event.state === 'before') { + interrupted = true; + throw new Error('leave complete pair'); + } + }, + }); + expect(applied.status).toBe('operational-failure'); + const leftWriter = new Database(left); + leftWriter.exec('PRAGMA foreign_keys = ON'); + leftWriter.query("DELETE FROM boards WHERE id = 'right-only'").run(); + leftWriter.close(); + + let recoveryBusy = false; + const recovered = recoverDatabaseReconciliation(request, { + onEvent: (event) => { + if (event.phase !== 'recovery-restore' || event.state !== 'before') return; + const older = new Database(event.role === 'left' ? left : right); + older.exec('PRAGMA busy_timeout = 20'); + try { + older.query("INSERT INTO meta (key, value) VALUES ('older-recovery', 'unexpected')").run(); + } catch (caught) { + recoveryBusy = caught instanceof Error && /locked/i.test(caught.message); + } finally { + older.close(); + } + }, + }); + expect(recovered.status).toBe('recovered'); + expect(recoveryBusy).toBe(true); + expect(boardNames(left)).toEqual(['left-only']); + expect(boardNames(right)).toEqual(['right-only']); + + insertBoard(left, 'arbitrary-left'); + insertBoard(right, 'arbitrary-right'); + let rollbackBusy = false; + const rollback = rollbackDatabaseReconciliation(request, applied.generationId ?? '', { + onEvent: (event) => { + if (event.phase !== 'rollback-restore' || event.state !== 'before' || rollbackBusy) return; + const older = new Database(event.role === 'left' ? left : right); + older.exec('PRAGMA busy_timeout = 20'); + try { + older.query("INSERT INTO meta (key, value) VALUES ('older-rollback', 'unexpected')").run(); + } catch (caught) { + rollbackBusy = caught instanceof Error && /locked/i.test(caught.message); + } finally { + older.close(); + } + }, + }); + expect(rollback.status).toBe('rolled-back'); + expect(rollbackBusy).toBe(true); + }); + + test('a rollback restore cut rolls live transactions back and leaves its safety generation recoverable', () => { + const left = currentDb('rollback-cut-left'); + const right = currentDb('rollback-cut-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const applied = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request)); + insertBoard(left, 'arbitrary-left'); + insertBoard(right, 'arbitrary-right'); + let injected = false; + const rollback = rollbackDatabaseReconciliation(request, applied.generationId ?? '', { + onEvent: (event) => { + if (!injected && event.phase === 'rollback-restore' && event.state === 'after') { + injected = true; + throw new Error('rollback cut'); + } + }, + }); + expect(rollback.status).toBe('operational-failure'); + expect(rollback.safetyGenerationId).not.toBeNull(); + expect(boardNames(left)).toEqual(['arbitrary-left', 'planned']); + expect(boardNames(right)).toEqual(['arbitrary-right', 'planned']); + expect(recoverDatabaseReconciliation(request).status).toBe('recovered'); + expect(boardNames(left)).toEqual(['arbitrary-left', 'planned']); + expect(boardNames(right)).toEqual(['arbitrary-right', 'planned']); + }); + + test('a rollback commit cut is reversed from the new safety generation on the next recovery', () => { + const left = currentDb('rollback-commit-left'); + const right = currentDb('rollback-commit-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const applied = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request)); + insertBoard(left, 'arbitrary-left'); + insertBoard(right, 'arbitrary-right'); + let committed = false; + let rollbackStarted = false; + const rollback = rollbackDatabaseReconciliation(request, applied.generationId ?? '', { + onEvent: (event) => { + if (event.phase === 'rollback-restore') rollbackStarted = true; + }, + onLockedOperationEvent: (event) => { + if (rollbackStarted && !committed && event.state === 'after') { + committed = true; + throw new Error('rollback commit cut'); + } + }, + }); + expect(rollback.status).toBe('operational-failure'); + expect(rollback.safetyGenerationId).not.toBeNull(); + expect(recoverDatabaseReconciliation(request).status).toBe('recovered'); + expect(boardNames(left)).toEqual(['arbitrary-left', 'planned']); + expect(boardNames(right)).toEqual(['arbitrary-right', 'planned']); + }); + + test('retention keeps the newest arbitrary count and stale staging is removed after success', () => { + const left = currentDb('retain-left'); + const right = currentDb('retain-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const identity = databaseSyncSnapshotIdentity(request); + mkdirSync(join(identity.root, '.staging-abandoned'), { recursive: true }); + writeFileSync(join(identity.root, '.staging-abandoned', 'partial'), 'incomplete'); + for (let index = 0; index < 4; index++) { + insertBoard(right, `board-${index}`); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 2, + now: () => new Date(1_700_000_000_000 + index), + }); + expect(report.recovery.status).toBe('converged'); + } + expect(generationDirectories(identity.root)).toHaveLength(2); + expect(existsSync(join(identity.root, '.staging-abandoned'))).toBe(false); + }); + + test('default retention keeps three complete generations', () => { + const left = currentDb('default-retain-left'); + const right = currentDb('default-retain-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + for (let index = 0; index < 4; index++) { + insertBoard(right, `default-${index}`); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + now: () => new Date(1_710_000_000_000 + index), + }).recovery.status, + ).toBe('converged'); + } + expect(generationDirectories(databaseSyncSnapshotIdentity(request).root)).toHaveLength(3); + }); + + test('pruning cuts are reported, never affect the committed databases, and retry on the next success', () => { + for (const state of ['before', 'after'] as const) { + const left = currentDb(`prune-${state}-left`); + const right = currentDb(`prune-${state}-right`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + insertBoard(right, 'first'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + now: () => new Date(1_720_000_000_000), + }).status, + ).toBe('changed'); + insertBoard(right, 'second'); + let injected = false; + const cut = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + now: () => new Date(1_720_000_000_001), + onEvent: (event) => { + if (!injected && event.phase === 'prune' && event.state === state) { + injected = true; + throw new Error(`prune ${state} cut`); + } + }, + }); + expect(cut.status).toBe('changed'); + expect(cut.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(boardNames(left)).toEqual(['first', 'second']); + expect(boardNames(right)).toEqual(['first', 'second']); + insertBoard(right, 'third'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + now: () => new Date(1_720_000_000_002), + }).status, + ).toBe('changed'); + expect(generationDirectories(databaseSyncSnapshotIdentity(request).root)).toHaveLength(1); + } + }); + + test('zero retention uses private 0700 state, never publishes, cleans on success, and cannot recover later', () => { + const left = currentDb('private-left'); + const right = currentDb('private-right'); + insertBoard(right, 'private'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const identity = databaseSyncSnapshotIdentity(request); + let privateRoot: string | undefined; + let privateMode: number | undefined; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + onEvent: (event) => { + if (event.phase === 'payload-write' && event.path === undefined && privateRoot === undefined) { + // The generation ID is intentionally not sufficient to discover the OS-private root. + privateRoot = readdirSync(tmpdir()) + .filter((name) => name.startsWith('genie-db-sync-private-')) + .map((name) => join(tmpdir(), name)) + .find((path) => statSync(path).isDirectory()); + privateMode = privateRoot === undefined ? undefined : statSync(privateRoot).mode & 0o777; + } + }, + }); + expect(report.recovery.status).toBe('converged'); + expect(existsSync(identity.root)).toBe(false); + expect(privateRoot).toBeDefined(); + expect(privateMode).toBe(0o700); + expect(privateRoot === undefined ? true : existsSync(privateRoot)).toBe(false); + expect(recoverDatabaseReconciliation(request, { keepSnapshots: 0 })).toMatchObject({ + status: 'none', + generationId: null, + }); + }); + + test('zero-retention cleanup failure is reported and leaked private state is never discoverable', () => { + const left = currentDb('private-failure-left'); + const right = currentDb('private-failure-right'); + insertBoard(right, 'private'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + let leakedRoot: string | undefined; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + onEvent: (event) => { + if (event.phase === 'payload-write' && leakedRoot === undefined) { + leakedRoot = readdirSync(tmpdir()) + .filter((name) => name.startsWith('genie-db-sync-private-')) + .map((name) => join(tmpdir(), name)) + .find((path) => statSync(path).isDirectory()); + } + }, + removeTree: () => { + throw new Error('cleanup denied'); + }, + }); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(leakedRoot).toBeDefined(); + expect(leakedRoot === undefined ? false : existsSync(leakedRoot)).toBe(true); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ status: 'none', generationId: null }); + if (leakedRoot !== undefined) rmSync(leakedRoot, { recursive: true, force: true }); + }); + + test('negative, fractional, and unsafe retention fail before snapshots or writes', () => { + for (const [index, retention] of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1].entries()) { + const left = currentDb(`invalid-retention-${index}-left`); + const right = currentDb(`invalid-retention-${index}-right`); + insertBoard(right, 'planned'); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { keepSnapshots: retention }, + ); + expect(report).toMatchObject({ + status: 'operational-failure', + failure: 'invalid-snapshot-option', + apply: null, + }); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); +}); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts new file mode 100644 index 000000000..8a48ca9d4 --- /dev/null +++ b/src/lib/v5/db-sync-snapshots.ts @@ -0,0 +1,1406 @@ +import { Database } from 'bun:sqlite'; +import { createHash, randomUUID } from 'node:crypto'; +import { + closeSync, + constants as fsConstants, + fsyncSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import type { Dirent } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { + type ReconciliationApplyEvent, + type ReconciliationApplyOptions, + type ReconciliationApplyReport, + type ReconciliationDatabaseObservation, + type ReconciliationInputRole, + type ReconciliationLockedDatabaseInput, + ReconciliationLockedOperationError, + type ReconciliationLockedOperationEvent, + type ReconciliationPlan, + type ReconciliationRequest, + type ReconciliationTargetRole, + applyDatabaseReconciliation, + inspectReconciliationDatabase, + withLockedReconciliationDatabases, +} from './db-reconciliation.js'; + +const SQLITE_HEADER_BYTES = new TextEncoder().encode('SQLite format 3\0'); +const SQLITE_MINIMUM_HEADER_BYTES = 100; +const SQLITE_WRITE_VERSION_OFFSET = 18; +const SQLITE_READ_VERSION_OFFSET = 19; +const SQLITE_ROLLBACK_FORMAT = 1; +const SQLITE_WAL_FORMAT = 2; +const MANIFEST_FORMAT_VERSION = 1 as const; +const RECONCILIATION_OPERATION_VERSION = 1 as const; +const SNAPSHOT_REPORT_VERSION = 1 as const; +const DEFAULT_SNAPSHOT_RETENTION = 3; +const MAX_MANIFEST_BYTES = 1024 * 1024; +const SNAPSHOT_FILE_PREFIX = 'snapshot-'; +const MANIFEST_FILE = 'manifest.json'; +const STAGING_PREFIX = '.staging-'; +const GENERATION_SEPARATOR = '--'; +const SHA256_PATTERN = /^[a-f0-9]{64}$/; +const GENERATION_ID_PATTERN = /^[a-f0-9]{64}--[0-9]{16}--[a-f0-9-]{36}$/; +const UUID_PATTERN = /^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/; + +export type SnapshotFailureCode = + | 'snapshot-invalid-header' + | 'snapshot-unsupported-header-mode' + | 'invalid-snapshot-option' + | 'snapshot-publication-failed' + | 'manifest-invalid' + | 'snapshot-hash-mismatch' + | 'snapshot-image-mismatch' + | 'generation-not-found' + | 'recovery-uncertain' + | 'locked-operation-failed' + | 'snapshot-cleanup-failed'; + +export class SnapshotError extends Error { + readonly code: SnapshotFailureCode; + + constructor(code: SnapshotFailureCode, message: string) { + super(message); + this.name = 'SnapshotError'; + this.code = code; + } +} + +/** + * Return private recovery bytes accepted by sqlite3_deserialize(). + * + * sqlite3_serialize() returns the complete logical image but preserves the + * source header's WAL read/write versions. SQLite explicitly requires an + * in-memory deserialized database to use rollback mode, so normalize only the + * documented header bytes on a copy. The caller's buffer is never modified. + */ +export function normalizeSerializedSqliteForDeserialize(serialized: Uint8Array): Uint8Array { + if (serialized.byteLength < SQLITE_MINIMUM_HEADER_BYTES) { + throw new SnapshotError('snapshot-invalid-header', 'Serialized SQLite snapshot is shorter than its file header.'); + } + for (let index = 0; index < SQLITE_HEADER_BYTES.length; index++) { + if (serialized[index] !== SQLITE_HEADER_BYTES[index]) { + throw new SnapshotError('snapshot-invalid-header', 'Serialized snapshot does not have a SQLite 3 header.'); + } + } + const writeVersion = serialized[SQLITE_WRITE_VERSION_OFFSET]; + const readVersion = serialized[SQLITE_READ_VERSION_OFFSET]; + const rollbackMode = writeVersion === SQLITE_ROLLBACK_FORMAT && readVersion === SQLITE_ROLLBACK_FORMAT; + const walMode = writeVersion === SQLITE_WAL_FORMAT && readVersion === SQLITE_WAL_FORMAT; + if (!rollbackMode && !walMode) { + throw new SnapshotError( + 'snapshot-unsupported-header-mode', + 'Serialized SQLite snapshot has unsupported or inconsistent read/write format versions.', + ); + } + + const normalized = serialized.slice(); + normalized[SQLITE_WRITE_VERSION_OFFSET] = SQLITE_ROLLBACK_FORMAT; + normalized[SQLITE_READ_VERSION_OFFSET] = SQLITE_ROLLBACK_FORMAT; + return normalized; +} + +export function deserializeSnapshotBytes(serialized: Uint8Array): Database { + return Database.deserialize(normalizeSerializedSqliteForDeserialize(serialized), { + readonly: true, + strict: true, + safeIntegers: true, + }); +} + +export type SnapshotGenerationState = + | 'provisional' + | 'complete' + | 'converged' + | 'recovered' + | 'uncertain' + | 'rolled-back'; + +export interface SnapshotManifestTarget { + readonly role: ReconciliationTargetRole; + readonly identity: string; + readonly path: string; + readonly preimage_digest: string; + readonly postimage_digest: string; + readonly snapshot_file: string; + readonly snapshot_sha256: string | null; +} + +export interface SnapshotManifestV1 { + readonly format_version: typeof MANIFEST_FORMAT_VERSION; + readonly operation_version: typeof RECONCILIATION_OPERATION_VERSION; + readonly generation_id: string; + readonly operation_id: string; + readonly mode: ReconciliationPlan['mode']; + readonly schema_fingerprint: string; + readonly created_at: string; + readonly state: SnapshotGenerationState; + readonly targets: readonly SnapshotManifestTarget[]; +} + +export interface SnapshotStoreIdentity { + readonly operationId: string; + readonly root: string; + readonly mode: ReconciliationPlan['mode']; + readonly canonicalPaths: readonly string[]; +} + +export type SnapshotLifecyclePhase = + | 'payload-write' + | 'provisional-manifest-write' + | 'payload-fsync' + | 'complete-manifest-write' + | 'complete-manifest-fsync' + | 'staging-fsync' + | 'generation-rename' + | 'root-fsync' + | 'state-rewrite-write' + | 'state-rewrite-fsync' + | 'state-rewrite-rename' + | 'generation-fsync' + | 'recovery-classify' + | 'recovery-restore' + | 'rollback-restore' + | 'staging-cleanup' + | 'prune'; + +export interface SnapshotLifecycleEvent { + readonly phase: SnapshotLifecyclePhase; + readonly state: 'before' | 'after'; + readonly generationId?: string; + readonly role?: ReconciliationTargetRole; + readonly path?: string; +} + +export interface DatabaseSyncSnapshotOptions { + readonly snapshotRoot?: string; + readonly keepSnapshots?: number; + readonly busyTimeoutMs?: number; + readonly onEvent?: (event: SnapshotLifecycleEvent) => void; + readonly onApplyEvent?: (event: ReconciliationApplyEvent) => void; + readonly onLockedOperationEvent?: (event: ReconciliationLockedOperationEvent) => void; + readonly now?: () => Date; + readonly randomId?: () => string; + readonly removeTree?: (path: string) => void; + readonly applyOptions?: Omit; +} + +export type SnapshotRecoveryStatus = 'none' | 'converged' | 'recovered' | 'uncertain' | 'operational-failure'; + +export interface SnapshotRecoveryReport { + readonly reportVersion: typeof SNAPSHOT_REPORT_VERSION; + readonly operation: 'recovery'; + readonly status: SnapshotRecoveryStatus; + readonly generationId: string | null; + readonly restoredPaths: readonly string[]; + readonly failure: SnapshotFailureCode | null; + readonly cleanupFailures: readonly SnapshotFailureCode[]; +} + +export interface SnapshotApplyReport { + readonly reportVersion: typeof SNAPSHOT_REPORT_VERSION; + readonly operation: 'apply'; + readonly status: + | ReconciliationApplyReport['status'] + | 'recovered' + | 'converged' + | 'uncertain' + | 'operational-failure'; + readonly generationId: string | null; + readonly recovery: SnapshotRecoveryReport; + readonly apply: ReconciliationApplyReport | null; + readonly failure: SnapshotFailureCode | null; + readonly cleanupFailures: readonly SnapshotFailureCode[]; +} + +export interface SnapshotRollbackReport { + readonly reportVersion: typeof SNAPSHOT_REPORT_VERSION; + readonly operation: 'rollback'; + readonly status: 'rolled-back' | 'uncertain' | 'operational-failure'; + readonly selectedGenerationId: string; + readonly safetyGenerationId: string | null; + readonly failure: SnapshotFailureCode | null; + readonly cleanupFailures: readonly SnapshotFailureCode[]; +} + +interface GenerationCapture { + readonly role: ReconciliationTargetRole; + readonly canonicalPath: string; + readonly preimageDigest: string; + readonly postimageDigest: string; + readonly normalizedBytes: Uint8Array; + readonly sha256: string; +} + +interface PublishedGeneration { + readonly manifest: SnapshotManifestV1; + readonly directory: string; + readonly privateRoot: string | null; +} + +interface ValidatedGeneration { + readonly manifest: SnapshotManifestV1; + readonly directory: string; + readonly snapshots: ReadonlyMap; +} + +interface RecoveryDecision { + readonly status: Exclude; + readonly generationId: string | null; + readonly restoredPaths: string[]; + readonly cleanupFailures: SnapshotFailureCode[]; +} + +function sha256(value: string | Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} + +function encodeIdentity(domain: string, values: readonly string[]): string { + const hash = createHash('sha256').update(`${domain}\0`); + for (const value of values) { + const bytes = Buffer.from(value); + hash.update(`${bytes.byteLength}:`); + hash.update(bytes); + } + return hash.digest('hex'); +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function targetIdentity(path: string): string { + return encodeIdentity('genie-db-sync-target-v1', [path]); +} + +function canonicalPathsFromPlan(plan: ReconciliationPlan): string[] { + return plan.inputs.map((input) => input.canonicalPath); +} + +function identityFromCanonical( + mode: ReconciliationPlan['mode'], + canonicalPaths: readonly string[], + snapshotRoot?: string, +): SnapshotStoreIdentity { + if (canonicalPaths.length !== 2 || canonicalPaths[0] === canonicalPaths[1]) { + throw new SnapshotError('invalid-snapshot-option', 'Database snapshot identity requires two distinct inputs.'); + } + if (mode === 'bidirectional') { + const sorted = [...canonicalPaths].sort(compareText); + const operationId = encodeIdentity('genie-db-sync-bidirectional-pair-v1', sorted); + return { + operationId, + root: + snapshotRoot === undefined ? join(dirname(sorted[0]), 'sync-snapshots', operationId) : resolve(snapshotRoot), + mode, + canonicalPaths: sorted, + }; + } + const [source, destination] = canonicalPaths; + const operationId = encodeIdentity('genie-db-sync-directional-pair-v1', [source, destination]); + return { + operationId, + root: snapshotRoot === undefined ? join(dirname(destination), 'sync-snapshots') : resolve(snapshotRoot), + mode, + canonicalPaths: [source, destination], + }; +} + +export function databaseSyncSnapshotIdentity( + request: ReconciliationRequest, + snapshotRoot?: string, +): SnapshotStoreIdentity { + const paths = + request.mode === 'bidirectional' + ? [request.leftPath, request.rightPath] + : [request.sourcePath, request.destinationPath]; + const canonical = paths.map((path) => { + const absolute = isAbsolute(path) ? path : resolve(path); + return realpathSync(absolute); + }); + return identityFromCanonical(request.mode, canonical, snapshotRoot); +} + +function requestFromPlan(plan: ReconciliationPlan): ReconciliationRequest { + const input = (role: ReconciliationInputRole): string => { + const found = plan.inputs.find((candidate) => candidate.role === role); + if (found === undefined) throw new SnapshotError('invalid-snapshot-option', `Plan is missing its ${role} input.`); + return found.canonicalPath; + }; + return plan.mode === 'bidirectional' + ? { mode: 'bidirectional', leftPath: input('left'), rightPath: input('right') } + : { mode: 'directional', sourcePath: input('source'), destinationPath: input('destination') }; +} + +function validateRetention(value: number | undefined): number { + const retention = value ?? DEFAULT_SNAPSHOT_RETENTION; + if (!Number.isSafeInteger(retention) || retention < 0) { + throw new SnapshotError('invalid-snapshot-option', 'Snapshot retention must be a nonnegative safe integer.'); + } + return retention; +} + +function emit(options: DatabaseSyncSnapshotOptions, event: SnapshotLifecycleEvent): void { + options.onEvent?.(event); +} + +function around( + options: DatabaseSyncSnapshotOptions, + event: Omit, + operation: () => void, +): void { + emit(options, { ...event, state: 'before' }); + operation(); + emit(options, { ...event, state: 'after' }); +} + +function fsyncPath(path: string): void { + const descriptor = openSync(path, fsConstants.O_RDONLY); + try { + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +} + +function writeJson(path: string, manifest: SnapshotManifestV1): void { + writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); +} + +function overwriteJson(path: string, manifest: SnapshotManifestV1): void { + writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'w', mode: 0o600 }); +} + +function generationId( + identity: SnapshotStoreIdentity, + options: DatabaseSyncSnapshotOptions, +): { + id: string; + createdAt: string; +} { + const now = options.now?.() ?? new Date(); + if (!Number.isFinite(now.getTime())) { + throw new SnapshotError('invalid-snapshot-option', 'Snapshot timestamp must be a valid date.'); + } + const timestamp = String(now.getTime()).padStart(16, '0'); + const random = (options.randomId?.() ?? randomUUID()).toLowerCase(); + const id = `${identity.operationId}${GENERATION_SEPARATOR}${timestamp}${GENERATION_SEPARATOR}${random}`; + if (!GENERATION_ID_PATTERN.test(id) || !UUID_PATTERN.test(random)) { + throw new SnapshotError('invalid-snapshot-option', 'Snapshot generation ID contains unsupported characters.'); + } + return { id, createdAt: now.toISOString() }; +} + +function validateCapturedImage( + bytes: Uint8Array, + schemaFingerprint: string, + logicalDigest: string, +): { normalizedBytes: Uint8Array; sha256: string } { + const normalizedBytes = normalizeSerializedSqliteForDeserialize(bytes); + let db: Database; + try { + db = Database.deserialize(normalizedBytes, { readonly: true, strict: true, safeIntegers: true }); + } catch { + throw new SnapshotError('snapshot-image-mismatch', 'Serialized snapshot cannot be deserialized as SQLite.'); + } + try { + let image: ReconciliationDatabaseObservation; + try { + image = inspectReconciliationDatabase(db); + } catch { + throw new SnapshotError('snapshot-image-mismatch', 'Serialized snapshot failed schema or integrity validation.'); + } + if (image.schemaFingerprint !== schemaFingerprint || image.logicalDigest !== logicalDigest) { + throw new SnapshotError( + 'snapshot-image-mismatch', + 'Serialized snapshot does not match its expected logical image.', + ); + } + } finally { + db.close(); + } + return { normalizedBytes, sha256: sha256(normalizedBytes) }; +} + +function makeManifest( + identity: SnapshotStoreIdentity, + schemaFingerprint: string, + captures: readonly GenerationCapture[], + id: string, + createdAt: string, + state: SnapshotGenerationState, + hashes: boolean, +): SnapshotManifestV1 { + return { + format_version: MANIFEST_FORMAT_VERSION, + operation_version: RECONCILIATION_OPERATION_VERSION, + generation_id: id, + operation_id: identity.operationId, + mode: identity.mode, + schema_fingerprint: schemaFingerprint, + created_at: createdAt, + state, + targets: captures.map((capture, index) => ({ + role: capture.role, + identity: targetIdentity(capture.canonicalPath), + path: capture.canonicalPath, + preimage_digest: capture.preimageDigest, + postimage_digest: capture.postimageDigest, + snapshot_file: `${SNAPSHOT_FILE_PREFIX}${index}.sqlite`, + snapshot_sha256: hashes ? capture.sha256 : null, + })), + }; +} + +function createPrivateRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'genie-db-sync-private-')); + const mode = statSync(root).mode & 0o777; + if (mode !== 0o700) { + rmSync(root, { recursive: true, force: true }); + throw new SnapshotError( + 'snapshot-publication-failed', + 'Private snapshot directory was not created with mode 0700.', + ); + } + return root; +} + +function publishGeneration( + identity: SnapshotStoreIdentity, + schemaFingerprint: string, + captures: readonly GenerationCapture[], + options: DatabaseSyncSnapshotOptions, + privateRoot: string | null = null, +): PublishedGeneration { + const root = privateRoot ?? identity.root; + mkdirSync(root, { recursive: true, mode: 0o700 }); + const generated = generationId(identity, options); + const staging = join(root, `${STAGING_PREFIX}${generated.id}-${randomUUID()}`); + const finalDirectory = join(root, generated.id); + mkdirSync(staging, { mode: 0o700 }); + const provisional = makeManifest( + identity, + schemaFingerprint, + captures, + generated.id, + generated.createdAt, + 'provisional', + false, + ); + try { + for (let index = 0; index < captures.length; index++) { + const capture = captures[index]; + around(options, { phase: 'payload-write', generationId: generated.id, role: capture.role }, () => + writeFileSync(join(staging, provisional.targets[index].snapshot_file), capture.normalizedBytes, { + flag: 'wx', + mode: 0o600, + }), + ); + } + around(options, { phase: 'provisional-manifest-write', generationId: generated.id }, () => + writeJson(join(staging, MANIFEST_FILE), provisional), + ); + for (const target of provisional.targets) { + around(options, { phase: 'payload-fsync', generationId: generated.id, role: target.role }, () => + fsyncPath(join(staging, target.snapshot_file)), + ); + } + const complete = makeManifest( + identity, + schemaFingerprint, + captures, + generated.id, + generated.createdAt, + 'complete', + true, + ); + around(options, { phase: 'complete-manifest-write', generationId: generated.id }, () => + overwriteJson(join(staging, MANIFEST_FILE), complete), + ); + around(options, { phase: 'complete-manifest-fsync', generationId: generated.id }, () => + fsyncPath(join(staging, MANIFEST_FILE)), + ); + around(options, { phase: 'staging-fsync', generationId: generated.id }, () => fsyncPath(staging)); + around(options, { phase: 'generation-rename', generationId: generated.id }, () => + renameSync(staging, finalDirectory), + ); + around(options, { phase: 'root-fsync', generationId: generated.id }, () => fsyncPath(root)); + return { manifest: complete, directory: finalDirectory, privateRoot }; + } catch (caught) { + if (caught instanceof SnapshotError) throw caught; + throw new SnapshotError( + 'snapshot-publication-failed', + caught instanceof Error ? caught.message : 'Snapshot publication failed.', + ); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function exactKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(compareText); + return ( + actual.length === expected.length && actual.every((key, index) => key === [...expected].sort(compareText)[index]) + ); +} + +function requireString(value: unknown): string { + if (typeof value !== 'string') + throw new SnapshotError('manifest-invalid', 'Snapshot manifest field is not a string.'); + return value; +} + +function parseTarget(value: unknown): SnapshotManifestTarget { + if ( + !isRecord(value) || + !exactKeys(value, [ + 'role', + 'identity', + 'path', + 'preimage_digest', + 'postimage_digest', + 'snapshot_file', + 'snapshot_sha256', + ]) + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest target shape is invalid.'); + } + const role = value.role; + if (role !== 'left' && role !== 'right' && role !== 'destination') { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest target role is invalid.'); + } + const path = requireString(value.path); + const identity = requireString(value.identity); + const preimage = requireString(value.preimage_digest); + const postimage = requireString(value.postimage_digest); + const file = requireString(value.snapshot_file); + const hash = value.snapshot_sha256; + if ( + !isAbsolute(path) || + identity !== targetIdentity(path) || + !SHA256_PATTERN.test(preimage) || + !SHA256_PATTERN.test(postimage) || + basename(file) !== file || + !file.startsWith(SNAPSHOT_FILE_PREFIX) || + (hash !== null && (typeof hash !== 'string' || !SHA256_PATTERN.test(hash))) + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest target values are invalid.'); + } + return { + role, + identity, + path, + preimage_digest: preimage, + postimage_digest: postimage, + snapshot_file: file, + snapshot_sha256: hash, + }; +} + +function parseManifest(value: unknown): SnapshotManifestV1 { + if ( + !isRecord(value) || + !exactKeys(value, [ + 'format_version', + 'operation_version', + 'generation_id', + 'operation_id', + 'mode', + 'schema_fingerprint', + 'created_at', + 'state', + 'targets', + ]) + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest shape is invalid.'); + } + if ( + value.format_version !== MANIFEST_FORMAT_VERSION || + value.operation_version !== RECONCILIATION_OPERATION_VERSION + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest version is unsupported.'); + } + const generation = requireString(value.generation_id); + const operation = requireString(value.operation_id); + const mode = value.mode; + const schema = requireString(value.schema_fingerprint); + const createdAt = requireString(value.created_at); + const state = value.state; + const generationParts = generation.split(GENERATION_SEPARATOR); + const parsedCreatedAt = Date.parse(createdAt); + const canonicalCreatedAt = !Number.isNaN(parsedCreatedAt) && new Date(parsedCreatedAt).toISOString() === createdAt; + if ( + !GENERATION_ID_PATTERN.test(generation) || + generationParts.length !== 3 || + !UUID_PATTERN.test(generationParts[2]) || + String(parsedCreatedAt).padStart(16, '0') !== generationParts[1] || + !canonicalCreatedAt || + !SHA256_PATTERN.test(operation) || + (mode !== 'bidirectional' && mode !== 'directional') || + !SHA256_PATTERN.test(schema) || + Number.isNaN(parsedCreatedAt) || + (state !== 'provisional' && + state !== 'complete' && + state !== 'converged' && + state !== 'recovered' && + state !== 'uncertain' && + state !== 'rolled-back') || + !Array.isArray(value.targets) + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest values are invalid.'); + } + const targets = value.targets.map(parseTarget); + const expectedCount = mode === 'bidirectional' ? 2 : 1; + const roles = targets.map((target) => target.role); + if ( + targets.length !== expectedCount || + new Set(targets.map((target) => target.path)).size !== targets.length || + (mode === 'bidirectional' && (!roles.includes('left') || !roles.includes('right'))) || + (mode === 'directional' && roles[0] !== 'destination') + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest target inventory is invalid.'); + } + return { + format_version: MANIFEST_FORMAT_VERSION, + operation_version: RECONCILIATION_OPERATION_VERSION, + generation_id: generation, + operation_id: operation, + mode, + schema_fingerprint: schema, + created_at: createdAt, + state, + targets, + }; +} + +function readManifest(directory: string): SnapshotManifestV1 { + const manifestPath = join(directory, MANIFEST_FILE); + const directoryStat = lstatSync(directory); + const manifestStat = lstatSync(manifestPath); + if ( + !directoryStat.isDirectory() || + directoryStat.isSymbolicLink() || + !manifestStat.isFile() || + manifestStat.isSymbolicLink() || + manifestStat.size > MAX_MANIFEST_BYTES + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot generation or manifest is not a bounded physical file.'); + } + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(manifestPath, 'utf8')) as unknown; + } catch { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest is not valid JSON.'); + } + const manifest = parseManifest(parsed); + if (basename(directory) !== manifest.generation_id) { + throw new SnapshotError('manifest-invalid', 'Snapshot generation directory does not match its manifest.'); + } + return manifest; +} + +function validateManifestIdentity(manifest: SnapshotManifestV1, identity: SnapshotStoreIdentity): void { + const manifestPaths = manifest.targets.map((target) => target.path).sort(compareText); + const expectedTargets = + identity.mode === 'bidirectional' ? [...identity.canonicalPaths].sort(compareText) : [identity.canonicalPaths[1]]; + if ( + manifest.operation_id !== identity.operationId || + manifest.mode !== identity.mode || + manifestPaths.length !== expectedTargets.length || + manifestPaths.some((path, index) => path !== expectedTargets[index]) + ) { + throw new SnapshotError('manifest-invalid', 'Snapshot manifest does not identify the locked database operation.'); + } +} + +function matchingGenerationDirectories(identity: SnapshotStoreIdentity): string[] { + try { + return readdirSync(identity.root, { withFileTypes: true }) + .filter( + (entry) => + entry.isDirectory() && + !entry.isSymbolicLink() && + entry.name.startsWith(`${identity.operationId}${GENERATION_SEPARATOR}`), + ) + .map((entry) => join(identity.root, entry.name)) + .sort((left, right) => compareText(right, left)); + } catch (caught) { + const code = isRecord(caught) ? caught.code : undefined; + if (code === 'ENOENT') return []; + throw caught; + } +} + +function newestUnresolved(identity: SnapshotStoreIdentity): { manifest: SnapshotManifestV1; directory: string } | null { + for (const directory of matchingGenerationDirectories(identity)) { + const manifest = readManifest(directory); + validateManifestIdentity(manifest, identity); + if (manifest.state === 'complete' || manifest.state === 'uncertain') return { manifest, directory }; + } + return null; +} + +function selectedGeneration( + identity: SnapshotStoreIdentity, + selectedGenerationId: string, +): { manifest: SnapshotManifestV1; directory: string } { + if (!GENERATION_ID_PATTERN.test(selectedGenerationId) || !selectedGenerationId.startsWith(identity.operationId)) { + throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not belong to this operation.'); + } + const directory = join(identity.root, selectedGenerationId); + let manifest: SnapshotManifestV1; + try { + manifest = readManifest(directory); + } catch (caught) { + const code = isRecord(caught) ? caught.code : undefined; + if (code === 'ENOENT') { + throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not exist.'); + } + throw caught; + } + validateManifestIdentity(manifest, identity); + if (manifest.state === 'provisional' || manifest.targets.some((target) => target.snapshot_sha256 === null)) { + throw new SnapshotError('manifest-invalid', 'Selected snapshot generation is incomplete.'); + } + return { manifest, directory }; +} + +function validateGeneration(generation: { manifest: SnapshotManifestV1; directory: string }): ValidatedGeneration { + const snapshots = new Map(); + try { + for (const target of generation.manifest.targets) { + if (target.snapshot_sha256 === null) { + throw new SnapshotError('manifest-invalid', 'Complete snapshot manifest is missing a payload hash.'); + } + const payloadPath = join(generation.directory, target.snapshot_file); + const payloadStat = lstatSync(payloadPath); + if (!payloadStat.isFile() || payloadStat.isSymbolicLink()) { + throw new SnapshotError('manifest-invalid', 'Snapshot payload is not a physical regular file.'); + } + const bytes = new Uint8Array(readFileSync(payloadPath)); + if (sha256(bytes) !== target.snapshot_sha256) { + throw new SnapshotError('snapshot-hash-mismatch', 'Snapshot payload hash does not match its manifest.'); + } + let db: Database; + try { + db = deserializeSnapshotBytes(bytes); + } catch { + throw new SnapshotError('snapshot-image-mismatch', 'Snapshot payload cannot be deserialized as SQLite.'); + } + let image: ReconciliationDatabaseObservation; + try { + image = inspectReconciliationDatabase(db); + } catch { + db.close(); + throw new SnapshotError('snapshot-image-mismatch', 'Snapshot payload failed schema or integrity validation.'); + } + if ( + image.schemaFingerprint !== generation.manifest.schema_fingerprint || + image.logicalDigest !== target.preimage_digest + ) { + db.close(); + throw new SnapshotError('snapshot-image-mismatch', 'Snapshot payload does not match its manifest image.'); + } + snapshots.set(target.path, db); + } + return { ...generation, snapshots }; + } catch (caught) { + for (const snapshot of snapshots.values()) snapshot.close(); + throw caught; + } +} + +function closeValidatedGeneration(generation: ValidatedGeneration): void { + for (const db of generation.snapshots.values()) db.close(); +} + +function rewriteManifestState( + generation: { manifest: SnapshotManifestV1; directory: string }, + state: Exclude, + options: DatabaseSyncSnapshotOptions, +): SnapshotManifestV1 { + const updated = { ...generation.manifest, state }; + const temporary = join(generation.directory, `.manifest-${randomUUID()}.tmp`); + around(options, { phase: 'state-rewrite-write', generationId: generation.manifest.generation_id }, () => + writeJson(temporary, updated), + ); + around(options, { phase: 'state-rewrite-fsync', generationId: generation.manifest.generation_id }, () => + fsyncPath(temporary), + ); + around(options, { phase: 'state-rewrite-rename', generationId: generation.manifest.generation_id }, () => + renameSync(temporary, join(generation.directory, MANIFEST_FILE)), + ); + around(options, { phase: 'generation-fsync', generationId: generation.manifest.generation_id }, () => + fsyncPath(generation.directory), + ); + return updated; +} + +function cleanupStaging( + root: string, + options: DatabaseSyncSnapshotOptions, + cleanupFailures: SnapshotFailureCode[], +): void { + let entries: Dirent[]; + try { + entries = readdirSync(root, { withFileTypes: true }); + } catch (caught) { + const code = isRecord(caught) ? caught.code : undefined; + if (code === 'ENOENT') return; + cleanupFailures.push('snapshot-cleanup-failed'); + return; + } + for (const entry of entries) { + if (!entry.name.startsWith(STAGING_PREFIX) || !entry.isDirectory() || entry.isSymbolicLink()) continue; + const path = join(root, entry.name); + try { + around(options, { phase: 'staging-cleanup', path }, () => + (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(path), + ); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } + } +} + +function pruneGenerations( + identity: SnapshotStoreIdentity, + retention: number, + options: DatabaseSyncSnapshotOptions, + cleanupFailures: SnapshotFailureCode[], +): void { + const directories = matchingGenerationDirectories(identity); + const retained = new Set(directories.slice(0, retention)); + let removed = false; + for (const directory of directories) { + if (retained.has(directory)) continue; + try { + const manifest = readManifest(directory); + validateManifestIdentity(manifest, identity); + if (manifest.state === 'complete' || manifest.state === 'uncertain') continue; + around(options, { phase: 'prune', generationId: manifest.generation_id, path: directory }, () => + (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(directory), + ); + removed = true; + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } + } + if (removed) { + try { + fsyncPath(identity.root); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } + } +} + +function targetInput( + inputs: readonly ReconciliationLockedDatabaseInput[], + path: string, +): ReconciliationLockedDatabaseInput { + const input = inputs.find((candidate) => candidate.canonicalPath === path); + if (input === undefined) { + throw new SnapshotError('manifest-invalid', 'Snapshot target is not one of the locked databases.'); + } + return input; +} + +function recoverValidatedGeneration( + identity: SnapshotStoreIdentity, + generation: { manifest: SnapshotManifestV1; directory: string }, + inputs: readonly ReconciliationLockedDatabaseInput[], + options: DatabaseSyncSnapshotOptions, + retention: number, +): { value: RecoveryDecision; afterCommit: () => void } { + if (generation.manifest.state === 'uncertain') { + return { + value: { + status: 'uncertain', + generationId: generation.manifest.generation_id, + restoredPaths: [], + cleanupFailures: [], + }, + afterCommit: () => {}, + }; + } + const validated = validateGeneration(generation); + const observed = new Map(inputs.map((input) => [input.canonicalPath, input.observe().logicalDigest])); + const classifications = generation.manifest.targets.map((target) => ({ + target, + current: observed.get(target.path), + pre: observed.get(target.path) === target.preimage_digest, + post: observed.get(target.path) === target.postimage_digest, + })); + emit(options, { phase: 'recovery-classify', state: 'before', generationId: generation.manifest.generation_id }); + let status: RecoveryDecision['status']; + const restoreTargets: SnapshotManifestTarget[] = []; + if (classifications.every((item) => item.post)) { + status = 'converged'; + } else if (classifications.every((item) => item.pre)) { + status = 'recovered'; + } else if ( + identity.mode === 'bidirectional' && + classifications.length === 2 && + classifications.every((item) => item.pre || item.post) && + classifications.filter((item) => item.pre).length === 1 && + classifications.filter((item) => item.post).length === 1 + ) { + status = 'recovered'; + restoreTargets.push(...classifications.filter((item) => item.post).map((item) => item.target)); + } else { + status = 'uncertain'; + } + emit(options, { phase: 'recovery-classify', state: 'after', generationId: generation.manifest.generation_id }); + + const restoredPaths: string[] = []; + try { + if (status !== 'uncertain') { + for (const target of restoreTargets) { + const snapshot = validated.snapshots.get(target.path); + if (snapshot === undefined) { + throw new SnapshotError('manifest-invalid', 'Validated snapshot payload is missing.'); + } + around( + options, + { phase: 'recovery-restore', generationId: generation.manifest.generation_id, role: target.role }, + () => { + const restored = targetInput(inputs, target.path).restoreFrom(snapshot); + if ( + restored.schemaFingerprint !== generation.manifest.schema_fingerprint || + restored.logicalDigest !== target.preimage_digest + ) { + throw new SnapshotError('snapshot-image-mismatch', 'Recovery did not reproduce the preimage.'); + } + }, + ); + restoredPaths.push(target.path); + } + } + } finally { + closeValidatedGeneration(validated); + } + + const cleanupFailures: SnapshotFailureCode[] = []; + return { + value: { + status, + generationId: generation.manifest.generation_id, + restoredPaths, + cleanupFailures, + }, + afterCommit: () => { + rewriteManifestState(generation, status === 'uncertain' ? 'uncertain' : status, options); + cleanupStaging(identity.root, options, cleanupFailures); + if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); + }, + }; +} + +function recoveryReport(decision: RecoveryDecision): SnapshotRecoveryReport { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'recovery', + status: decision.status, + generationId: decision.generationId, + restoredPaths: decision.restoredPaths, + failure: decision.status === 'uncertain' ? 'recovery-uncertain' : null, + cleanupFailures: decision.cleanupFailures, + }; +} + +function failedRecovery(caught: unknown): SnapshotRecoveryReport { + const failure = + caught instanceof SnapshotError + ? caught.code + : caught instanceof ReconciliationLockedOperationError + ? caught.operationCause instanceof SnapshotError + ? caught.operationCause.code + : 'locked-operation-failed' + : 'locked-operation-failed'; + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'recovery', + status: 'operational-failure', + generationId: null, + restoredPaths: [], + failure, + cleanupFailures: [], + }; +} + +export function recoverDatabaseReconciliation( + request: ReconciliationRequest, + options: DatabaseSyncSnapshotOptions = {}, +): SnapshotRecoveryReport { + let retention: number; + try { + retention = validateRetention(options.keepSnapshots); + if (retention === 0) { + return recoveryReport({ + status: 'none', + generationId: null, + restoredPaths: [], + cleanupFailures: [], + }); + } + const decision = withLockedReconciliationDatabases( + request, + (inputs) => { + const identity = identityFromCanonical( + request.mode, + inputs.map((input) => input.canonicalPath), + options.snapshotRoot, + ); + const unresolved = newestUnresolved(identity); + if (unresolved === null) { + const cleanupFailures: SnapshotFailureCode[] = []; + return { + value: { + status: 'none' as const, + generationId: null, + restoredPaths: [], + cleanupFailures, + }, + afterCommit: () => cleanupStaging(identity.root, options, cleanupFailures), + }; + } + return recoverValidatedGeneration(identity, unresolved, inputs, options, retention); + }, + { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onLockedOperationEvent, + ...options.applyOptions, + }, + ); + return recoveryReport(decision); + } catch (caught) { + return failedRecovery(caught); + } +} + +function capturesForPlan( + plan: ReconciliationPlan, + inputs: readonly { + readonly role: ReconciliationInputRole; + readonly canonicalPath: string; + serialize(): Uint8Array; + }[], +): GenerationCapture[] { + return plan.targets.map((target) => { + if (target.postimageDigest === null) { + throw new SnapshotError('snapshot-image-mismatch', 'A mutating snapshot target is missing its postimage digest.'); + } + const input = inputs.find((candidate) => candidate.canonicalPath === target.canonicalPath); + if (input === undefined) { + throw new SnapshotError('snapshot-image-mismatch', 'A snapshot target is missing from the locked input set.'); + } + const validated = validateCapturedImage(input.serialize(), plan.schemaFingerprint, target.preimageDigest); + return { + role: target.role, + canonicalPath: target.canonicalPath, + preimageDigest: target.preimageDigest, + postimageDigest: target.postimageDigest, + ...validated, + }; + }); +} + +function privateCleanup( + root: string | null, + options: DatabaseSyncSnapshotOptions, + cleanupFailures: SnapshotFailureCode[], +): void { + if (root === null) return; + try { + (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(root); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } +} + +function publishedGenerationId(published: PublishedGeneration | null): string | null { + return published === null ? null : published.manifest.generation_id; +} + +function recoverPrivateGeneration( + published: PublishedGeneration, + request: ReconciliationRequest, + identity: SnapshotStoreIdentity, + options: DatabaseSyncSnapshotOptions, +): SnapshotRecoveryReport { + try { + const decision = withLockedReconciliationDatabases( + request, + (inputs) => + recoverValidatedGeneration( + identity, + { manifest: readManifest(published.directory), directory: published.directory }, + inputs, + options, + 0, + ), + { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onLockedOperationEvent, + ...options.applyOptions, + }, + ); + return recoveryReport(decision); + } catch (caught) { + return failedRecovery(caught); + } +} + +export function applyDatabaseReconciliationWithSnapshots( + plan: ReconciliationPlan, + options: DatabaseSyncSnapshotOptions = {}, +): SnapshotApplyReport { + let retention: number; + try { + retention = validateRetention(options.keepSnapshots); + } catch (caught) { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status: 'operational-failure', + generationId: null, + recovery: failedRecovery(caught), + apply: null, + failure: caught instanceof SnapshotError ? caught.code : 'invalid-snapshot-option', + cleanupFailures: [], + }; + } + const request = requestFromPlan(plan); + if (plan.status !== 'changed') { + const apply = applyDatabaseReconciliation(plan, { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onApplyEvent, + ...options.applyOptions, + }); + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status: apply.status, + generationId: null, + recovery: recoveryReport({ + status: 'none', + generationId: null, + restoredPaths: [], + cleanupFailures: [], + }), + apply, + failure: null, + cleanupFailures: [], + }; + } + + const priorRecovery = recoverDatabaseReconciliation(request, options); + if (priorRecovery.status !== 'none') { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status: priorRecovery.status === 'operational-failure' ? 'operational-failure' : priorRecovery.status, + generationId: priorRecovery.generationId, + recovery: priorRecovery, + apply: null, + failure: priorRecovery.failure, + cleanupFailures: priorRecovery.cleanupFailures, + }; + } + + const identity = identityFromCanonical(plan.mode, canonicalPathsFromPlan(plan), options.snapshotRoot); + let published: PublishedGeneration | null = null; + let privateRoot: string | null = null; + const cleanupFailures: SnapshotFailureCode[] = [...priorRecovery.cleanupFailures]; + try { + if (retention === 0) privateRoot = createPrivateRoot(); + const apply = applyDatabaseReconciliation(plan, { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onApplyEvent, + ...options.applyOptions, + onLocked: (inputs) => { + const captures = capturesForPlan(plan, inputs); + published = publishGeneration(identity, plan.schemaFingerprint, captures, options, privateRoot); + }, + }); + const recovery = + retention === 0 + ? published === null + ? recoveryReport({ status: 'none', generationId: null, restoredPaths: [], cleanupFailures: [] }) + : recoverPrivateGeneration(published, request, identity, options) + : recoverDatabaseReconciliation(request, options); + privateCleanup(privateRoot, options, cleanupFailures); + const status = + recovery.status === 'none' + ? apply.status + : recovery.status === 'converged' && apply.status === 'changed' + ? 'changed' + : recovery.status === 'operational-failure' + ? 'operational-failure' + : recovery.status; + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status, + generationId: publishedGenerationId(published) ?? recovery.generationId, + recovery, + apply, + failure: recovery.failure, + cleanupFailures: [...recovery.cleanupFailures, ...cleanupFailures], + }; + } catch (caught) { + privateCleanup(privateRoot, options, cleanupFailures); + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status: 'operational-failure', + generationId: publishedGenerationId(published), + recovery: failedRecovery(caught), + apply: null, + failure: caught instanceof SnapshotError ? caught.code : 'snapshot-publication-failed', + cleanupFailures, + }; + } +} + +export function rollbackDatabaseReconciliation( + request: ReconciliationRequest, + selectedGenerationId: string, + options: DatabaseSyncSnapshotOptions = {}, +): SnapshotRollbackReport { + let retention: number; + try { + retention = validateRetention(options.keepSnapshots); + } catch (caught) { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'rollback', + status: 'operational-failure', + selectedGenerationId, + safetyGenerationId: null, + failure: caught instanceof SnapshotError ? caught.code : 'invalid-snapshot-option', + cleanupFailures: [], + }; + } + const recovery = recoverDatabaseReconciliation(request, options); + if (recovery.status === 'uncertain' || recovery.status === 'operational-failure') { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'rollback', + status: recovery.status === 'uncertain' ? 'uncertain' : 'operational-failure', + selectedGenerationId, + safetyGenerationId: null, + failure: recovery.failure, + cleanupFailures: recovery.cleanupFailures, + }; + } + + let privateRoot: string | null = null; + let published: PublishedGeneration | null = null; + const cleanupFailures: SnapshotFailureCode[] = [...recovery.cleanupFailures]; + try { + if (retention === 0) privateRoot = createPrivateRoot(); + const value = withLockedReconciliationDatabases( + request, + (inputs) => { + const identity = identityFromCanonical( + request.mode, + inputs.map((input) => input.canonicalPath), + options.snapshotRoot, + ); + const selected = validateGeneration(selectedGeneration(identity, selectedGenerationId)); + try { + const captures = selected.manifest.targets.map((target) => { + const input = targetInput(inputs, target.path); + const current = input.observe(); + if (current.schemaFingerprint !== selected.manifest.schema_fingerprint) { + throw new SnapshotError('snapshot-image-mismatch', 'Rollback target schema does not match the snapshot.'); + } + const validated = validateCapturedImage( + input.serialize(), + current.schemaFingerprint, + current.logicalDigest, + ); + return { + role: target.role, + canonicalPath: target.path, + preimageDigest: current.logicalDigest, + postimageDigest: target.preimage_digest, + ...validated, + }; + }); + published = publishGeneration(identity, selected.manifest.schema_fingerprint, captures, options, privateRoot); + for (const target of selected.manifest.targets) { + const source = selected.snapshots.get(target.path); + if (source === undefined) + throw new SnapshotError('manifest-invalid', 'Rollback snapshot payload is missing.'); + around( + options, + { phase: 'rollback-restore', generationId: selectedGenerationId, role: target.role }, + () => { + const restored = targetInput(inputs, target.path).restoreFrom(source); + if (restored.logicalDigest !== target.preimage_digest) { + throw new SnapshotError('snapshot-image-mismatch', 'Rollback did not reproduce the selected image.'); + } + }, + ); + } + return { + value: { identity, generation: published }, + afterCommit: () => { + if (published === null) return; + rewriteManifestState(published, 'rolled-back', options); + if (retention > 0) { + cleanupStaging(identity.root, options, cleanupFailures); + pruneGenerations(identity, retention, options, cleanupFailures); + } + }, + }; + } finally { + closeValidatedGeneration(selected); + } + }, + { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onLockedOperationEvent, + ...options.applyOptions, + }, + ); + privateCleanup(privateRoot, options, cleanupFailures); + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'rollback', + status: 'rolled-back', + selectedGenerationId, + safetyGenerationId: value.generation.manifest.generation_id, + failure: null, + cleanupFailures, + }; + } catch (caught) { + privateCleanup(privateRoot, options, cleanupFailures); + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'rollback', + status: 'operational-failure', + selectedGenerationId, + safetyGenerationId: publishedGenerationId(published), + failure: caught instanceof SnapshotError ? caught.code : 'locked-operation-failed', + cleanupFailures, + }; + } +} From 063b73a8a809b9f722fa9e0585904ab270a1ee64 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 14:14:35 +0000 Subject: [PATCH 09/21] fix(db-sync): harden persistent snapshot recovery --- src/lib/v5/db-reconciliation.test.ts | 49 +++ src/lib/v5/db-reconciliation.ts | 24 +- src/lib/v5/db-sync-snapshots.test.ts | 321 ++++++++++++++- src/lib/v5/db-sync-snapshots.ts | 575 ++++++++++++++++++++------- 4 files changed, 820 insertions(+), 149 deletions(-) diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index b8bfcd0df..2106c1c3c 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -22,12 +22,14 @@ import { IDENTICAL_HISTORY_ADDITION_LIMITATION, type ReconciliationApplyEvent, ReconciliationError, + ReconciliationLockedOperationError, type ReconciliationRequest, type TaskEventReconciliationValue, applyDatabaseReconciliation, dryRunDatabaseReconciliation, planDatabaseReconciliation, reconciliationAdvisoryLockPath, + withLockedReconciliationDatabases, } from './db-reconciliation.js'; import { openDb } from './genie-db.js'; @@ -2090,6 +2092,53 @@ describe('canonical locking and transactional apply', () => { expect(taskTitle(destination, 'planned')).toBeNull(); }); + test('locked operations report close and advisory-release failures on success and preserve them on failure', () => { + const openWithFailingClose = (path: string, options: ConstructorParameters[1]): Database => { + const db = new Database(path, options); + return new Proxy(db, { + get(target, property) { + if (property === 'close') { + return () => { + target.close(); + throw new Error('injected close failure'); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }; + + for (const primaryFailure of [false, true]) { + const left = currentDb(`locked-cleanup-${primaryFailure}-left`, null); + const right = currentDb(`locked-cleanup-${primaryFailure}-right`, null); + const request = bidirectional(left, right); + let caught: unknown; + try { + withLockedReconciliationDatabases( + request, + () => { + if (primaryFailure) throw new Error('primary locked-operation failure'); + return { value: 'success' }; + }, + { + openDatabase: openWithFailingClose, + advisoryUnlock: () => -1, + }, + ); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(ReconciliationLockedOperationError); + const lockedError = caught as ReconciliationLockedOperationError; + expect(lockedError.failure.code).toBe(primaryFailure ? 'unexpected-failure' : 'close-failed'); + expect(lockedError.cleanupFailures).toEqual([ + { code: 'close-failed', phase: 'cleanup' }, + { code: 'advisory-lock-release-failed', phase: 'cleanup' }, + ]); + } + }); + test('directional board-name swaps use a transaction-local parking order', () => { const source = currentDb('swap-source', null); const destination = currentDb('swap-destination', null); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index aae982205..45f61ef7c 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -36,7 +36,7 @@ const MAX_BUSY_TIMEOUT_MS = 2_147_483_647; const ADVISORY_RETRY_MS = 10; const LOCK_EXCLUSIVE_NONBLOCKING = 2 | 4; const LOCK_UNLOCK = 8; -const MAX_DATABASE_BYTES = 256 * 1024 * 1024; +export const MAX_RECONCILIATION_DATABASE_BYTES = 256 * 1024 * 1024; const MAX_ROWS_PER_TABLE = 1_000_000; const MAX_TOTAL_ROWS = 2_000_000; const MAX_CONFLICTS = 10_000; @@ -777,7 +777,7 @@ function resolvePhysicalInput(path: string): PhysicalInput { throw new Error('SHM is not a regular file'); } const logicalBytes = stats.size + walBytes; - if (logicalBytes > MAX_DATABASE_BYTES) { + if (logicalBytes > MAX_RECONCILIATION_DATABASE_BYTES) { throw error('input-too-large', 'The database exceeds the bounded reconciliation input size.'); } return { @@ -3359,6 +3359,8 @@ export function withLockedReconciliationDatabases( let advisoryLocks: AdvisoryLock[] = []; let locked: LockedDatabase[] = []; const cleanupFailures: ReconciliationApplyFailure[] = []; + let result: ReconciliationLockedOperationResult | undefined; + let operationFailure: { readonly failure: ReconciliationApplyFailure; readonly cause: unknown } | undefined; const lockOptions: ReconciliationApplyOptions = { busyTimeoutMs: options.busyTimeoutMs, openDatabase: options.openDatabase, @@ -3385,16 +3387,13 @@ export function withLockedReconciliationDatabases( physical, }; }); - if (samePhysicalInput(requested[0].physical, requested[1].physical)) { - throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); - } const deadline = Date.now() + timeoutMs; advisoryLocks = acquirePlannedAdvisoryLocks(requested, deadline, lockOptions); locked = openLockedDatabases(requested, deadline, lockOptions); const inputs = lockedOperationInputs(requested, locked); const initial = inputs.map((input) => input.observe()); if (initial[0].schemaFingerprint !== initial[1].schemaFingerprint) unsupportedSchema(); - const result = operation(inputs); + result = operation(inputs); const final = inputs.map((input) => input.observe()); if (final.some((image) => image.schemaFingerprint !== initial[0].schemaFingerprint)) unsupportedSchema(); for (const item of locked) { @@ -3408,14 +3407,13 @@ export function withLockedReconciliationDatabases( emitLockedOperationEvent(options, { phase: 'commit', role, state: 'after' }); } result.afterCommit?.(); - return result.value; } catch (caught) { if (caught instanceof ApplyBoundaryError) cleanupFailures.push(...caught.cleanupFailures); if (!rollbackOpenTransactions(locked)) { cleanupFailures.push({ code: 'rollback-failed', phase: 'rollback' }); } const failure = failureFrom(caught, { code: 'apply-failed', phase: 'mutation' }); - throw new ReconciliationLockedOperationError(failure, caught, cleanupFailures); + operationFailure = { failure, cause: caught }; } finally { if (!closeLockedDatabases(locked)) { cleanupFailures.push({ code: 'close-failed', phase: 'cleanup' }); @@ -3424,4 +3422,14 @@ export function withLockedReconciliationDatabases( cleanupFailures.push({ code: 'advisory-lock-release-failed', phase: 'cleanup' }); } } + if (operationFailure !== undefined) { + throw new ReconciliationLockedOperationError(operationFailure.failure, operationFailure.cause, cleanupFailures); + } + if (cleanupFailures.length > 0) { + throw new ReconciliationLockedOperationError(cleanupFailures[0], undefined, cleanupFailures); + } + if (result === undefined) { + throw new ReconciliationLockedOperationError({ code: 'apply-failed', phase: 'mutation' }); + } + return result.value; } diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index 87213f409..227b6cf8c 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -3,17 +3,25 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { createHash } from 'node:crypto'; import { existsSync, + linkSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, + renameSync, rmSync, statSync, + symlinkSync, + truncateSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { applyDatabaseReconciliation, planDatabaseReconciliation } from './db-reconciliation.js'; +import { + MAX_RECONCILIATION_DATABASE_BYTES, + applyDatabaseReconciliation, + planDatabaseReconciliation, +} from './db-reconciliation.js'; import { SnapshotError, applyDatabaseReconciliationWithSnapshots, @@ -273,6 +281,26 @@ describe('database sync snapshots', () => { expect(existsSync(databaseSyncSnapshotIdentity(request).root)).toBe(false); }); + test('persistent roots reject root and ancestor symlinks before publication or mutation', () => { + for (const kind of ['root', 'ancestor'] as const) { + const left = currentDb(`unsafe-${kind}-left`); + const right = currentDb(`unsafe-${kind}-right`); + insertBoard(right, 'planned'); + const physical = join(fixtureRoot, `${kind}-physical`); + mkdirSync(physical, { mode: 0o700 }); + const linked = join(fixtureRoot, `${kind}-linked`); + symlinkSync(physical, linked); + const snapshotRoot = kind === 'root' ? linked : join(linked, 'snapshots'); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { snapshotRoot }, + ); + expect(report).toMatchObject({ status: 'operational-failure', apply: null }); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); + test('publishes normalized complete payloads in the durable order and finalizes both-post as converged', () => { const left = currentDb('publish-left'); const right = currentDb('publish-right'); @@ -372,6 +400,113 @@ describe('database sync snapshots', () => { expect(boardNames(right)).toEqual(['right-only']); }); + test('unresolved persistent recovery runs before no-op, conflict, and same-database plan statuses', () => { + { + const left = currentDb('recovery-first-noop-left'); + const right = currentDb('recovery-first-noop-right'); + leaveCompleteGeneration(left, right); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + ); + expect(report).toMatchObject({ status: 'converged', apply: null, recovery: { status: 'converged' } }); + } + { + const left = currentDb('recovery-first-conflict-left'); + const right = currentDb('recovery-first-conflict-right'); + leaveCompleteGeneration(left, right); + const leftDb = new Database(left); + const rightDb = new Database(right); + leftDb.query("UPDATE boards SET name = 'left' WHERE id = 'planned'").run(); + rightDb.query("UPDATE boards SET name = 'right' WHERE id = 'planned'").run(); + leftDb.close(); + rightDb.close(); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + expect(plan.status).toBe('conflict'); + const report = applyDatabaseReconciliationWithSnapshots(plan); + expect(report).toMatchObject({ status: 'uncertain', apply: null, recovery: { status: 'uncertain' } }); + } + { + const left = currentDb('recovery-first-alias-left'); + const right = currentDb('recovery-first-alias-right'); + leaveCompleteGeneration(left, right); + for (const path of [left, right]) { + const db = new Database(path); + db.query('PRAGMA wal_checkpoint(TRUNCATE)').get(); + db.close(); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); + } + rmSync(right); + linkSync(left, right); + const plan = planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }); + expect(plan.status).toBe('same-database'); + const report = applyDatabaseReconciliationWithSnapshots(plan); + expect(report).toMatchObject({ status: 'converged', apply: null, recovery: { status: 'converged' } }); + } + }); + + test('zero retention recovers retained generations before changed apply and rollback work', () => { + const left = currentDb('zero-existing-left'); + const right = currentDb('zero-existing-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const first = applyDatabaseReconciliationWithSnapshots( + (() => { + insertBoard(right, 'first'); + return planDatabaseReconciliation(request); + })(), + ); + expect(first.generationId).not.toBeNull(); + + insertBoard(right, 'second'); + let leftComplete = false; + const interrupted = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (!leftComplete && event.phase === 'recovery-classify' && event.state === 'before') { + leftComplete = true; + throw new Error('retain unresolved generation'); + } + }, + }); + expect(interrupted.status).toBe('operational-failure'); + + insertBoard(right, 'third'); + const changed = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + }); + expect(changed).toMatchObject({ status: 'uncertain', apply: null, recovery: { status: 'uncertain' } }); + + const rollbackLeft = currentDb('zero-rollback-left'); + const rollbackRight = currentDb('zero-rollback-right'); + const rollbackRequest = { + mode: 'bidirectional' as const, + leftPath: rollbackLeft, + rightPath: rollbackRight, + }; + insertBoard(rollbackRight, 'rollback-first'); + const selected = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(rollbackRequest)); + insertBoard(rollbackRight, 'rollback-second'); + let rollbackComplete = false; + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(rollbackRequest), { + onEvent: (event) => { + if (!rollbackComplete && event.phase === 'recovery-classify' && event.state === 'before') { + rollbackComplete = true; + throw new Error('retain rollback generation'); + } + }, + }).status, + ).toBe('operational-failure'); + const phases: string[] = []; + const rolledBack = rollbackDatabaseReconciliation(rollbackRequest, selected.generationId ?? '', { + keepSnapshots: 0, + onEvent: (event) => { + if (event.state === 'before') phases.push(event.phase); + }, + }); + expect(rolledBack.status).toBe('rolled-back'); + expect(phases.indexOf('recovery-classify')).toBeLessThan(phases.indexOf('rollback-restore')); + }); + test('unexpected current digests persist uncertain and later recovery never overwrites them', () => { const left = currentDb('uncertain-left'); const right = currentDb('uncertain-right'); @@ -457,6 +592,134 @@ describe('database sync snapshots', () => { expect(boardNames(right)).toEqual(['planned']); }); + test('payload reads reject oversized sparse, symlink, and FIFO inputs without blocking or allocation', () => { + for (const kind of ['sparse', 'symlink', 'fifo'] as const) { + const left = currentDb(`bounded-${kind}-left`); + const right = currentDb(`bounded-${kind}-right`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + const manifest = JSON.parse(readFileSync(join(directory, 'manifest.json'), 'utf8')); + const payload = join(directory, manifest.targets[0].snapshot_file); + rmSync(payload); + if (kind === 'sparse') { + writeFileSync(payload, ''); + truncateSync(payload, MAX_RECONCILIATION_DATABASE_BYTES + 1); + } else if (kind === 'symlink') { + symlinkSync(left, payload); + } else { + expect(Bun.spawnSync(['mkfifo', payload]).exitCode).toBe(0); + } + const started = Date.now(); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'manifest-invalid', + }); + expect(Date.now() - started).toBeLessThan(1_000); + } + }); + + test('generation and manifest substitutions refuse state rewrite without redirecting it', () => { + for (const kind of ['generation', 'manifest'] as const) { + const left = currentDb(`substitute-${kind}-left`); + const right = currentDb(`substitute-${kind}-right`); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + let substituted = false; + const report = recoverDatabaseReconciliation(request, { + onEvent: (event) => { + if (substituted || event.phase !== 'recovery-classify' || event.state !== 'before') return; + substituted = true; + if (kind === 'generation') { + renameSync(directory, `${directory}.owned`); + mkdirSync(directory, { mode: 0o700 }); + writeFileSync(join(directory, 'sentinel'), 'replacement'); + } else { + const manifest = join(directory, 'manifest.json'); + renameSync(manifest, `${manifest}.owned`); + symlinkSync('/dev/null', manifest); + } + }, + }); + expect(report.status).toBe('operational-failure'); + if (kind === 'generation') expect(readFileSync(join(directory, 'sentinel'), 'utf8')).toBe('replacement'); + else expect(statSync(join(directory, 'manifest.json.owned')).isFile()).toBe(true); + } + }); + + test('manifest reads are bounded and failed rewrites clean only their exact temporary inode', () => { + { + const left = currentDb('manifest-bounded-left'); + const right = currentDb('manifest-bounded-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + truncateSync(join(directory, 'manifest.json'), 1024 * 1024 + 1); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ + status: 'operational-failure', + failure: 'manifest-invalid', + }); + } + { + const left = currentDb('manifest-temp-left'); + const right = currentDb('manifest-temp-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const { directory } = leaveCompleteGeneration(left, right); + let replacement: string | null = null; + const recovery = recoverDatabaseReconciliation(request, { + onEvent: (event) => { + if (replacement !== null || event.phase !== 'state-rewrite-fsync' || event.state !== 'before') return; + const temporary = readdirSync(directory).find( + (name) => name.startsWith('.manifest-') && name.endsWith('.tmp'), + ); + expect(temporary).toBeDefined(); + replacement = join(directory, temporary ?? ''); + renameSync(replacement, `${replacement}.owned`); + writeFileSync(replacement, 'replacement'); + }, + }); + expect(recovery.status).toBe('operational-failure'); + expect(recovery.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(readFileSync(replacement ?? '', 'utf8')).toBe('replacement'); + } + }); + + test('snapshot recovery reports locked close and advisory-release cleanup evidence', () => { + const openWithFailingClose = (path: string, options: ConstructorParameters[1]): Database => { + const db = new Database(path, options); + return new Proxy(db, { + get(target, property) { + if (property === 'close') { + return () => { + target.close(); + throw new Error('injected close failure'); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }; + for (const primaryFailure of [false, true]) { + const left = currentDb(`snapshot-cleanup-${primaryFailure}-left`); + const right = currentDb(`snapshot-cleanup-${primaryFailure}-right`); + const report = recoverDatabaseReconciliation( + { mode: 'bidirectional', leftPath: left, rightPath: right }, + { + applyOptions: { + openDatabase: openWithFailingClose, + advisoryUnlock: () => -1, + }, + onLockedOperationEvent: primaryFailure + ? (event) => { + if (event.state === 'before') throw new Error('primary commit failure'); + } + : undefined, + }, + ); + expect(report.status).toBe('operational-failure'); + expect(report.cleanupFailures).toEqual(['locked-close-failed', 'locked-advisory-release-failed']); + } + }); + test('manifest version, shape, digest, identity, and state validation refuse recovery authority', () => { const mutations: Array<(manifest: Record) => void> = [ (manifest) => { @@ -832,6 +1095,62 @@ describe('database sync snapshots', () => { } }); + test('staging cleanup and pruning refuse directory substitutions instead of deleting replacements', () => { + { + const left = currentDb('cleanup-race-left'); + const right = currentDb('cleanup-race-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const root = databaseSyncSnapshotIdentity(request).root; + const staging = join(root, '.staging-owned'); + const victim = join(fixtureRoot, 'cleanup-victim'); + mkdirSync(staging, { recursive: true }); + writeFileSync(join(staging, 'owned'), 'owned'); + mkdirSync(victim); + writeFileSync(join(victim, 'sentinel'), 'victim'); + let raced = false; + const recovery = recoverDatabaseReconciliation(request, { + onEvent: (event) => { + if (!raced && event.phase === 'staging-cleanup' && event.state === 'before') { + raced = true; + renameSync(staging, `${staging}.original`); + renameSync(victim, staging); + } + }, + }); + expect(recovery.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(readFileSync(join(staging, 'sentinel'), 'utf8')).toBe('victim'); + } + { + const left = currentDb('prune-race-left'); + const right = currentDb('prune-race-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + for (const id of ['one', 'two']) { + insertBoard(right, id); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { keepSnapshots: 3 }).status, + ).toBe('changed'); + } + const victim = join(fixtureRoot, 'prune-victim'); + mkdirSync(victim); + writeFileSync(join(victim, 'sentinel'), 'victim'); + insertBoard(right, 'three'); + let replacement: string | null = null; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + onEvent: (event) => { + if (replacement !== null || event.phase !== 'prune' || event.state !== 'before' || event.path === undefined) + return; + replacement = event.path; + renameSync(event.path, `${event.path}.original`); + renameSync(victim, event.path); + }, + }); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(replacement).not.toBeNull(); + expect(readFileSync(join(replacement ?? '', 'sentinel'), 'utf8')).toBe('victim'); + } + }); + test('zero retention uses private 0700 state, never publishes, cleans on success, and cannot recover later', () => { const left = currentDb('private-left'); const right = currentDb('private-right'); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index 8a48ca9d4..60d368ead 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -3,24 +3,28 @@ import { createHash, randomUUID } from 'node:crypto'; import { closeSync, constants as fsConstants, + fstatSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, - readFileSync, + readSync, readdirSync, realpathSync, renameSync, - rmSync, + rmdirSync, statSync, + unlinkSync, writeFileSync, } from 'node:fs'; -import type { Dirent } from 'node:fs'; +import type { Dirent, Stats } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { basename, dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; import { + MAX_RECONCILIATION_DATABASE_BYTES, type ReconciliationApplyEvent, + type ReconciliationApplyFailure, type ReconciliationApplyOptions, type ReconciliationApplyReport, type ReconciliationDatabaseObservation, @@ -66,15 +70,20 @@ export type SnapshotFailureCode = | 'generation-not-found' | 'recovery-uncertain' | 'locked-operation-failed' + | 'locked-rollback-failed' + | 'locked-close-failed' + | 'locked-advisory-release-failed' | 'snapshot-cleanup-failed'; export class SnapshotError extends Error { readonly code: SnapshotFailureCode; + readonly cleanupFailures: readonly SnapshotFailureCode[]; - constructor(code: SnapshotFailureCode, message: string) { + constructor(code: SnapshotFailureCode, message: string, cleanupFailures: readonly SnapshotFailureCode[] = []) { super(message); this.name = 'SnapshotError'; this.code = code; + this.cleanupFailures = cleanupFailures; } } @@ -247,9 +256,18 @@ interface GenerationCapture { interface PublishedGeneration { readonly manifest: SnapshotManifestV1; readonly directory: string; + readonly directoryIdentity: FileIdentity; + readonly manifestIdentity: FileIdentity; readonly privateRoot: string | null; } +interface GenerationReference { + readonly manifest: SnapshotManifestV1; + readonly directory: string; + readonly directoryIdentity: FileIdentity; + readonly manifestIdentity: FileIdentity; +} + interface ValidatedGeneration { readonly manifest: SnapshotManifestV1; readonly directory: string; @@ -263,6 +281,17 @@ interface RecoveryDecision { readonly cleanupFailures: SnapshotFailureCode[]; } +interface FileIdentity { + readonly dev: number; + readonly ino: number; +} + +interface BoundDirectory { + readonly path: string; + readonly descriptor: number; + readonly identity: FileIdentity; +} + function sha256(value: string | Uint8Array): string { return createHash('sha256').update(value).digest('hex'); } @@ -294,8 +323,8 @@ function identityFromCanonical( canonicalPaths: readonly string[], snapshotRoot?: string, ): SnapshotStoreIdentity { - if (canonicalPaths.length !== 2 || canonicalPaths[0] === canonicalPaths[1]) { - throw new SnapshotError('invalid-snapshot-option', 'Database snapshot identity requires two distinct inputs.'); + if (canonicalPaths.length !== 2) { + throw new SnapshotError('invalid-snapshot-option', 'Database snapshot identity requires exactly two inputs.'); } if (mode === 'bidirectional') { const sorted = [...canonicalPaths].sort(compareText); @@ -366,8 +395,121 @@ function around( emit(options, { ...event, state: 'after' }); } +function fileIdentity(stats: Stats): FileIdentity { + return { dev: stats.dev, ino: stats.ino }; +} + +function sameFileIdentity(left: FileIdentity, right: Stats | FileIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function filesystemCode(caught: unknown): unknown { + return isRecord(caught) ? caught.code : undefined; +} + +function assertSafeDirectoryAncestors(path: string): void { + const absolute = resolve(path); + const root = parsePath(absolute).root; + const parts = absolute.slice(root.length).split('/').filter(Boolean); + let current = root; + for (const part of parts) { + current = join(current, part); + let stats: Stats; + try { + stats = lstatSync(current); + } catch (caught) { + if (filesystemCode(caught) === 'ENOENT') return; + throw caught; + } + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new SnapshotError( + 'manifest-invalid', + 'Snapshot root and its existing ancestors must be physical directories.', + ); + } + } +} + +function assertPathIdentity(path: string, identity: FileIdentity, kind: 'directory' | 'file'): void { + let stats: Stats; + try { + stats = lstatSync(path); + } catch { + throw new SnapshotError('manifest-invalid', `Snapshot ${kind} identity changed during the operation.`); + } + const validType = kind === 'directory' ? stats.isDirectory() : stats.isFile(); + if (stats.isSymbolicLink() || !validType || !sameFileIdentity(identity, stats)) { + throw new SnapshotError('manifest-invalid', `Snapshot ${kind} identity changed during the operation.`); + } +} + +function openBoundDirectory(path: string, create: boolean, requirePrivate = false): BoundDirectory { + assertSafeDirectoryAncestors(path); + if (create) mkdirSync(path, { recursive: true, mode: 0o700 }); + assertSafeDirectoryAncestors(path); + let descriptor: number; + try { + descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW); + } catch (caught) { + if (!create && filesystemCode(caught) === 'ENOENT') throw caught; + throw new SnapshotError('manifest-invalid', 'Snapshot root is not a safe physical directory.'); + } + try { + const stats = fstatSync(descriptor); + const currentUid = typeof process.getuid === 'function' ? process.getuid() : stats.uid; + const mode = stats.mode & 0o777; + if (!stats.isDirectory() || stats.uid !== currentUid || (requirePrivate ? mode !== 0o700 : (mode & 0o022) !== 0)) { + throw new SnapshotError('manifest-invalid', 'Snapshot root must be a private directory owned by this user.'); + } + const identity = fileIdentity(stats); + assertPathIdentity(path, identity, 'directory'); + return { path, descriptor, identity }; + } catch (caught) { + closeSync(descriptor); + throw caught; + } +} + +function closeBoundDirectory(directory: BoundDirectory): void { + closeSync(directory.descriptor); +} + +function readBoundedRegularFile( + path: string, + maximumBytes: number, +): { readonly bytes: Uint8Array; readonly identity: FileIdentity } { + let descriptor: number; + try { + descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0)); + } catch { + throw new SnapshotError('manifest-invalid', 'Snapshot input must be a no-follow physical regular file.'); + } + try { + const initial = fstatSync(descriptor); + if (!initial.isFile() || initial.size > maximumBytes) { + throw new SnapshotError('manifest-invalid', 'Snapshot input is not a bounded physical regular file.'); + } + const identity = fileIdentity(initial); + const bytes = new Uint8Array(initial.size); + let offset = 0; + while (offset < bytes.byteLength) { + const count = readSync(descriptor, bytes, offset, bytes.byteLength - offset, offset); + if (count === 0) break; + offset += count; + } + const final = fstatSync(descriptor); + if (offset !== bytes.byteLength || final.size !== initial.size || !sameFileIdentity(identity, final)) { + throw new SnapshotError('manifest-invalid', 'Snapshot input changed while it was read.'); + } + assertPathIdentity(path, identity, 'file'); + return { bytes, identity }; + } finally { + closeSync(descriptor); + } +} + function fsyncPath(path: string): void { - const descriptor = openSync(path, fsConstants.O_RDONLY); + const descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); try { fsyncSync(descriptor); } finally { @@ -380,7 +522,12 @@ function writeJson(path: string, manifest: SnapshotManifestV1): void { } function overwriteJson(path: string, manifest: SnapshotManifestV1): void { - writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'w', mode: 0o600 }); + const descriptor = openSync(path, fsConstants.O_WRONLY | fsConstants.O_TRUNC | fsConstants.O_NOFOLLOW); + try { + writeFileSync(descriptor, `${JSON.stringify(manifest, null, 2)}\n`); + } finally { + closeSync(descriptor); + } } function generationId( @@ -464,17 +611,22 @@ function makeManifest( }; } -function createPrivateRoot(): string { +function createPrivateRoot(): { readonly path: string; readonly identity: FileIdentity } { const root = mkdtempSync(join(tmpdir(), 'genie-db-sync-private-')); const mode = statSync(root).mode & 0o777; if (mode !== 0o700) { - rmSync(root, { recursive: true, force: true }); + rmdirSync(root); throw new SnapshotError( 'snapshot-publication-failed', 'Private snapshot directory was not created with mode 0700.', ); } - return root; + const binding = openBoundDirectory(root, false, true); + try { + return { path: root, identity: binding.identity }; + } finally { + closeBoundDirectory(binding); + } } function publishGeneration( @@ -485,11 +637,12 @@ function publishGeneration( privateRoot: string | null = null, ): PublishedGeneration { const root = privateRoot ?? identity.root; - mkdirSync(root, { recursive: true, mode: 0o700 }); + const rootBinding = openBoundDirectory(root, true, privateRoot !== null); const generated = generationId(identity, options); const staging = join(root, `${STAGING_PREFIX}${generated.id}-${randomUUID()}`); const finalDirectory = join(root, generated.id); mkdirSync(staging, { mode: 0o700 }); + const stagingBinding = openBoundDirectory(staging, false, true); const provisional = makeManifest( identity, schemaFingerprint, @@ -503,18 +656,29 @@ function publishGeneration( for (let index = 0; index < captures.length; index++) { const capture = captures[index]; around(options, { phase: 'payload-write', generationId: generated.id, role: capture.role }, () => - writeFileSync(join(staging, provisional.targets[index].snapshot_file), capture.normalizedBytes, { - flag: 'wx', - mode: 0o600, - }), + (() => { + assertPathIdentity(root, rootBinding.identity, 'directory'); + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + writeFileSync(join(staging, provisional.targets[index].snapshot_file), capture.normalizedBytes, { + flag: 'wx', + mode: 0o600, + }); + })(), ); } around(options, { phase: 'provisional-manifest-write', generationId: generated.id }, () => - writeJson(join(staging, MANIFEST_FILE), provisional), + (() => { + assertPathIdentity(root, rootBinding.identity, 'directory'); + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + writeJson(join(staging, MANIFEST_FILE), provisional); + })(), ); for (const target of provisional.targets) { around(options, { phase: 'payload-fsync', generationId: generated.id, role: target.role }, () => - fsyncPath(join(staging, target.snapshot_file)), + (() => { + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + fsyncPath(join(staging, target.snapshot_file)); + })(), ); } const complete = makeManifest( @@ -527,23 +691,49 @@ function publishGeneration( true, ); around(options, { phase: 'complete-manifest-write', generationId: generated.id }, () => - overwriteJson(join(staging, MANIFEST_FILE), complete), + (() => { + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + overwriteJson(join(staging, MANIFEST_FILE), complete); + })(), ); around(options, { phase: 'complete-manifest-fsync', generationId: generated.id }, () => - fsyncPath(join(staging, MANIFEST_FILE)), + (() => { + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + fsyncPath(join(staging, MANIFEST_FILE)); + })(), ); - around(options, { phase: 'staging-fsync', generationId: generated.id }, () => fsyncPath(staging)); - around(options, { phase: 'generation-rename', generationId: generated.id }, () => - renameSync(staging, finalDirectory), - ); - around(options, { phase: 'root-fsync', generationId: generated.id }, () => fsyncPath(root)); - return { manifest: complete, directory: finalDirectory, privateRoot }; + around(options, { phase: 'staging-fsync', generationId: generated.id }, () => { + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + fsyncSync(stagingBinding.descriptor); + }); + around(options, { phase: 'generation-rename', generationId: generated.id }, () => { + assertPathIdentity(root, rootBinding.identity, 'directory'); + assertPathIdentity(staging, stagingBinding.identity, 'directory'); + renameSync(staging, finalDirectory); + assertPathIdentity(finalDirectory, stagingBinding.identity, 'directory'); + }); + around(options, { phase: 'root-fsync', generationId: generated.id }, () => { + assertPathIdentity(root, rootBinding.identity, 'directory'); + fsyncSync(rootBinding.descriptor); + }); + const manifestPath = join(finalDirectory, MANIFEST_FILE); + const manifestIdentity = readBoundedRegularFile(manifestPath, MAX_MANIFEST_BYTES).identity; + return { + manifest: complete, + directory: finalDirectory, + directoryIdentity: stagingBinding.identity, + manifestIdentity, + privateRoot, + }; } catch (caught) { if (caught instanceof SnapshotError) throw caught; throw new SnapshotError( 'snapshot-publication-failed', caught instanceof Error ? caught.message : 'Snapshot publication failed.', ); + } finally { + closeBoundDirectory(stagingBinding); + closeBoundDirectory(rootBinding); } } @@ -687,30 +877,29 @@ function parseManifest(value: unknown): SnapshotManifestV1 { }; } -function readManifest(directory: string): SnapshotManifestV1 { +function readManifest(directory: string): GenerationReference { const manifestPath = join(directory, MANIFEST_FILE); - const directoryStat = lstatSync(directory); - const manifestStat = lstatSync(manifestPath); - if ( - !directoryStat.isDirectory() || - directoryStat.isSymbolicLink() || - !manifestStat.isFile() || - manifestStat.isSymbolicLink() || - manifestStat.size > MAX_MANIFEST_BYTES - ) { - throw new SnapshotError('manifest-invalid', 'Snapshot generation or manifest is not a bounded physical file.'); - } + const directoryBinding = openBoundDirectory(directory, false, true); let parsed: unknown; try { - parsed = JSON.parse(readFileSync(manifestPath, 'utf8')) as unknown; + const loaded = readBoundedRegularFile(manifestPath, MAX_MANIFEST_BYTES); + parsed = JSON.parse(new TextDecoder().decode(loaded.bytes)) as unknown; + const manifest = parseManifest(parsed); + if (basename(directory) !== manifest.generation_id) { + throw new SnapshotError('manifest-invalid', 'Snapshot generation directory does not match its manifest.'); + } + assertPathIdentity(directory, directoryBinding.identity, 'directory'); + return { + manifest, + directory, + directoryIdentity: directoryBinding.identity, + manifestIdentity: loaded.identity, + }; } catch { throw new SnapshotError('manifest-invalid', 'Snapshot manifest is not valid JSON.'); + } finally { + closeBoundDirectory(directoryBinding); } - const manifest = parseManifest(parsed); - if (basename(directory) !== manifest.generation_id) { - throw new SnapshotError('manifest-invalid', 'Snapshot generation directory does not match its manifest.'); - } - return manifest; } function validateManifestIdentity(manifest: SnapshotManifestV1, identity: SnapshotStoreIdentity): void { @@ -728,8 +917,15 @@ function validateManifestIdentity(manifest: SnapshotManifestV1, identity: Snapsh } function matchingGenerationDirectories(identity: SnapshotStoreIdentity): string[] { + let rootBinding: BoundDirectory; + try { + rootBinding = openBoundDirectory(identity.root, false); + } catch (caught) { + if (filesystemCode(caught) === 'ENOENT') return []; + throw caught; + } try { - return readdirSync(identity.root, { withFileTypes: true }) + const directories = readdirSync(identity.root, { withFileTypes: true }) .filter( (entry) => entry.isDirectory() && @@ -738,33 +934,30 @@ function matchingGenerationDirectories(identity: SnapshotStoreIdentity): string[ ) .map((entry) => join(identity.root, entry.name)) .sort((left, right) => compareText(right, left)); - } catch (caught) { - const code = isRecord(caught) ? caught.code : undefined; - if (code === 'ENOENT') return []; - throw caught; + assertPathIdentity(identity.root, rootBinding.identity, 'directory'); + return directories; + } finally { + closeBoundDirectory(rootBinding); } } -function newestUnresolved(identity: SnapshotStoreIdentity): { manifest: SnapshotManifestV1; directory: string } | null { +function newestUnresolved(identity: SnapshotStoreIdentity): GenerationReference | null { for (const directory of matchingGenerationDirectories(identity)) { - const manifest = readManifest(directory); - validateManifestIdentity(manifest, identity); - if (manifest.state === 'complete' || manifest.state === 'uncertain') return { manifest, directory }; + const generation = readManifest(directory); + validateManifestIdentity(generation.manifest, identity); + if (generation.manifest.state === 'complete' || generation.manifest.state === 'uncertain') return generation; } return null; } -function selectedGeneration( - identity: SnapshotStoreIdentity, - selectedGenerationId: string, -): { manifest: SnapshotManifestV1; directory: string } { +function selectedGeneration(identity: SnapshotStoreIdentity, selectedGenerationId: string): GenerationReference { if (!GENERATION_ID_PATTERN.test(selectedGenerationId) || !selectedGenerationId.startsWith(identity.operationId)) { throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not belong to this operation.'); } const directory = join(identity.root, selectedGenerationId); - let manifest: SnapshotManifestV1; + let generation: GenerationReference; try { - manifest = readManifest(directory); + generation = readManifest(directory); } catch (caught) { const code = isRecord(caught) ? caught.code : undefined; if (code === 'ENOENT') { @@ -772,14 +965,17 @@ function selectedGeneration( } throw caught; } - validateManifestIdentity(manifest, identity); - if (manifest.state === 'provisional' || manifest.targets.some((target) => target.snapshot_sha256 === null)) { + validateManifestIdentity(generation.manifest, identity); + if ( + generation.manifest.state === 'provisional' || + generation.manifest.targets.some((target) => target.snapshot_sha256 === null) + ) { throw new SnapshotError('manifest-invalid', 'Selected snapshot generation is incomplete.'); } - return { manifest, directory }; + return generation; } -function validateGeneration(generation: { manifest: SnapshotManifestV1; directory: string }): ValidatedGeneration { +function validateGeneration(generation: GenerationReference): ValidatedGeneration { const snapshots = new Map(); try { for (const target of generation.manifest.targets) { @@ -787,11 +983,9 @@ function validateGeneration(generation: { manifest: SnapshotManifestV1; director throw new SnapshotError('manifest-invalid', 'Complete snapshot manifest is missing a payload hash.'); } const payloadPath = join(generation.directory, target.snapshot_file); - const payloadStat = lstatSync(payloadPath); - if (!payloadStat.isFile() || payloadStat.isSymbolicLink()) { - throw new SnapshotError('manifest-invalid', 'Snapshot payload is not a physical regular file.'); - } - const bytes = new Uint8Array(readFileSync(payloadPath)); + assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); + assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); + const bytes = readBoundedRegularFile(payloadPath, MAX_RECONCILIATION_DATABASE_BYTES).bytes; if (sha256(bytes) !== target.snapshot_sha256) { throw new SnapshotError('snapshot-hash-mismatch', 'Snapshot payload hash does not match its manifest.'); } @@ -816,6 +1010,8 @@ function validateGeneration(generation: { manifest: SnapshotManifestV1; director throw new SnapshotError('snapshot-image-mismatch', 'Snapshot payload does not match its manifest image.'); } snapshots.set(target.path, db); + assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); + assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); } return { ...generation, snapshots }; } catch (caught) { @@ -828,26 +1024,99 @@ function closeValidatedGeneration(generation: ValidatedGeneration): void { for (const db of generation.snapshots.values()) db.close(); } +function safeRemoveTree(path: string, expected: FileIdentity): void { + assertPathIdentity(path, expected, 'directory'); + const directory = openBoundDirectory(path, false); + try { + if (!sameFileIdentity(expected, directory.identity)) { + throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup target identity changed.'); + } + const descriptorPath = `/proc/self/fd/${directory.descriptor}`; + for (const entry of readdirSync(descriptorPath, { withFileTypes: true })) { + const descriptorChild = join(descriptorPath, entry.name); + const child = join(path, entry.name); + const stats = lstatSync(descriptorChild); + const identity = fileIdentity(stats); + const current = lstatSync(child); + if (!sameFileIdentity(identity, current)) { + throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup child identity changed.'); + } + if (stats.isDirectory() && !stats.isSymbolicLink()) { + safeRemoveTree(child, identity); + } else { + assertPathIdentity(child, identity, 'file'); + unlinkSync(child); + } + } + fsyncSync(directory.descriptor); + } finally { + closeBoundDirectory(directory); + } + assertPathIdentity(path, expected, 'directory'); + rmdirSync(path); +} + +function removeBoundTree(path: string, expected: FileIdentity, options: DatabaseSyncSnapshotOptions): void { + assertPathIdentity(path, expected, 'directory'); + if (options.removeTree !== undefined) { + options.removeTree(path); + try { + lstatSync(path); + } catch (caught) { + if (filesystemCode(caught) === 'ENOENT') return; + throw caught; + } + throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup did not remove its exact target.'); + } + safeRemoveTree(path, expected); +} + function rewriteManifestState( - generation: { manifest: SnapshotManifestV1; directory: string }, + generation: GenerationReference, state: Exclude, options: DatabaseSyncSnapshotOptions, ): SnapshotManifestV1 { const updated = { ...generation.manifest, state }; const temporary = join(generation.directory, `.manifest-${randomUUID()}.tmp`); - around(options, { phase: 'state-rewrite-write', generationId: generation.manifest.generation_id }, () => - writeJson(temporary, updated), - ); - around(options, { phase: 'state-rewrite-fsync', generationId: generation.manifest.generation_id }, () => - fsyncPath(temporary), - ); - around(options, { phase: 'state-rewrite-rename', generationId: generation.manifest.generation_id }, () => - renameSync(temporary, join(generation.directory, MANIFEST_FILE)), - ); - around(options, { phase: 'generation-fsync', generationId: generation.manifest.generation_id }, () => - fsyncPath(generation.directory), - ); - return updated; + let temporaryIdentity: FileIdentity | null = null; + try { + assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); + assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); + around(options, { phase: 'state-rewrite-write', generationId: generation.manifest.generation_id }, () => + writeJson(temporary, updated), + ); + temporaryIdentity = readBoundedRegularFile(temporary, MAX_MANIFEST_BYTES).identity; + around(options, { phase: 'state-rewrite-fsync', generationId: generation.manifest.generation_id }, () => { + assertPathIdentity(temporary, temporaryIdentity as FileIdentity, 'file'); + fsyncPath(temporary); + }); + around(options, { phase: 'state-rewrite-rename', generationId: generation.manifest.generation_id }, () => { + assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); + assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); + assertPathIdentity(temporary, temporaryIdentity as FileIdentity, 'file'); + renameSync(temporary, join(generation.directory, MANIFEST_FILE)); + assertPathIdentity(join(generation.directory, MANIFEST_FILE), temporaryIdentity as FileIdentity, 'file'); + }); + around(options, { phase: 'generation-fsync', generationId: generation.manifest.generation_id }, () => { + assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); + fsyncPath(generation.directory); + }); + return updated; + } catch (caught) { + const cleanupFailures: SnapshotFailureCode[] = []; + if (temporaryIdentity !== null) { + try { + assertPathIdentity(temporary, temporaryIdentity, 'file'); + unlinkSync(temporary); + } catch (cleanupCaught) { + if (filesystemCode(cleanupCaught) !== 'ENOENT') cleanupFailures.push('snapshot-cleanup-failed'); + } + } + if (caught instanceof SnapshotError) { + throw new SnapshotError(caught.code, caught.message, [...caught.cleanupFailures, ...cleanupFailures]); + } + throw new SnapshotError('snapshot-publication-failed', 'Snapshot manifest state rewrite failed.', cleanupFailures); + } } function cleanupStaging( @@ -868,9 +1137,8 @@ function cleanupStaging( if (!entry.name.startsWith(STAGING_PREFIX) || !entry.isDirectory() || entry.isSymbolicLink()) continue; const path = join(root, entry.name); try { - around(options, { phase: 'staging-cleanup', path }, () => - (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(path), - ); + const identity = fileIdentity(lstatSync(path)); + around(options, { phase: 'staging-cleanup', path }, () => removeBoundTree(path, identity, options)); } catch { cleanupFailures.push('snapshot-cleanup-failed'); } @@ -890,10 +1158,10 @@ function pruneGenerations( if (retained.has(directory)) continue; try { const manifest = readManifest(directory); - validateManifestIdentity(manifest, identity); - if (manifest.state === 'complete' || manifest.state === 'uncertain') continue; - around(options, { phase: 'prune', generationId: manifest.generation_id, path: directory }, () => - (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(directory), + validateManifestIdentity(manifest.manifest, identity); + if (manifest.manifest.state === 'complete' || manifest.manifest.state === 'uncertain') continue; + around(options, { phase: 'prune', generationId: manifest.manifest.generation_id, path: directory }, () => + removeBoundTree(directory, manifest.directoryIdentity, options), ); removed = true; } catch { @@ -922,7 +1190,7 @@ function targetInput( function recoverValidatedGeneration( identity: SnapshotStoreIdentity, - generation: { manifest: SnapshotManifestV1; directory: string }, + generation: GenerationReference, inputs: readonly ReconciliationLockedDatabaseInput[], options: DatabaseSyncSnapshotOptions, retention: number, @@ -1023,6 +1291,27 @@ function recoveryReport(decision: RecoveryDecision): SnapshotRecoveryReport { }; } +function snapshotCleanupCode(failure: ReconciliationApplyFailure): SnapshotFailureCode { + switch (failure.code) { + case 'rollback-failed': + return 'locked-rollback-failed'; + case 'close-failed': + return 'locked-close-failed'; + case 'advisory-lock-release-failed': + return 'locked-advisory-release-failed'; + default: + return 'locked-operation-failed'; + } +} + +function snapshotCleanupFailures(caught: unknown): SnapshotFailureCode[] { + if (caught instanceof SnapshotError) return [...caught.cleanupFailures]; + if (caught instanceof ReconciliationLockedOperationError) { + return [...snapshotCleanupFailures(caught.operationCause), ...caught.cleanupFailures.map(snapshotCleanupCode)]; + } + return []; +} + function failedRecovery(caught: unknown): SnapshotRecoveryReport { const failure = caught instanceof SnapshotError @@ -1039,7 +1328,7 @@ function failedRecovery(caught: unknown): SnapshotRecoveryReport { generationId: null, restoredPaths: [], failure, - cleanupFailures: [], + cleanupFailures: snapshotCleanupFailures(caught), }; } @@ -1050,14 +1339,7 @@ export function recoverDatabaseReconciliation( let retention: number; try { retention = validateRetention(options.keepSnapshots); - if (retention === 0) { - return recoveryReport({ - status: 'none', - generationId: null, - restoredPaths: [], - cleanupFailures: [], - }); - } + const persistentRetention = retention === 0 ? DEFAULT_SNAPSHOT_RETENTION : retention; const decision = withLockedReconciliationDatabases( request, (inputs) => { @@ -1079,7 +1361,7 @@ export function recoverDatabaseReconciliation( afterCommit: () => cleanupStaging(identity.root, options, cleanupFailures), }; } - return recoverValidatedGeneration(identity, unresolved, inputs, options, retention); + return recoverValidatedGeneration(identity, unresolved, inputs, options, persistentRetention); }, { busyTimeoutMs: options.busyTimeoutMs, @@ -1122,12 +1404,13 @@ function capturesForPlan( function privateCleanup( root: string | null, + identity: FileIdentity | null, options: DatabaseSyncSnapshotOptions, cleanupFailures: SnapshotFailureCode[], ): void { - if (root === null) return; + if (root === null || identity === null) return; try { - (options.removeTree ?? ((target) => rmSync(target, { recursive: true, force: true })))(root); + removeBoundTree(root, identity, options); } catch { cleanupFailures.push('snapshot-cleanup-failed'); } @@ -1146,14 +1429,7 @@ function recoverPrivateGeneration( try { const decision = withLockedReconciliationDatabases( request, - (inputs) => - recoverValidatedGeneration( - identity, - { manifest: readManifest(published.directory), directory: published.directory }, - inputs, - options, - 0, - ), + (inputs) => recoverValidatedGeneration(identity, readManifest(published.directory), inputs, options, 0), { busyTimeoutMs: options.busyTimeoutMs, onEvent: options.onLockedOperationEvent, @@ -1186,6 +1462,19 @@ export function applyDatabaseReconciliationWithSnapshots( }; } const request = requestFromPlan(plan); + const priorRecovery = recoverDatabaseReconciliation(request, options); + if (priorRecovery.status !== 'none') { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'apply', + status: priorRecovery.status === 'operational-failure' ? 'operational-failure' : priorRecovery.status, + generationId: priorRecovery.generationId, + recovery: priorRecovery, + apply: null, + failure: priorRecovery.failure, + cleanupFailures: priorRecovery.cleanupFailures, + }; + } if (plan.status !== 'changed') { const apply = applyDatabaseReconciliation(plan, { busyTimeoutMs: options.busyTimeoutMs, @@ -1197,54 +1486,48 @@ export function applyDatabaseReconciliationWithSnapshots( operation: 'apply', status: apply.status, generationId: null, - recovery: recoveryReport({ - status: 'none', - generationId: null, - restoredPaths: [], - cleanupFailures: [], - }), + recovery: priorRecovery, apply, failure: null, - cleanupFailures: [], - }; - } - - const priorRecovery = recoverDatabaseReconciliation(request, options); - if (priorRecovery.status !== 'none') { - return { - reportVersion: SNAPSHOT_REPORT_VERSION, - operation: 'apply', - status: priorRecovery.status === 'operational-failure' ? 'operational-failure' : priorRecovery.status, - generationId: priorRecovery.generationId, - recovery: priorRecovery, - apply: null, - failure: priorRecovery.failure, - cleanupFailures: priorRecovery.cleanupFailures, + cleanupFailures: apply.cleanupFailures.map(snapshotCleanupCode), }; } const identity = identityFromCanonical(plan.mode, canonicalPathsFromPlan(plan), options.snapshotRoot); let published: PublishedGeneration | null = null; let privateRoot: string | null = null; + let privateRootIdentity: FileIdentity | null = null; const cleanupFailures: SnapshotFailureCode[] = [...priorRecovery.cleanupFailures]; try { - if (retention === 0) privateRoot = createPrivateRoot(); + if (retention === 0) { + const created = createPrivateRoot(); + privateRoot = created.path; + privateRootIdentity = created.identity; + } const apply = applyDatabaseReconciliation(plan, { busyTimeoutMs: options.busyTimeoutMs, onEvent: options.onApplyEvent, ...options.applyOptions, onLocked: (inputs) => { + const unresolved = newestUnresolved(identity); + if (unresolved !== null) { + throw new SnapshotError( + 'recovery-uncertain', + 'A persistent snapshot generation became unresolved after recovery and before apply.', + ); + } const captures = capturesForPlan(plan, inputs); published = publishGeneration(identity, plan.schemaFingerprint, captures, options, privateRoot); }, }); + cleanupFailures.push(...apply.cleanupFailures.map(snapshotCleanupCode)); const recovery = retention === 0 ? published === null ? recoveryReport({ status: 'none', generationId: null, restoredPaths: [], cleanupFailures: [] }) : recoverPrivateGeneration(published, request, identity, options) : recoverDatabaseReconciliation(request, options); - privateCleanup(privateRoot, options, cleanupFailures); + privateCleanup(privateRoot, privateRootIdentity, options, cleanupFailures); const status = recovery.status === 'none' ? apply.status @@ -1264,7 +1547,7 @@ export function applyDatabaseReconciliationWithSnapshots( cleanupFailures: [...recovery.cleanupFailures, ...cleanupFailures], }; } catch (caught) { - privateCleanup(privateRoot, options, cleanupFailures); + privateCleanup(privateRoot, privateRootIdentity, options, cleanupFailures); return { reportVersion: SNAPSHOT_REPORT_VERSION, operation: 'apply', @@ -1273,7 +1556,7 @@ export function applyDatabaseReconciliationWithSnapshots( recovery: failedRecovery(caught), apply: null, failure: caught instanceof SnapshotError ? caught.code : 'snapshot-publication-failed', - cleanupFailures, + cleanupFailures: [...cleanupFailures, ...snapshotCleanupFailures(caught)], }; } } @@ -1311,10 +1594,15 @@ export function rollbackDatabaseReconciliation( } let privateRoot: string | null = null; + let privateRootIdentity: FileIdentity | null = null; let published: PublishedGeneration | null = null; const cleanupFailures: SnapshotFailureCode[] = [...recovery.cleanupFailures]; try { - if (retention === 0) privateRoot = createPrivateRoot(); + if (retention === 0) { + const created = createPrivateRoot(); + privateRoot = created.path; + privateRootIdentity = created.identity; + } const value = withLockedReconciliationDatabases( request, (inputs) => { @@ -1323,6 +1611,13 @@ export function rollbackDatabaseReconciliation( inputs.map((input) => input.canonicalPath), options.snapshotRoot, ); + const racedUnresolved = newestUnresolved(identity); + if (racedUnresolved !== null) { + throw new SnapshotError( + 'recovery-uncertain', + 'A persistent snapshot generation became unresolved after recovery and before rollback.', + ); + } const selected = validateGeneration(selectedGeneration(identity, selectedGenerationId)); try { const captures = selected.manifest.targets.map((target) => { @@ -1381,7 +1676,7 @@ export function rollbackDatabaseReconciliation( ...options.applyOptions, }, ); - privateCleanup(privateRoot, options, cleanupFailures); + privateCleanup(privateRoot, privateRootIdentity, options, cleanupFailures); return { reportVersion: SNAPSHOT_REPORT_VERSION, operation: 'rollback', @@ -1392,7 +1687,7 @@ export function rollbackDatabaseReconciliation( cleanupFailures, }; } catch (caught) { - privateCleanup(privateRoot, options, cleanupFailures); + privateCleanup(privateRoot, privateRootIdentity, options, cleanupFailures); return { reportVersion: SNAPSHOT_REPORT_VERSION, operation: 'rollback', @@ -1400,7 +1695,7 @@ export function rollbackDatabaseReconciliation( selectedGenerationId, safetyGenerationId: publishedGenerationId(published), failure: caught instanceof SnapshotError ? caught.code : 'locked-operation-failed', - cleanupFailures, + cleanupFailures: [...cleanupFailures, ...snapshotCleanupFailures(caught)], }; } } From c19578c47e79ab2e10a96987cc04ed6ba88cae41 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 14:49:53 +0000 Subject: [PATCH 10/21] fix(db-sync): bind snapshot filesystem mutations --- src/lib/v5/db-sync-snapshots.test.ts | 269 ++++++++ src/lib/v5/db-sync-snapshots.ts | 950 ++++++++++++++++++++------- 2 files changed, 993 insertions(+), 226 deletions(-) diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index 227b6cf8c..5ed78c5d9 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -24,11 +24,13 @@ import { } from './db-reconciliation.js'; import { SnapshotError, + type SnapshotPosixDirectoryApi, applyDatabaseReconciliationWithSnapshots, databaseSyncSnapshotIdentity, deserializeSnapshotBytes, normalizeSerializedSqliteForDeserialize, recoverDatabaseReconciliation, + resolveSnapshotPosixDirectory, rollbackDatabaseReconciliation, } from './db-sync-snapshots.js'; import { openDb } from './genie-db.js'; @@ -111,6 +113,34 @@ function generationDirectories(root: string): string[] { .sort(); } +function nativePosixDirectory(): SnapshotPosixDirectoryApi { + const api = resolveSnapshotPosixDirectory(); + expect(api).not.toBeNull(); + return api as SnapshotPosixDirectoryApi; +} + +function delegatePosix( + api: SnapshotPosixDirectoryApi, + overrides: Partial, +): SnapshotPosixDirectoryApi { + return { + openAt: overrides.openAt ?? ((...args) => api.openAt(...args)), + mkdirAt: overrides.mkdirAt ?? ((...args) => api.mkdirAt(...args)), + renameAt: overrides.renameAt ?? ((...args) => api.renameAt(...args)), + unlinkAt: overrides.unlinkAt ?? ((...args) => api.unlinkAt(...args)), + list: overrides.list ?? ((...args) => api.list(...args)), + }; +} + +function descriptorCount(): number { + const directory = process.platform === 'darwin' ? '/dev/fd' : '/proc/self/fd'; + return readdirSync(directory).length; +} + +function privateSnapshotDirectories(): string[] { + return readdirSync(tmpdir()).filter((name) => name.startsWith('genie-db-sync-private-')); +} + function leaveCompleteGeneration(left: string, right: string): { root: string; directory: string } { insertBoard(right, 'planned'); const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; @@ -301,6 +331,46 @@ describe('database sync snapshots', () => { } }); + test('root and ancestor substitution at descriptor-relative creation cannot redirect publication writes', () => { + for (const kind of ['root', 'ancestor'] as const) { + const left = currentDb(`bound-${kind}-left`); + const right = currentDb(`bound-${kind}-right`); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const ancestor = join(fixtureRoot, `bound-${kind}-ancestor`); + const root = join(ancestor, 'snapshots'); + const moved = kind === 'root' ? `${root}.owned` : `${ancestor}.owned`; + let substituted = false; + const native = nativePosixDirectory(); + const api = delegatePosix(native, { + mkdirAt: (descriptor, name, mode) => { + if (!substituted && name.startsWith('.staging-')) { + substituted = true; + if (kind === 'root') { + renameSync(root, moved); + mkdirSync(root, { mode: 0o700 }); + } else { + renameSync(ancestor, moved); + mkdirSync(root, { recursive: true, mode: 0o700 }); + } + writeFileSync(join(root, 'sentinel'), 'replacement'); + } + native.mkdirAt(descriptor, name, mode); + }, + }); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + snapshotRoot: root, + posixDirectory: { api }, + }); + expect(substituted).toBe(true); + expect(['rolled-back', 'operational-failure']).toContain(report.status); + expect(readFileSync(join(root, 'sentinel'), 'utf8')).toBe('replacement'); + expect(readdirSync(root)).toEqual(['sentinel']); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); + test('publishes normalized complete payloads in the durable order and finalizes both-post as converged', () => { const left = currentDb('publish-left'); const right = currentDb('publish-right'); @@ -360,6 +430,49 @@ describe('database sync snapshots', () => { expect(events).toContain('generation-fsync:after'); }); + test('publication staging, generation, and manifest substitution cannot redirect a write', () => { + for (const kind of ['staging', 'generation', 'manifest'] as const) { + const left = currentDb(`publish-substitute-${kind}-left`); + const right = currentDb(`publish-substitute-${kind}-right`); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const root = databaseSyncSnapshotIdentity(request).root; + const victim = join(fixtureRoot, `publish-substitute-${kind}-victim`); + writeFileSync(victim, 'victim'); + let substituted = false; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + onEvent: (event) => { + if (substituted || event.state !== 'before') return; + if (kind === 'manifest' && event.phase === 'complete-manifest-write') { + const stagingName = readdirSync(root).find((name) => name.startsWith('.staging-')) as string; + const manifest = join(root, stagingName, 'manifest.json'); + renameSync(manifest, `${manifest}.owned`); + symlinkSync(victim, manifest); + substituted = true; + } else if (kind === 'staging' && event.phase === 'generation-rename') { + const stagingName = readdirSync(root).find((name) => name.startsWith('.staging-')) as string; + const staging = join(root, stagingName); + renameSync(staging, `${staging}.owned`); + mkdirSync(staging, { mode: 0o700 }); + writeFileSync(join(staging, 'sentinel'), 'replacement'); + substituted = true; + } else if (kind === 'generation' && event.phase === 'root-fsync') { + const generation = generationDirectories(root)[0]; + renameSync(generation, `${generation}.owned`); + mkdirSync(generation, { mode: 0o700 }); + writeFileSync(join(generation, 'sentinel'), 'replacement'); + substituted = true; + } + }, + }); + expect(substituted).toBe(true); + expect(['rolled-back', 'operational-failure']).toContain(report.status); + expect(readFileSync(victim, 'utf8')).toBe('victim'); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); + test('a first-commit cut is recovered immediately by restoring the post side to its preimage', () => { const left = currentDb('mixed-left'); const right = currentDb('mixed-right'); @@ -1151,6 +1264,55 @@ describe('database sync snapshots', () => { } }); + test('injected Darwin descriptor operations clean stale staging, prune retention, and delete private N=0 state', () => { + const native = nativePosixDirectory(); + let directoryRemovals = 0; + const api = delegatePosix(native, { + unlinkAt: (descriptor, name, directory) => { + if (directory) directoryRemovals++; + native.unlinkAt(descriptor, name, directory); + }, + }); + const candidates: string[] = []; + const darwin = { + platform: 'darwin' as const, + darwinOpener: (candidate: string, platform: 'darwin') => { + candidates.push(`${platform}:${candidate}`); + return api; + }, + }; + const left = currentDb('darwin-cleanup-left'); + const right = currentDb('darwin-cleanup-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const root = databaseSyncSnapshotIdentity(request).root; + mkdirSync(join(root, '.staging-darwin'), { recursive: true }); + writeFileSync(join(root, '.staging-darwin', 'partial'), 'partial'); + + for (const id of ['first', 'second']) { + insertBoard(right, id); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + posixDirectory: darwin, + }).status, + ).toBe('changed'); + } + expect(existsSync(join(root, '.staging-darwin'))).toBe(false); + expect(generationDirectories(root)).toHaveLength(1); + + const privateBefore = privateSnapshotDirectories(); + insertBoard(right, 'private'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + posixDirectory: darwin, + }).status, + ).toBe('changed'); + expect(privateSnapshotDirectories()).toEqual(privateBefore); + expect(directoryRemovals).toBeGreaterThanOrEqual(3); + expect(candidates).toContain('darwin:/usr/lib/libSystem.B.dylib'); + }); + test('zero retention uses private 0700 state, never publishes, cleans on success, and cannot recover later', () => { const left = currentDb('private-left'); const right = currentDb('private-right'); @@ -1210,6 +1372,113 @@ describe('database sync snapshots', () => { if (leakedRoot !== undefined) rmSync(leakedRoot, { recursive: true, force: true }); }); + test('POSIX resolution falls through Linux libc candidates and selects Darwin libSystem lazily', () => { + const native = nativePosixDirectory(); + const linuxAttempts: string[] = []; + expect( + resolveSnapshotPosixDirectory({ + platform: 'linux', + architecture: 'x64', + linuxCandidates: ['missing-glibc', 'working-musl'], + linuxOpener: (candidate, platform) => { + linuxAttempts.push(`${platform}:${candidate}`); + return candidate === 'working-musl' ? native : null; + }, + }), + ).toBe(native); + expect(linuxAttempts).toEqual(['linux:missing-glibc', 'linux:working-musl']); + + const darwinAttempts: string[] = []; + expect( + resolveSnapshotPosixDirectory({ + platform: 'darwin', + architecture: 'arm64', + darwinOpener: (candidate, platform) => { + darwinAttempts.push(`${platform}:${candidate}`); + return native; + }, + }), + ).toBe(native); + expect(darwinAttempts).toEqual(['darwin:/usr/lib/libSystem.B.dylib']); + expect(resolveSnapshotPosixDirectory({ platform: 'win32', architecture: 'x64' })).toBeNull(); + }); + + test('root and child descriptors close on invalid options, staging faults, and reacquisition', () => { + const native = nativePosixDirectory(); + const left = currentDb('descriptor-lifetime-left'); + const right = currentDb('descriptor-lifetime-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const plan = planDatabaseReconciliation(request); + const root = join(fixtureRoot, 'descriptor-lifetime-root'); + Bun.gc(true); + const before = descriptorCount(); + + for (let attempt = 0; attempt < 8; attempt++) { + expect( + applyDatabaseReconciliationWithSnapshots(plan, { + snapshotRoot: root, + randomId: () => 'invalid', + }), + ).toMatchObject({ status: 'rolled-back', apply: { failure: { code: 'apply-failed', phase: 'locked' } } }); + } + const mkdirFault = delegatePosix(native, { + mkdirAt: () => { + throw new Error('mkdirat fault'); + }, + }); + for (let attempt = 0; attempt < 8; attempt++) { + expect( + applyDatabaseReconciliationWithSnapshots(plan, { + snapshotRoot: root, + posixDirectory: { api: mkdirFault }, + }).status, + ).toBe('rolled-back'); + } + const bindFault = delegatePosix(native, { + openAt: (descriptor, name, flags, mode) => { + if (name.startsWith('.staging-')) throw new Error('staging bind fault'); + return native.openAt(descriptor, name, flags, mode); + }, + }); + for (let attempt = 0; attempt < 8; attempt++) { + expect( + applyDatabaseReconciliationWithSnapshots(plan, { + snapshotRoot: root, + posixDirectory: { api: bindFault }, + }).status, + ).toBe('rolled-back'); + } + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + + const recovered = applyDatabaseReconciliationWithSnapshots(plan, { + snapshotRoot: root, + posixDirectory: { api: native }, + }); + expect(recovered.status).toBe('changed'); + expect(boardNames(left)).toEqual(['planned']); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + + const validationLeft = currentDb('descriptor-validation-left'); + const validationRight = currentDb('descriptor-validation-right'); + const complete = leaveCompleteGeneration(validationLeft, validationRight); + const manifest = JSON.parse(readFileSync(join(complete.directory, 'manifest.json'), 'utf8')); + writeFileSync(join(complete.directory, manifest.targets[0].snapshot_file), 'invalid'); + const validationRequest = { + mode: 'bidirectional' as const, + leftPath: validationLeft, + rightPath: validationRight, + }; + const validationBefore = descriptorCount(); + for (let attempt = 0; attempt < 8; attempt++) { + expect(recoverDatabaseReconciliation(validationRequest).status).toBe('operational-failure'); + } + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(validationBefore + 1); + }); + test('negative, fractional, and unsafe retention fail before snapshots or writes', () => { for (const [index, retention] of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1].entries()) { const left = currentDb(`invalid-retention-${index}-left`); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index 60d368ead..02a907558 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -1,3 +1,4 @@ +import { CString, FFIType, dlopen } from 'bun:ffi'; import { Database } from 'bun:sqlite'; import { createHash, randomUUID } from 'node:crypto'; import { @@ -5,22 +6,22 @@ import { constants as fsConstants, fstatSync, fsyncSync, + ftruncateSync, lstatSync, mkdirSync, mkdtempSync, openSync, readSync, - readdirSync, realpathSync, - renameSync, rmdirSync, statSync, - unlinkSync, writeFileSync, + writeSync, } from 'node:fs'; -import type { Dirent, Stats } from 'node:fs'; +import type { Stats } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; +import { linuxLibcCandidates } from '../install-transaction.js'; import { MAX_RECONCILIATION_DATABASE_BYTES, type ReconciliationApplyEvent, @@ -58,6 +59,7 @@ const GENERATION_SEPARATOR = '--'; const SHA256_PATTERN = /^[a-f0-9]{64}$/; const GENERATION_ID_PATTERN = /^[a-f0-9]{64}--[0-9]{16}--[a-f0-9-]{36}$/; const UUID_PATTERN = /^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/; +const POSIX_CLOSE_ON_EXEC = process.platform === 'darwin' ? 0x1000000 : 0x80000; export type SnapshotFailureCode = | 'snapshot-invalid-header' @@ -203,9 +205,33 @@ export interface DatabaseSyncSnapshotOptions { readonly now?: () => Date; readonly randomId?: () => string; readonly removeTree?: (path: string) => void; + /** Lazy native descriptor-relative filesystem resolution seam for portability and fault tests. */ + readonly posixDirectory?: SnapshotPosixDirectoryDependencies; readonly applyOptions?: Omit; } +export interface SnapshotPosixDirectoryApi { + openAt(directoryDescriptor: number, name: string, flags: number, mode?: number): number; + mkdirAt(directoryDescriptor: number, name: string, mode: number): void; + renameAt( + sourceDirectoryDescriptor: number, + sourceName: string, + destinationDirectoryDescriptor: number, + destinationName: string, + ): void; + unlinkAt(directoryDescriptor: number, name: string, directory: boolean): void; + list(directoryDescriptor: number): readonly string[]; +} + +export interface SnapshotPosixDirectoryDependencies { + readonly platform?: NodeJS.Platform; + readonly architecture?: NodeJS.Architecture; + readonly linuxCandidates?: readonly string[]; + readonly linuxOpener?: (candidate: string, platform: 'linux') => SnapshotPosixDirectoryApi | null; + readonly darwinOpener?: (candidate: string, platform: 'darwin') => SnapshotPosixDirectoryApi | null; + readonly api?: SnapshotPosixDirectoryApi; +} + export type SnapshotRecoveryStatus = 'none' | 'converged' | 'recovered' | 'uncertain' | 'operational-failure'; export interface SnapshotRecoveryReport { @@ -266,11 +292,13 @@ interface GenerationReference { readonly directory: string; readonly directoryIdentity: FileIdentity; readonly manifestIdentity: FileIdentity; + readonly binding: BoundDirectory; } interface ValidatedGeneration { readonly manifest: SnapshotManifestV1; readonly directory: string; + readonly binding: BoundDirectory; readonly snapshots: ReadonlyMap; } @@ -292,6 +320,166 @@ interface BoundDirectory { readonly identity: FileIdentity; } +const MAX_DIRECTORY_ENTRIES = 4096; +const POSIX_DIRECTORY_SYMBOLS = { + openat: { + args: [FFIType.i32, FFIType.cstring, FFIType.i32, FFIType.u32], + returns: FFIType.i32, + }, + mkdirat: { + args: [FFIType.i32, FFIType.cstring, FFIType.u32], + returns: FFIType.i32, + }, + renameat: { + args: [FFIType.i32, FFIType.cstring, FFIType.i32, FFIType.cstring], + returns: FFIType.i32, + }, + unlinkat: { + args: [FFIType.i32, FFIType.cstring, FFIType.i32], + returns: FFIType.i32, + }, + dup: { + args: [FFIType.i32], + returns: FFIType.i32, + }, + fdopendir: { + args: [FFIType.i32], + returns: FFIType.ptr, + }, + readdir: { + args: [FFIType.ptr], + returns: FFIType.ptr, + }, + closedir: { + args: [FFIType.ptr], + returns: FFIType.i32, + }, +} as const; +let defaultPosixDirectoryApi: SnapshotPosixDirectoryApi | null | undefined; + +function componentBytes(name: string): Uint8Array { + if (name.length === 0 || name === '.' || name === '..' || name.includes('/') || name.includes('\0')) { + throw new SnapshotError('manifest-invalid', 'Snapshot entry name is not a safe path component.'); + } + return new TextEncoder().encode(`${name}\0`); +} + +function openPosixDirectoryApi(candidate: string, platform: 'linux' | 'darwin'): SnapshotPosixDirectoryApi | null { + try { + const library = dlopen(candidate, POSIX_DIRECTORY_SYMBOLS); + const nameOffset = platform === 'darwin' ? 21 : 19; + const removedDirectoryFlag = platform === 'darwin' ? 0x80 : 0x200; + return { + openAt(directoryDescriptor, name, flags, mode = 0) { + const descriptor = library.symbols.openat(directoryDescriptor, componentBytes(name), flags, mode); + if (descriptor < 0) throw new Error('openat failed'); + return descriptor; + }, + mkdirAt(directoryDescriptor, name, mode) { + if (library.symbols.mkdirat(directoryDescriptor, componentBytes(name), mode) !== 0) { + throw new Error('mkdirat failed'); + } + }, + renameAt(sourceDirectoryDescriptor, sourceName, destinationDirectoryDescriptor, destinationName) { + if ( + library.symbols.renameat( + sourceDirectoryDescriptor, + componentBytes(sourceName), + destinationDirectoryDescriptor, + componentBytes(destinationName), + ) !== 0 + ) { + throw new Error('renameat failed'); + } + }, + unlinkAt(directoryDescriptor, name, directory) { + if ( + library.symbols.unlinkat(directoryDescriptor, componentBytes(name), directory ? removedDirectoryFlag : 0) !== + 0 + ) { + throw new Error('unlinkat failed'); + } + }, + list(directoryDescriptor) { + const duplicate = library.symbols.dup(directoryDescriptor); + if (duplicate < 0) throw new Error('dup failed'); + const stream = library.symbols.fdopendir(duplicate); + if (stream === null) { + closeSync(duplicate); + throw new Error('fdopendir failed'); + } + const entries: string[] = []; + try { + for (;;) { + const entry = library.symbols.readdir(stream); + if (entry === null) break; + const name = new CString(entry, nameOffset).toString(); + if (name === '.' || name === '..') continue; + componentBytes(name); + entries.push(name); + if (entries.length > MAX_DIRECTORY_ENTRIES) { + throw new SnapshotError('manifest-invalid', 'Snapshot directory exceeds the bounded entry limit.'); + } + } + } catch (caught) { + library.symbols.closedir(stream); + throw caught; + } + if (library.symbols.closedir(stream) !== 0) throw new Error('closedir failed'); + return entries; + }, + }; + } catch { + return null; + } +} + +/** + * Resolve the supported descriptor-relative POSIX surface lazily. + * + * No native library is loaded at module import. Linux tries the repository's + * architecture-aware glibc/musl candidates; Darwin uses libSystem. + */ +export function resolveSnapshotPosixDirectory( + dependencies?: SnapshotPosixDirectoryDependencies, +): SnapshotPosixDirectoryApi | null { + if (dependencies?.api !== undefined) return dependencies.api; + if (dependencies === undefined && defaultPosixDirectoryApi !== undefined) return defaultPosixDirectoryApi; + const platform = dependencies?.platform ?? process.platform; + const architecture = dependencies?.architecture ?? process.arch; + let resolved: SnapshotPosixDirectoryApi | null = null; + if (platform === 'linux') { + const opener = dependencies?.linuxOpener ?? openPosixDirectoryApi; + for (const candidate of dependencies?.linuxCandidates ?? linuxLibcCandidates(architecture)) { + try { + resolved = opener(candidate, 'linux'); + } catch { + resolved = null; + } + if (resolved !== null) break; + } + } else if (platform === 'darwin') { + try { + resolved = (dependencies?.darwinOpener ?? openPosixDirectoryApi)('/usr/lib/libSystem.B.dylib', 'darwin'); + } catch { + resolved = null; + } + } + if (dependencies === undefined) defaultPosixDirectoryApi = resolved; + return resolved; +} + +function posixDirectory(options: DatabaseSyncSnapshotOptions): SnapshotPosixDirectoryApi { + const api = resolveSnapshotPosixDirectory(options.posixDirectory); + if (api === null) { + throw new SnapshotError( + 'snapshot-publication-failed', + 'Descriptor-relative snapshot filesystem operations are unavailable on this platform.', + ); + } + return api; +} + function sha256(value: string | Uint8Array): string { return createHash('sha256').update(value).digest('hex'); } @@ -474,62 +662,140 @@ function closeBoundDirectory(directory: BoundDirectory): void { closeSync(directory.descriptor); } -function readBoundedRegularFile( +function openBoundDirectoryAt( + parent: BoundDirectory, + name: string, path: string, - maximumBytes: number, -): { readonly bytes: Uint8Array; readonly identity: FileIdentity } { + options: DatabaseSyncSnapshotOptions, + requirePrivate = false, +): BoundDirectory { + const api = posixDirectory(options); let descriptor: number; try { - descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0)); + descriptor = api.openAt( + parent.descriptor, + name, + fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, + ); } catch { - throw new SnapshotError('manifest-invalid', 'Snapshot input must be a no-follow physical regular file.'); + throw new SnapshotError('manifest-invalid', 'Snapshot child is not a safe physical directory.'); } try { - const initial = fstatSync(descriptor); - if (!initial.isFile() || initial.size > maximumBytes) { - throw new SnapshotError('manifest-invalid', 'Snapshot input is not a bounded physical regular file.'); - } - const identity = fileIdentity(initial); - const bytes = new Uint8Array(initial.size); - let offset = 0; - while (offset < bytes.byteLength) { - const count = readSync(descriptor, bytes, offset, bytes.byteLength - offset, offset); - if (count === 0) break; - offset += count; - } - const final = fstatSync(descriptor); - if (offset !== bytes.byteLength || final.size !== initial.size || !sameFileIdentity(identity, final)) { - throw new SnapshotError('manifest-invalid', 'Snapshot input changed while it was read.'); + const stats = fstatSync(descriptor); + const currentUid = typeof process.getuid === 'function' ? process.getuid() : stats.uid; + const mode = stats.mode & 0o777; + if (!stats.isDirectory() || stats.uid !== currentUid || (requirePrivate ? mode !== 0o700 : (mode & 0o022) !== 0)) { + throw new SnapshotError('manifest-invalid', 'Snapshot child must be a private directory owned by this user.'); } - assertPathIdentity(path, identity, 'file'); - return { bytes, identity }; - } finally { + return { path, descriptor, identity: fileIdentity(stats) }; + } catch (caught) { closeSync(descriptor); + throw caught; } } -function fsyncPath(path: string): void { - const descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); +function openRegularFileAt( + directory: BoundDirectory, + name: string, + flags: number, + options: DatabaseSyncSnapshotOptions, + mode = 0, +): number { try { - fsyncSync(descriptor); - } finally { - closeSync(descriptor); + return posixDirectory(options).openAt( + directory.descriptor, + name, + flags | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, + mode, + ); + } catch { + throw new SnapshotError('manifest-invalid', 'Snapshot input must be a no-follow physical regular file.'); + } +} + +function readBoundedRegularDescriptor( + descriptor: number, + maximumBytes: number, +): { readonly bytes: Uint8Array; readonly identity: FileIdentity } { + const initial = fstatSync(descriptor); + if (!initial.isFile() || initial.size > maximumBytes) { + throw new SnapshotError('manifest-invalid', 'Snapshot input is not a bounded physical regular file.'); } + const identity = fileIdentity(initial); + const bytes = new Uint8Array(initial.size); + let offset = 0; + while (offset < bytes.byteLength) { + const count = readSync(descriptor, bytes, offset, bytes.byteLength - offset, offset); + if (count === 0) break; + offset += count; + } + const final = fstatSync(descriptor); + if (offset !== bytes.byteLength || final.size !== initial.size || !sameFileIdentity(identity, final)) { + throw new SnapshotError('manifest-invalid', 'Snapshot input changed while it was read.'); + } + return { bytes, identity }; } -function writeJson(path: string, manifest: SnapshotManifestV1): void { - writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); +function readBoundedRegularFileAt( + directory: BoundDirectory, + name: string, + maximumBytes: number, + options: DatabaseSyncSnapshotOptions, +): { readonly bytes: Uint8Array; readonly identity: FileIdentity } { + const descriptor = openRegularFileAt(directory, name, fsConstants.O_RDONLY | (fsConstants.O_NONBLOCK ?? 0), options); + try { + return readBoundedRegularDescriptor(descriptor, maximumBytes); + } finally { + closeSync(descriptor); + } } -function overwriteJson(path: string, manifest: SnapshotManifestV1): void { - const descriptor = openSync(path, fsConstants.O_WRONLY | fsConstants.O_TRUNC | fsConstants.O_NOFOLLOW); +function assertEntryIdentityAt( + directory: BoundDirectory, + name: string, + identity: FileIdentity, + kind: 'directory' | 'file', + options: DatabaseSyncSnapshotOptions, +): void { + let descriptor: number; + try { + descriptor = + kind === 'directory' + ? posixDirectory(options).openAt( + directory.descriptor, + name, + fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, + ) + : openRegularFileAt(directory, name, fsConstants.O_RDONLY | (fsConstants.O_NONBLOCK ?? 0), options); + } catch { + throw new SnapshotError('manifest-invalid', `Snapshot ${kind} identity changed during the operation.`); + } try { - writeFileSync(descriptor, `${JSON.stringify(manifest, null, 2)}\n`); + const stats = fstatSync(descriptor); + const validType = kind === 'directory' ? stats.isDirectory() : stats.isFile(); + if (!validType || !sameFileIdentity(identity, stats)) { + throw new SnapshotError('manifest-invalid', `Snapshot ${kind} identity changed during the operation.`); + } } finally { closeSync(descriptor); } } +function assertBoundDirectory(directory: BoundDirectory): void { + const stats = fstatSync(directory.descriptor); + if (!stats.isDirectory() || !sameFileIdentity(directory.identity, stats)) { + throw new SnapshotError('manifest-invalid', 'Snapshot directory binding changed during the operation.'); + } +} + +function writeJsonDescriptor(descriptor: number, manifest: SnapshotManifestV1): void { + const bytes = Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`); + let offset = 0; + while (offset < bytes.byteLength) { + offset += writeSync(descriptor, bytes, offset, bytes.byteLength - offset, offset); + } +} + function generationId( identity: SnapshotStoreIdentity, options: DatabaseSyncSnapshotOptions, @@ -638,47 +904,58 @@ function publishGeneration( ): PublishedGeneration { const root = privateRoot ?? identity.root; const rootBinding = openBoundDirectory(root, true, privateRoot !== null); - const generated = generationId(identity, options); - const staging = join(root, `${STAGING_PREFIX}${generated.id}-${randomUUID()}`); - const finalDirectory = join(root, generated.id); - mkdirSync(staging, { mode: 0o700 }); - const stagingBinding = openBoundDirectory(staging, false, true); - const provisional = makeManifest( - identity, - schemaFingerprint, - captures, - generated.id, - generated.createdAt, - 'provisional', - false, - ); + let stagingBinding: BoundDirectory | null = null; + let manifestDescriptor: number | null = null; + const payloadDescriptors: number[] = []; try { + const api = posixDirectory(options); + const generated = generationId(identity, options); + const stagingName = `${STAGING_PREFIX}${generated.id}-${randomUUID()}`; + const staging = join(root, stagingName); + const finalDirectory = join(root, generated.id); + api.mkdirAt(rootBinding.descriptor, stagingName, 0o700); + stagingBinding = openBoundDirectoryAt(rootBinding, stagingName, staging, options, true); + const provisional = makeManifest( + identity, + schemaFingerprint, + captures, + generated.id, + generated.createdAt, + 'provisional', + false, + ); for (let index = 0; index < captures.length; index++) { const capture = captures[index]; around(options, { phase: 'payload-write', generationId: generated.id, role: capture.role }, () => (() => { - assertPathIdentity(root, rootBinding.identity, 'directory'); - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - writeFileSync(join(staging, provisional.targets[index].snapshot_file), capture.normalizedBytes, { - flag: 'wx', - mode: 0o600, - }); + const descriptor = openRegularFileAt( + stagingBinding as BoundDirectory, + provisional.targets[index].snapshot_file, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + options, + 0o600, + ); + payloadDescriptors.push(descriptor); + writeFileSync(descriptor, capture.normalizedBytes); })(), ); } around(options, { phase: 'provisional-manifest-write', generationId: generated.id }, () => (() => { - assertPathIdentity(root, rootBinding.identity, 'directory'); - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - writeJson(join(staging, MANIFEST_FILE), provisional); + manifestDescriptor = openRegularFileAt( + stagingBinding as BoundDirectory, + MANIFEST_FILE, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + options, + 0o600, + ); + writeJsonDescriptor(manifestDescriptor, provisional); })(), ); - for (const target of provisional.targets) { + for (let index = 0; index < provisional.targets.length; index++) { + const target = provisional.targets[index]; around(options, { phase: 'payload-fsync', generationId: generated.id, role: target.role }, () => - (() => { - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - fsyncPath(join(staging, target.snapshot_file)); - })(), + fsyncSync(payloadDescriptors[index]), ); } const complete = makeManifest( @@ -692,36 +969,70 @@ function publishGeneration( ); around(options, { phase: 'complete-manifest-write', generationId: generated.id }, () => (() => { - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - overwriteJson(join(staging, MANIFEST_FILE), complete); + ftruncateSync(manifestDescriptor as number, 0); + writeJsonDescriptor(manifestDescriptor as number, complete); })(), ); around(options, { phase: 'complete-manifest-fsync', generationId: generated.id }, () => - (() => { - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - fsyncPath(join(staging, MANIFEST_FILE)); - })(), + fsyncSync(manifestDescriptor as number), ); around(options, { phase: 'staging-fsync', generationId: generated.id }, () => { - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - fsyncSync(stagingBinding.descriptor); + fsyncSync((stagingBinding as BoundDirectory).descriptor); }); around(options, { phase: 'generation-rename', generationId: generated.id }, () => { assertPathIdentity(root, rootBinding.identity, 'directory'); - assertPathIdentity(staging, stagingBinding.identity, 'directory'); - renameSync(staging, finalDirectory); - assertPathIdentity(finalDirectory, stagingBinding.identity, 'directory'); + assertEntryIdentityAt( + rootBinding, + stagingName, + (stagingBinding as BoundDirectory).identity, + 'directory', + options, + ); + assertEntryIdentityAt( + stagingBinding as BoundDirectory, + MANIFEST_FILE, + fileIdentity(fstatSync(manifestDescriptor as number)), + 'file', + options, + ); + for (let index = 0; index < provisional.targets.length; index++) { + assertEntryIdentityAt( + stagingBinding as BoundDirectory, + provisional.targets[index].snapshot_file, + fileIdentity(fstatSync(payloadDescriptors[index])), + 'file', + options, + ); + } + api.renameAt(rootBinding.descriptor, stagingName, rootBinding.descriptor, generated.id); + assertEntryIdentityAt( + rootBinding, + generated.id, + (stagingBinding as BoundDirectory).identity, + 'directory', + options, + ); }); around(options, { phase: 'root-fsync', generationId: generated.id }, () => { assertPathIdentity(root, rootBinding.identity, 'directory'); + assertEntryIdentityAt( + rootBinding, + generated.id, + (stagingBinding as BoundDirectory).identity, + 'directory', + options, + ); fsyncSync(rootBinding.descriptor); + assertPathIdentity(root, rootBinding.identity, 'directory'); }); - const manifestPath = join(finalDirectory, MANIFEST_FILE); - const manifestIdentity = readBoundedRegularFile(manifestPath, MAX_MANIFEST_BYTES).identity; + if (manifestDescriptor === null) { + throw new SnapshotError('snapshot-publication-failed', 'Snapshot manifest descriptor was not created.'); + } + const manifestIdentity = fileIdentity(fstatSync(manifestDescriptor)); return { manifest: complete, directory: finalDirectory, - directoryIdentity: stagingBinding.identity, + directoryIdentity: (stagingBinding as BoundDirectory).identity, manifestIdentity, privateRoot, }; @@ -732,7 +1043,9 @@ function publishGeneration( caught instanceof Error ? caught.message : 'Snapshot publication failed.', ); } finally { - closeBoundDirectory(stagingBinding); + if (manifestDescriptor !== null) closeSync(manifestDescriptor); + for (const descriptor of payloadDescriptors) closeSync(descriptor); + if (stagingBinding !== null) closeBoundDirectory(stagingBinding); closeBoundDirectory(rootBinding); } } @@ -877,28 +1190,70 @@ function parseManifest(value: unknown): SnapshotManifestV1 { }; } -function readManifest(directory: string): GenerationReference { - const manifestPath = join(directory, MANIFEST_FILE); - const directoryBinding = openBoundDirectory(directory, false, true); +function readManifestAt( + rootBinding: BoundDirectory, + name: string, + options: DatabaseSyncSnapshotOptions, +): GenerationReference { + const directory = join(rootBinding.path, name); + const directoryBinding = openBoundDirectoryAt(rootBinding, name, directory, options, true); let parsed: unknown; try { - const loaded = readBoundedRegularFile(manifestPath, MAX_MANIFEST_BYTES); + const loaded = readBoundedRegularFileAt(directoryBinding, MANIFEST_FILE, MAX_MANIFEST_BYTES, options); parsed = JSON.parse(new TextDecoder().decode(loaded.bytes)) as unknown; const manifest = parseManifest(parsed); - if (basename(directory) !== manifest.generation_id) { + if (name !== manifest.generation_id) { throw new SnapshotError('manifest-invalid', 'Snapshot generation directory does not match its manifest.'); } - assertPathIdentity(directory, directoryBinding.identity, 'directory'); return { manifest, directory, directoryIdentity: directoryBinding.identity, manifestIdentity: loaded.identity, + binding: directoryBinding, }; } catch { + closeBoundDirectory(directoryBinding); throw new SnapshotError('manifest-invalid', 'Snapshot manifest is not valid JSON.'); + } +} + +function readPublishedGeneration( + published: PublishedGeneration, + options: DatabaseSyncSnapshotOptions, +): GenerationReference { + const rootPath = dirname(published.directory); + const rootBinding = openBoundDirectory(rootPath, false, published.privateRoot !== null); + try { + const generation = readManifestAt(rootBinding, published.manifest.generation_id, options); + if ( + !sameFileIdentity(published.directoryIdentity, generation.directoryIdentity) || + !sameFileIdentity(published.manifestIdentity, generation.manifestIdentity) + ) { + closeBoundDirectory(generation.binding); + throw new SnapshotError('manifest-invalid', 'Published snapshot generation identity changed.'); + } + return generation; } finally { - closeBoundDirectory(directoryBinding); + closeBoundDirectory(rootBinding); + } +} + +function reopenGeneration(generation: GenerationReference, options: DatabaseSyncSnapshotOptions): GenerationReference { + const rootPath = dirname(generation.directory); + const rootBinding = openBoundDirectory(rootPath, false); + try { + const reopened = readManifestAt(rootBinding, basename(generation.directory), options); + if ( + !sameFileIdentity(generation.directoryIdentity, reopened.directoryIdentity) || + !sameFileIdentity(generation.manifestIdentity, reopened.manifestIdentity) + ) { + closeBoundDirectory(reopened.binding); + throw new SnapshotError('manifest-invalid', 'Snapshot generation identity changed before state rewrite.'); + } + return reopened; + } finally { + closeBoundDirectory(rootBinding); } } @@ -916,76 +1271,103 @@ function validateManifestIdentity(manifest: SnapshotManifestV1, identity: Snapsh } } -function matchingGenerationDirectories(identity: SnapshotStoreIdentity): string[] { +function matchingGenerationNames( + identity: SnapshotStoreIdentity, + rootBinding: BoundDirectory, + options: DatabaseSyncSnapshotOptions, +): string[] { + return posixDirectory(options) + .list(rootBinding.descriptor) + .filter((name) => name.startsWith(`${identity.operationId}${GENERATION_SEPARATOR}`)) + .sort((left, right) => compareText(right, left)); +} + +function newestUnresolved( + identity: SnapshotStoreIdentity, + options: DatabaseSyncSnapshotOptions, +): GenerationReference | null { let rootBinding: BoundDirectory; try { rootBinding = openBoundDirectory(identity.root, false); } catch (caught) { - if (filesystemCode(caught) === 'ENOENT') return []; + if (filesystemCode(caught) === 'ENOENT') return null; throw caught; } try { - const directories = readdirSync(identity.root, { withFileTypes: true }) - .filter( - (entry) => - entry.isDirectory() && - !entry.isSymbolicLink() && - entry.name.startsWith(`${identity.operationId}${GENERATION_SEPARATOR}`), - ) - .map((entry) => join(identity.root, entry.name)) - .sort((left, right) => compareText(right, left)); - assertPathIdentity(identity.root, rootBinding.identity, 'directory'); - return directories; + for (const name of matchingGenerationNames(identity, rootBinding, options)) { + const generation = readManifestAt(rootBinding, name, options); + try { + validateManifestIdentity(generation.manifest, identity); + if (generation.manifest.state === 'complete' || generation.manifest.state === 'uncertain') { + return generation; + } + } catch (caught) { + closeBoundDirectory(generation.binding); + throw caught; + } + closeBoundDirectory(generation.binding); + } + return null; } finally { closeBoundDirectory(rootBinding); } } -function newestUnresolved(identity: SnapshotStoreIdentity): GenerationReference | null { - for (const directory of matchingGenerationDirectories(identity)) { - const generation = readManifest(directory); - validateManifestIdentity(generation.manifest, identity); - if (generation.manifest.state === 'complete' || generation.manifest.state === 'uncertain') return generation; - } - return null; -} - -function selectedGeneration(identity: SnapshotStoreIdentity, selectedGenerationId: string): GenerationReference { +function selectedGeneration( + identity: SnapshotStoreIdentity, + selectedGenerationId: string, + options: DatabaseSyncSnapshotOptions, +): GenerationReference { if (!GENERATION_ID_PATTERN.test(selectedGenerationId) || !selectedGenerationId.startsWith(identity.operationId)) { throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not belong to this operation.'); } - const directory = join(identity.root, selectedGenerationId); - let generation: GenerationReference; + let rootBinding: BoundDirectory; try { - generation = readManifest(directory); - } catch (caught) { - const code = isRecord(caught) ? caught.code : undefined; - if (code === 'ENOENT') { + rootBinding = openBoundDirectory(identity.root, false); + } catch { + throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not exist.'); + } + try { + if (!matchingGenerationNames(identity, rootBinding, options).includes(selectedGenerationId)) { throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not exist.'); } - throw caught; - } - validateManifestIdentity(generation.manifest, identity); - if ( - generation.manifest.state === 'provisional' || - generation.manifest.targets.some((target) => target.snapshot_sha256 === null) - ) { - throw new SnapshotError('manifest-invalid', 'Selected snapshot generation is incomplete.'); + const generation = readManifestAt(rootBinding, selectedGenerationId, options); + try { + validateManifestIdentity(generation.manifest, identity); + if ( + generation.manifest.state === 'provisional' || + generation.manifest.targets.some((target) => target.snapshot_sha256 === null) + ) { + throw new SnapshotError('manifest-invalid', 'Selected snapshot generation is incomplete.'); + } + return generation; + } catch (caught) { + closeBoundDirectory(generation.binding); + throw caught; + } + } finally { + closeBoundDirectory(rootBinding); } - return generation; } -function validateGeneration(generation: GenerationReference): ValidatedGeneration { +function validateGeneration( + generation: GenerationReference, + options: DatabaseSyncSnapshotOptions, +): ValidatedGeneration { const snapshots = new Map(); try { for (const target of generation.manifest.targets) { if (target.snapshot_sha256 === null) { throw new SnapshotError('manifest-invalid', 'Complete snapshot manifest is missing a payload hash.'); } - const payloadPath = join(generation.directory, target.snapshot_file); - assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); - assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); - const bytes = readBoundedRegularFile(payloadPath, MAX_RECONCILIATION_DATABASE_BYTES).bytes; + assertBoundDirectory(generation.binding); + assertEntryIdentityAt(generation.binding, MANIFEST_FILE, generation.manifestIdentity, 'file', options); + const bytes = readBoundedRegularFileAt( + generation.binding, + target.snapshot_file, + MAX_RECONCILIATION_DATABASE_BYTES, + options, + ).bytes; if (sha256(bytes) !== target.snapshot_sha256) { throw new SnapshotError('snapshot-hash-mismatch', 'Snapshot payload hash does not match its manifest.'); } @@ -1010,8 +1392,8 @@ function validateGeneration(generation: GenerationReference): ValidatedGeneratio throw new SnapshotError('snapshot-image-mismatch', 'Snapshot payload does not match its manifest image.'); } snapshots.set(target.path, db); - assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); - assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); + assertBoundDirectory(generation.binding); + assertEntryIdentityAt(generation.binding, MANIFEST_FILE, generation.manifestIdentity, 'file', options); } return { ...generation, snapshots }; } catch (caught) { @@ -1024,51 +1406,78 @@ function closeValidatedGeneration(generation: ValidatedGeneration): void { for (const db of generation.snapshots.values()) db.close(); } -function safeRemoveTree(path: string, expected: FileIdentity): void { - assertPathIdentity(path, expected, 'directory'); - const directory = openBoundDirectory(path, false); +function removeBoundTreeAt( + parent: BoundDirectory, + name: string, + path: string, + expected: FileIdentity, + options: DatabaseSyncSnapshotOptions, +): void { + const api = posixDirectory(options); + const directory = openBoundDirectoryAt(parent, name, path, options); try { if (!sameFileIdentity(expected, directory.identity)) { throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup target identity changed.'); } - const descriptorPath = `/proc/self/fd/${directory.descriptor}`; - for (const entry of readdirSync(descriptorPath, { withFileTypes: true })) { - const descriptorChild = join(descriptorPath, entry.name); - const child = join(path, entry.name); - const stats = lstatSync(descriptorChild); - const identity = fileIdentity(stats); - const current = lstatSync(child); - if (!sameFileIdentity(identity, current)) { - throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup child identity changed.'); + for (const childName of api.list(directory.descriptor)) { + const childPath = join(path, childName); + let childDescriptor: number | null = null; + try { + childDescriptor = api.openAt( + directory.descriptor, + childName, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0) | POSIX_CLOSE_ON_EXEC, + ); + } catch { + // A no-follow open rejects symlinks. unlinkat removes the link itself, + // never the object it names. + } + if (childDescriptor === null) { + api.unlinkAt(directory.descriptor, childName, false); + continue; } - if (stats.isDirectory() && !stats.isSymbolicLink()) { - safeRemoveTree(child, identity); - } else { - assertPathIdentity(child, identity, 'file'); - unlinkSync(child); + try { + const stats = fstatSync(childDescriptor); + const childIdentity = fileIdentity(stats); + if (stats.isDirectory()) { + closeSync(childDescriptor); + childDescriptor = null; + removeBoundTreeAt(directory, childName, childPath, childIdentity, options); + } else { + assertEntryIdentityAt(directory, childName, childIdentity, 'file', options); + api.unlinkAt(directory.descriptor, childName, false); + } + } finally { + if (childDescriptor !== null) closeSync(childDescriptor); } } fsyncSync(directory.descriptor); } finally { closeBoundDirectory(directory); } - assertPathIdentity(path, expected, 'directory'); - rmdirSync(path); + assertEntryIdentityAt(parent, name, expected, 'directory', options); + api.unlinkAt(parent.descriptor, name, true); } function removeBoundTree(path: string, expected: FileIdentity, options: DatabaseSyncSnapshotOptions): void { assertPathIdentity(path, expected, 'directory'); - if (options.removeTree !== undefined) { - options.removeTree(path); - try { - lstatSync(path); - } catch (caught) { - if (filesystemCode(caught) === 'ENOENT') return; - throw caught; - } - throw new SnapshotError('snapshot-cleanup-failed', 'Snapshot cleanup did not remove its exact target.'); + options.removeTree?.(path); + const parentPath = dirname(path); + assertSafeDirectoryAncestors(parentPath); + const descriptor = openSync( + parentPath, + fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, + ); + try { + const parent: BoundDirectory = { + path: parentPath, + descriptor, + identity: fileIdentity(fstatSync(descriptor)), + }; + removeBoundTreeAt(parent, basename(path), path, expected, options); + } finally { + closeSync(descriptor); } - safeRemoveTree(path, expected); } function rewriteManifestState( @@ -1077,45 +1486,61 @@ function rewriteManifestState( options: DatabaseSyncSnapshotOptions, ): SnapshotManifestV1 { const updated = { ...generation.manifest, state }; - const temporary = join(generation.directory, `.manifest-${randomUUID()}.tmp`); + const temporaryName = `.manifest-${randomUUID()}.tmp`; let temporaryIdentity: FileIdentity | null = null; + let temporaryDescriptor: number | null = null; + let temporaryPublished = false; try { - assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); - assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); - around(options, { phase: 'state-rewrite-write', generationId: generation.manifest.generation_id }, () => - writeJson(temporary, updated), - ); - temporaryIdentity = readBoundedRegularFile(temporary, MAX_MANIFEST_BYTES).identity; + assertBoundDirectory(generation.binding); + assertEntryIdentityAt(generation.binding, MANIFEST_FILE, generation.manifestIdentity, 'file', options); + around(options, { phase: 'state-rewrite-write', generationId: generation.manifest.generation_id }, () => { + temporaryDescriptor = openRegularFileAt( + generation.binding, + temporaryName, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + options, + 0o600, + ); + writeJsonDescriptor(temporaryDescriptor, updated); + temporaryIdentity = fileIdentity(fstatSync(temporaryDescriptor)); + }); around(options, { phase: 'state-rewrite-fsync', generationId: generation.manifest.generation_id }, () => { - assertPathIdentity(temporary, temporaryIdentity as FileIdentity, 'file'); - fsyncPath(temporary); + fsyncSync(temporaryDescriptor as number); }); around(options, { phase: 'state-rewrite-rename', generationId: generation.manifest.generation_id }, () => { - assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); - assertPathIdentity(join(generation.directory, MANIFEST_FILE), generation.manifestIdentity, 'file'); - assertPathIdentity(temporary, temporaryIdentity as FileIdentity, 'file'); - renameSync(temporary, join(generation.directory, MANIFEST_FILE)); - assertPathIdentity(join(generation.directory, MANIFEST_FILE), temporaryIdentity as FileIdentity, 'file'); + assertBoundDirectory(generation.binding); + assertEntryIdentityAt(generation.binding, MANIFEST_FILE, generation.manifestIdentity, 'file', options); + assertEntryIdentityAt(generation.binding, temporaryName, temporaryIdentity as FileIdentity, 'file', options); + posixDirectory(options).renameAt( + generation.binding.descriptor, + temporaryName, + generation.binding.descriptor, + MANIFEST_FILE, + ); + temporaryPublished = true; + assertEntryIdentityAt(generation.binding, MANIFEST_FILE, temporaryIdentity as FileIdentity, 'file', options); }); around(options, { phase: 'generation-fsync', generationId: generation.manifest.generation_id }, () => { - assertPathIdentity(generation.directory, generation.directoryIdentity, 'directory'); - fsyncPath(generation.directory); + assertBoundDirectory(generation.binding); + fsyncSync(generation.binding.descriptor); }); return updated; } catch (caught) { const cleanupFailures: SnapshotFailureCode[] = []; - if (temporaryIdentity !== null) { + if (temporaryIdentity !== null && !temporaryPublished) { try { - assertPathIdentity(temporary, temporaryIdentity, 'file'); - unlinkSync(temporary); - } catch (cleanupCaught) { - if (filesystemCode(cleanupCaught) !== 'ENOENT') cleanupFailures.push('snapshot-cleanup-failed'); + assertEntryIdentityAt(generation.binding, temporaryName, temporaryIdentity, 'file', options); + posixDirectory(options).unlinkAt(generation.binding.descriptor, temporaryName, false); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); } } if (caught instanceof SnapshotError) { throw new SnapshotError(caught.code, caught.message, [...caught.cleanupFailures, ...cleanupFailures]); } throw new SnapshotError('snapshot-publication-failed', 'Snapshot manifest state rewrite failed.', cleanupFailures); + } finally { + if (temporaryDescriptor !== null) closeSync(temporaryDescriptor); } } @@ -1124,24 +1549,38 @@ function cleanupStaging( options: DatabaseSyncSnapshotOptions, cleanupFailures: SnapshotFailureCode[], ): void { - let entries: Dirent[]; + let rootBinding: BoundDirectory; try { - entries = readdirSync(root, { withFileTypes: true }); + rootBinding = openBoundDirectory(root, false); } catch (caught) { - const code = isRecord(caught) ? caught.code : undefined; - if (code === 'ENOENT') return; + if (filesystemCode(caught) === 'ENOENT') return; cleanupFailures.push('snapshot-cleanup-failed'); return; } - for (const entry of entries) { - if (!entry.name.startsWith(STAGING_PREFIX) || !entry.isDirectory() || entry.isSymbolicLink()) continue; - const path = join(root, entry.name); - try { - const identity = fileIdentity(lstatSync(path)); - around(options, { phase: 'staging-cleanup', path }, () => removeBoundTree(path, identity, options)); - } catch { - cleanupFailures.push('snapshot-cleanup-failed'); + try { + for (const name of posixDirectory(options).list(rootBinding.descriptor)) { + if (!name.startsWith(STAGING_PREFIX)) continue; + const path = join(root, name); + let staging: BoundDirectory; + try { + staging = openBoundDirectoryAt(rootBinding, name, path, options); + } catch { + continue; + } + try { + around(options, { phase: 'staging-cleanup', path }, () => + removeBoundTreeAt(rootBinding, name, path, staging.identity, options), + ); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } finally { + closeBoundDirectory(staging); + } } + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } finally { + closeBoundDirectory(rootBinding); } } @@ -1151,29 +1590,53 @@ function pruneGenerations( options: DatabaseSyncSnapshotOptions, cleanupFailures: SnapshotFailureCode[], ): void { - const directories = matchingGenerationDirectories(identity); - const retained = new Set(directories.slice(0, retention)); + let rootBinding: BoundDirectory; + try { + rootBinding = openBoundDirectory(identity.root, false); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + return; + } let removed = false; - for (const directory of directories) { - if (retained.has(directory)) continue; - try { - const manifest = readManifest(directory); - validateManifestIdentity(manifest.manifest, identity); - if (manifest.manifest.state === 'complete' || manifest.manifest.state === 'uncertain') continue; - around(options, { phase: 'prune', generationId: manifest.manifest.generation_id, path: directory }, () => - removeBoundTree(directory, manifest.directoryIdentity, options), - ); - removed = true; - } catch { - cleanupFailures.push('snapshot-cleanup-failed'); + try { + const names = matchingGenerationNames(identity, rootBinding, options); + const retained = new Set(names.slice(0, retention)); + for (const name of names) { + if (retained.has(name)) continue; + let manifest: GenerationReference | null = null; + const directory = join(identity.root, name); + try { + manifest = readManifestAt(rootBinding, name, options); + validateManifestIdentity(manifest.manifest, identity); + if (manifest.manifest.state === 'complete' || manifest.manifest.state === 'uncertain') continue; + around(options, { phase: 'prune', generationId: manifest.manifest.generation_id, path: directory }, () => + removeBoundTreeAt(rootBinding, name, directory, (manifest as GenerationReference).directoryIdentity, options), + ); + removed = true; + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } finally { + if (manifest !== null) closeBoundDirectory(manifest.binding); + } } + if (removed) fsyncSync(rootBinding.descriptor); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } finally { + closeBoundDirectory(rootBinding); } - if (removed) { - try { - fsyncPath(identity.root); - } catch { - cleanupFailures.push('snapshot-cleanup-failed'); - } +} + +/* + * Keep this pathname wrapper only for private-temp cleanup and the existing + * deterministic cleanup-failure seam. Persistent cleanup and pruning call the + * parent-descriptor form above. + */ +function removePrivateTree(path: string, expected: FileIdentity, options: DatabaseSyncSnapshotOptions): void { + try { + removeBoundTree(path, expected, options); + } catch { + throw new SnapshotError('snapshot-cleanup-failed', 'Private snapshot cleanup failed.'); } } @@ -1196,6 +1659,7 @@ function recoverValidatedGeneration( retention: number, ): { value: RecoveryDecision; afterCommit: () => void } { if (generation.manifest.state === 'uncertain') { + closeBoundDirectory(generation.binding); return { value: { status: 'uncertain', @@ -1206,7 +1670,7 @@ function recoverValidatedGeneration( afterCommit: () => {}, }; } - const validated = validateGeneration(generation); + const validated = validateGeneration(generation, options); const observed = new Map(inputs.map((input) => [input.canonicalPath, input.observe().logicalDigest])); const classifications = generation.manifest.targets.map((target) => ({ target, @@ -1264,6 +1728,7 @@ function recoverValidatedGeneration( } const cleanupFailures: SnapshotFailureCode[] = []; + closeBoundDirectory(generation.binding); return { value: { status, @@ -1272,9 +1737,14 @@ function recoverValidatedGeneration( cleanupFailures, }, afterCommit: () => { - rewriteManifestState(generation, status === 'uncertain' ? 'uncertain' : status, options); - cleanupStaging(identity.root, options, cleanupFailures); - if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); + const rebound = reopenGeneration(generation, options); + try { + rewriteManifestState(rebound, status === 'uncertain' ? 'uncertain' : status, options); + cleanupStaging(identity.root, options, cleanupFailures); + if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); + } finally { + closeBoundDirectory(rebound.binding); + } }, }; } @@ -1348,7 +1818,7 @@ export function recoverDatabaseReconciliation( inputs.map((input) => input.canonicalPath), options.snapshotRoot, ); - const unresolved = newestUnresolved(identity); + const unresolved = newestUnresolved(identity, options); if (unresolved === null) { const cleanupFailures: SnapshotFailureCode[] = []; return { @@ -1361,7 +1831,12 @@ export function recoverDatabaseReconciliation( afterCommit: () => cleanupStaging(identity.root, options, cleanupFailures), }; } - return recoverValidatedGeneration(identity, unresolved, inputs, options, persistentRetention); + try { + return recoverValidatedGeneration(identity, unresolved, inputs, options, persistentRetention); + } catch (caught) { + closeBoundDirectory(unresolved.binding); + throw caught; + } }, { busyTimeoutMs: options.busyTimeoutMs, @@ -1410,7 +1885,7 @@ function privateCleanup( ): void { if (root === null || identity === null) return; try { - removeBoundTree(root, identity, options); + removePrivateTree(root, identity, options); } catch { cleanupFailures.push('snapshot-cleanup-failed'); } @@ -1429,7 +1904,15 @@ function recoverPrivateGeneration( try { const decision = withLockedReconciliationDatabases( request, - (inputs) => recoverValidatedGeneration(identity, readManifest(published.directory), inputs, options, 0), + (inputs) => { + const generation = readPublishedGeneration(published, options); + try { + return recoverValidatedGeneration(identity, generation, inputs, options, 0); + } catch (caught) { + closeBoundDirectory(generation.binding); + throw caught; + } + }, { busyTimeoutMs: options.busyTimeoutMs, onEvent: options.onLockedOperationEvent, @@ -1509,8 +1992,9 @@ export function applyDatabaseReconciliationWithSnapshots( onEvent: options.onApplyEvent, ...options.applyOptions, onLocked: (inputs) => { - const unresolved = newestUnresolved(identity); + const unresolved = newestUnresolved(identity, options); if (unresolved !== null) { + closeBoundDirectory(unresolved.binding); throw new SnapshotError( 'recovery-uncertain', 'A persistent snapshot generation became unresolved after recovery and before apply.', @@ -1611,14 +2095,22 @@ export function rollbackDatabaseReconciliation( inputs.map((input) => input.canonicalPath), options.snapshotRoot, ); - const racedUnresolved = newestUnresolved(identity); + const racedUnresolved = newestUnresolved(identity, options); if (racedUnresolved !== null) { + closeBoundDirectory(racedUnresolved.binding); throw new SnapshotError( 'recovery-uncertain', 'A persistent snapshot generation became unresolved after recovery and before rollback.', ); } - const selected = validateGeneration(selectedGeneration(identity, selectedGenerationId)); + const selectedReference = selectedGeneration(identity, selectedGenerationId, options); + let selected: ValidatedGeneration; + try { + selected = validateGeneration(selectedReference, options); + } catch (caught) { + closeBoundDirectory(selectedReference.binding); + throw caught; + } try { const captures = selected.manifest.targets.map((target) => { const input = targetInput(inputs, target.path); @@ -1659,7 +2151,12 @@ export function rollbackDatabaseReconciliation( value: { identity, generation: published }, afterCommit: () => { if (published === null) return; - rewriteManifestState(published, 'rolled-back', options); + const generation = readPublishedGeneration(published, options); + try { + rewriteManifestState(generation, 'rolled-back', options); + } finally { + closeBoundDirectory(generation.binding); + } if (retention > 0) { cleanupStaging(identity.root, options, cleanupFailures); pruneGenerations(identity, retention, options, cleanupFailures); @@ -1668,6 +2165,7 @@ export function rollbackDatabaseReconciliation( }; } finally { closeValidatedGeneration(selected); + closeBoundDirectory(selected.binding); } }, { From 7a2cd4bf2cf8fb1343d5b36a993d56ae0e4b0ec8 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Tue, 28 Jul 2026 15:12:37 +0000 Subject: [PATCH 11/21] fix(db-sync): fail closed on snapshot discovery --- src/lib/v5/db-sync-snapshots.test.ts | 249 +++++++++++++++++++ src/lib/v5/db-sync-snapshots.ts | 358 ++++++++++++++++++--------- 2 files changed, 485 insertions(+), 122 deletions(-) diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index 5ed78c5d9..ffade8a4d 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -2,6 +2,7 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { createHash } from 'node:crypto'; import { + chmodSync, existsSync, linkSync, mkdirSync, @@ -119,6 +120,16 @@ function nativePosixDirectory(): SnapshotPosixDirectoryApi { return api as SnapshotPosixDirectoryApi; } +function nativePosixWithReaddirError(at: number, list?: number): SnapshotPosixDirectoryApi { + const api = resolveSnapshotPosixDirectory({ + platform: process.platform, + architecture: process.arch, + readdirError: { at, errno: 5, list }, + }); + expect(api).not.toBeNull(); + return api as SnapshotPosixDirectoryApi; +} + function delegatePosix( api: SnapshotPosixDirectoryApi, overrides: Partial, @@ -371,6 +382,103 @@ describe('database sync snapshots', () => { } }); + test('bootstrap walks and binds every absolute root component before it can authorize publication', () => { + for (const boundary of ['bootstrap', 'one', 'two']) { + const left = currentDb(`bootstrap-open-${boundary}-left`); + const right = currentDb(`bootstrap-open-${boundary}-right`); + insertBoard(right, 'planned'); + const base = join(fixtureRoot, `bootstrap-open-${boundary}`); + const root = join(base, 'bootstrap', 'one', 'two'); + const boundaryPath = join( + base, + ...['bootstrap', 'one', 'two'].slice(0, ['bootstrap', 'one', 'two'].indexOf(boundary) + 1), + ); + const moved = `${boundaryPath}.bound`; + let swapped = false; + const native = nativePosixDirectory(); + const api = delegatePosix(native, { + openAt: (descriptor, name, flags, mode) => { + const opened = native.openAt(descriptor, name, flags, mode); + if (!swapped && name === boundary && existsSync(boundaryPath)) { + swapped = true; + renameSync(boundaryPath, moved); + mkdirSync(boundaryPath, { mode: 0o700 }); + writeFileSync(join(boundaryPath, 'sentinel'), 'replacement'); + } + return opened; + }, + }); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { snapshotRoot: root, posixDirectory: { api } }, + ); + expect(swapped).toBe(true); + expect(['rolled-back', 'operational-failure']).toContain(report.status); + expect(readFileSync(join(boundaryPath, 'sentinel'), 'utf8')).toBe('replacement'); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); + + test('bootstrap rejects replacement immediately after each missing-component creation', () => { + for (const boundary of ['bootstrap', 'one', 'two']) { + const left = currentDb(`bootstrap-create-${boundary}-left`); + const right = currentDb(`bootstrap-create-${boundary}-right`); + insertBoard(right, 'planned'); + const base = join(fixtureRoot, `bootstrap-create-${boundary}`); + const parts = ['bootstrap', 'one', 'two']; + const root = join(base, ...parts); + const boundaryPath = join(base, ...parts.slice(0, parts.indexOf(boundary) + 1)); + let swapped = false; + const native = nativePosixDirectory(); + const api = delegatePosix(native, { + mkdirAt: (descriptor, name, mode) => { + native.mkdirAt(descriptor, name, mode); + if (!swapped && name === boundary) { + swapped = true; + renameSync(boundaryPath, `${boundaryPath}.bound`); + mkdirSync(boundaryPath, { mode: 0o755 }); + chmodSync(boundaryPath, 0o755); + writeFileSync(join(boundaryPath, 'sentinel'), 'replacement'); + } + }, + }); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { snapshotRoot: root, posixDirectory: { api } }, + ); + expect(swapped).toBe(true); + expect(['rolled-back', 'operational-failure']).toContain(report.status); + expect(readFileSync(join(boundaryPath, 'sentinel'), 'utf8')).toBe('replacement'); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + } + }); + + test('bootstrap creates nested physical roots and rejects non-directories and dot-dot roots', () => { + const left = currentDb('bootstrap-shapes-left'); + const right = currentDb('bootstrap-shapes-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const nested = join(fixtureRoot, 'nested', 'snapshot', 'root'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { snapshotRoot: nested }).status, + ).toBe('changed'); + expect(statSync(nested).isDirectory()).toBe(true); + + const anotherRight = currentDb('bootstrap-file-right'); + insertBoard(anotherRight, 'planned'); + const fileAncestor = join(fixtureRoot, 'not-a-directory'); + writeFileSync(fileAncestor, 'file'); + const invalid = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: anotherRight }), + { snapshotRoot: join(fileAncestor, 'root') }, + ); + expect(invalid.status).toBe('operational-failure'); + expect(boardNames(anotherRight)).toEqual(['planned']); + expect(() => databaseSyncSnapshotIdentity(request, `${fixtureRoot}/nested/../escape`)).toThrow(SnapshotError); + }); + test('publishes normalized complete payloads in the durable order and finalizes both-post as converged', () => { const left = currentDb('publish-left'); const right = currentDb('publish-right'); @@ -558,6 +666,119 @@ describe('database sync snapshots', () => { } }); + test('first-read and mid-stream readdir failures block newest-generation discovery without descriptor leaks', () => { + { + const left = currentDb('readdir-first-left'); + const right = currentDb('readdir-first-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const root = databaseSyncSnapshotIdentity(request).root; + mkdirSync(root, { recursive: true, mode: 0o700 }); + chmodSync(root, 0o700); + const api = nativePosixWithReaddirError(0); + Bun.gc(true); + const before = descriptorCount(); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + posixDirectory: { api }, + }); + expect(report).toMatchObject({ status: 'operational-failure', apply: null }); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['planned']); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + expect(applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request)).status).toBe('changed'); + } + + { + const left = currentDb('readdir-mid-left'); + const right = currentDb('readdir-mid-right'); + const complete = leaveCompleteGeneration(left, right); + mkdirSync(join(complete.root, 'aaa'), { mode: 0o700 }); + mkdirSync(join(complete.root, 'bbb'), { mode: 0o700 }); + const api = nativePosixWithReaddirError(3); + Bun.gc(true); + const before = descriptorCount(); + const report = recoverDatabaseReconciliation( + { mode: 'bidirectional', leftPath: left, rightPath: right }, + { posixDirectory: { api } }, + ); + expect(report.status).toBe('operational-failure'); + expect(boardNames(left)).toEqual(['planned']); + expect(boardNames(right)).toEqual(['planned']); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + expect(recoverDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }).status).toBe( + 'converged', + ); + } + }); + + test('readdir failures during staging cleanup and pruning are reported without leaks or unsafe removal', () => { + { + const left = currentDb('readdir-cleanup-left'); + const right = currentDb('readdir-cleanup-right'); + insertBoard(right, 'planned'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const root = databaseSyncSnapshotIdentity(request).root; + const stale = join(root, '.staging-stale'); + const native = nativePosixDirectory(); + const faulty = nativePosixWithReaddirError(0); + let cleanupArmed = false; + const api = delegatePosix(native, { + list: (descriptor) => (cleanupArmed ? faulty.list(descriptor) : native.list(descriptor)), + }); + Bun.gc(true); + const before = descriptorCount(); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + posixDirectory: { api }, + onEvent: (event) => { + if (event.phase === 'recovery-classify' && event.state === 'before' && !existsSync(stale)) { + mkdirSync(stale, { mode: 0o700 }); + writeFileSync(join(stale, 'sentinel'), 'keep'); + cleanupArmed = true; + } + }, + }); + expect(report.status).toBe('changed'); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(readFileSync(join(stale, 'sentinel'), 'utf8')).toBe('keep'); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + } + + { + const left = currentDb('readdir-prune-left'); + const right = currentDb('readdir-prune-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + for (let index = 0; index < 4; index++) { + insertBoard(right, `generation-${index}`); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 10, + }).status, + ).toBe('changed'); + } + insertBoard(right, 'generation-final'); + const native = nativePosixDirectory(); + const faulty = nativePosixWithReaddirError(3); + let listCalls = 0; + const api = delegatePosix(native, { + list: (descriptor) => (listCalls++ === 5 ? faulty.list(descriptor) : native.list(descriptor)), + }); + Bun.gc(true); + const before = descriptorCount(); + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 1, + posixDirectory: { api }, + }); + expect(report.status).toBe('changed'); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(generationDirectories(databaseSyncSnapshotIdentity(request).root).length).toBeGreaterThan(1); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + } + }); + test('zero retention recovers retained generations before changed apply and rollback work', () => { const left = currentDb('zero-existing-left'); const right = currentDb('zero-existing-right'); @@ -996,6 +1217,34 @@ describe('database sync snapshots', () => { expect(boardNames(right)).toEqual(['right-only']); }); + test('classification hook failure closes validated snapshots and leaves recovery retryable', () => { + const left = currentDb('classification-close-left'); + const right = currentDb('classification-close-right'); + leaveCompleteGeneration(left, right); + const before = descriptorCount(); + let injected = false; + const failed = recoverDatabaseReconciliation( + { mode: 'bidirectional', leftPath: left, rightPath: right }, + { + onEvent: (event) => { + if (!injected && event.phase === 'recovery-classify' && event.state === 'before') { + injected = true; + throw new Error('classification failed'); + } + }, + }, + ); + expect(injected).toBe(true); + expect(failed.status).toBe('operational-failure'); + Bun.gc(true); + expect(descriptorCount()).toBeLessThanOrEqual(before + 1); + expect(boardNames(left)).toEqual(['planned']); + expect(boardNames(right)).toEqual(['planned']); + expect(recoverDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }).status).toBe( + 'converged', + ); + }); + test('explicit rollback snapshots arbitrary current state before restoring the selected preimages', () => { const left = currentDb('rollback-left'); const right = currentDb('rollback-right'); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index 02a907558..5c2eb9053 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -1,4 +1,5 @@ -import { CString, FFIType, dlopen } from 'bun:ffi'; +import { CString, FFIType, dlopen, toArrayBuffer } from 'bun:ffi'; +import type { Pointer } from 'bun:ffi'; import { Database } from 'bun:sqlite'; import { createHash, randomUUID } from 'node:crypto'; import { @@ -8,7 +9,6 @@ import { fsyncSync, ftruncateSync, lstatSync, - mkdirSync, mkdtempSync, openSync, readSync, @@ -20,7 +20,7 @@ import { } from 'node:fs'; import type { Stats } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; import { linuxLibcCandidates } from '../install-transaction.js'; import { MAX_RECONCILIATION_DATABASE_BYTES, @@ -230,6 +230,8 @@ export interface SnapshotPosixDirectoryDependencies { readonly linuxOpener?: (candidate: string, platform: 'linux') => SnapshotPosixDirectoryApi | null; readonly darwinOpener?: (candidate: string, platform: 'darwin') => SnapshotPosixDirectoryApi | null; readonly api?: SnapshotPosixDirectoryApi; + /** Native fault seam: make the indexed readdir call return null with this errno. */ + readonly readdirError?: { readonly at: number; readonly errno: number; readonly list?: number }; } export type SnapshotRecoveryStatus = 'none' | 'converged' | 'recovered' | 'uncertain' | 'operational-failure'; @@ -355,8 +357,35 @@ const POSIX_DIRECTORY_SYMBOLS = { returns: FFIType.i32, }, } as const; +const ERRNO_ACCESSOR_SYMBOL = { + args: [], + returns: FFIType.ptr, +} as const; let defaultPosixDirectoryApi: SnapshotPosixDirectoryApi | null | undefined; +interface NativeDirectorySymbols { + openat(directoryDescriptor: number, name: Uint8Array, flags: number, mode: number): number; + mkdirat(directoryDescriptor: number, name: Uint8Array, mode: number): number; + renameat( + sourceDirectoryDescriptor: number, + sourceName: Uint8Array, + destinationDirectoryDescriptor: number, + destinationName: Uint8Array, + ): number; + unlinkat(directoryDescriptor: number, name: Uint8Array, flags: number): number; + dup(descriptor: number): number; + fdopendir(descriptor: number): Pointer | null; + readdir(stream: Pointer): Pointer | null; + closedir(stream: Pointer): number; +} + +function nativeError(operation: string, errno: number): Error & { code?: string; errno: number } { + const error = new Error(`${operation} failed with errno ${errno}`) as Error & { code?: string; errno: number }; + error.errno = errno; + if (errno === 2) error.code = 'ENOENT'; + return error; +} + function componentBytes(name: string): Uint8Array { if (name.length === 0 || name === '.' || name === '..' || name.includes('/') || name.includes('\0')) { throw new SnapshotError('manifest-invalid', 'Snapshot entry name is not a safe path component.'); @@ -364,25 +393,70 @@ function componentBytes(name: string): Uint8Array { return new TextEncoder().encode(`${name}\0`); } -function openPosixDirectoryApi(candidate: string, platform: 'linux' | 'darwin'): SnapshotPosixDirectoryApi | null { +function readNativeDirectory( + stream: Pointer, + symbols: NativeDirectorySymbols, + errno: Int32Array, + nameOffset: number, + fault: SnapshotPosixDirectoryDependencies['readdirError'], + listIndex: number, +): string[] { + const entries: string[] = []; + for (let readIndex = 0; ; readIndex++) { + errno[0] = 0; + const injected = fault?.at === readIndex && (fault.list === undefined || fault.list === listIndex); + const entry = injected + ? (() => { + errno[0] = fault.errno; + return null; + })() + : symbols.readdir(stream); + if (entry === null) { + if (errno[0] !== 0) throw nativeError('readdir', errno[0]); + return entries; + } + const name = new CString(entry, nameOffset).toString(); + if (name === '.' || name === '..') continue; + componentBytes(name); + entries.push(name); + if (entries.length > MAX_DIRECTORY_ENTRIES) { + throw new SnapshotError('manifest-invalid', 'Snapshot directory exceeds the bounded entry limit.'); + } + } +} + +function openPosixDirectoryApi( + candidate: string, + platform: 'linux' | 'darwin', + readdirError?: SnapshotPosixDirectoryDependencies['readdirError'], +): SnapshotPosixDirectoryApi | null { try { - const library = dlopen(candidate, POSIX_DIRECTORY_SYMBOLS); + const library = + platform === 'darwin' + ? dlopen(candidate, { ...POSIX_DIRECTORY_SYMBOLS, __error: ERRNO_ACCESSOR_SYMBOL }) + : dlopen(candidate, { ...POSIX_DIRECTORY_SYMBOLS, __errno_location: ERRNO_ACCESSOR_SYMBOL }); + const symbols = library.symbols as unknown as NativeDirectorySymbols & + Partial<{ __error(): Pointer | null; __errno_location(): Pointer | null }>; + const errnoPointer = platform === 'darwin' ? symbols.__error?.() : symbols.__errno_location?.(); + if (errnoPointer == null) return null; + const errno = new Int32Array(toArrayBuffer(errnoPointer, 0, Int32Array.BYTES_PER_ELEMENT)); const nameOffset = platform === 'darwin' ? 21 : 19; const removedDirectoryFlag = platform === 'darwin' ? 0x80 : 0x200; + let listIndex = 0; return { openAt(directoryDescriptor, name, flags, mode = 0) { - const descriptor = library.symbols.openat(directoryDescriptor, componentBytes(name), flags, mode); - if (descriptor < 0) throw new Error('openat failed'); + const descriptor = symbols.openat(directoryDescriptor, componentBytes(name), flags, mode); + if (descriptor < 0) throw nativeError('openat', errno[0]); return descriptor; }, mkdirAt(directoryDescriptor, name, mode) { - if (library.symbols.mkdirat(directoryDescriptor, componentBytes(name), mode) !== 0) { - throw new Error('mkdirat failed'); + if (symbols.mkdirat(directoryDescriptor, componentBytes(name), mode) !== 0) { + throw nativeError('mkdirat', errno[0]); } }, renameAt(sourceDirectoryDescriptor, sourceName, destinationDirectoryDescriptor, destinationName) { if ( - library.symbols.renameat( + symbols.renameat( sourceDirectoryDescriptor, componentBytes(sourceName), destinationDirectoryDescriptor, @@ -393,39 +467,27 @@ function openPosixDirectoryApi(candidate: string, platform: 'linux' | 'darwin'): } }, unlinkAt(directoryDescriptor, name, directory) { - if ( - library.symbols.unlinkat(directoryDescriptor, componentBytes(name), directory ? removedDirectoryFlag : 0) !== - 0 - ) { - throw new Error('unlinkat failed'); + if (symbols.unlinkat(directoryDescriptor, componentBytes(name), directory ? removedDirectoryFlag : 0) !== 0) { + throw nativeError('unlinkat', errno[0]); } }, list(directoryDescriptor) { - const duplicate = library.symbols.dup(directoryDescriptor); - if (duplicate < 0) throw new Error('dup failed'); - const stream = library.symbols.fdopendir(duplicate); + const currentList = listIndex++; + const duplicate = symbols.dup(directoryDescriptor); + if (duplicate < 0) throw nativeError('dup', errno[0]); + const stream = symbols.fdopendir(duplicate); if (stream === null) { closeSync(duplicate); - throw new Error('fdopendir failed'); + throw nativeError('fdopendir', errno[0]); } - const entries: string[] = []; + let entries: string[]; try { - for (;;) { - const entry = library.symbols.readdir(stream); - if (entry === null) break; - const name = new CString(entry, nameOffset).toString(); - if (name === '.' || name === '..') continue; - componentBytes(name); - entries.push(name); - if (entries.length > MAX_DIRECTORY_ENTRIES) { - throw new SnapshotError('manifest-invalid', 'Snapshot directory exceeds the bounded entry limit.'); - } - } + entries = readNativeDirectory(stream, symbols, errno, nameOffset, readdirError, currentList); } catch (caught) { - library.symbols.closedir(stream); + symbols.closedir(stream); throw caught; } - if (library.symbols.closedir(stream) !== 0) throw new Error('closedir failed'); + if (symbols.closedir(stream) !== 0) throw nativeError('closedir', errno[0]); return entries; }, }; @@ -449,7 +511,9 @@ export function resolveSnapshotPosixDirectory( const architecture = dependencies?.architecture ?? process.arch; let resolved: SnapshotPosixDirectoryApi | null = null; if (platform === 'linux') { - const opener = dependencies?.linuxOpener ?? openPosixDirectoryApi; + const opener = + dependencies?.linuxOpener ?? + ((candidate: string, target: 'linux') => openPosixDirectoryApi(candidate, target, dependencies?.readdirError)); for (const candidate of dependencies?.linuxCandidates ?? linuxLibcCandidates(architecture)) { try { resolved = opener(candidate, 'linux'); @@ -460,7 +524,10 @@ export function resolveSnapshotPosixDirectory( } } else if (platform === 'darwin') { try { - resolved = (dependencies?.darwinOpener ?? openPosixDirectoryApi)('/usr/lib/libSystem.B.dylib', 'darwin'); + const opener = + dependencies?.darwinOpener ?? + ((candidate: string, target: 'darwin') => openPosixDirectoryApi(candidate, target, dependencies?.readdirError)); + resolved = opener('/usr/lib/libSystem.B.dylib', 'darwin'); } catch { resolved = null; } @@ -520,7 +587,9 @@ function identityFromCanonical( return { operationId, root: - snapshotRoot === undefined ? join(dirname(sorted[0]), 'sync-snapshots', operationId) : resolve(snapshotRoot), + snapshotRoot === undefined + ? join(dirname(sorted[0]), 'sync-snapshots', operationId) + : normalizedSnapshotRoot(snapshotRoot), mode, canonicalPaths: sorted, }; @@ -529,12 +598,20 @@ function identityFromCanonical( const operationId = encodeIdentity('genie-db-sync-directional-pair-v1', [source, destination]); return { operationId, - root: snapshotRoot === undefined ? join(dirname(destination), 'sync-snapshots') : resolve(snapshotRoot), + root: + snapshotRoot === undefined ? join(dirname(destination), 'sync-snapshots') : normalizedSnapshotRoot(snapshotRoot), mode, canonicalPaths: [source, destination], }; } +function normalizedSnapshotRoot(path: string): string { + if (!isAbsolute(path) || path.split('/').some((component) => component === '.' || component === '..')) { + throw new SnapshotError('invalid-snapshot-option', 'Snapshot root must be a normalized absolute path.'); + } + return resolve(path); +} + export function databaseSyncSnapshotIdentity( request: ReconciliationRequest, snapshotRoot?: string, @@ -595,29 +672,6 @@ function filesystemCode(caught: unknown): unknown { return isRecord(caught) ? caught.code : undefined; } -function assertSafeDirectoryAncestors(path: string): void { - const absolute = resolve(path); - const root = parsePath(absolute).root; - const parts = absolute.slice(root.length).split('/').filter(Boolean); - let current = root; - for (const part of parts) { - current = join(current, part); - let stats: Stats; - try { - stats = lstatSync(current); - } catch (caught) { - if (filesystemCode(caught) === 'ENOENT') return; - throw caught; - } - if (stats.isSymbolicLink() || !stats.isDirectory()) { - throw new SnapshotError( - 'manifest-invalid', - 'Snapshot root and its existing ancestors must be physical directories.', - ); - } - } -} - function assertPathIdentity(path: string, identity: FileIdentity, kind: 'directory' | 'file'): void { let stats: Stats; try { @@ -631,27 +685,83 @@ function assertPathIdentity(path: string, identity: FileIdentity, kind: 'directo } } -function openBoundDirectory(path: string, create: boolean, requirePrivate = false): BoundDirectory { - assertSafeDirectoryAncestors(path); - if (create) mkdirSync(path, { recursive: true, mode: 0o700 }); - assertSafeDirectoryAncestors(path); - let descriptor: number; +function rootComponents(path: string): readonly string[] { + if (!isAbsolute(path) || resolve(path) !== path || path.includes('\0')) { + throw new SnapshotError('manifest-invalid', 'Snapshot root must be a normalized absolute path.'); + } + const components = path.split('/').filter(Boolean); + if (components.some((component) => component === '.' || component === '..')) { + throw new SnapshotError('manifest-invalid', 'Snapshot root contains an unsafe path component.'); + } + return components; +} + +function openRootComponent( + api: SnapshotPosixDirectoryApi, + parent: number, + component: string, + create: boolean, +): { readonly descriptor: number; readonly created: boolean } { + const flags = fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC; try { - descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW); + return { descriptor: api.openAt(parent, component, flags), created: false }; } catch (caught) { - if (!create && filesystemCode(caught) === 'ENOENT') throw caught; + if (!create) throw caught; + if (filesystemCode(caught) !== 'ENOENT') { + throw new SnapshotError('manifest-invalid', 'Snapshot root is not a safe physical directory.'); + } + } + try { + api.mkdirAt(parent, component, 0o700); + return { descriptor: api.openAt(parent, component, flags), created: true }; + } catch { throw new SnapshotError('manifest-invalid', 'Snapshot root is not a safe physical directory.'); } +} + +function assertCreatedRootComponent(descriptor: number): void { + const stats = fstatSync(descriptor); + const currentUid = typeof process.getuid === 'function' ? process.getuid() : stats.uid; + if (!stats.isDirectory() || stats.uid !== currentUid || (stats.mode & 0o777) !== 0o700) { + throw new SnapshotError('manifest-invalid', 'Created snapshot ancestor changed before descriptor binding.'); + } +} + +function openBoundDirectory( + path: string, + create: boolean, + options: DatabaseSyncSnapshotOptions, + requirePrivate = false, + allowShared = false, +): BoundDirectory { + const api = posixDirectory(options); + const components = rootComponents(path); + let descriptor = openSync( + '/', + fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, + ); try { + for (const component of components) { + const child = openRootComponent(api, descriptor, component, create); + try { + if (child.created) assertCreatedRootComponent(child.descriptor); + } catch (caught) { + closeSync(child.descriptor); + throw caught; + } + closeSync(descriptor); + descriptor = child.descriptor; + } const stats = fstatSync(descriptor); const currentUid = typeof process.getuid === 'function' ? process.getuid() : stats.uid; const mode = stats.mode & 0o777; - if (!stats.isDirectory() || stats.uid !== currentUid || (requirePrivate ? mode !== 0o700 : (mode & 0o022) !== 0)) { + if ( + !stats.isDirectory() || + (!allowShared && (stats.uid !== currentUid || (requirePrivate ? mode !== 0o700 : (mode & 0o022) !== 0))) + ) { throw new SnapshotError('manifest-invalid', 'Snapshot root must be a private directory owned by this user.'); } - const identity = fileIdentity(stats); - assertPathIdentity(path, identity, 'directory'); - return { path, descriptor, identity }; + return { path, descriptor, identity: fileIdentity(stats) }; } catch (caught) { closeSync(descriptor); throw caught; @@ -788,6 +898,17 @@ function assertBoundDirectory(directory: BoundDirectory): void { } } +function assertBoundRoot(directory: BoundDirectory, options: DatabaseSyncSnapshotOptions): void { + const reopened = openBoundDirectory(directory.path, false, options); + try { + if (!sameFileIdentity(directory.identity, reopened.identity)) { + throw new SnapshotError('manifest-invalid', 'Snapshot root identity changed during the operation.'); + } + } finally { + closeBoundDirectory(reopened); + } +} + function writeJsonDescriptor(descriptor: number, manifest: SnapshotManifestV1): void { const bytes = Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`); let offset = 0; @@ -877,7 +998,10 @@ function makeManifest( }; } -function createPrivateRoot(): { readonly path: string; readonly identity: FileIdentity } { +function createPrivateRoot(options: DatabaseSyncSnapshotOptions): { + readonly path: string; + readonly identity: FileIdentity; +} { const root = mkdtempSync(join(tmpdir(), 'genie-db-sync-private-')); const mode = statSync(root).mode & 0o777; if (mode !== 0o700) { @@ -887,7 +1011,7 @@ function createPrivateRoot(): { readonly path: string; readonly identity: FileId 'Private snapshot directory was not created with mode 0700.', ); } - const binding = openBoundDirectory(root, false, true); + const binding = openBoundDirectory(root, false, options, true); try { return { path: root, identity: binding.identity }; } finally { @@ -903,7 +1027,7 @@ function publishGeneration( privateRoot: string | null = null, ): PublishedGeneration { const root = privateRoot ?? identity.root; - const rootBinding = openBoundDirectory(root, true, privateRoot !== null); + const rootBinding = openBoundDirectory(root, true, options, privateRoot !== null); let stagingBinding: BoundDirectory | null = null; let manifestDescriptor: number | null = null; const payloadDescriptors: number[] = []; @@ -980,7 +1104,7 @@ function publishGeneration( fsyncSync((stagingBinding as BoundDirectory).descriptor); }); around(options, { phase: 'generation-rename', generationId: generated.id }, () => { - assertPathIdentity(root, rootBinding.identity, 'directory'); + assertBoundRoot(rootBinding, options); assertEntryIdentityAt( rootBinding, stagingName, @@ -1014,7 +1138,7 @@ function publishGeneration( ); }); around(options, { phase: 'root-fsync', generationId: generated.id }, () => { - assertPathIdentity(root, rootBinding.identity, 'directory'); + assertBoundRoot(rootBinding, options); assertEntryIdentityAt( rootBinding, generated.id, @@ -1023,7 +1147,7 @@ function publishGeneration( options, ); fsyncSync(rootBinding.descriptor); - assertPathIdentity(root, rootBinding.identity, 'directory'); + assertBoundRoot(rootBinding, options); }); if (manifestDescriptor === null) { throw new SnapshotError('snapshot-publication-failed', 'Snapshot manifest descriptor was not created.'); @@ -1223,7 +1347,7 @@ function readPublishedGeneration( options: DatabaseSyncSnapshotOptions, ): GenerationReference { const rootPath = dirname(published.directory); - const rootBinding = openBoundDirectory(rootPath, false, published.privateRoot !== null); + const rootBinding = openBoundDirectory(rootPath, false, options, published.privateRoot !== null); try { const generation = readManifestAt(rootBinding, published.manifest.generation_id, options); if ( @@ -1241,7 +1365,7 @@ function readPublishedGeneration( function reopenGeneration(generation: GenerationReference, options: DatabaseSyncSnapshotOptions): GenerationReference { const rootPath = dirname(generation.directory); - const rootBinding = openBoundDirectory(rootPath, false); + const rootBinding = openBoundDirectory(rootPath, false, options); try { const reopened = readManifestAt(rootBinding, basename(generation.directory), options); if ( @@ -1288,7 +1412,7 @@ function newestUnresolved( ): GenerationReference | null { let rootBinding: BoundDirectory; try { - rootBinding = openBoundDirectory(identity.root, false); + rootBinding = openBoundDirectory(identity.root, false, options); } catch (caught) { if (filesystemCode(caught) === 'ENOENT') return null; throw caught; @@ -1323,7 +1447,7 @@ function selectedGeneration( } let rootBinding: BoundDirectory; try { - rootBinding = openBoundDirectory(identity.root, false); + rootBinding = openBoundDirectory(identity.root, false, options); } catch { throw new SnapshotError('generation-not-found', 'Selected snapshot generation does not exist.'); } @@ -1463,20 +1587,11 @@ function removeBoundTree(path: string, expected: FileIdentity, options: Database assertPathIdentity(path, expected, 'directory'); options.removeTree?.(path); const parentPath = dirname(path); - assertSafeDirectoryAncestors(parentPath); - const descriptor = openSync( - parentPath, - fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW | POSIX_CLOSE_ON_EXEC, - ); + const parent = openBoundDirectory(parentPath, false, options, false, true); try { - const parent: BoundDirectory = { - path: parentPath, - descriptor, - identity: fileIdentity(fstatSync(descriptor)), - }; removeBoundTreeAt(parent, basename(path), path, expected, options); } finally { - closeSync(descriptor); + closeBoundDirectory(parent); } } @@ -1551,7 +1666,7 @@ function cleanupStaging( ): void { let rootBinding: BoundDirectory; try { - rootBinding = openBoundDirectory(root, false); + rootBinding = openBoundDirectory(root, false, options); } catch (caught) { if (filesystemCode(caught) === 'ENOENT') return; cleanupFailures.push('snapshot-cleanup-failed'); @@ -1592,7 +1707,7 @@ function pruneGenerations( ): void { let rootBinding: BoundDirectory; try { - rootBinding = openBoundDirectory(identity.root, false); + rootBinding = openBoundDirectory(identity.root, false, options); } catch { cleanupFailures.push('snapshot-cleanup-failed'); return; @@ -1671,36 +1786,35 @@ function recoverValidatedGeneration( }; } const validated = validateGeneration(generation, options); - const observed = new Map(inputs.map((input) => [input.canonicalPath, input.observe().logicalDigest])); - const classifications = generation.manifest.targets.map((target) => ({ - target, - current: observed.get(target.path), - pre: observed.get(target.path) === target.preimage_digest, - post: observed.get(target.path) === target.postimage_digest, - })); - emit(options, { phase: 'recovery-classify', state: 'before', generationId: generation.manifest.generation_id }); let status: RecoveryDecision['status']; const restoreTargets: SnapshotManifestTarget[] = []; - if (classifications.every((item) => item.post)) { - status = 'converged'; - } else if (classifications.every((item) => item.pre)) { - status = 'recovered'; - } else if ( - identity.mode === 'bidirectional' && - classifications.length === 2 && - classifications.every((item) => item.pre || item.post) && - classifications.filter((item) => item.pre).length === 1 && - classifications.filter((item) => item.post).length === 1 - ) { - status = 'recovered'; - restoreTargets.push(...classifications.filter((item) => item.post).map((item) => item.target)); - } else { - status = 'uncertain'; - } - emit(options, { phase: 'recovery-classify', state: 'after', generationId: generation.manifest.generation_id }); - const restoredPaths: string[] = []; try { + const observed = new Map(inputs.map((input) => [input.canonicalPath, input.observe().logicalDigest])); + const classifications = generation.manifest.targets.map((target) => ({ + target, + current: observed.get(target.path), + pre: observed.get(target.path) === target.preimage_digest, + post: observed.get(target.path) === target.postimage_digest, + })); + emit(options, { phase: 'recovery-classify', state: 'before', generationId: generation.manifest.generation_id }); + if (classifications.every((item) => item.post)) { + status = 'converged'; + } else if (classifications.every((item) => item.pre)) { + status = 'recovered'; + } else if ( + identity.mode === 'bidirectional' && + classifications.length === 2 && + classifications.every((item) => item.pre || item.post) && + classifications.filter((item) => item.pre).length === 1 && + classifications.filter((item) => item.post).length === 1 + ) { + status = 'recovered'; + restoreTargets.push(...classifications.filter((item) => item.post).map((item) => item.target)); + } else { + status = 'uncertain'; + } + emit(options, { phase: 'recovery-classify', state: 'after', generationId: generation.manifest.generation_id }); if (status !== 'uncertain') { for (const target of restoreTargets) { const snapshot = validated.snapshots.get(target.path); @@ -1983,7 +2097,7 @@ export function applyDatabaseReconciliationWithSnapshots( const cleanupFailures: SnapshotFailureCode[] = [...priorRecovery.cleanupFailures]; try { if (retention === 0) { - const created = createPrivateRoot(); + const created = createPrivateRoot(options); privateRoot = created.path; privateRootIdentity = created.identity; } @@ -2083,7 +2197,7 @@ export function rollbackDatabaseReconciliation( const cleanupFailures: SnapshotFailureCode[] = [...recovery.cleanupFailures]; try { if (retention === 0) { - const created = createPrivateRoot(); + const created = createPrivateRoot(options); privateRoot = created.path; privateRootIdentity = created.identity; } From a0cfc27ec9f4da8ca8b532c1f00cef18ba53db0e Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 02:47:32 +0000 Subject: [PATCH 12/21] fix(db-sync): canonicalize private snapshot roots --- src/lib/v5/db-sync-snapshots.test.ts | 162 +++++++++++++++++++++++++++ src/lib/v5/db-sync-snapshots.ts | 88 ++++++++++++--- 2 files changed, 235 insertions(+), 15 deletions(-) diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index ffade8a4d..e08a4f39b 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -9,6 +9,7 @@ import { mkdtempSync, readFileSync, readdirSync, + realpathSync, renameSync, rmSync, statSync, @@ -1621,6 +1622,167 @@ describe('database sync snapshots', () => { if (leakedRoot !== undefined) rmSync(leakedRoot, { recursive: true, force: true }); }); + test('zero retention canonicalizes a symlink-aliased temp parent and deletes its private root', () => { + const physicalTemp = join(fixtureRoot, 'physical-temp-success'); + const aliasedTemp = join(fixtureRoot, 'aliased-temp-success'); + mkdirSync(physicalTemp, { mode: 0o700 }); + symlinkSync(physicalTemp, aliasedTemp); + const left = currentDb('aliased-private-left'); + const right = currentDb('aliased-private-right'); + insertBoard(right, 'private'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const identity = databaseSyncSnapshotIdentity(request); + let privateRoot: string | undefined; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + privateRoot: { temporaryDirectory: aliasedTemp }, + onEvent: (event) => { + if (event.phase === 'payload-write' && privateRoot === undefined) { + privateRoot = readdirSync(physicalTemp) + .filter((name) => name.startsWith('genie-db-sync-private-')) + .map((name) => join(physicalTemp, name)) + .find((path) => statSync(path).isDirectory()); + } + }, + }); + expect(report.status).toBe('changed'); + expect(report.recovery.status).toBe('converged'); + expect(privateRoot).toBeDefined(); + expect(privateRoot === undefined ? true : existsSync(privateRoot)).toBe(false); + expect(existsSync(identity.root)).toBe(false); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ status: 'none', generationId: null }); + }); + + test('private-root canonicalization failure removes the exact newly owned directory', () => { + const physicalTemp = join(fixtureRoot, 'physical-temp-canonical-failure'); + const aliasedTemp = join(fixtureRoot, 'aliased-temp-canonical-failure'); + mkdirSync(physicalTemp, { mode: 0o700 }); + symlinkSync(physicalTemp, aliasedTemp); + const left = currentDb('canonical-failure-left'); + const right = currentDb('canonical-failure-right'); + insertBoard(right, 'private'); + let created = false; + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { + keepSnapshots: 0, + privateRoot: { + temporaryDirectory: aliasedTemp, + canonicalize: () => { + created = true; + throw new Error('canonicalization failed'); + }, + }, + }, + ); + expect(created).toBe(true); + expect(report.status).toBe('operational-failure'); + expect(report.cleanupFailures).toEqual([]); + expect(readdirSync(physicalTemp)).toEqual([]); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['private']); + }); + + test('private-root binding failure removes the exact newly owned directory', () => { + const physicalTemp = join(fixtureRoot, 'physical-temp-binding-failure'); + const aliasedTemp = join(fixtureRoot, 'aliased-temp-binding-failure'); + mkdirSync(physicalTemp, { mode: 0o700 }); + symlinkSync(physicalTemp, aliasedTemp); + const left = currentDb('binding-failure-left'); + const right = currentDb('binding-failure-right'); + insertBoard(right, 'private'); + const native = nativePosixDirectory(); + let injected = false; + const api = delegatePosix(native, { + openAt: (descriptor, name, flags, mode) => { + if (!injected && name.startsWith('genie-db-sync-private-')) { + injected = true; + throw new Error('private bind failed'); + } + return native.openAt(descriptor, name, flags, mode); + }, + }); + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { + keepSnapshots: 0, + privateRoot: { temporaryDirectory: aliasedTemp }, + posixDirectory: { api }, + }, + ); + expect(injected).toBe(true); + expect(report.status).toBe('operational-failure'); + expect(report.cleanupFailures).toEqual([]); + expect(readdirSync(physicalTemp)).toEqual([]); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['private']); + }); + + test('private-root setup cleanup failure is reported and remains undiscoverable', () => { + const physicalTemp = join(fixtureRoot, 'physical-temp-cleanup-failure'); + const aliasedTemp = join(fixtureRoot, 'aliased-temp-cleanup-failure'); + mkdirSync(physicalTemp, { mode: 0o700 }); + symlinkSync(physicalTemp, aliasedTemp); + const left = currentDb('setup-cleanup-failure-left'); + const right = currentDb('setup-cleanup-failure-right'); + insertBoard(right, 'private'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + privateRoot: { + temporaryDirectory: aliasedTemp, + canonicalize: () => { + throw new Error('canonicalization failed'); + }, + }, + removeTree: () => { + throw new Error('cleanup denied'); + }, + }); + expect(report.status).toBe('operational-failure'); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(readdirSync(physicalTemp)).toHaveLength(1); + expect(existsSync(databaseSyncSnapshotIdentity(request).root)).toBe(false); + expect(recoverDatabaseReconciliation(request)).toMatchObject({ status: 'none', generationId: null }); + rmSync(join(physicalTemp, readdirSync(physicalTemp)[0]), { recursive: true }); + }); + + test('private-root substitution never deletes or authorizes the replacement victim', () => { + const physicalTemp = join(fixtureRoot, 'physical-temp-substitution'); + const aliasedTemp = join(fixtureRoot, 'aliased-temp-substitution'); + mkdirSync(physicalTemp, { mode: 0o700 }); + symlinkSync(physicalTemp, aliasedTemp); + const left = currentDb('private-substitution-left'); + const right = currentDb('private-substitution-right'); + insertBoard(right, 'private'); + let victim: string | undefined; + let moved: string | undefined; + const report = applyDatabaseReconciliationWithSnapshots( + planDatabaseReconciliation({ mode: 'bidirectional', leftPath: left, rightPath: right }), + { + keepSnapshots: 0, + privateRoot: { + temporaryDirectory: aliasedTemp, + canonicalize: (lexicalRoot) => { + moved = `${lexicalRoot}.owned`; + victim = lexicalRoot; + renameSync(lexicalRoot, moved); + mkdirSync(victim, { mode: 0o700 }); + writeFileSync(join(victim, 'sentinel'), 'victim'); + return realpathSync(victim); + }, + }, + }, + ); + expect(report.status).toBe('operational-failure'); + expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(victim === undefined ? '' : readFileSync(join(victim, 'sentinel'), 'utf8')).toBe('victim'); + expect(boardNames(left)).toEqual([]); + expect(boardNames(right)).toEqual(['private']); + if (victim !== undefined) rmSync(victim, { recursive: true }); + if (moved !== undefined) rmSync(moved, { recursive: true }); + }); + test('POSIX resolution falls through Linux libc candidates and selects Darwin libSystem lazily', () => { const native = nativePosixDirectory(); const linuxAttempts: string[] = []; diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index 5c2eb9053..a4bc21092 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -13,8 +13,6 @@ import { openSync, readSync, realpathSync, - rmdirSync, - statSync, writeFileSync, writeSync, } from 'node:fs'; @@ -205,6 +203,11 @@ export interface DatabaseSyncSnapshotOptions { readonly now?: () => Date; readonly randomId?: () => string; readonly removeTree?: (path: string) => void; + /** Private N=0 root seams for aliased-temp and canonicalization fault tests. */ + readonly privateRoot?: { + readonly temporaryDirectory?: string; + readonly canonicalize?: (path: string) => string; + }; /** Lazy native descriptor-relative filesystem resolution seam for portability and fault tests. */ readonly posixDirectory?: SnapshotPosixDirectoryDependencies; readonly applyOptions?: Omit; @@ -998,24 +1001,79 @@ function makeManifest( }; } +function cleanupFailedPrivateRoot( + lexicalPath: string, + physicalParent: string, + name: string, + identity: FileIdentity, + options: DatabaseSyncSnapshotOptions, +): SnapshotFailureCode[] { + try { + options.removeTree?.(lexicalPath); + const parent = openBoundDirectory(physicalParent, false, options, false, true); + try { + removeBoundTreeAt(parent, name, join(physicalParent, name), identity, options); + } finally { + closeBoundDirectory(parent); + } + return []; + } catch { + return ['snapshot-cleanup-failed']; + } +} + +function privateRootFailure(caught: unknown, cleanupFailures: readonly SnapshotFailureCode[]): SnapshotError { + if (caught instanceof SnapshotError) { + return new SnapshotError(caught.code, caught.message, [...caught.cleanupFailures, ...cleanupFailures]); + } + return new SnapshotError( + 'snapshot-publication-failed', + caught instanceof Error ? caught.message : 'Private snapshot root setup failed.', + cleanupFailures, + ); +} + function createPrivateRoot(options: DatabaseSyncSnapshotOptions): { readonly path: string; readonly identity: FileIdentity; } { - const root = mkdtempSync(join(tmpdir(), 'genie-db-sync-private-')); - const mode = statSync(root).mode & 0o777; - if (mode !== 0o700) { - rmdirSync(root); - throw new SnapshotError( - 'snapshot-publication-failed', - 'Private snapshot directory was not created with mode 0700.', - ); - } - const binding = openBoundDirectory(root, false, options, true); + const temporaryDirectory = options.privateRoot?.temporaryDirectory ?? tmpdir(); + const physicalParent = realpathSync(temporaryDirectory); + const lexicalRoot = mkdtempSync(join(temporaryDirectory, 'genie-db-sync-private-')); + const initial = lstatSync(lexicalRoot); + const identity = fileIdentity(initial); + const name = basename(lexicalRoot); + let cleanupParent = physicalParent; try { - return { path: root, identity: binding.identity }; - } finally { - closeBoundDirectory(binding); + const currentUid = typeof process.getuid === 'function' ? process.getuid() : initial.uid; + if ( + !initial.isDirectory() || + initial.isSymbolicLink() || + initial.uid !== currentUid || + (initial.mode & 0o777) !== 0o700 + ) { + throw new SnapshotError( + 'snapshot-publication-failed', + 'Private snapshot directory was not created with mode 0700.', + ); + } + const canonicalize = options.privateRoot?.canonicalize ?? realpathSync; + const physicalRoot = canonicalize(lexicalRoot); + if (!isAbsolute(physicalRoot) || resolve(physicalRoot) !== physicalRoot || basename(physicalRoot) !== name) { + throw new SnapshotError('snapshot-publication-failed', 'Private snapshot directory canonical identity changed.'); + } + cleanupParent = dirname(physicalRoot); + const binding = openBoundDirectory(physicalRoot, false, options, true); + try { + if (!sameFileIdentity(identity, binding.identity)) { + throw new SnapshotError('snapshot-publication-failed', 'Private snapshot directory identity changed.'); + } + return { path: physicalRoot, identity }; + } finally { + closeBoundDirectory(binding); + } + } catch (caught) { + throw privateRootFailure(caught, cleanupFailedPrivateRoot(lexicalRoot, cleanupParent, name, identity, options)); } } From 3b93bc78d3719dcf0d9f0e9dd52bc9b0f60d1d26 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 03:09:07 +0000 Subject: [PATCH 13/21] fix(db-sync): avoid persistent cleanup for private snapshots --- src/lib/v5/db-sync-snapshots.test.ts | 3 ++- src/lib/v5/db-sync-snapshots.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index e08a4f39b..f1649f27e 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -1584,6 +1584,7 @@ describe('database sync snapshots', () => { } }, }); + expect(report.cleanupFailures).toEqual([]); expect(report.recovery.status).toBe('converged'); expect(existsSync(identity.root)).toBe(false); expect(privateRoot).toBeDefined(); @@ -1615,7 +1616,7 @@ describe('database sync snapshots', () => { throw new Error('cleanup denied'); }, }); - expect(report.cleanupFailures).toContain('snapshot-cleanup-failed'); + expect(report.cleanupFailures).toEqual(['snapshot-cleanup-failed']); expect(leakedRoot).toBeDefined(); expect(leakedRoot === undefined ? false : existsSync(leakedRoot)).toBe(true); expect(recoverDatabaseReconciliation(request)).toMatchObject({ status: 'none', generationId: null }); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index a4bc21092..b36d5a0d7 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -1830,6 +1830,7 @@ function recoverValidatedGeneration( inputs: readonly ReconciliationLockedDatabaseInput[], options: DatabaseSyncSnapshotOptions, retention: number, + cleanupPersistentStore = true, ): { value: RecoveryDecision; afterCommit: () => void } { if (generation.manifest.state === 'uncertain') { closeBoundDirectory(generation.binding); @@ -1912,8 +1913,10 @@ function recoverValidatedGeneration( const rebound = reopenGeneration(generation, options); try { rewriteManifestState(rebound, status === 'uncertain' ? 'uncertain' : status, options); - cleanupStaging(identity.root, options, cleanupFailures); - if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); + if (cleanupPersistentStore) { + cleanupStaging(identity.root, options, cleanupFailures); + if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); + } } finally { closeBoundDirectory(rebound.binding); } @@ -2079,7 +2082,7 @@ function recoverPrivateGeneration( (inputs) => { const generation = readPublishedGeneration(published, options); try { - return recoverValidatedGeneration(identity, generation, inputs, options, 0); + return recoverValidatedGeneration(identity, generation, inputs, options, 0, false); } catch (caught) { closeBoundDirectory(generation.binding); throw caught; From f9b02d9dd4fa7ea96df850628fedbe7680e8a33b Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 03:13:57 +0000 Subject: [PATCH 14/21] fix(db-sync): prune retained snapshots at zero retention --- src/lib/v5/db-sync-snapshots.test.ts | 70 ++++++++++++++++++++++++++++ src/lib/v5/db-sync-snapshots.ts | 17 +++---- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index f1649f27e..f7df63226 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -1596,6 +1596,76 @@ describe('database sync snapshots', () => { }); }); + test('zero retention removes finalized persistent generations after apply and explicit rollback', () => { + const applyLeft = currentDb('private-prune-apply-left'); + const applyRight = currentDb('private-prune-apply-right'); + const applyRequest = { + mode: 'bidirectional' as const, + leftPath: applyLeft, + rightPath: applyRight, + }; + insertBoard(applyRight, 'retained'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(applyRequest), { + keepSnapshots: 3, + }).status, + ).toBe('changed'); + insertBoard(applyRight, 'private'); + const applied = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(applyRequest), { + keepSnapshots: 0, + }); + expect(applied).toMatchObject({ status: 'changed', cleanupFailures: [] }); + expect(generationDirectories(databaseSyncSnapshotIdentity(applyRequest).root)).toEqual([]); + + const rollbackLeft = currentDb('private-prune-rollback-left'); + const rollbackRight = currentDb('private-prune-rollback-right'); + const rollbackRequest = { + mode: 'bidirectional' as const, + leftPath: rollbackLeft, + rightPath: rollbackRight, + }; + insertBoard(rollbackRight, 'retained'); + const retained = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(rollbackRequest), { + keepSnapshots: 3, + }); + const rolledBack = rollbackDatabaseReconciliation(rollbackRequest, retained.generationId ?? '', { + keepSnapshots: 0, + }); + expect(rolledBack).toMatchObject({ status: 'rolled-back', cleanupFailures: [] }); + expect(generationDirectories(databaseSyncSnapshotIdentity(rollbackRequest).root)).toEqual([]); + }); + + test('zero-retention persistent prune failures are reported without leaking private state', () => { + const left = currentDb('private-prune-failure-left'); + const right = currentDb('private-prune-failure-right'); + const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + insertBoard(right, 'retained'); + expect( + applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 3, + }).status, + ).toBe('changed'); + const privateBefore = privateSnapshotDirectories(); + insertBoard(right, 'private'); + let injected = false; + const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { + keepSnapshots: 0, + onEvent: (event) => { + if (!injected && event.phase === 'prune' && event.state === 'before') { + injected = true; + throw new Error('persistent prune denied'); + } + }, + }); + expect(injected).toBe(true); + expect(report).toMatchObject({ + status: 'changed', + cleanupFailures: ['snapshot-cleanup-failed'], + }); + expect(privateSnapshotDirectories()).toEqual(privateBefore); + expect(generationDirectories(databaseSyncSnapshotIdentity(request).root)).toHaveLength(1); + }); + test('zero-retention cleanup failure is reported and leaked private state is never discoverable', () => { const left = currentDb('private-failure-left'); const right = currentDb('private-failure-right'); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index b36d5a0d7..febdc686d 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -1766,7 +1766,8 @@ function pruneGenerations( let rootBinding: BoundDirectory; try { rootBinding = openBoundDirectory(identity.root, false, options); - } catch { + } catch (caught) { + if (filesystemCode(caught) === 'ENOENT') return; cleanupFailures.push('snapshot-cleanup-failed'); return; } @@ -1830,7 +1831,7 @@ function recoverValidatedGeneration( inputs: readonly ReconciliationLockedDatabaseInput[], options: DatabaseSyncSnapshotOptions, retention: number, - cleanupPersistentStore = true, + cleanupPersistentStaging = true, ): { value: RecoveryDecision; afterCommit: () => void } { if (generation.manifest.state === 'uncertain') { closeBoundDirectory(generation.binding); @@ -1913,10 +1914,8 @@ function recoverValidatedGeneration( const rebound = reopenGeneration(generation, options); try { rewriteManifestState(rebound, status === 'uncertain' ? 'uncertain' : status, options); - if (cleanupPersistentStore) { - cleanupStaging(identity.root, options, cleanupFailures); - if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); - } + if (cleanupPersistentStaging) cleanupStaging(identity.root, options, cleanupFailures); + if (status !== 'uncertain') pruneGenerations(identity, retention, options, cleanupFailures); } finally { closeBoundDirectory(rebound.binding); } @@ -2332,10 +2331,8 @@ export function rollbackDatabaseReconciliation( } finally { closeBoundDirectory(generation.binding); } - if (retention > 0) { - cleanupStaging(identity.root, options, cleanupFailures); - pruneGenerations(identity, retention, options, cleanupFailures); - } + if (retention > 0) cleanupStaging(identity.root, options, cleanupFailures); + pruneGenerations(identity, retention, options, cleanupFailures); }, }; } finally { From ff507e3c0aaa142885ea1d83d39c115c8b86ef54 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 03:18:07 +0000 Subject: [PATCH 15/21] feat(db-sync): add standalone reconciliation command --- CLAUDE.md | 3 +- README.md | 148 +++++++- scripts/release-docs.test.ts | 7 +- src/genie.ts | 8 +- src/term-commands/v5-db-sync.test.ts | 482 +++++++++++++++++++++++++ src/term-commands/v5-db-sync.ts | 512 +++++++++++++++++++++++++++ 6 files changed, 1154 insertions(+), 6 deletions(-) create mode 100644 src/term-commands/v5-db-sync.test.ts create mode 100644 src/term-commands/v5-db-sync.ts diff --git a/CLAUDE.md b/CLAUDE.md index 26d0f369f..a8991891b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,11 +72,12 @@ skills/ Skill prompt files (brainstorm, wish, work, revi ## CLI Commands -Fourteen top-level commands (run `genie --help` for detail): +Fifteen top-level commands (run `genie --help` for detail): | Command | Purpose | |---------|---------| | `board` | Kanban view derived by query (no stored view state); `--board`, `--wish`, `--json` | +| `db` | Standalone database reconciliation (`genie db sync`) with snapshots, recovery, and rollback | | `doctor` | Diagnostic checks on the genie installation | | `hook` | Hook middleware for Claude Code (`genie hook dispatch` runs in-process) | | `init` | Scaffold per-repo state and reconcile `.mcp.json`, `.warp/.mcp.json`, plus the marker-owned `.codex/config.toml` stable-facade route | diff --git a/README.md b/README.md index 340678491..cdbcd8353 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Re-run `genie board` any time for a current snapshot of task state on the kanban - **Skills** carry the methodology — `brainstorm → design review → wish → plan review → work → implementation review`, authored once for native Claude and Codex surfaces. - **Documents in git.** Wishes, designs, and brainstorms are plain markdown under `.genie/wishes//` and `.genie/brainstorms//`; you diff, review, and version them like any other code. - **One file of state.** Tasks, boards, dependency edges, and wish-group execution state live in a single per-repo SQLite file (`.genie/genie.db`), on Bun's built-in engine. -- **Small.** 14 CLI commands, 4 runtime dependencies (`@inquirer/prompts`, `commander`, `zod`, `nats`) — `nats` initializes only when the omni runner starts. A ~0.9 MB single-file bundle. Bun-powered. +- **Small.** 15 CLI commands, 4 runtime dependencies (`@inquirer/prompts`, `commander`, `zod`, `nats`) — `nats` initializes only when the omni runner starts. A ~0.9 MB single-file bundle. Bun-powered. - **Warp cockpit (optional).** `genie launch ` turns a wish's ready groups into a Warp window — one pane per group, each in its own git worktree running that group's agent on a kickoff prompt. Emitting the launch config works on any platform; opening it needs Warp (macOS/Linux). Everywhere else the config is still written for you to open by hand. - **Zero daemons, no Postgres.** Nothing runs in the background between invocations. @@ -70,6 +70,7 @@ genie --help | `genie launch` | Open a Warp cockpit for a wish — one pane per ready group, each in its own worktree | | `genie board` | Kanban view of task state, derived live by query | | `genie task` | Inspect and drive task state (SQLite, zero-daemon) | +| `genie db sync` | Reconcile two exact-current Genie databases with snapshots and rollback | | `genie install` | Finish a verified install and deliver selected integrations; Codex activation is deferred to setup | | `genie mcp` | Serve read-only Genie task/board state over stdio MCP | | `genie omni` | Bridge agents to WhatsApp via Omni — remote approvals + inbound one-shots (`serve`, `status`, `inbox`, `handshake`) | @@ -163,6 +164,151 @@ Documents live in git; operational state lives in one SQLite file. `work` fans a All linked worktrees of a repository share one `genie.db`, resolved from the git common directory, so a task created in one worktree is immediately visible in another with no sync step. +### Database reconciliation + +Use `genie db sync` when two separate Genie repositories or copied +`.genie/genie.db` files have diverged. The command is noninteractive and takes +explicit database paths. It never discovers or connects a live shared database. + +Choose exactly one mode: + +```bash +# Conservative bidirectional reconciliation +genie db sync /path/to/left.db /path/to/right.db + +# Directional reconciliation with explicit source authority +genie db sync \ + --source /path/to/source.db \ + --destination /path/to/destination.db +``` + +Bidirectional mode unions additions. It reports a conflict when the same +mutable key differs because the databases do not carry ancestry that could +prove which row is newer. Directional mode makes the source authoritative for +shared mutable task, board, wish-group, hire-roster, and unknown metadata +keys. It writes only the destination. Both modes preserve destination-only +rows: absence never means deletion, and every report fixes the deletion count +at zero. + +Dependency edges are unioned. Task events and legacy stage-log entries preserve +the maximum occurrence count observed on either applicable side. Independent +byte-identical history additions are indistinguishable without ancestry, so +they can be undercounted rather than guessed or duplicated. + +Preview the complete schema validation, conflict detection, and logical plan +without write locks, snapshots, or database writes: + +```bash +genie db sync /path/to/left.db /path/to/right.db --dry-run --json +``` + +Dry-run rejects snapshot, retention, rollback, and busy-timeout options because +they have no read-only effect. Ambiguous positional and directional forms also +fail before opening either database for mutation. Run +`genie db sync --help` for the two accepted forms and all options. + +#### Schema compatibility + +Reconciliation accepts the complete current Genie schema only. It compares all +eight user tables, their named columns, foreign keys, indexes, uniqueness, +checks, and `user_version`; the version number alone is not enough. It creates +no table, column, index, trigger, or migration. + +An older supported additive shape can still have the current `user_version`. +Open that database once through a normal current Genie command, such as +`genie board`, to add the supported current columns and tables. Then retry +reconciliation. Unknown or extra schema objects remain unsupported and are not +normalized. + +#### Snapshots, recovery, and rollback + +Every mutating run serializes and validates each destination preimage before it +writes. The generation manifest records its format and operation versions, +generation ID, canonical database identities and roles, preimage and planned +postimage digests, payload hashes, creation time, and recovery state. Snapshot +payloads are recovery inputs; Genie restores their logical rows through live +SQLite transactions and never replaces a live database, WAL, or SHM file. + +Bidirectional runs derive a stable pair identity from the sorted canonical +paths. Their default root is: + +```text +/sync-snapshots/ +``` + +Directional runs are role-sensitive. Their default root is: + +```text +/sync-snapshots +``` + +Use `--snapshot-root /normalized/absolute/path` to override either default. +Incomplete staging directories are not rollback points. Only a fully published +generation with a complete manifest can be recovered or selected. + +Before a new mutation, Genie classifies the newest unresolved generation: + +- `converged` means every database already has its recorded postimage. +- `recovered` means every database is back at its preimage, including a + transactionally restored mixed preimage/postimage pair. +- `uncertain` means at least one database matches neither recorded image. Genie + overwrites nothing and requires manual inspection. +- `operational-failure` means recovery could not complete or be classified. + +When an apply reports `converged` or `recovered` for a prior generation, rerun +the same command. That invocation performed recovery instead of starting the +new requested mutation. + +Rollback is explicit source authority over history. Pass the same database +pair and mode used by the generation: + +```bash +genie db sync /path/to/left.db /path/to/right.db \ + --rollback +``` + +Genie validates the manifest and canonical targets, snapshots the arbitrary +current state into a new safety generation, then restores the selected older +logical images. Rollback is reported as `rolled-back`, never as automatic crash +recovery. + +The default `--keep-snapshots` value is `3`. Any nonnegative integer is valid. +With `--keep-snapshots 0`, Genie uses a private mode-`0700` OS temporary +directory only for same-process recovery and removes it best-effort on exit. +No generation is published to the normal snapshot root, and finalized +persistent generations for that operation are pruned after success. A later +process therefore cannot recover or roll back the zero-retention operation. A +cleanup failure is reported and returns a nonzero exit even when the logical +apply succeeded. + +`--busy-timeout-ms N` bounds the total wait for advisory and SQLite write locks. +`N` must be between `0` and `2147483647`. A timeout returns a bounded +operational report; the command does not wait indefinitely. + +#### Automation report and exit codes + +`--json` emits report version `1`. The stable envelope identifies the operation, +mode, status, hashed database identities, logical and schema digests, per-table +change counts, conflict counts, generation IDs, recovery state, bounded failure +codes, and cleanup failures. It never includes task titles, notes, or other row +content. Human output uses the same statuses and abbreviated identities. + +| Exit | Meaning | +|------|---------| +| `0` | The requested dry-run, apply, no-op, same-database check, or rollback completed | +| `1` | Commander rejected command syntax or an unknown option | +| `2` | The mode or option combination is invalid; no mutation started | +| `3` | Conservative reconciliation found conflicts; no mutation started | +| `4` | An operational, lock, rollback, observation, or cleanup failure occurred | +| `5` | Recovery or commit state is uncertain; manual inspection is required | +| `6` | A prior generation was recovered or classified converged; rerun the request | + +Crashes can leave a complete unresolved manifest. Automatic recovery changes +data only when every current digest matches that manifest's recorded preimage +or postimage. It never guesses after an unrelated write. A crash before a +complete manifest is published leaves no recovery point, and retention zero +cannot support recovery after process loss. + ## Omni (WhatsApp bridge) `genie omni` wires a running agent to WhatsApp through an [Omni](https://automagik.dev) hub, so you can drive approvals and short tasks from your phone. diff --git a/scripts/release-docs.test.ts b/scripts/release-docs.test.ts index 0f721c00a..1671f49ae 100644 --- a/scripts/release-docs.test.ts +++ b/scripts/release-docs.test.ts @@ -817,9 +817,10 @@ describe('Group E release and documentation contracts', () => { ).toContain('legacy workspace-write grants are forbidden'); }); - test('README and contributor command inventories match the 14-command source surface', () => { + test('README and contributor command inventories match the 15-command source surface', () => { const expected = [ 'board', + 'db', 'doctor', 'help', 'hook', @@ -836,8 +837,8 @@ describe('Group E release and documentation contracts', () => { ]; const readme = read('README.md'); const contributor = read('CLAUDE.md'); - expect(readme).toContain('14 CLI commands'); - expect(contributor).toContain('Fourteen top-level commands'); + expect(readme).toContain('15 CLI commands'); + expect(contributor).toContain('Fifteen top-level commands'); const readmeCommands = [...readme.matchAll(/^\| `genie ([a-z-]+)/gm)].map((match) => match[1]).sort(); const contributorCommands = [...contributor.matchAll(/^\| `([a-z-]+)/gm)].map((match) => match[1]).sort(); expect(readmeCommands).toEqual(expected); diff --git a/src/genie.ts b/src/genie.ts index 590b886e4..7c2dfd8f7 100644 --- a/src/genie.ts +++ b/src/genie.ts @@ -32,6 +32,7 @@ import { registerMcpCommand } from './term-commands/mcp.js'; import { registerOmniCommands } from './term-commands/omni.js'; import { registerUiBridgeCommand } from './term-commands/ui-bridge.js'; import { registerV5BoardCommands } from './term-commands/v5-board.js'; +import { registerV5DatabaseSyncCommand } from './term-commands/v5-db-sync.js'; import { registerV5TaskCommands } from './term-commands/v5-task.js'; const program = new Command(); @@ -191,6 +192,7 @@ registerMcpCommand(program); registerUiBridgeCommand(program); registerV5TaskCommands(program); registerV5BoardCommands(program); +registerV5DatabaseSyncCommand(program); registerIdeaCommand(program); registerOmniCommands(program); @@ -198,6 +200,10 @@ registerOmniCommands(program); // Universal workspace check — ensures workspace exists before commands that need it // ============================================================================ -installWorkspaceCheck(program); +// `db` takes explicit database paths and must validate them before any legacy +// workspace prompt can create state. It is therefore standalone like task and +// board, without broadening the legacy middleware's shared exemption list. +const requestedRootCommand = process.argv.slice(2).find((argument) => !argument.startsWith('-')); +if (requestedRootCommand !== 'db') installWorkspaceCheck(program); await program.parseAsync(process.argv); diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts new file mode 100644 index 000000000..3efbcfda7 --- /dev/null +++ b/src/term-commands/v5-db-sync.test.ts @@ -0,0 +1,482 @@ +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { openDb, resolveDbPath } from '../lib/v5/genie-db.js'; +import { createTask, getTask, listTasks } from '../lib/v5/task-state.js'; +import { DATABASE_SYNC_EXIT_CODES } from './v5-db-sync.js'; + +const GENIE = join(import.meta.dir, '..', 'genie.ts'); + +interface CliResult { + stdout: string; + stderr: string; + code: number; +} + +interface JsonReport { + reportVersion: number; + operation: string; + mode: string; + status: string; + inputs: Array<{ role: string; databaseIdentity: string; logicalDigest: string | null }>; + plan: { + status: string; + schemaFingerprint: string | null; + targets: Array<{ role: string; changes: Record }>; + conflicts: { total: number; byTable: Record }; + operationalFailure: { code: string; guidance?: string } | null; + } | null; + apply: { + generationId: string | null; + recovery: { status: string }; + cleanupFailures: string[]; + } | null; + rollback: { + selectedGenerationId: string; + safetyGenerationId: string | null; + cleanupFailures: string[]; + } | null; +} + +let root: string; + +function git(cwd: string, ...args: string[]): void { + execFileSync('git', args, { + cwd, + stdio: 'ignore', + env: { + ...process.env, + GIT_AUTHOR_NAME: 'Test', + GIT_AUTHOR_EMAIL: 'test@example.com', + GIT_COMMITTER_NAME: 'Test', + GIT_COMMITTER_EMAIL: 'test@example.com', + }, + }); +} + +function createRepo(name: string, current = true): { repo: string; database: string } { + const repo = join(root, name); + mkdirSync(repo); + git(repo, 'init', '-b', 'main'); + git(repo, 'commit', '--allow-empty', '-m', 'init'); + const database = resolveDbPath(repo); + if (current) { + const db = openDb({ cwd: repo }); + db.close(); + } + return { repo, database }; +} + +async function cli(cwd: string, ...args: string[]): Promise { + const proc = Bun.spawn([process.execPath, GENIE, ...args], { + cwd, + stdout: 'pipe', + stderr: 'pipe', + env: { + ...process.env, + NO_COLOR: '1', + GENIE_TEST_SKIP_PGSERVE: '1', + }, + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + return { stdout, stderr, code: await proc.exited }; +} + +async function sync(...args: string[]): Promise { + return cli(root, 'db', 'sync', ...args); +} + +function parseJson(result: CliResult): JsonReport { + return JSON.parse(result.stdout) as JsonReport; +} + +function insertTask(path: string, id: string, title: string): void { + const db = openDb({ path }); + db.query('INSERT INTO tasks (id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)').run( + id, + title, + 'ready', + 1, + 1, + ); + db.close(); +} + +function taskTitles(path: string): string[] { + const db = openDb({ path }); + try { + return listTasks(db) + .map((task) => task.title) + .sort(); + } finally { + db.close(); + } +} + +/** Exact pre-lanes, pre-runtime user_version=1 shape accepted by normal current open. */ +function seedPriorDatabase(path: string): void { + mkdirSync(dirname(path), { recursive: true }); + const db = new Database(path); + db.exec('PRAGMA user_version = 1'); + db.exec(` + CREATE TABLE boards (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at INTEGER NOT NULL); + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + board_id TEXT REFERENCES boards(id) ON DELETE SET NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + claimed_by TEXT, claimed_at INTEGER, wish TEXT, group_name TEXT, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE task_dependencies ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) + ); + CREATE TABLE stage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + stage TEXT NOT NULL, note TEXT, created_at INTEGER NOT NULL + ); + CREATE TABLE wish_groups ( + wish TEXT NOT NULL, name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('blocked','ready','in_progress','done')), + depends_on TEXT NOT NULL DEFAULT '[]', assignee TEXT, + started_at INTEGER, completed_at INTEGER, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, + PRIMARY KEY (wish, name) + ); + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + `); + db.query('INSERT INTO tasks (id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)').run( + 't_prior', + 'prior task', + 'ready', + 1, + 1, + ); + db.close(); +} + +function schemaInventory(path: string): string[] { + const db = new Database(path, { readonly: true }); + try { + const objects = ( + db + .query("SELECT type, name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name") + .all() as Array<{ type: string; name: string }> + ).map((row) => `${row.type}:${row.name}`); + const tables = ( + db + .query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() as Array<{ name: string }> + ).map((row) => row.name); + const columns = tables.flatMap((table) => + ( + db.query(`PRAGMA table_info("${table}")`).all() as Array<{ + name: string; + type: string; + notnull: number; + dflt_value: string | null; + pk: number; + }> + ) + .sort((left, right) => left.name.localeCompare(right.name)) + .map( + (column) => + `${table}.${column.name}:${column.type}:${column.notnull}:${column.dflt_value ?? '-'}:${column.pk}`, + ), + ); + return [...objects, ...columns]; + } finally { + db.close(); + } +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'genie-v5-db-sync-')); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe('database sync CLI contract', () => { + test('help publishes both explicit modes, recovery options, JSON, and exit codes', async () => { + const result = await sync('--help'); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(result.stderr).toBe(''); + for (const text of [ + ' ', + '--source ', + '--destination ', + '--dry-run', + '--rollback ', + '--snapshot-root ', + '--keep-snapshots ', + '--busy-timeout-ms ', + '--json', + '6 recovery handled', + ]) { + expect(result.stdout).toContain(text); + } + }); + + test('ambiguous and invalid forms fail before database or snapshot mutation', async () => { + const left = createRepo('left'); + const right = createRepo('right'); + insertTask(left.database, 't_left', 'left'); + const beforeLeft = readFileSync(left.database); + const beforeRight = readFileSync(right.database); + const snapshotRoot = join(root, 'snapshots'); + const cases = [ + [left.database, '--snapshot-root', snapshotRoot], + ['--source', left.database, '--snapshot-root', snapshotRoot], + [ + left.database, + right.database, + '--source', + left.database, + '--destination', + right.database, + '--snapshot-root', + snapshotRoot, + ], + [left.database, right.database, '--dry-run', '--keep-snapshots', '3'], + [left.database, right.database, '--keep-snapshots', '-1', '--snapshot-root', snapshotRoot], + [left.database, right.database, '--busy-timeout-ms', '2147483648', '--snapshot-root', snapshotRoot], + [left.database, right.database, '--snapshot-root', 'relative'], + [left.database, right.database, '--rollback', 'not-a-generation', '--snapshot-root', snapshotRoot], + ]; + + for (const args of cases) { + const result = await sync(...args); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.usage); + expect(result.stdout).toBe(''); + expect(result.stderr).toContain('genie db sync --help'); + } + expect(readFileSync(left.database)).toEqual(beforeLeft); + expect(readFileSync(right.database)).toEqual(beforeRight); + expect(existsSync(snapshotRoot)).toBe(false); + }); + + test('directional dry-run reports bounded identities and counts without row content or writes', async () => { + const source = createRepo('source'); + const destination = createRepo('destination'); + const sourceDb = openDb({ path: source.database }); + createTask(sourceDb, { title: 'HOSTILE ROW CONTENT\nSHOULD NOT LEAK' }); + sourceDb.close(); + const beforeSource = readFileSync(source.database); + const beforeDestination = readFileSync(destination.database); + + const result = await sync( + '--source', + source.database, + '--destination', + destination.database, + '--dry-run', + '--json', + ); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(result.stderr).toBe(''); + expect(result.stdout).not.toContain('HOSTILE ROW CONTENT'); + const report = parseJson(result); + expect(report).toMatchObject({ + reportVersion: 1, + operation: 'dry-run', + mode: 'directional', + status: 'changed', + }); + expect(report.inputs.map((input) => input.role)).toEqual(['source', 'destination']); + expect(report.inputs.every((input) => /^[a-f0-9]{64}$/.test(input.databaseIdentity))).toBe(true); + expect(report.plan?.targets[0].changes.tasks).toBe(1); + expect(readFileSync(source.database)).toEqual(beforeSource); + expect(readFileSync(destination.database)).toEqual(beforeDestination); + expect(existsSync(join(destination.repo, '.genie', 'sync-snapshots'))).toBe(false); + }); + + test('bidirectional apply converges disjoint additions, never deletes, and reports a durable generation', async () => { + const left = createRepo('left'); + const right = createRepo('right'); + insertTask(left.database, 't_left', 'LEFT HOSTILE TITLE'); + insertTask(right.database, 't_right', 'RIGHT HOSTILE TITLE'); + + const result = await sync(left.database, right.database, '--json'); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(result.stderr).toBe(''); + expect(result.stdout).not.toContain('HOSTILE TITLE'); + const report = parseJson(result); + expect(report.status).toBe('changed'); + expect(report.apply?.generationId).toMatch(/^[a-f0-9]{64}--[0-9]{16}--[a-f0-9-]{36}$/); + expect(report.apply?.recovery.status).toBe('converged'); + expect(report.plan?.targets.map((target) => target.changes.deletions)).toEqual([0, 0]); + expect(taskTitles(left.database)).toEqual(['LEFT HOSTILE TITLE', 'RIGHT HOSTILE TITLE']); + expect(taskTitles(right.database)).toEqual(['LEFT HOSTILE TITLE', 'RIGHT HOSTILE TITLE']); + + const noOp = await sync(right.database, left.database, '--json'); + expect(noOp.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(noOp).status).toBe('no-op'); + }); + + test('bidirectional mutable conflicts use exit 3, bounded digests/counts, and zero mutation', async () => { + const left = createRepo('left'); + const right = createRepo('right'); + insertTask(left.database, 't_shared', 'LEFT SECRET'); + insertTask(right.database, 't_shared', 'RIGHT SECRET'); + const beforeLeft = readFileSync(left.database); + const beforeRight = readFileSync(right.database); + + const result = await sync(left.database, right.database, '--json'); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.conflict); + expect(result.stderr).toBe(''); + expect(result.stdout).not.toContain('LEFT SECRET'); + expect(result.stdout).not.toContain('RIGHT SECRET'); + const report = parseJson(result); + expect(report.status).toBe('conflict'); + expect(report.plan?.conflicts.total).toBe(1); + expect(report.plan?.conflicts.byTable.tasks).toBe(1); + expect(readFileSync(left.database)).toEqual(beforeLeft); + expect(readFileSync(right.database)).toEqual(beforeRight); + }); + + test('directional source authority overwrites shared rows while preserving destination-only rows and source bytes', async () => { + const source = createRepo('source'); + const destination = createRepo('destination'); + insertTask(source.database, 't_shared', 'source wins'); + insertTask(destination.database, 't_shared', 'destination loses'); + insertTask(destination.database, 't_destination', 'destination remains'); + const beforeSource = readFileSync(source.database); + + const result = await sync( + '--source', + source.database, + '--destination', + destination.database, + '--keep-snapshots', + '2', + '--busy-timeout-ms', + '1000', + '--json', + ); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(result).status).toBe('changed'); + expect(readFileSync(source.database)).toEqual(beforeSource); + expect(taskTitles(destination.database)).toEqual(['destination remains', 'source wins']); + }); + + test('rollback restores both pre-sync images and current task/database commands remain compatible', async () => { + const left = createRepo('left'); + const right = createRepo('right'); + insertTask(left.database, 't_left', 'left only'); + insertTask(right.database, 't_right', 'right only'); + + const applied = await sync(left.database, right.database, '--json'); + const generation = parseJson(applied).apply?.generationId; + if (generation == null) throw new Error('Expected apply to publish a generation.'); + expect(taskTitles(left.database)).toEqual(['left only', 'right only']); + + const rolledBack = await sync(left.database, right.database, '--rollback', generation, '--json'); + expect(rolledBack.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + const report = parseJson(rolledBack); + expect(report.status).toBe('rolled-back'); + expect(report.rollback?.selectedGenerationId).toBe(generation); + expect(report.rollback?.safetyGenerationId).toMatch(/^[a-f0-9]{64}--/); + expect(taskTitles(left.database)).toEqual(['left only']); + expect(taskTitles(right.database)).toEqual(['right only']); + + for (const repo of [left.repo, right.repo]) { + const exported = await cli(repo, 'task', 'export'); + expect(exported.code).toBe(0); + expect(() => JSON.parse(exported.stdout)).not.toThrow(); + const board = await cli(repo, 'board', '--json'); + expect(board.code).toBe(0); + expect(() => JSON.parse(board.stdout)).not.toThrow(); + } + }); + + test('stale prior shape fails read-only, then normal current commands normalize it to the identical complete schema', async () => { + const prior = createRepo('prior', false); + const current = createRepo('current'); + seedPriorDatabase(prior.database); + const staleBytes = readFileSync(prior.database); + + const rejected = await sync(prior.database, current.database, '--dry-run', '--json'); + expect(rejected.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + const rejectedReport = parseJson(rejected); + expect(rejectedReport.plan?.operationalFailure?.code).toBe('stale-current-schema'); + expect(rejectedReport.plan?.operationalFailure?.guidance).toContain('normal current open path'); + expect(readFileSync(prior.database)).toEqual(staleBytes); + + const priorBoard = await cli(prior.repo, 'board', '--json'); + expect(priorBoard.code).toBe(0); + const priorCreate = await cli(prior.repo, 'task', 'create', '--title', 'current command task'); + expect(priorCreate.code).toBe(0); + + const reconciled = await sync(prior.database, current.database, '--json'); + expect(reconciled.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(reconciled).status).toBe('changed'); + expect(schemaInventory(prior.database)).toEqual(schemaInventory(current.database)); + + for (const repo of [prior.repo, current.repo]) { + const listed = await cli(repo, 'task', 'list', '--json'); + expect(listed.code).toBe(0); + expect(JSON.parse(listed.stdout)).toHaveLength(2); + const db = openDb({ cwd: repo }); + expect(getTask(db, 't_prior')?.title).toBe('prior task'); + db.close(); + } + }); + + test('retention zero uses only same-process recovery and publishes no persistent root', async () => { + const source = createRepo('source'); + const destination = createRepo('destination'); + insertTask(source.database, 't_source', 'source'); + + const result = await sync( + '--source', + source.database, + '--destination', + destination.database, + '--keep-snapshots', + '0', + '--json', + ); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + const report = parseJson(result); + expect(report.status).toBe('changed'); + expect(report.apply?.generationId).toMatch(/^[a-f0-9]{64}--/); + expect(report.apply?.cleanupFailures).toEqual([]); + expect(existsSync(join(destination.repo, '.genie', 'sync-snapshots'))).toBe(false); + }); + + test('busy timeout is forwarded as a bounded operational result', async () => { + const source = createRepo('source'); + const destination = createRepo('destination'); + insertTask(source.database, 't_source', 'source'); + const held = new Database(destination.database); + held.exec('BEGIN IMMEDIATE'); + const started = Date.now(); + try { + const result = await sync( + '--source', + source.database, + '--destination', + destination.database, + '--busy-timeout-ms', + '0', + '--json', + ); + expect(Date.now() - started).toBeLessThan(2_000); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + expect(parseJson(result).status).toBe('operational-failure'); + } finally { + held.exec('ROLLBACK'); + held.close(); + } + expect(taskTitles(destination.database)).toEqual([]); + }); +}); diff --git a/src/term-commands/v5-db-sync.ts b/src/term-commands/v5-db-sync.ts new file mode 100644 index 000000000..39ec369bb --- /dev/null +++ b/src/term-commands/v5-db-sync.ts @@ -0,0 +1,512 @@ +/** + * genie db sync — thin standalone CLI over the v5 reconciliation and + * snapshot modules. + * + * The command deliberately owns only argument validation, bounded report + * shaping, human rendering, and exit-code classification. Planning, locking, + * mutation, recovery, rollback, and retention remain in src/lib/v5. + */ + +import { createHash } from 'node:crypto'; +import { realpathSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import type { Command } from 'commander'; +import { + IDENTICAL_HISTORY_ADDITION_LIMITATION, + type ReconciliationApplyReport, + type ReconciliationConflict, + ReconciliationError, + type ReconciliationInputRole, + type ReconciliationRequest, + type ReconciliationTableName, + type ReconciliationTargetReport, + planDatabaseReconciliation, +} from '../lib/v5/db-reconciliation.js'; +import { + type SnapshotApplyReport, + type SnapshotRecoveryReport, + type SnapshotRollbackReport, + applyDatabaseReconciliationWithSnapshots, + rollbackDatabaseReconciliation, +} from '../lib/v5/db-sync-snapshots.js'; + +const CLI_REPORT_VERSION = 1 as const; +const MAX_BUSY_TIMEOUT_MS = 2_147_483_647; +const GENERATION_ID_PATTERN = /^[a-f0-9]{64}--[0-9]{16}--[a-f0-9-]{36}$/; + +export const DATABASE_SYNC_EXIT_CODES = { + success: 0, + parserError: 1, + usage: 2, + conflict: 3, + operationalFailure: 4, + uncertain: 5, + recoveryHandled: 6, +} as const; + +type DatabaseSyncExitCode = (typeof DATABASE_SYNC_EXIT_CODES)[keyof typeof DATABASE_SYNC_EXIT_CODES]; +type DatabaseSyncOperation = 'dry-run' | 'apply' | 'rollback'; + +interface DatabaseSyncOptions { + source?: string; + destination?: string; + dryRun?: boolean; + rollback?: string; + snapshotRoot?: string; + keepSnapshots?: string; + busyTimeoutMs?: string; + json?: boolean; +} + +interface DatabaseIdentityReport { + role: ReconciliationInputRole; + databaseIdentity: string; + logicalDigest: string | null; +} + +interface ConflictCountReport { + total: number; + byTable: Record; +} + +interface PlanReport { + status: 'no-op' | 'changed' | 'conflict' | 'operational-failure'; + sameDatabase: boolean; + schemaFingerprint: string | null; + targets: readonly ReconciliationTargetReport[]; + conflicts: ConflictCountReport; + operationalFailure: { + code: string; + guidance?: string; + } | null; + historyLimitation: string; +} + +interface RecoveryReport { + status: SnapshotRecoveryReport['status']; + generationId: string | null; + restoredDatabaseIdentities: readonly string[]; + failure: SnapshotRecoveryReport['failure']; + cleanupFailures: SnapshotRecoveryReport['cleanupFailures']; +} + +interface ApplyReport { + status: SnapshotApplyReport['status']; + generationId: string | null; + recovery: RecoveryReport; + apply: ReconciliationApplyReport | null; + failure: SnapshotApplyReport['failure']; + cleanupFailures: SnapshotApplyReport['cleanupFailures']; +} + +interface RollbackReport { + status: SnapshotRollbackReport['status']; + selectedGenerationId: string; + safetyGenerationId: string | null; + failure: SnapshotRollbackReport['failure']; + cleanupFailures: SnapshotRollbackReport['cleanupFailures']; +} + +interface DatabaseSyncCliReport { + reportVersion: typeof CLI_REPORT_VERSION; + command: 'database-sync'; + operation: DatabaseSyncOperation; + mode: ReconciliationRequest['mode']; + status: string; + inputs: readonly DatabaseIdentityReport[]; + plan: PlanReport | null; + apply: ApplyReport | null; + rollback: RollbackReport | null; +} + +class DatabaseSyncUsageError extends Error {} + +function databaseIdentity(path: string): string { + const absolute = resolve(path); + let canonical = absolute; + try { + canonical = realpathSync(absolute); + } catch { + // Failure reports still need a stable bounded identity for an unavailable + // input. The reconciliation layer owns the actionable availability code. + } + return createHash('sha256').update(`genie-db-sync-cli-identity-v1\0${canonical}`).digest('hex'); +} + +function requestInputs(request: ReconciliationRequest): Array<{ role: ReconciliationInputRole; path: string }> { + return request.mode === 'bidirectional' + ? [ + { role: 'left', path: request.leftPath }, + { role: 'right', path: request.rightPath }, + ] + : [ + { role: 'source', path: request.sourcePath }, + { role: 'destination', path: request.destinationPath }, + ]; +} + +function unresolvedInputs(request: ReconciliationRequest): DatabaseIdentityReport[] { + return requestInputs(request).map((input) => ({ + role: input.role, + databaseIdentity: databaseIdentity(input.path), + logicalDigest: null, + })); +} + +function parseRequest(databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions) { + const positionalCount = Number(databaseA !== undefined) + Number(databaseB !== undefined); + const directionalCount = Number(options.source !== undefined) + Number(options.destination !== undefined); + if (positionalCount === 2 && directionalCount === 0) { + return { + mode: 'bidirectional', + leftPath: databaseA as string, + rightPath: databaseB as string, + } satisfies ReconciliationRequest; + } + if (positionalCount === 0 && directionalCount === 2) { + return { + mode: 'directional', + sourcePath: options.source as string, + destinationPath: options.destination as string, + } satisfies ReconciliationRequest; + } + throw new DatabaseSyncUsageError( + 'Choose exactly one mode: `genie db sync ` or ' + + '`genie db sync --source --destination `.', + ); +} + +function parseNonnegativeInteger(name: string, raw: string | undefined, maximum = Number.MAX_SAFE_INTEGER) { + if (raw === undefined) return undefined; + if (!/^(0|[1-9][0-9]*)$/.test(raw)) { + throw new DatabaseSyncUsageError(`${name} must be a nonnegative integer.`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value > maximum) { + throw new DatabaseSyncUsageError(`${name} must be at most ${maximum}.`); + } + return value; +} + +function parseSnapshotRoot(raw: string | undefined): string | undefined { + if (raw === undefined) return undefined; + if (!isAbsolute(raw) || raw.split('/').some((component) => component === '.' || component === '..')) { + throw new DatabaseSyncUsageError('--snapshot-root must be a normalized absolute path.'); + } + return resolve(raw); +} + +function parseOperation(options: DatabaseSyncOptions): DatabaseSyncOperation { + if (options.dryRun && options.rollback !== undefined) { + throw new DatabaseSyncUsageError('--dry-run and --rollback cannot be used together.'); + } + if (options.dryRun && (options.snapshotRoot || options.keepSnapshots || options.busyTimeoutMs)) { + throw new DatabaseSyncUsageError( + '--dry-run is read-only; omit --snapshot-root, --keep-snapshots, and --busy-timeout-ms.', + ); + } + if (options.rollback !== undefined && !GENERATION_ID_PATTERN.test(options.rollback)) { + throw new DatabaseSyncUsageError('--rollback must be a complete generation ID from an earlier report.'); + } + return options.dryRun ? 'dry-run' : options.rollback === undefined ? 'apply' : 'rollback'; +} + +function emptyConflictCounts(): Record { + return { + boards: 0, + tasks: 0, + wish_groups: 0, + hire_roster: 0, + meta: 0, + task_dependencies: 0, + stage_log: 0, + task_events: 0, + }; +} + +function conflictCounts(conflicts: readonly ReconciliationConflict[]): ConflictCountReport { + const byTable = emptyConflictCounts(); + for (const conflict of conflicts) byTable[conflict.table]++; + return { total: conflicts.length, byTable }; +} + +function summarizePlan(plan: ReturnType): PlanReport { + return { + status: plan.report.status, + sameDatabase: plan.report.sameDatabase, + schemaFingerprint: plan.report.schemaFingerprint, + targets: plan.report.targets, + conflicts: conflictCounts(plan.conflicts), + operationalFailure: null, + historyLimitation: plan.report.historyLimitation, + }; +} + +function planningFailure(caught: unknown): PlanReport { + const operationalFailure = + caught instanceof ReconciliationError + ? { code: caught.code, ...(caught.guidance === undefined ? {} : { guidance: caught.guidance }) } + : { code: 'unexpected-failure' }; + return { + status: 'operational-failure', + sameDatabase: false, + schemaFingerprint: null, + targets: [], + conflicts: { total: 0, byTable: emptyConflictCounts() }, + operationalFailure, + historyLimitation: IDENTICAL_HISTORY_ADDITION_LIMITATION, + }; +} + +function resolvedInputs(plan: ReturnType): DatabaseIdentityReport[] { + return plan.inputs.map((input) => ({ + role: input.role, + databaseIdentity: databaseIdentity(input.canonicalPath), + logicalDigest: input.logicalDigest, + })); +} + +function summarizeRecovery(recovery: SnapshotRecoveryReport): RecoveryReport { + return { + status: recovery.status, + generationId: recovery.generationId, + restoredDatabaseIdentities: recovery.restoredPaths.map(databaseIdentity), + failure: recovery.failure, + cleanupFailures: recovery.cleanupFailures, + }; +} + +function summarizeApply(report: SnapshotApplyReport): ApplyReport { + return { + status: report.status, + generationId: report.generationId, + recovery: summarizeRecovery(report.recovery), + apply: report.apply, + failure: report.failure, + cleanupFailures: report.cleanupFailures, + }; +} + +function summarizeRollback(report: SnapshotRollbackReport): RollbackReport { + return { + status: report.status, + selectedGenerationId: report.selectedGenerationId, + safetyGenerationId: report.safetyGenerationId, + failure: report.failure, + cleanupFailures: report.cleanupFailures, + }; +} + +function reportForPlanningFailure( + request: ReconciliationRequest, + operation: DatabaseSyncOperation, + caught: unknown, +): DatabaseSyncCliReport { + return { + reportVersion: CLI_REPORT_VERSION, + command: 'database-sync', + operation, + mode: request.mode, + status: 'operational-failure', + inputs: unresolvedInputs(request), + plan: planningFailure(caught), + apply: null, + rollback: null, + }; +} + +function execute( + request: ReconciliationRequest, + operation: DatabaseSyncOperation, + options: { snapshotRoot?: string; keepSnapshots?: number; busyTimeoutMs?: number; rollback?: string }, +): DatabaseSyncCliReport { + if (operation === 'rollback') { + const result = rollbackDatabaseReconciliation(request, options.rollback as string, options); + return { + reportVersion: CLI_REPORT_VERSION, + command: 'database-sync', + operation, + mode: request.mode, + status: result.status, + inputs: unresolvedInputs(request), + plan: null, + apply: null, + rollback: summarizeRollback(result), + }; + } + + let plan: ReturnType; + try { + plan = planDatabaseReconciliation(request); + } catch (caught) { + return reportForPlanningFailure(request, operation, caught); + } + const planReport = summarizePlan(plan); + if (operation === 'dry-run') { + return { + reportVersion: CLI_REPORT_VERSION, + command: 'database-sync', + operation, + mode: request.mode, + status: planReport.status, + inputs: resolvedInputs(plan), + plan: planReport, + apply: null, + rollback: null, + }; + } + + const result = applyDatabaseReconciliationWithSnapshots(plan, options); + return { + reportVersion: CLI_REPORT_VERSION, + command: 'database-sync', + operation, + mode: request.mode, + status: result.status, + inputs: resolvedInputs(plan), + plan: planReport, + apply: summarizeApply(result), + rollback: null, + }; +} + +function cleanupFailureCount(report: DatabaseSyncCliReport): number { + if (report.apply !== null) return report.apply.cleanupFailures.length; + return report.rollback?.cleanupFailures.length ?? 0; +} + +function exitCode(report: DatabaseSyncCliReport): DatabaseSyncExitCode { + if (report.status === 'conflict') return DATABASE_SYNC_EXIT_CODES.conflict; + if (report.status === 'uncertain') return DATABASE_SYNC_EXIT_CODES.uncertain; + if (report.status === 'converged' || report.status === 'recovered') { + return DATABASE_SYNC_EXIT_CODES.recoveryHandled; + } + if ( + report.status === 'operational-failure' || + report.status === 'lock-timeout' || + report.status === 'preimage-changed' || + (report.status === 'rolled-back' && report.operation !== 'rollback') || + report.status === 'expected-postimage' || + report.status === 'partial-commit' || + report.status === 'unexpected-intervening-write' || + cleanupFailureCount(report) > 0 + ) { + return DATABASE_SYNC_EXIT_CODES.operationalFailure; + } + return DATABASE_SYNC_EXIT_CODES.success; +} + +function shortDigest(value: string | null): string { + return value === null ? '-' : value.slice(0, 12); +} + +function changesLine(target: ReconciliationTargetReport): string { + const changed = Object.entries(target.changes) + .filter(([name, count]) => name !== 'deletions' && count > 0) + .map(([name, count]) => `${name}=${count}`); + return ` ${target.role}: ${changed.length === 0 ? 'no changes' : changed.join(', ')}; deletions=0`; +} + +function humanLines(report: DatabaseSyncCliReport): string[] { + const lines = [ + `Database reconciliation ${report.operation}: ${report.status}`, + `Mode: ${report.mode}`, + 'Inputs:', + ...report.inputs.map( + (input) => + ` ${input.role}: database=${shortDigest(input.databaseIdentity)} logical=${shortDigest(input.logicalDigest)}`, + ), + ]; + if (report.plan !== null) { + lines.push(`Plan: ${report.plan.status}; conflicts=${report.plan.conflicts.total}`); + lines.push(...report.plan.targets.map(changesLine)); + if (report.plan.operationalFailure !== null) { + lines.push(`Failure: ${report.plan.operationalFailure.code}`); + if (report.plan.operationalFailure.guidance) lines.push(`Guidance: ${report.plan.operationalFailure.guidance}`); + } + } + if (report.apply !== null) { + lines.push(`Snapshot generation: ${report.apply.generationId ?? '-'}`); + lines.push( + `Recovery: ${report.apply.recovery.status}${report.apply.recovery.generationId === null ? '' : ` (${report.apply.recovery.generationId})`}`, + ); + if (report.apply.failure !== null) lines.push(`Failure: ${report.apply.failure}`); + } + if (report.rollback !== null) { + lines.push(`Selected generation: ${report.rollback.selectedGenerationId}`); + lines.push(`Safety generation: ${report.rollback.safetyGenerationId ?? '-'}`); + if (report.rollback.failure !== null) lines.push(`Failure: ${report.rollback.failure}`); + } + if (report.status === 'converged' || report.status === 'recovered') { + lines.push('Recovery was handled before the requested apply; rerun the same command.'); + } + if (cleanupFailureCount(report) > 0) lines.push(`Cleanup failures: ${cleanupFailureCount(report)}`); + return lines; +} + +function writeReport(report: DatabaseSyncCliReport, json: boolean): void { + process.stdout.write(json ? `${JSON.stringify(report, null, 2)}\n` : `${humanLines(report).join('\n')}\n`); +} + +function failUsage(message: string): never { + process.stderr.write(`Error: ${message}\nRun \`genie db sync --help\` for noninteractive usage.\n`); + process.exit(DATABASE_SYNC_EXIT_CODES.usage); +} + +function handleSync(databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions): void { + try { + const request = parseRequest(databaseA, databaseB, options); + const operation = parseOperation(options); + const snapshotRoot = parseSnapshotRoot(options.snapshotRoot); + const keepSnapshots = parseNonnegativeInteger('--keep-snapshots', options.keepSnapshots); + const busyTimeoutMs = parseNonnegativeInteger('--busy-timeout-ms', options.busyTimeoutMs, MAX_BUSY_TIMEOUT_MS); + const report = execute(request, operation, { + snapshotRoot, + keepSnapshots, + busyTimeoutMs, + rollback: options.rollback, + }); + writeReport(report, options.json ?? false); + process.exitCode = exitCode(report); + } catch (caught) { + if (caught instanceof DatabaseSyncUsageError) failUsage(caught.message); + const report: DatabaseSyncCliReport = { + reportVersion: CLI_REPORT_VERSION, + command: 'database-sync', + operation: options.dryRun ? 'dry-run' : options.rollback === undefined ? 'apply' : 'rollback', + mode: options.source === undefined ? 'bidirectional' : 'directional', + status: 'operational-failure', + inputs: [], + plan: planningFailure(caught), + apply: null, + rollback: null, + }; + writeReport(report, options.json ?? false); + process.exitCode = DATABASE_SYNC_EXIT_CODES.operationalFailure; + } +} + +export function registerV5DatabaseSyncCommand(program: Command): void { + const db = program.command('db').description('Standalone Genie database operations'); + db.command('sync [database-a] [database-b]') + .description('Reconcile two exact-current Genie databases') + .option('--source ', 'Directional source database (authoritative for shared mutable rows)') + .option('--destination ', 'Directional destination database (requires --source)') + .option('--dry-run', 'Validate and plan without locks, snapshots, or writes') + .option('--rollback ', 'Restore a retained generation after snapshotting current state') + .option('--snapshot-root ', 'Override snapshot discovery and publication root') + .option('--keep-snapshots ', 'Retain newest complete generations (default: 3; 0 is same-process only)') + .option('--busy-timeout-ms ', 'Bound the total advisory and SQLite lock wait') + .option('--json', 'Emit the stable bounded JSON report') + .addHelpText( + 'after', + '\nModes:\n' + + ' genie db sync \n' + + ' genie db sync --source --destination \n\n' + + 'Exit codes: 0 success, 1 parser error, 2 usage, 3 conflict, ' + + '4 operational failure, 5 uncertain/manual, 6 recovery handled (rerun).\n', + ) + .action((databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions) => + handleSync(databaseA, databaseB, options), + ); +} From 3fcca5062178e1af05442a0f1cbc4d45a5549308 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 03:42:02 +0000 Subject: [PATCH 16/21] fix(db-sync): reject repeated singleton options --- README.md | 16 ++- scripts/release-docs.test.ts | 10 ++ src/term-commands/v5-db-sync.test.ts | 191 ++++++++++++++++++++++++++- src/term-commands/v5-db-sync.ts | 46 ++++++- 4 files changed, 248 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index cdbcd8353..4917cef25 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,8 @@ All linked worktrees of a repository share one `genie.db`, resolved from the git Use `genie db sync` when two separate Genie repositories or copied `.genie/genie.db` files have diverged. The command is noninteractive and takes -explicit database paths. It never discovers or connects a live shared database. +explicit database paths; it does not silently discover databases. An explicit +path may name a database that active Genie processes also use. Choose exactly one mode: @@ -204,7 +205,9 @@ genie db sync /path/to/left.db /path/to/right.db --dry-run --json Dry-run rejects snapshot, retention, rollback, and busy-timeout options because they have no read-only effect. Ambiguous positional and directional forms also -fail before opening either database for mutation. Run +fail before opening either database for mutation. Every named option is a +singleton: repeating one is an actionable usage error rather than +last-value-wins behavior. Run `genie db sync --help` for the two accepted forms and all options. #### Schema compatibility @@ -281,9 +284,12 @@ process therefore cannot recover or roll back the zero-retention operation. A cleanup failure is reported and returns a nonzero exit even when the logical apply succeeded. -`--busy-timeout-ms N` bounds the total wait for advisory and SQLite write locks. -`N` must be between `0` and `2147483647`. A timeout returns a bounded -operational report; the command does not wait indefinitely. +Mutating runs take canonical-path advisory locks for reconciliation-aware +writers, then SQLite write locks for the databases themselves. These locks let +explicit paths safely name active databases without claiming that unrelated +writers are discovered or refused. `--busy-timeout-ms N` bounds the combined +wait for both lock layers. `N` must be between `0` and `2147483647`. A timeout +returns a bounded operational report; the command does not wait indefinitely. #### Automation report and exit codes diff --git a/scripts/release-docs.test.ts b/scripts/release-docs.test.ts index 1671f49ae..3699af15f 100644 --- a/scripts/release-docs.test.ts +++ b/scripts/release-docs.test.ts @@ -749,6 +749,16 @@ describe('Group E release and documentation contracts', () => { expect(archiveSmoke).toBeGreaterThan(extract); }); + test('database reconciliation docs describe explicit live paths and both bounded lock layers', () => { + const readme = read('README.md'); + expect(readme).toContain('An explicit\npath may name a database that active Genie processes also use.'); + expect(readme).toContain('canonical-path advisory locks'); + expect(readme).toContain('SQLite write locks'); + expect(readme).toContain('bounds the combined\nwait for both lock layers'); + expect(readme).toContain('repeating one is an actionable usage error'); + expect(readme).not.toContain('It never discovers or connects a live shared database.'); + }); + test('shipped Codex integration doc carries the exit matrix, trailer, lease, and candidate-channel contract', () => { const doc = read('plugins/genie/references/codex-integration-map.md'); // Exit matrix (per-command 0/1/2) with the busy code. diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts index 3efbcfda7..0bf3451b2 100644 --- a/src/term-commands/v5-db-sync.test.ts +++ b/src/term-commands/v5-db-sync.test.ts @@ -1,7 +1,8 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { openDb, resolveDbPath } from '../lib/v5/genie-db.js'; @@ -117,6 +118,70 @@ function taskTitles(path: string): string[] { } } +function logicalInventory(path: string): string[] { + const db = new Database(path, { readonly: true }); + try { + const tables = ( + db + .query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() as Array<{ name: string }> + ).map((row) => row.name); + return tables.map( + (table) => `${table}:${JSON.stringify(db.query(`SELECT * FROM "${table}" ORDER BY rowid`).all())}`, + ); + } finally { + db.close(); + } +} + +function sqliteFileInventory(path: string): Record { + return Object.fromEntries( + ['', '-wal', '-shm'].map((suffix) => { + const file = `${path}${suffix}`; + return [ + suffix === '' ? 'database' : suffix.slice(1), + existsSync(file) ? createHash('sha256').update(readFileSync(file)).digest('hex') : null, + ]; + }), + ); +} + +function databaseInventory(path: string): { logical: string[]; files: Record } { + return { logical: logicalInventory(path), files: sqliteFileInventory(path) }; +} + +function rewriteGenerationState(snapshotRoot: string, generationId: string, state: string): void { + const manifestPath = join(snapshotRoot, generationId, 'manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Record; + manifest.state = state; + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); +} + +async function syncWithInjectedReport(report: unknown, ...args: string[]): Promise { + const harness = join(root, 'db-sync-runner.ts'); + const commandUrl = new URL('./v5-db-sync.ts', import.meta.url).href; + writeFileSync( + harness, + [ + "import { Command } from 'commander';", + `import { registerV5DatabaseSyncCommand } from ${JSON.stringify(commandUrl)};`, + `const report = ${JSON.stringify(report)};`, + "const program = new Command().name('genie');", + 'registerV5DatabaseSyncCommand(program, { execute: () => report });', + "await program.parseAsync(['node', 'genie', 'db', 'sync', ...process.argv.slice(2)]);", + ].join('\n'), + ); + const proc = Bun.spawn([process.execPath, harness, ...args], { + cwd: root, + stdout: 'pipe', + stderr: 'pipe', + env: { ...process.env, NO_COLOR: '1' }, + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + return { stdout, stderr, code: await proc.exited }; +} + /** Exact pre-lanes, pre-runtime user_version=1 shape accepted by normal current open. */ function seedPriorDatabase(path: string): void { mkdirSync(dirname(path), { recursive: true }); @@ -230,9 +295,12 @@ describe('database sync CLI contract', () => { const left = createRepo('left'); const right = createRepo('right'); insertTask(left.database, 't_left', 'left'); - const beforeLeft = readFileSync(left.database); - const beforeRight = readFileSync(right.database); + const beforeLeft = databaseInventory(left.database); + const beforeRight = databaseInventory(right.database); const snapshotRoot = join(root, 'snapshots'); + const otherSnapshotRoot = join(root, 'other-snapshots'); + const generationA = `${'a'.repeat(64)}--0000000000000001--00000000-0000-4000-8000-000000000001`; + const generationB = `${'b'.repeat(64)}--0000000000000002--00000000-0000-4000-8000-000000000002`; const cases = [ [left.database, '--snapshot-root', snapshotRoot], ['--source', left.database, '--snapshot-root', snapshotRoot], @@ -251,17 +319,114 @@ describe('database sync CLI contract', () => { [left.database, right.database, '--busy-timeout-ms', '2147483648', '--snapshot-root', snapshotRoot], [left.database, right.database, '--snapshot-root', 'relative'], [left.database, right.database, '--rollback', 'not-a-generation', '--snapshot-root', snapshotRoot], + ['--source', left.database, '--source', right.database, '--destination', right.database], + ['--source', left.database, '--destination', right.database, '--destination', left.database], + [left.database, right.database, '--snapshot-root', snapshotRoot, '--snapshot-root', otherSnapshotRoot], + [left.database, right.database, '--rollback', generationA, '--rollback', generationB], + [left.database, right.database, '--keep-snapshots', '1', '--keep-snapshots', '2'], + [left.database, right.database, '--busy-timeout-ms', '0', '--busy-timeout-ms', '1'], + [left.database, right.database, '--dry-run', '--dry-run'], + [left.database, right.database, '--json', '--json'], ]; - for (const args of cases) { + for (const [index, args] of cases.entries()) { const result = await sync(...args); expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.usage); expect(result.stdout).toBe(''); expect(result.stderr).toContain('genie db sync --help'); + if (index >= 8) expect(result.stderr).toContain('may be specified only once'); } - expect(readFileSync(left.database)).toEqual(beforeLeft); - expect(readFileSync(right.database)).toEqual(beforeRight); + expect(databaseInventory(left.database)).toEqual(beforeLeft); + expect(databaseInventory(right.database)).toEqual(beforeRight); expect(existsSync(snapshotRoot)).toBe(false); + expect(existsSync(otherSnapshotRoot)).toBe(false); + }); + + test('Commander syntax errors retain parser exit 1', async () => { + const result = await sync('--unknown-option'); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.parserError); + expect(result.stdout).toBe(''); + expect(result.stderr).toContain("unknown option '--unknown-option'"); + }); + + test('human output and same-database success remain stable and path-free', async () => { + const database = createRepo('same').database; + insertTask(database, 't_hostile', 'HOSTILE SAME-DATABASE TITLE'); + + const result = await sync(database, database, '--dry-run'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(result.stderr).toBe(''); + expect(result.stdout).toContain('Database reconciliation dry-run: no-op'); + expect(result.stdout).toContain('Mode: bidirectional'); + expect(result.stdout).toContain('Plan: no-op; conflicts=0'); + expect(result.stdout).not.toContain(database); + expect(result.stdout).not.toContain('HOSTILE SAME-DATABASE TITLE'); + }); + + test('persistent recovery maps converged to exit 6 and hostile uncertain state to exit 5', async () => { + const left = createRepo('recovery-left'); + const right = createRepo('recovery-right'); + insertTask(left.database, 't_left', 'left'); + insertTask(right.database, 't_right', 'right'); + const snapshotRoot = join(root, 'recovery-snapshots'); + + const applied = await sync(left.database, right.database, '--snapshot-root', snapshotRoot, '--json'); + const generationId = parseJson(applied).apply?.generationId; + if (generationId == null) throw new Error('Expected a durable generation.'); + rewriteGenerationState(snapshotRoot, generationId, 'complete'); + + const converged = await sync(left.database, right.database, '--snapshot-root', snapshotRoot, '--json'); + expect(converged.code).toBe(DATABASE_SYNC_EXIT_CODES.recoveryHandled); + expect(parseJson(converged).status).toBe('converged'); + + rewriteGenerationState(snapshotRoot, generationId, 'complete'); + insertTask(left.database, 't_hostile', 'HOSTILE RECOVERY CONTENT\nMUST NOT LEAK'); + const beforeLeft = databaseInventory(left.database); + const beforeRight = databaseInventory(right.database); + + const uncertain = await sync(left.database, right.database, '--snapshot-root', snapshotRoot, '--json'); + expect(uncertain.code).toBe(DATABASE_SYNC_EXIT_CODES.uncertain); + expect(uncertain.stderr).toBe(''); + expect(uncertain.stdout).not.toContain('HOSTILE RECOVERY CONTENT'); + expect(uncertain.stdout.length).toBeLessThan(8_000); + expect(parseJson(uncertain).status).toBe('uncertain'); + expect(databaseInventory(left.database)).toEqual(beforeLeft); + expect(databaseInventory(right.database)).toEqual(beforeRight); + }); + + test('cleanup failures map an otherwise successful subprocess report to exit 4', async () => { + const report = { + reportVersion: 1, + command: 'database-sync', + operation: 'apply', + mode: 'bidirectional', + status: 'changed', + inputs: [], + plan: null, + apply: { + status: 'changed', + generationId: null, + recovery: { + status: 'none', + generationId: null, + restoredDatabaseIdentities: [], + failure: null, + cleanupFailures: [], + }, + apply: null, + failure: null, + cleanupFailures: ['snapshot-cleanup-failed'], + }, + rollback: null, + }; + + const result = await syncWithInjectedReport(report, '/unused/left.db', '/unused/right.db'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + expect(result.stderr).toBe(''); + expect(result.stdout).toContain('Database reconciliation apply: changed'); + expect(result.stdout).toContain('Cleanup failures: 1'); }); test('directional dry-run reports bounded identities and counts without row content or writes', async () => { @@ -299,6 +464,20 @@ describe('database sync CLI contract', () => { expect(existsSync(join(destination.repo, '.genie', 'sync-snapshots'))).toBe(false); }); + test('operational reports omit hostile input paths and remain bounded', async () => { + const destination = createRepo('operational-destination'); + const hostile = join(root, 'HOSTILE PATH\nMUST NOT LEAK.db'); + + const result = await sync('--source', hostile, '--destination', destination.database, '--dry-run', '--json'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + expect(result.stderr).toBe(''); + expect(result.stdout).not.toContain('HOSTILE PATH'); + expect(result.stdout).not.toContain(hostile); + expect(result.stdout.length).toBeLessThan(8_000); + expect(parseJson(result).plan?.operationalFailure?.code).toBe('input-unavailable'); + }); + test('bidirectional apply converges disjoint additions, never deletes, and reports a durable generation', async () => { const left = createRepo('left'); const right = createRepo('right'); diff --git a/src/term-commands/v5-db-sync.ts b/src/term-commands/v5-db-sync.ts index 39ec369bb..9c76c7069 100644 --- a/src/term-commands/v5-db-sync.ts +++ b/src/term-commands/v5-db-sync.ts @@ -119,8 +119,37 @@ interface DatabaseSyncCliReport { rollback: RollbackReport | null; } +interface DatabaseSyncCommandDependencies { + readonly execute?: typeof execute; +} + class DatabaseSyncUsageError extends Error {} +const SINGLETON_OPTIONS = new Set([ + '--source', + '--destination', + '--dry-run', + '--rollback', + '--snapshot-root', + '--keep-snapshots', + '--busy-timeout-ms', + '--json', +]); + +function rejectRepeatedOptions(args: readonly string[]): void { + const counts = new Map(); + for (const argument of args) { + if (argument === '--') break; + const name = argument.split('=', 1)[0]; + if (!SINGLETON_OPTIONS.has(name)) continue; + const count = (counts.get(name) ?? 0) + 1; + counts.set(name, count); + if (count > 1) { + throw new DatabaseSyncUsageError(`${name} may be specified only once.`); + } + } +} + function databaseIdentity(path: string): string { const absolute = resolve(path); let canonical = absolute; @@ -453,14 +482,20 @@ function failUsage(message: string): never { process.exit(DATABASE_SYNC_EXIT_CODES.usage); } -function handleSync(databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions): void { +function handleSync( + databaseA: string | undefined, + databaseB: string | undefined, + options: DatabaseSyncOptions, + dependencies: DatabaseSyncCommandDependencies, +): void { try { + rejectRepeatedOptions(process.argv.slice(2)); const request = parseRequest(databaseA, databaseB, options); const operation = parseOperation(options); const snapshotRoot = parseSnapshotRoot(options.snapshotRoot); const keepSnapshots = parseNonnegativeInteger('--keep-snapshots', options.keepSnapshots); const busyTimeoutMs = parseNonnegativeInteger('--busy-timeout-ms', options.busyTimeoutMs, MAX_BUSY_TIMEOUT_MS); - const report = execute(request, operation, { + const report = (dependencies.execute ?? execute)(request, operation, { snapshotRoot, keepSnapshots, busyTimeoutMs, @@ -486,7 +521,10 @@ function handleSync(databaseA: string | undefined, databaseB: string | undefined } } -export function registerV5DatabaseSyncCommand(program: Command): void { +export function registerV5DatabaseSyncCommand( + program: Command, + dependencies: DatabaseSyncCommandDependencies = {}, +): void { const db = program.command('db').description('Standalone Genie database operations'); db.command('sync [database-a] [database-b]') .description('Reconcile two exact-current Genie databases') @@ -507,6 +545,6 @@ export function registerV5DatabaseSyncCommand(program: Command): void { '4 operational failure, 5 uncertain/manual, 6 recovery handled (rerun).\n', ) .action((databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions) => - handleSync(databaseA, databaseB, options), + handleSync(databaseA, databaseB, options, dependencies), ); } From 0f2c799dddbad61280e2a791c9e8b0f77e8f63bf Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 16:07:29 +0800 Subject: [PATCH 17/21] feat(db-sync): propagate roster deletion tombstones --- README.md | 14 ++++-- src/lib/v5/db-reconciliation.test.ts | 62 ++++++++++++++++++++++++ src/lib/v5/db-reconciliation.ts | 66 ++++++++++++++++++++++++-- src/lib/v5/reconciliation-tombstone.ts | 66 ++++++++++++++++++++++++++ src/lib/v5/task-state.test.ts | 28 +++++++++++ src/lib/v5/task-state.ts | 43 +++++++++++------ src/term-commands/ui-bridge.ts | 2 +- src/term-commands/v5-db-sync.test.ts | 28 ++++++++++- src/term-commands/v5-db-sync.ts | 2 +- 9 files changed, 285 insertions(+), 26 deletions(-) create mode 100644 src/lib/v5/reconciliation-tombstone.ts diff --git a/README.md b/README.md index 4917cef25..df17520cc 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,17 @@ Bidirectional mode unions additions. It reports a conflict when the same mutable key differs because the databases do not carry ancestry that could prove which row is newer. Directional mode makes the source authoritative for shared mutable task, board, wish-group, hire-roster, and unknown metadata -keys. It writes only the destination. Both modes preserve destination-only -rows: absence never means deletion, and every report fixes the deletion count -at zero. +keys. It writes only the destination. Both modes preserve destination-only rows: +ordinary absence never means deletion. + +An explicit roster unhire (the `roster_unhire` UI-bridge operation) is the +exception. It records a versioned tombstone in the existing metadata table, so +a later sync removes the matching hire-roster row instead of resurrecting it. +This changes neither the SQLite schema nor its `user_version`; reconciliation +reports the explicit removal in its deletion count. Tombstones are durable and +deletion wins while any replica retains one. Re-hiring the same agent clears +the local tombstone, but an offline replica with the old tombstone can reassert +the deletion when it returns. Dependency edges are unioned. Task events and legacy stage-log entries preserve the maximum occurrence count observed on either applicable side. Independent diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index 2106c1c3c..85c681a16 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -32,6 +32,8 @@ import { withLockedReconciliationDatabases, } from './db-reconciliation.js'; import { openDb } from './genie-db.js'; +import { RECONCILIATION_TOMBSTONE_PREFIX, reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; +import { unhireAgent } from './task-state.js'; let fixtureRoot: string; @@ -840,6 +842,66 @@ describe('keyed, edge-set, and history-multiset planning', () => { expect(plan.report.targets[0].changes.deletions).toBe(0); }); + test('a roster tombstone propagates a guest deletion instead of resurrecting the live host row', () => { + const host = currentDb('host'); + const guest = currentDb('guest'); + for (const path of [host, guest]) { + mutate(path, (db) => { + db.query( + `INSERT INTO hire_roster + (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('w', 'agent', 'worker', '/wt/agent', 1, 'hired')`, + ).run(); + }); + } + mutate(guest, (db) => expect(unhireAgent(db, 'w', 'agent')).toBe(true)); + + const plan = planDatabaseReconciliation(bidirectional(host, guest)); + + expect(plan.status).toBe('changed'); + expect(target(plan, 'left').changes.deletions).toEqual([ + { table: 'hire_roster', wish: 'w', agentAdapterId: 'agent' }, + ]); + expect(target(plan, 'right').changes.deletions).toEqual([]); + expect(plan.report.targets.map(({ changes }) => changes.deletions)).toEqual([1, 0]); + expect(applyDatabaseReconciliation(plan)).toMatchObject({ status: 'changed', converged: true }); + for (const path of [host, guest]) { + mutate(path, (db) => { + expect(db.query('SELECT * FROM hire_roster').all()).toEqual([]); + expect( + db.query('SELECT count(*) AS count FROM meta WHERE key LIKE ?').get(`${RECONCILIATION_TOMBSTONE_PREFIX}%`), + ).toEqual({ count: 1 }); + }); + } + expect(planDatabaseReconciliation(bidirectional(host, guest)).status).toBe('no-op'); + }); + + test('malformed tombstones and a live-row contradiction fail closed before planning', () => { + const peer = currentDb('peer'); + const malformed = currentDb('malformed-tombstone'); + mutate(malformed, (db) => { + db.query('INSERT INTO meta (key, value) VALUES (?, ?)').run( + `${RECONCILIATION_TOMBSTONE_PREFIX}hire_roster:not-canonical-base64`, + 'deleted', + ); + }); + expect(dryRunDatabaseReconciliation(bidirectional(malformed, peer)).operationalFailure?.code).toBe('invalid-data'); + + const contradictory = currentDb('contradictory-tombstone'); + mutate(contradictory, (db) => { + db.query( + `INSERT INTO hire_roster + (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('w', 'agent', NULL, '/wt/agent', 1, 'hired')`, + ).run(); + const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish: 'w', agentAdapterId: 'agent' }); + db.query('INSERT INTO meta (key, value) VALUES (?, ?)').run(tombstone.key, tombstone.value); + }); + expect(dryRunDatabaseReconciliation(bidirectional(contradictory, peer)).operationalFailure?.code).toBe( + 'invalid-data', + ); + }); + test('a cross-image uniqueness collision conflicts instead of planning an invalid target', () => { const source = currentDb('source'); const destination = currentDb('destination'); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 45f61ef7c..91e4c3128 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -27,6 +27,7 @@ import { tmpdir } from 'node:os'; import { join, normalize } from 'node:path'; import { linuxLibcCandidates } from '../install-transaction.js'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; +import { type ReconciliationTombstone, parseReconciliationTombstoneMeta } from './reconciliation-tombstone.js'; import { BUSY_TIMEOUT_MS, isBusyError } from './sqlite-open.js'; const PLAN_VERSION = 1 as const; @@ -225,6 +226,7 @@ export interface ReconciliationTargetChanges { readonly taskDependencies: readonly TaskDependencyReconciliationRow[]; readonly stageLog: readonly HistoryAddition[]; readonly taskEvents: readonly HistoryAddition[]; + readonly deletions: readonly ReconciliationTombstone[]; } export interface ReconciliationConflict { @@ -262,7 +264,7 @@ export interface ReconciliationChangeCounts { readonly taskDependencies: number; readonly stageLog: number; readonly taskEvents: number; - readonly deletions: 0; + readonly deletions: number; } export interface ReconciliationTargetReport { @@ -1629,6 +1631,7 @@ function readLogicalState(db: Database): LogicalState { stageLog: countValues(stages, stageLogKey), taskEvents: countValues(events, taskEventKey), }; + validateInputTombstones(state); if (!dependencyGraphIsAcyclic(state.taskDependencies)) { throw error('invalid-data', 'A reconciliation input contains an invalid task dependency graph.'); } @@ -1732,6 +1735,39 @@ function taskEventKey(row: TaskEventReconciliationValue): string { return canonicalTuple([row.taskId, row.kind, row.note, row.authorKind, row.author, row.createdAt]); } +function reconciliationTombstones(state: LogicalState): ReconciliationTombstone[] { + const tombstones: ReconciliationTombstone[] = []; + try { + for (const row of state.meta.values()) { + const tombstone = parseReconciliationTombstoneMeta(row.key, row.value); + if (tombstone !== null) tombstones.push(tombstone); + } + } catch { + throw error('invalid-data', 'A reconciliation input contains invalid tombstone metadata.'); + } + return tombstones.sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); +} + +function tombstoneKey(tombstone: ReconciliationTombstone): string { + return canonicalTuple([tombstone.wish, tombstone.agentAdapterId]); +} + +function tombstoneHasLiveRow(state: LogicalState, tombstone: ReconciliationTombstone): boolean { + return state.hireRoster.has(tombstoneKey(tombstone)); +} + +function validateInputTombstones(state: LogicalState): void { + if (reconciliationTombstones(state).some((tombstone) => tombstoneHasLiveRow(state, tombstone))) { + throw error('invalid-data', 'A reconciliation input contains both a tombstone and its live row.'); + } +} + +function applyTombstonesToState(state: LogicalState): void { + for (const tombstone of reconciliationTombstones(state)) { + state.hireRoster.delete(tombstoneKey(tombstone)); + } +} + function boardValues(row: BoardReconciliationRow): readonly (string | bigint | null)[] { return [row.id, row.name, row.createdAt, row.lanes]; } @@ -2143,6 +2179,8 @@ function reconcileStates( } if (conflicts.length === 0) { + applyTombstonesToState(left); + applyTombstonesToState(right); if (mode === 'bidirectional') validatePlannedTargetIntegrity(left, 'left', conflicts); validatePlannedTargetIntegrity(right, mode === 'bidirectional' ? 'right' : 'destination', conflicts); } @@ -2181,6 +2219,12 @@ function historyAdditions( .map(({ value, count }) => ({ value, count })); } +function targetDeletions(current: LogicalState, target: LogicalState): ReconciliationTombstone[] { + return reconciliationTombstones(target).filter( + (tombstone) => tombstoneHasLiveRow(current, tombstone) && !tombstoneHasLiveRow(target, tombstone), + ); +} + function targetChanges(current: LogicalState, target: LogicalState): ReconciliationTargetChanges { return { boards: changedRows(current.boards, target.boards, boardValues), @@ -2191,6 +2235,7 @@ function targetChanges(current: LogicalState, target: LogicalState): Reconciliat taskDependencies: addedSetRows(current.taskDependencies, target.taskDependencies), stageLog: historyAdditions(current.stageLog, target.stageLog), taskEvents: historyAdditions(current.taskEvents, target.taskEvents), + deletions: targetDeletions(current, target), }; } @@ -2204,6 +2249,7 @@ function emptyChanges(): ReconciliationTargetChanges { taskDependencies: [], stageLog: [], taskEvents: [], + deletions: [], }; } @@ -2217,12 +2263,12 @@ function changeCounts(changes: ReconciliationTargetChanges): ReconciliationChang taskDependencies: changes.taskDependencies.length, stageLog: changes.stageLog.reduce((total, addition) => total + addition.count, 0), taskEvents: changes.taskEvents.reduce((total, addition) => total + addition.count, 0), - deletions: 0, + deletions: changes.deletions.length, }; } function hasChanges(changes: ReconciliationTargetChanges): boolean { - return Object.entries(changeCounts(changes)).some(([name, count]) => name !== 'deletions' && count > 0); + return Object.values(changeCounts(changes)).some((count) => count > 0); } function deepFreeze(value: T): T { @@ -2912,9 +2958,18 @@ function applyMeta(db: Database, rows: readonly MetaReconciliationRow[]): void { for (const row of rows) upsert.run(...metaValues(row)); } +function applyDeletions(db: Database, tombstones: readonly ReconciliationTombstone[]): void { + const deleteHire = db.query('DELETE FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?'); + for (const tombstone of tombstones) { + deleteHire.run(tombstone.wish, tombstone.agentAdapterId); + } +} + function applyTargetChanges(db: Database, changes: ReconciliationTargetChanges): void { - // Parent rows precede dependents. The backfill marker is written last so its - // logical coverage invariant is never transiently asserted ahead of history. + // Tombstoned independent rows are removed before upserts. Parent rows then + // precede dependents. The backfill marker is written last so its logical + // coverage invariant is never transiently asserted ahead of history. + applyDeletions(db, changes.deletions); applyBoards(db, changes.boards); applyTasks(db, changes.tasks); applyWishGroups(db, changes.wishGroups); @@ -2935,6 +2990,7 @@ function completeStateChanges(state: LogicalState): ReconciliationTargetChanges taskDependencies: [...state.taskDependencies.values()], stageLog: [...state.stageLog.values()].map(({ value, count }) => ({ value, count })), taskEvents: [...state.taskEvents.values()].map(({ value, count }) => ({ value, count })), + deletions: [], }; } diff --git a/src/lib/v5/reconciliation-tombstone.ts b/src/lib/v5/reconciliation-tombstone.ts new file mode 100644 index 000000000..0e718bc77 --- /dev/null +++ b/src/lib/v5/reconciliation-tombstone.ts @@ -0,0 +1,66 @@ +import { Buffer } from 'node:buffer'; + +export const RECONCILIATION_TOMBSTONE_PREFIX = 'genie:reconciliation-tombstone:v1:'; +export const RECONCILIATION_TOMBSTONE_VALUE = 'deleted'; + +const MAX_ENCODED_KEY_BYTES = 4_096; + +export interface ReconciliationTombstone { + readonly table: 'hire_roster'; + readonly wish: string; + readonly agentAdapterId: string; +} + +export interface ReconciliationTombstoneMeta { + readonly key: string; + readonly value: typeof RECONCILIATION_TOMBSTONE_VALUE; +} + +function tombstoneParts(tombstone: ReconciliationTombstone): readonly string[] { + return [tombstone.wish, tombstone.agentAdapterId]; +} + +function encodedParts(parts: readonly string[]): string { + const encoded = Buffer.from(JSON.stringify(parts), 'utf8').toString('base64url'); + if (Buffer.byteLength(encoded, 'utf8') > MAX_ENCODED_KEY_BYTES) invalidTombstone(); + return encoded; +} + +export function reconciliationTombstoneMeta(tombstone: ReconciliationTombstone): ReconciliationTombstoneMeta { + return { + key: `${RECONCILIATION_TOMBSTONE_PREFIX}${tombstone.table}:${encodedParts(tombstoneParts(tombstone))}`, + value: RECONCILIATION_TOMBSTONE_VALUE, + }; +} + +function invalidTombstone(): never { + throw new Error('Invalid reconciliation tombstone metadata.'); +} + +function decodedParts(encoded: string): readonly [string, string] { + if (encoded.length === 0 || Buffer.byteLength(encoded, 'utf8') > MAX_ENCODED_KEY_BYTES) invalidTombstone(); + let parsed: unknown; + try { + const decoded = Buffer.from(encoded, 'base64url').toString('utf8'); + parsed = JSON.parse(decoded); + if (Buffer.from(decoded, 'utf8').toString('base64url') !== encoded) invalidTombstone(); + } catch { + invalidTombstone(); + } + if (!Array.isArray(parsed) || parsed.length !== 2 || parsed.some((part) => typeof part !== 'string')) { + invalidTombstone(); + } + return parsed as [string, string]; +} + +export function parseReconciliationTombstoneMeta(key: string, value: string): ReconciliationTombstone | null { + if (!key.startsWith(RECONCILIATION_TOMBSTONE_PREFIX)) return null; + if (value !== RECONCILIATION_TOMBSTONE_VALUE) invalidTombstone(); + const suffix = key.slice(RECONCILIATION_TOMBSTONE_PREFIX.length); + const separator = suffix.indexOf(':'); + if (separator < 1) invalidTombstone(); + const table = suffix.slice(0, separator); + const [first, second] = decodedParts(suffix.slice(separator + 1)); + if (table === 'hire_roster') return { table, wish: first, agentAdapterId: second }; + return invalidTombstone(); +} diff --git a/src/lib/v5/task-state.test.ts b/src/lib/v5/task-state.test.ts index 0a466e99e..f24325c8b 100644 --- a/src/lib/v5/task-state.test.ts +++ b/src/lib/v5/task-state.test.ts @@ -4,6 +4,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { ensureSchema, openDb } from './genie-db.js'; +import { reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; import { CheckoutConflictError, CycleError, @@ -653,6 +654,33 @@ describe('hire roster (single-row upsert / delete)', () => { expect(unhireAgent(db, 'w', 'a')).toBe(false); }); + test('unhire records a schema-neutral tombstone and explicit re-hire clears it', () => { + const schemaVersion = db.query('PRAGMA user_version').get() as { user_version: number }; + const tableNames = ( + db + .query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() as Array<{ name: string }> + ).map(({ name }) => name); + const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish: 'w', agentAdapterId: 'a' }); + + expect(unhireAgent(db, 'w', 'a')).toBe(false); + expect(db.query('SELECT value FROM meta WHERE key = ?').get(tombstone.key)).toEqual({ + value: tombstone.value, + }); + expect(db.query('PRAGMA user_version').get()).toEqual(schemaVersion); + expect( + ( + db + .query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() as Array<{ name: string }> + ).map(({ name }) => name), + ).toEqual(tableNames); + + hireAgent(db, { wish: 'w', agentAdapterId: 'a', worktree: '/wt/re-hired' }); + expect(db.query('SELECT value FROM meta WHERE key = ?').get(tombstone.key)).toBeNull(); + expect(getHire(db, 'w', 'a')?.worktree).toBe('/wt/re-hired'); + }); + test('listHires scopes by wish and orders stably', () => { hireAgent(db, { wish: 'w1', agentAdapterId: 'b', worktree: '/wt/b' }); hireAgent(db, { wish: 'w1', agentAdapterId: 'a', worktree: '/wt/a' }); diff --git a/src/lib/v5/task-state.ts b/src/lib/v5/task-state.ts index 88a8f11f6..526a5f635 100644 --- a/src/lib/v5/task-state.ts +++ b/src/lib/v5/task-state.ts @@ -13,6 +13,7 @@ import type { Database } from 'bun:sqlite'; import { createHash, randomBytes } from 'node:crypto'; +import { reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; // ============================================================================ // Type boundaries @@ -1294,31 +1295,43 @@ function mapHire(row: RawHire): HireRosterRow { * `(wish, agent_adapter_id)`: a re-hire refreshes profile/worktree/state but * preserves the original `hired_at` by OMITTING `hired_at` from the `ON CONFLICT * DO UPDATE SET` list — an unset column keeps its stored value, so the first - * hire's timestamp survives every re-hire and the call converges on one row. A - * single statement is atomic on its own; the WAL + busy_timeout the handle - * carries (see sqlite-open.ts) serializes it against concurrent writers. + * hire's timestamp survives every re-hire and the call converges on one row. + * The transaction also clears this replica's reconciliation tombstone. */ export function hireAgent(db: Database, input: HireAgentInput): HireRosterRow { const now = Date.now(); const state = input.state ?? 'hired'; - db.query( - `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(wish, agent_adapter_id) DO UPDATE SET - profile = excluded.profile, - worktree = excluded.worktree, - state = excluded.state`, - ).run(input.wish, input.agentAdapterId, input.profile ?? null, input.worktree, now, state); + const tombstone = reconciliationTombstoneMeta({ + table: 'hire_roster', + wish: input.wish, + agentAdapterId: input.agentAdapterId, + }); + db.transaction(() => { + db.query('DELETE FROM meta WHERE key = ?').run(tombstone.key); + db.query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(wish, agent_adapter_id) DO UPDATE SET + profile = excluded.profile, + worktree = excluded.worktree, + state = excluded.state`, + ).run(input.wish, input.agentAdapterId, input.profile ?? null, input.worktree, now, state); + })(); return getHire(db, input.wish, input.agentAdapterId) as HireRosterRow; } /** - * Unhire an agent adapter from a wish. Idempotent single-row delete: removing an - * absent hire is a no-op that returns false; a real removal returns true. + * Unhire an agent adapter from a wish. The transaction records a durable + * reconciliation tombstone even when the live row is already absent. The + * boolean reports whether this call removed a live local row. */ export function unhireAgent(db: Database, wish: string, agentAdapterId: string): boolean { - const res = db.query('DELETE FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?').run(wish, agentAdapterId); - return res.changes > 0; + const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish, agentAdapterId }); + return db.transaction(() => { + const res = db.query('DELETE FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?').run(wish, agentAdapterId); + db.query('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)').run(tombstone.key, tombstone.value); + return res.changes > 0; + })(); } export function getHire(db: Database, wish: string, agentAdapterId: string): HireRosterRow | null { diff --git a/src/term-commands/ui-bridge.ts b/src/term-commands/ui-bridge.ts index ed9d8be4f..1e8927a16 100644 --- a/src/term-commands/ui-bridge.ts +++ b/src/term-commands/ui-bridge.ts @@ -156,7 +156,7 @@ export function buildRosterTools(getWriteDb: () => import('bun:sqlite').Database }, { name: 'roster_unhire', - description: 'Remove an agent adapter from a wish (idempotent; removed=false when absent).', + description: 'Remove an agent adapter and retain a sync tombstone (idempotent; removed=false when absent).', inputSchema: { type: 'object', properties: { diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts index 0bf3451b2..c800c5543 100644 --- a/src/term-commands/v5-db-sync.test.ts +++ b/src/term-commands/v5-db-sync.test.ts @@ -6,7 +6,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { openDb, resolveDbPath } from '../lib/v5/genie-db.js'; -import { createTask, getTask, listTasks } from '../lib/v5/task-state.js'; +import { createTask, getTask, hireAgent, listHires, listTasks, unhireAgent } from '../lib/v5/task-state.js'; import { DATABASE_SYNC_EXIT_CODES } from './v5-db-sync.js'; const GENIE = join(import.meta.dir, '..', 'genie.ts'); @@ -501,6 +501,32 @@ describe('database sync CLI contract', () => { expect(parseJson(noOp).status).toBe('no-op'); }); + test('bidirectional apply propagates an explicit roster tombstone without a schema migration', async () => { + const host = createRepo('tombstone-host'); + const guest = createRepo('tombstone-guest'); + const hostDb = openDb({ path: host.database }); + hireAgent(hostDb, { wish: 'w', agentAdapterId: 'agent', worktree: '/wt/host' }); + hostDb.close(); + const guestDb = openDb({ path: guest.database }); + expect(unhireAgent(guestDb, 'w', 'agent')).toBe(false); + guestDb.close(); + + const preview = await sync(host.database, guest.database, '--dry-run'); + expect(preview.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(preview.stdout).toContain('left: meta=1; deletions=1'); + + const result = await sync(host.database, guest.database, '--json'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(result).plan?.targets.map((target) => target.changes.deletions)).toEqual([1, 0]); + for (const path of [host.database, guest.database]) { + const db = openDb({ path }); + expect(listHires(db, 'w')).toEqual([]); + expect((db.query('PRAGMA user_version').get() as { user_version: number }).user_version).toBeGreaterThan(0); + db.close(); + } + }); + test('bidirectional mutable conflicts use exit 3, bounded digests/counts, and zero mutation', async () => { const left = createRepo('left'); const right = createRepo('right'); diff --git a/src/term-commands/v5-db-sync.ts b/src/term-commands/v5-db-sync.ts index 9c76c7069..cd294533d 100644 --- a/src/term-commands/v5-db-sync.ts +++ b/src/term-commands/v5-db-sync.ts @@ -433,7 +433,7 @@ function changesLine(target: ReconciliationTargetReport): string { const changed = Object.entries(target.changes) .filter(([name, count]) => name !== 'deletions' && count > 0) .map(([name, count]) => `${name}=${count}`); - return ` ${target.role}: ${changed.length === 0 ? 'no changes' : changed.join(', ')}; deletions=0`; + return ` ${target.role}: ${changed.length === 0 ? 'no changes' : changed.join(', ')}; deletions=${target.changes.deletions}`; } function humanLines(report: DatabaseSyncCliReport): string[] { From 9711b1597d6a9b7a4605925579fcd4d2a18d5cfe Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 29 Jul 2026 18:27:30 +0800 Subject: [PATCH 18/21] test(db-sync): normalize secure fixture modes --- .../update-command-publication.test.ts | 18 +++++------ src/genie-commands/__tests__/update.test.ts | 3 ++ .../local-delivery-repair.test.ts | 2 +- src/lib/agent-sync.test.ts | 30 +++++++++++++++++-- src/lib/v5/db-sync-snapshots.test.ts | 10 +++---- tests/support/codex-dogfood-harness.ts | 4 +-- .../support/update-current-boundary-runner.ts | 6 ++-- 7 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/genie-commands/__tests__/update-command-publication.test.ts b/src/genie-commands/__tests__/update-command-publication.test.ts index daca995f7..275bb36ba 100644 --- a/src/genie-commands/__tests__/update-command-publication.test.ts +++ b/src/genie-commands/__tests__/update-command-publication.test.ts @@ -46,15 +46,15 @@ function buildReleasePayload( } { const payload = join(root, 'payload'); for (const directory of ['.agents', '.claude-plugin', 'plugins/genie', 'skills/review', 'templates']) { - mkdirSync(join(payload, directory), { recursive: true }); + mkdirSync(join(payload, directory), { recursive: true, mode: 0o755 }); } - writeFileSync(join(payload, '.agents', 'plugin.json'), '{}\n'); - writeFileSync(join(payload, '.claude-plugin', 'marketplace.json'), '{}\n'); - writeFileSync(join(payload, 'LICENSE'), 'test fixture\n'); - writeFileSync(join(payload, 'VERSION'), `${version}\n`); - writeFileSync(join(payload, 'plugins', 'genie', 'plugin.txt'), 'authenticated plugin payload\n'); - writeFileSync(join(payload, 'skills', 'review', 'SKILL.md'), '# Review\n'); - writeFileSync(join(payload, 'templates', 'template.txt'), 'template\n'); + writeFileSync(join(payload, '.agents', 'plugin.json'), '{}\n', { mode: 0o644 }); + writeFileSync(join(payload, '.claude-plugin', 'marketplace.json'), '{}\n', { mode: 0o644 }); + writeFileSync(join(payload, 'LICENSE'), 'test fixture\n', { mode: 0o644 }); + writeFileSync(join(payload, 'VERSION'), `${version}\n`, { mode: 0o644 }); + writeFileSync(join(payload, 'plugins', 'genie', 'plugin.txt'), 'authenticated plugin payload\n', { mode: 0o644 }); + writeFileSync(join(payload, 'skills', 'review', 'SKILL.md'), '# Review\n', { mode: 0o644 }); + writeFileSync(join(payload, 'templates', 'template.txt'), 'template\n', { mode: 0o644 }); writeExecutable( join(payload, 'genie'), `#!/bin/sh\nif [ "\${1:-}" = "--version" ]; then printf 'genie ${version}\\n'; exit 0; fi\nexit 0\n`, @@ -77,7 +77,7 @@ describe('updateCommand publication boundary', () => { const bin = join(genieHome, 'bin'); const fakeBin = join(root, 'fake-bin'); const fixture = join(root, 'fixture'); - mkdirSync(bin, { recursive: true }); + mkdirSync(bin, { recursive: true, mode: 0o755 }); mkdirSync(fakeBin); mkdirSync(fixture); diff --git a/src/genie-commands/__tests__/update.test.ts b/src/genie-commands/__tests__/update.test.ts index 280f42410..1c864ed49 100644 --- a/src/genie-commands/__tests__/update.test.ts +++ b/src/genie-commands/__tests__/update.test.ts @@ -2455,6 +2455,9 @@ describe('manual post-update convergence (2026-07-11 cascade regression)', () => const result = runManualUpdateConvergence({ expectedVersion: '5.260711.3', bundleRoot: '/tmp/verified-bundle', + // Explicit selection: the default reads the host's persisted integration + // consent, so omitting it makes the test depend on machine state (#2732). + selection: 'all', runSync: () => calls.push('parent-safe-sync'), refreshPlugins: (options) => { calls.push(`parent-plugin-refresh:${options.expectedVersion}:${options.selection}`); diff --git a/src/genie-commands/local-delivery-repair.test.ts b/src/genie-commands/local-delivery-repair.test.ts index ee8b067bc..91d998965 100644 --- a/src/genie-commands/local-delivery-repair.test.ts +++ b/src/genie-commands/local-delivery-repair.test.ts @@ -113,7 +113,7 @@ function isolatedEnv(root: string, overrides: Record = {}): Reco const genieHome = join(root, 'genie-home'); const codexHome = join(root, 'codex-home'); const temp = join(root, 'tmp'); - for (const path of [home, genieHome, codexHome, temp]) mkdirSync(path, { recursive: true }); + for (const path of [home, genieHome, codexHome, temp]) mkdirSync(path, { recursive: true, mode: 0o700 }); return { ...env, HOME: home, diff --git a/src/lib/agent-sync.test.ts b/src/lib/agent-sync.test.ts index 8892bcf9b..fbbee2685 100644 --- a/src/lib/agent-sync.test.ts +++ b/src/lib/agent-sync.test.ts @@ -2760,6 +2760,9 @@ describe('stampWorkflow parity with council-stamp.cjs', () => { const nextTarget = readFileSync(join(targetDir, TARGET_NAME)); const nextManifest = readFileSync(join(targetDir, WORKFLOW_MANIFEST_NAME)); const transaction = join(targetDir, `.council.genie-txn-cross-${name}`); + for (const directory of [transaction, join(transaction, 'before'), join(transaction, 'staged')]) { + mkdirSync(directory, { recursive: true, mode: 0o755 }); + } writeFile(join(transaction, 'before', TARGET_NAME), oldTarget.toString()); writeFile(join(transaction, 'before', WORKFLOW_MANIFEST_NAME), oldManifest.toString()); writeFile(join(transaction, 'staged', TARGET_NAME), nextTarget.toString()); @@ -2776,6 +2779,15 @@ describe('stampWorkflow parity with council-stamp.cjs', () => { beforeManifestDigest: createHash('sha256').update(oldManifest).digest('hex'), })}\n`, ); + for (const file of [ + join(transaction, 'before', TARGET_NAME), + join(transaction, 'before', WORKFLOW_MANIFEST_NAME), + join(transaction, 'staged', TARGET_NAME), + join(transaction, 'staged', WORKFLOW_MANIFEST_NAME), + join(transaction, 'journal.json'), + ]) { + chmodSync(file, 0o644); + } recover(targetDir); @@ -3944,13 +3956,25 @@ function frozenHistoricalSkillsRoot(name: string): string { return join(release.payloadRoot, 'skills'); } +function copyFrozenHistoricalSkill(source: string, destination: string): void { + cpSync(source, destination, { recursive: true }); + const directories = [destination]; + while (directories.length > 0) { + const directory = directories.pop() as string; + chmodSync(directory, 0o755); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory()) directories.push(join(directory, entry.name)); + } + } +} + describe('Codex fallback ownership planning', () => { test('accepts all 23 committed historical name/version/physical tuples', () => { const fallback = join(fixture.root, 'historical-fallbacks'); const shippedSkills = frozenHistoricalSkillsRoot('verified-release-all'); for (const tuple of historicalCodexFallbackAllowlist) { const destination = join(fallback, tuple.skillName); - cpSync(join(shippedSkills, tuple.skillName), destination, { recursive: true }); + copyFrozenHistoricalSkill(join(shippedSkills, tuple.skillName), destination); expect(stampFallback(destination, tuple.markerVersion)).toBe(tuple.physicalDigest); } @@ -3969,7 +3993,7 @@ describe('Codex fallback ownership planning', () => { const tuple = historicalCodexFallbackAllowlist[0]; if (tuple === undefined) throw new Error('missing historical tuple'); const destination = join(fallback, tuple.skillName); - cpSync(join(shippedSkills, tuple.skillName), destination, { recursive: true }); + copyFrozenHistoricalSkill(join(shippedSkills, tuple.skillName), destination); stampFallback(destination, tuple.markerVersion); const marker = JSON.parse(readFileSync(join(destination, MANIFEST_NAME), 'utf8')) as Record; marker.syncedAt = 'not-authenticated-provenance'; @@ -3990,7 +4014,7 @@ describe('Codex fallback ownership planning', () => { const tuple = historicalCodexFallbackAllowlist[0]; if (tuple === undefined) throw new Error('missing historical tuple'); const destination = join(fallback, tuple.skillName); - cpSync(join(shippedSkills, tuple.skillName), destination, { recursive: true }); + copyFrozenHistoricalSkill(join(shippedSkills, tuple.skillName), destination); expect(stampFallback(destination, '5.260713.1')).toBe(tuple.physicalDigest); expect(tuple.markerVersion).not.toBe('5.260713.1'); diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index f7df63226..4b1e0f1a3 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -1390,7 +1390,7 @@ describe('database sync snapshots', () => { const right = currentDb('retain-right'); const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; const identity = databaseSyncSnapshotIdentity(request); - mkdirSync(join(identity.root, '.staging-abandoned'), { recursive: true }); + mkdirSync(join(identity.root, '.staging-abandoned'), { recursive: true, mode: 0o700 }); writeFileSync(join(identity.root, '.staging-abandoned', 'partial'), 'incomplete'); for (let index = 0; index < 4; index++) { insertBoard(right, `board-${index}`); @@ -1466,9 +1466,9 @@ describe('database sync snapshots', () => { const root = databaseSyncSnapshotIdentity(request).root; const staging = join(root, '.staging-owned'); const victim = join(fixtureRoot, 'cleanup-victim'); - mkdirSync(staging, { recursive: true }); + mkdirSync(staging, { recursive: true, mode: 0o700 }); writeFileSync(join(staging, 'owned'), 'owned'); - mkdirSync(victim); + mkdirSync(victim, { mode: 0o700 }); writeFileSync(join(victim, 'sentinel'), 'victim'); let raced = false; const recovery = recoverDatabaseReconciliation(request, { @@ -1494,7 +1494,7 @@ describe('database sync snapshots', () => { ).toBe('changed'); } const victim = join(fixtureRoot, 'prune-victim'); - mkdirSync(victim); + mkdirSync(victim, { mode: 0o700 }); writeFileSync(join(victim, 'sentinel'), 'victim'); insertBoard(right, 'three'); let replacement: string | null = null; @@ -1535,7 +1535,7 @@ describe('database sync snapshots', () => { const right = currentDb('darwin-cleanup-right'); const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; const root = databaseSyncSnapshotIdentity(request).root; - mkdirSync(join(root, '.staging-darwin'), { recursive: true }); + mkdirSync(join(root, '.staging-darwin'), { recursive: true, mode: 0o700 }); writeFileSync(join(root, '.staging-darwin', 'partial'), 'partial'); for (const id of ['first', 'second']) { diff --git a/tests/support/codex-dogfood-harness.ts b/tests/support/codex-dogfood-harness.ts index 7e5ecde40..694cbd706 100644 --- a/tests/support/codex-dogfood-harness.ts +++ b/tests/support/codex-dogfood-harness.ts @@ -231,7 +231,7 @@ function isolatedDogfoodEnv(root: string, overrides: Record = {} const codexHome = join(root, 'codex-home'); const bin = join(root, 'bin'); for (const path of [home, temp, xdgConfig, xdgCache, xdgData, xdgState, genieHome, codexHome, bin]) { - mkdirSync(path, { recursive: true }); + mkdirSync(path, { recursive: true, mode: 0o755 }); } const env: Record = { PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}`, @@ -268,7 +268,7 @@ export async function runDogfoodEntry( const input = normalizeInput(rawInput); const ownsRoot = dependencies.root === undefined; const root = dependencies.root ?? mkdtempSync(join(tmpdir(), 'genie-dogfood-entry-')); - mkdirSync(root, { recursive: true }); + mkdirSync(root, { recursive: true, mode: 0o755 }); try { const previous = verifyGeneration( 'previous', diff --git a/tests/support/update-current-boundary-runner.ts b/tests/support/update-current-boundary-runner.ts index e412b6b50..597a794d1 100644 --- a/tests/support/update-current-boundary-runner.ts +++ b/tests/support/update-current-boundary-runner.ts @@ -15,7 +15,7 @@ if ( const bin = join(genieHome, 'bin'); for (const directory of ['.agents', '.claude-plugin', 'plugins/genie', 'skills/review', 'templates']) { - mkdirSync(join(bin, directory), { recursive: true }); + mkdirSync(join(bin, directory), { recursive: true, mode: 0o755 }); } writeFileSync(join(bin, '.agents', 'plugin.json'), '{}\n'); writeFileSync(join(bin, '.claude-plugin', 'marketplace.json'), '{}\n'); @@ -27,8 +27,8 @@ writeFileSync(join(bin, 'templates', 'fixture.txt'), 'fixture\n'); const executable = join(bin, 'genie'); writeFileSync(executable, `#!/bin/sh\nif [ "\${1:-}" = "--version" ]; then printf 'genie ${VERSION}\\n'; fi\nexit 0\n`); chmodSync(executable, 0o755); -mkdirSync(process.env.HOME as string, { recursive: true }); -mkdirSync(process.env.CODEX_HOME as string, { recursive: true }); +mkdirSync(process.env.HOME as string, { recursive: true, mode: 0o755 }); +mkdirSync(process.env.CODEX_HOME as string, { recursive: true, mode: 0o755 }); const marker = join(genieHome, '.install-version'); writeFileSync(marker, 'prior-marker\n'); From ac3a1d0c32b33e8923d06172fc24caa4ac3cb00a Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Thu, 30 Jul 2026 12:16:58 +0800 Subject: [PATCH 19/21] feat(db-sync): bootstrap an absent database --- README.md | 6 + src/lib/v5/db-reconciliation.ts | 35 +- src/lib/v5/db-sync-snapshots.test.ts | 444 +++++++++++++ src/lib/v5/db-sync-snapshots.ts | 914 ++++++++++++++++++++++++++- src/term-commands/v5-db-sync.test.ts | 175 ++++- src/term-commands/v5-db-sync.ts | 87 ++- 6 files changed, 1622 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index df17520cc..c6dc0bd18 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ shared mutable task, board, wish-group, hire-roster, and unknown metadata keys. It writes only the destination. Both modes preserve destination-only rows: ordinary absence never means deletion. +When exactly one named database is absent and the other is an exact-current +Genie database, sync bootstraps the absent side from the complete logical image +of the existing side, including committed WAL content. Dry-run reports that +intent without creating the file or sidecars; apply uses the same bounded locks +and refuses unsafe parents, path substitution, or a target that appears first. + An explicit roster unhire (the `roster_unhire` UI-bridge operation) is the exception. It records a versioned tombstone in the existing metadata table, so a later sync removes the matching hire-roster row instead of resurrecting it. diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 91e4c3128..0e70920c5 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -24,7 +24,7 @@ import { statSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join, normalize } from 'node:path'; +import { isAbsolute, join, normalize } from 'node:path'; import { linuxLibcCandidates } from '../install-transaction.js'; import { CURRENT_SCHEMA_VERSION } from './genie-db.js'; import { type ReconciliationTombstone, parseReconciliationTombstoneMeta } from './reconciliation-tombstone.js'; @@ -370,6 +370,8 @@ export interface ReconciliationLockedOperationOptions { readonly advisoryFlock?: ReconciliationAdvisoryFlockDependencies; readonly onAdvisoryDescriptorOpened?: (descriptor: number) => void; readonly advisoryUnlock?: (descriptor: number) => number; + /** Canonical paths whose advisory identities must be reserved without opening them as SQLite inputs. */ + readonly advisoryOnlyPaths?: readonly string[]; } export interface ReconciliationLockedOperationResult { @@ -377,6 +379,11 @@ export interface ReconciliationLockedOperationResult { readonly afterCommit?: () => void; } +export interface ReconciliationLockedOperationContext { + /** Remaining milliseconds in the one total advisory + SQLite acquisition budget. */ + remainingWaitMs(): number; +} + export class ReconciliationLockedOperationError extends Error { readonly failure: ReconciliationApplyFailure; readonly cleanupFailures: readonly ReconciliationApplyFailure[]; @@ -2585,9 +2592,12 @@ function acquirePlannedAdvisoryLocks( inputs: readonly { readonly physical: PhysicalInput }[], deadline: number, options: ReconciliationApplyOptions, + advisoryOnlyPaths: readonly string[] = [], ): AdvisoryLock[] { const locks: AdvisoryLock[] = []; - const paths = [...new Set(inputs.map((input) => input.physical.canonicalPath))].sort(compareCanonical); + const paths = [...new Set([...inputs.map((input) => input.physical.canonicalPath), ...advisoryOnlyPaths])].sort( + compareCanonical, + ); try { for (const path of paths) locks.push(acquireAdvisoryLock(path, remainingLockWait(deadline), options)); return locks; @@ -3409,7 +3419,10 @@ function emitLockedOperationEvent( */ export function withLockedReconciliationDatabases( request: ReconciliationRequest, - operation: (inputs: readonly ReconciliationLockedDatabaseInput[]) => ReconciliationLockedOperationResult, + operation: ( + inputs: readonly ReconciliationLockedDatabaseInput[], + context: ReconciliationLockedOperationContext, + ) => ReconciliationLockedOperationResult, options: ReconciliationLockedOperationOptions = {}, ): T { let advisoryLocks: AdvisoryLock[] = []; @@ -3443,13 +3456,25 @@ export function withLockedReconciliationDatabases( physical, }; }); + const advisoryOnlyPaths = options.advisoryOnlyPaths ?? []; + if ( + advisoryOnlyPaths.length > 8 || + advisoryOnlyPaths.some((path) => !isAbsolute(path) || normalize(path) !== path || path.includes('\0')) + ) { + throw new ApplyBoundaryError({ code: 'invalid-plan', phase: 'plan-validation' }); + } const deadline = Date.now() + timeoutMs; - advisoryLocks = acquirePlannedAdvisoryLocks(requested, deadline, lockOptions); + advisoryLocks = acquirePlannedAdvisoryLocks(requested, deadline, lockOptions, advisoryOnlyPaths); locked = openLockedDatabases(requested, deadline, lockOptions); const inputs = lockedOperationInputs(requested, locked); const initial = inputs.map((input) => input.observe()); if (initial[0].schemaFingerprint !== initial[1].schemaFingerprint) unsupportedSchema(); - result = operation(inputs); + result = operation( + inputs, + Object.freeze({ + remainingWaitMs: () => remainingLockWait(deadline), + }), + ); const final = inputs.map((input) => input.observe()); if (final.some((image) => image.schemaFingerprint !== initial[0].schemaFingerprint)) unsupportedSchema(); for (const item of locked) { diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index 4b1e0f1a3..663ebee31 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -3,12 +3,15 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { createHash } from 'node:crypto'; import { chmodSync, + closeSync, existsSync, + fstatSync, linkSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, + readlinkSync, realpathSync, renameSync, rmSync, @@ -23,14 +26,19 @@ import { MAX_RECONCILIATION_DATABASE_BYTES, applyDatabaseReconciliation, planDatabaseReconciliation, + reconciliationAdvisoryLockPath, } from './db-reconciliation.js'; import { SnapshotError, type SnapshotPosixDirectoryApi, applyDatabaseReconciliationWithSnapshots, + applyDatabaseSyncExecution, + applyMissingDatabaseBootstrap, databaseSyncSnapshotIdentity, deserializeSnapshotBytes, normalizeSerializedSqliteForDeserialize, + planDatabaseSyncExecution, + planMissingDatabaseBootstrap, recoverDatabaseReconciliation, resolveSnapshotPosixDirectory, rollbackDatabaseReconciliation, @@ -138,6 +146,9 @@ function delegatePosix( return { openAt: overrides.openAt ?? ((...args) => api.openAt(...args)), mkdirAt: overrides.mkdirAt ?? ((...args) => api.mkdirAt(...args)), + linkAt: + overrides.linkAt ?? + (api.linkAt === undefined ? undefined : (...args) => (api.linkAt as NonNullable)(...args)), renameAt: overrides.renameAt ?? ((...args) => api.renameAt(...args)), unlinkAt: overrides.unlinkAt ?? ((...args) => api.unlinkAt(...args)), list: overrides.list ?? ((...args) => api.list(...args)), @@ -243,6 +254,439 @@ describe('database sync snapshots', () => { restored.close(); }); + test('missing-side bootstrap publishes the validated complete WAL-backed logical image and is idempotent', () => { + const source = currentDb('bootstrap-source'); + const target = join(fixtureRoot, 'bootstrap-target.db'); + const writer = new Database(source); + writer.exec('PRAGMA journal_mode = WAL'); + writer.exec('PRAGMA wal_autocheckpoint = 0'); + writer.query("INSERT INTO boards (id, name, created_at, lanes) VALUES ('wal', 'WAL', 1, NULL)").run(); + expect(existsSync(`${source}-wal`)).toBe(true); + expect(statSync(`${source}-wal`).size).toBeGreaterThan(0); + + const request = { mode: 'bidirectional' as const, leftPath: source, rightPath: target }; + const bootstrap = planMissingDatabaseBootstrap(request); + expect(bootstrap).toMatchObject({ existingRole: 'left', missingRole: 'right' }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + const report = applyMissingDatabaseBootstrap(bootstrap); + + expect(report).toMatchObject({ status: 'changed', failure: null, cleanupFailures: [] }); + expect(boardNames(target)).toEqual(['WAL']); + const repeated = planDatabaseReconciliation(request); + expect(repeated.status).toBe('no-op'); + expect(repeated.inputs[0].logicalDigest).toBe(repeated.inputs[1].logicalDigest); + expect(readdirSync(fixtureRoot).filter((name) => name.startsWith('.genie-db-bootstrap-'))).toEqual([]); + writer.close(); + }); + + test('missing-side bootstrap shares one sorted advisory wait deadline', () => { + const source = currentDb('bootstrap-budget-a-source'); + const target = join(fixtureRoot, 'bootstrap-budget-z-target.db'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + const descriptors: number[] = []; + const openedPaths: string[] = []; + const firstAttempt = new Map(); + const startedAt = Date.now(); + const fakeFlock = (descriptor: number, operation: number): number => { + if (operation === 8) return 0; + const index = descriptors.indexOf(descriptor); + const attemptedAt = firstAttempt.get(descriptor) ?? Date.now(); + firstAttempt.set(descriptor, attemptedAt); + const waitFor = index === 0 ? 30 : 200; + return Date.now() - attemptedAt >= waitFor ? 0 : -1; + }; + + const report = applyMissingDatabaseBootstrap(bootstrap, { + busyTimeoutMs: 80, + applyOptions: { + advisoryFlock: + process.platform === 'darwin' + ? { platform: 'darwin', darwinOpener: () => fakeFlock } + : { + platform: 'linux', + architecture: process.arch, + linuxCandidates: ['injected'], + linuxOpener: () => fakeFlock, + }, + onAdvisoryDescriptorOpened: (descriptor) => { + descriptors.push(descriptor); + const descriptorRoot = process.platform === 'darwin' ? '/dev/fd' : '/proc/self/fd'; + openedPaths.push(readlinkSync(`${descriptorRoot}/${descriptor}`)); + }, + }, + }); + const elapsed = Date.now() - startedAt; + + expect(report).toMatchObject({ + status: 'operational-failure', + failure: 'locked-operation-failed', + cleanupFailures: [], + }); + expect(existsSync(target)).toBe(false); + expect(openedPaths).toEqual([reconciliationAdvisoryLockPath(source), reconciliationAdvisoryLockPath(target)]); + expect(elapsed).toBeGreaterThanOrEqual(60); + expect(elapsed).toBeLessThan(300); + }); + + test('missing-side bootstrap gives target SQLite only the advisory deadline remainder', async () => { + const source = currentDb('bootstrap-sqlite-budget-a-source'); + const target = join(fixtureRoot, 'bootstrap-sqlite-budget-z-target.db'); + const ready = join(fixtureRoot, 'bootstrap-sqlite-budget.ready'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + const descriptors: number[] = []; + const openedPaths: string[] = []; + const firstAttempt = new Map(); + const busyTimeoutMs = 400; + const advisoryDelayMs = 250; + const targetHoldMs = 220; + let blocker: ReturnType | null = null; + let blockerReadyAt = 0; + const fakeFlock = (descriptor: number, operation: number): number => { + if (operation === 8) return 0; + if (descriptors.indexOf(descriptor) !== 0) return 0; + const attemptedAt = firstAttempt.get(descriptor) ?? Date.now(); + firstAttempt.set(descriptor, attemptedAt); + return Date.now() - attemptedAt >= advisoryDelayMs ? 0 : -1; + }; + + const report = applyMissingDatabaseBootstrap(bootstrap, { + busyTimeoutMs, + onEvent: (event) => { + if (event.phase !== 'bootstrap-publish' || event.state !== 'after') return; + const code = ` + import { Database } from 'bun:sqlite'; + const db = new Database(Bun.argv[1]); + db.exec('PRAGMA busy_timeout = 0'); + db.exec('BEGIN IMMEDIATE'); + await Bun.write(Bun.argv[2], 'ready'); + await Bun.sleep(Number(Bun.argv[3])); + db.exec('ROLLBACK'); + db.close(); + `; + blocker = Bun.spawn({ + cmd: [process.execPath, '-e', code, target, ready, String(targetHoldMs)], + stdout: 'ignore', + stderr: 'ignore', + }); + const readyDeadline = Date.now() + 5_000; + while (!existsSync(ready) && Date.now() < readyDeadline) Bun.sleepSync(5); + if (!existsSync(ready)) throw new Error('target SQLite blocker did not acquire its transaction'); + blockerReadyAt = Date.now(); + }, + applyOptions: { + advisoryFlock: + process.platform === 'darwin' + ? { platform: 'darwin', darwinOpener: () => fakeFlock } + : { + platform: 'linux', + architecture: process.arch, + linuxCandidates: ['injected'], + linuxOpener: () => fakeFlock, + }, + onAdvisoryDescriptorOpened: (descriptor) => { + descriptors.push(descriptor); + const descriptorRoot = process.platform === 'darwin' ? '/dev/fd' : '/proc/self/fd'; + openedPaths.push(readlinkSync(`${descriptorRoot}/${descriptor}`)); + }, + }, + }); + const applyFinishedAt = Date.now(); + const blockerProcess = blocker as ReturnType | null; + if (blockerProcess === null) throw new Error('target SQLite blocker was not started'); + const blockerExit = await blockerProcess.exited; + + expect(blockerExit).toBe(0); + expect(report).toMatchObject({ + status: 'operational-failure', + failure: 'snapshot-publication-failed', + cleanupFailures: [], + }); + expect(openedPaths).toEqual([reconciliationAdvisoryLockPath(source), reconciliationAdvisoryLockPath(target)]); + expect(blockerReadyAt).toBeGreaterThan(0); + expect(applyFinishedAt - blockerReadyAt).toBeLessThan(targetHoldMs); + }); + + test('missing-side bootstrap never clobbers a target that appears at the publication boundary', () => { + const source = currentDb('bootstrap-race-source'); + const target = join(fixtureRoot, 'bootstrap-race-target.db'); + const request = { mode: 'bidirectional' as const, leftPath: source, rightPath: target }; + const bootstrap = planMissingDatabaseBootstrap(request); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + const foreign = Buffer.from('foreign target bytes'); + + const report = applyMissingDatabaseBootstrap(bootstrap, { + onEvent: (event) => { + if (event.phase === 'bootstrap-publish' && event.state === 'before') { + writeFileSync(target, foreign, { mode: 0o600 }); + } + }, + }); + + expect(report.status).toBe('operational-failure'); + expect(readFileSync(target)).toEqual(foreign); + expect(readdirSync(fixtureRoot).filter((name) => name.startsWith('.genie-db-bootstrap-'))).toEqual([]); + }); + + test('missing-side bootstrap rejects a named-target substitution after final target validation', () => { + const source = currentDb('bootstrap-final-target-source'); + const parent = join(fixtureRoot, 'bootstrap-final-target-parent'); + const target = join(parent, 'target.db'); + const retained = join(parent, 'retained-owned-target.db'); + const foreign = Buffer.from('foreign replacement must remain undamaged'); + mkdirSync(parent, { mode: 0o775 }); + chmodSync(parent, 0o775); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + + const report = applyMissingDatabaseBootstrap(bootstrap, { + onEvent: (event) => { + if (event.phase === 'bootstrap-target-validate' && event.state === 'after') { + renameSync(target, retained); + writeFileSync(target, foreign, { mode: 0o600 }); + } + }, + }); + + expect(report).toMatchObject({ + status: 'operational-failure', + converged: false, + failure: 'snapshot-image-mismatch', + cleanupFailures: ['locked-operation-failed'], + }); + expect(readFileSync(target)).toEqual(foreign); + expect(existsSync(retained)).toBe(true); + }); + + test('missing-side bootstrap rejects target-parent identity substitution before publication', () => { + const source = currentDb('bootstrap-parent-source'); + const parent = join(fixtureRoot, 'bootstrap-parent'); + const target = join(parent, 'target.db'); + mkdirSync(parent, { mode: 0o700 }); + const request = { mode: 'directional' as const, sourcePath: source, destinationPath: target }; + const bootstrap = planMissingDatabaseBootstrap(request); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + + const report = applyMissingDatabaseBootstrap(bootstrap, { + onEvent: (event) => { + if (event.phase === 'bootstrap-publish' && event.state === 'before') { + renameSync(parent, `${parent}.original`); + mkdirSync(parent, { mode: 0o700 }); + } + }, + }); + + expect(report.status).toBe('operational-failure'); + expect(existsSync(target)).toBe(false); + expect(existsSync(join(`${parent}.original`, 'target.db'))).toBe(false); + }); + + test('missing-side bootstrap cleans its exact stage after a pre-validation failure', () => { + const source = currentDb('bootstrap-stage-source'); + const target = join(fixtureRoot, 'bootstrap-stage-target.db'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + + const report = applyMissingDatabaseBootstrap(bootstrap, { + onEvent: (event) => { + if (event.phase === 'bootstrap-stage-validate' && event.state === 'before') { + throw new Error('injected pre-validation failure'); + } + }, + }); + + expect(report).toMatchObject({ + operation: 'bootstrap', + status: 'operational-failure', + converged: false, + cleanupFailures: [], + }); + expect(existsSync(target)).toBe(false); + expect(readdirSync(fixtureRoot).filter((name) => name.startsWith('.genie-db-bootstrap-'))).toEqual([]); + }); + + test('missing-side bootstrap reacquires exact stage ownership after first-fstat failure', () => { + const source = currentDb('bootstrap-first-fstat-source'); + const target = join(fixtureRoot, 'bootstrap-first-fstat-target.db'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + let calls = 0; + + const report = applyMissingDatabaseBootstrap(bootstrap, { + bootstrapStageFstat: (descriptor) => { + if (calls++ === 0) throw new Error('injected first fstat failure'); + return fstatSync(descriptor); + }, + }); + + expect(report).toMatchObject({ + status: 'operational-failure', + converged: false, + cleanupFailures: [], + }); + expect(existsSync(target)).toBe(false); + expect(readdirSync(fixtureRoot).filter((name) => name.startsWith('.genie-db-bootstrap-'))).toEqual([]); + }); + + test('missing-side bootstrap rolls target back when parent close finalization fails', () => { + const source = currentDb('bootstrap-parent-close-source'); + const target = join(fixtureRoot, 'bootstrap-parent-close-target.db'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + + const report = applyMissingDatabaseBootstrap(bootstrap, { + bootstrapCloseDirectory: (descriptor) => { + closeSync(descriptor); + throw new Error('injected parent close failure'); + }, + }); + + expect(report).toMatchObject({ + status: 'operational-failure', + converged: false, + cleanupFailures: ['locked-close-failed'], + }); + const writer = new Database(target); + writer.exec('PRAGMA busy_timeout = 0'); + writer.exec('BEGIN IMMEDIATE'); + writer.exec('ROLLBACK'); + writer.close(); + }); + + test('missing-side bootstrap reports a real stage unlink failure without false success', () => { + const source = currentDb('bootstrap-unlink-source'); + const target = join(fixtureRoot, 'bootstrap-unlink-target.db'); + const bootstrap = planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: source, + rightPath: target, + }); + if (bootstrap === null) throw new Error('expected bootstrap plan'); + const native = nativePosixDirectory(); + const api = delegatePosix(native, { + unlinkAt: (descriptor, name, directory) => { + if (name.startsWith('.genie-db-bootstrap-')) throw new Error('injected unlink failure'); + native.unlinkAt(descriptor, name, directory); + }, + }); + + const report = applyMissingDatabaseBootstrap(bootstrap, { posixDirectory: { api } }); + + expect(report).toMatchObject({ + status: 'operational-failure', + converged: false, + failure: 'snapshot-cleanup-failed', + cleanupFailures: ['snapshot-cleanup-failed'], + }); + expect(readdirSync(fixtureRoot).filter((name) => name.startsWith('.genie-db-bootstrap-'))).toHaveLength(1); + }); + + test('library-owned bootstrap convergence rejects a writer mutation after publication', () => { + const source = currentDb('bootstrap-postimage-source'); + const target = join(fixtureRoot, 'bootstrap-postimage-target.db'); + const request = { mode: 'bidirectional' as const, leftPath: source, rightPath: target }; + const execution = planDatabaseSyncExecution(request); + expect(execution.kind).toBe('bootstrap'); + + const result = applyDatabaseSyncExecution(execution, { + onEvent: (event) => { + if (event.phase === 'bootstrap-publish' && event.state === 'after') { + const writer = new Database(target); + writer.query("INSERT INTO boards (id, name, created_at, lanes) VALUES ('racer', 'racer', 1, NULL)").run(); + writer.close(); + } + }, + }); + + expect(result.kind).toBe('bootstrap'); + if (result.kind !== 'bootstrap') throw new Error('expected bootstrap result'); + expect(result.report).toMatchObject({ + operation: 'bootstrap', + status: 'operational-failure', + converged: false, + failure: 'snapshot-image-mismatch', + }); + expect(result.report.inputs.find((input) => input.role === 'right')?.logicalDigest).toBeNull(); + expect(boardNames(target)).toEqual(['racer']); + }); + + test('missing-side bootstrap grants no authority through symlinks, non-files, or unsafe parents', () => { + const source = currentDb('bootstrap-unsafe-source'); + const sourceAlias = join(fixtureRoot, 'bootstrap-source-alias.db'); + symlinkSync(source, sourceAlias); + expect( + planMissingDatabaseBootstrap({ + mode: 'bidirectional', + leftPath: sourceAlias, + rightPath: join(fixtureRoot, 'missing-from-alias.db'), + }), + ).toBeNull(); + + const dangling = join(fixtureRoot, 'bootstrap-dangling.db'); + symlinkSync(join(fixtureRoot, 'absent-target.db'), dangling); + expect(planMissingDatabaseBootstrap({ mode: 'bidirectional', leftPath: source, rightPath: dangling })).toBeNull(); + + const directory = join(fixtureRoot, 'bootstrap-directory.db'); + mkdirSync(directory, { mode: 0o700 }); + expect(planMissingDatabaseBootstrap({ mode: 'bidirectional', leftPath: source, rightPath: directory })).toBeNull(); + + const unsafeParent = join(fixtureRoot, 'bootstrap-unsafe-parent'); + mkdirSync(unsafeParent, { mode: 0o777 }); + chmodSync(unsafeParent, 0o777); + expect( + planMissingDatabaseBootstrap({ + mode: 'directional', + sourcePath: source, + destinationPath: join(unsafeParent, 'missing.db'), + }), + ).toBeNull(); + }); + + test('missing-side bootstrap rejects a non-owner 0775 target parent', () => { + const source = currentDb('bootstrap-non-owner-source'); + const parent = join(fixtureRoot, 'bootstrap-non-owner-parent'); + mkdirSync(parent, { mode: 0o775 }); + chmodSync(parent, 0o775); + const ownerUid = statSync(parent).uid; + expect(statSync(parent).mode & 0o777).toBe(0o775); + + const bootstrap = planMissingDatabaseBootstrap( + { + mode: 'directional', + sourcePath: source, + destinationPath: join(parent, 'missing.db'), + }, + { bootstrapCurrentUid: () => (ownerUid === 0 ? 1 : 0) }, + ); + + expect(bootstrap).toBeNull(); + }); + test('serialized snapshot normalization rejects short, non-SQLite, and inconsistent format headers', () => { expect(() => normalizeSerializedSqliteForDeserialize(new Uint8Array(99))).toThrow(SnapshotError); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index febdc686d..248d35081 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -1,9 +1,10 @@ import { CString, FFIType, dlopen, toArrayBuffer } from 'bun:ffi'; import type { Pointer } from 'bun:ffi'; -import { Database } from 'bun:sqlite'; +import { Database, constants as sqliteConstants } from 'bun:sqlite'; import { createHash, randomUUID } from 'node:crypto'; import { closeSync, + existsSync, constants as fsConstants, fstatSync, fsyncSync, @@ -13,6 +14,7 @@ import { openSync, readSync, realpathSync, + rmSync, writeFileSync, writeSync, } from 'node:fs'; @@ -27,6 +29,7 @@ import { type ReconciliationApplyOptions, type ReconciliationApplyReport, type ReconciliationDatabaseObservation, + ReconciliationError, type ReconciliationInputRole, type ReconciliationLockedDatabaseInput, ReconciliationLockedOperationError, @@ -36,8 +39,10 @@ import { type ReconciliationTargetRole, applyDatabaseReconciliation, inspectReconciliationDatabase, + planDatabaseReconciliation, withLockedReconciliationDatabases, } from './db-reconciliation.js'; +import { isBusyError } from './sqlite-open.js'; const SQLITE_HEADER_BYTES = new TextEncoder().encode('SQLite format 3\0'); const SQLITE_MINIMUM_HEADER_BYTES = 100; @@ -183,7 +188,10 @@ export type SnapshotLifecyclePhase = | 'recovery-restore' | 'rollback-restore' | 'staging-cleanup' - | 'prune'; + | 'prune' + | 'bootstrap-publish' + | 'bootstrap-stage-validate' + | 'bootstrap-target-validate'; export interface SnapshotLifecycleEvent { readonly phase: SnapshotLifecyclePhase; @@ -210,12 +218,24 @@ export interface DatabaseSyncSnapshotOptions { }; /** Lazy native descriptor-relative filesystem resolution seam for portability and fault tests. */ readonly posixDirectory?: SnapshotPosixDirectoryDependencies; + /** Private bootstrap-only close fault seam. */ + readonly bootstrapCloseDirectory?: (descriptor: number) => void; + /** Private bootstrap-only first-fstat fault seam. */ + readonly bootstrapStageFstat?: (descriptor: number) => Stats; + /** Private bootstrap-only effective UID seam for owner-rejection tests. */ + readonly bootstrapCurrentUid?: () => number; readonly applyOptions?: Omit; } export interface SnapshotPosixDirectoryApi { openAt(directoryDescriptor: number, name: string, flags: number, mode?: number): number; mkdirAt(directoryDescriptor: number, name: string, mode: number): void; + linkAt?( + sourceDirectoryDescriptor: number, + sourceName: string, + destinationDirectoryDescriptor: number, + destinationName: string, + ): void; renameAt( sourceDirectoryDescriptor: number, sourceName: string, @@ -319,6 +339,58 @@ interface FileIdentity { readonly ino: number; } +export interface MissingDatabaseBootstrapPlan { + readonly request: ReconciliationRequest; + readonly existingRole: ReconciliationInputRole; + readonly missingRole: ReconciliationInputRole; + readonly existingCanonicalPath: string; + readonly missingPath: string; + readonly missingParentPath: string; + readonly missingParentIdentity: FileIdentity; + readonly schemaFingerprint: string; + readonly logicalDigest: string; +} + +export type DatabaseSyncExecutionPlan = + | { + readonly kind: 'reconciliation'; + readonly request: ReconciliationRequest; + readonly reconciliation: ReconciliationPlan; + } + | { + readonly kind: 'bootstrap'; + readonly request: ReconciliationRequest; + readonly bootstrap: MissingDatabaseBootstrapPlan; + }; + +export interface MissingDatabaseBootstrapInputReport { + readonly role: ReconciliationInputRole; + readonly canonicalPath: string; + readonly logicalDigest: string | null; +} + +export interface MissingDatabaseBootstrapApplyReport { + readonly reportVersion: typeof SNAPSHOT_REPORT_VERSION; + readonly operation: 'bootstrap'; + readonly status: 'changed' | 'operational-failure'; + readonly converged: boolean; + readonly inputs: readonly MissingDatabaseBootstrapInputReport[]; + readonly failure: SnapshotFailureCode | null; + readonly cleanupFailures: readonly SnapshotFailureCode[]; +} + +export type DatabaseSyncExecutionResult = + | { + readonly kind: 'reconciliation'; + readonly plan: ReconciliationPlan; + readonly report: SnapshotApplyReport; + } + | { + readonly kind: 'bootstrap'; + readonly plan: MissingDatabaseBootstrapPlan; + readonly report: MissingDatabaseBootstrapApplyReport; + }; + interface BoundDirectory { readonly path: string; readonly descriptor: number; @@ -335,6 +407,10 @@ const POSIX_DIRECTORY_SYMBOLS = { args: [FFIType.i32, FFIType.cstring, FFIType.u32], returns: FFIType.i32, }, + linkat: { + args: [FFIType.i32, FFIType.cstring, FFIType.i32, FFIType.cstring, FFIType.i32], + returns: FFIType.i32, + }, renameat: { args: [FFIType.i32, FFIType.cstring, FFIType.i32, FFIType.cstring], returns: FFIType.i32, @@ -369,6 +445,13 @@ let defaultPosixDirectoryApi: SnapshotPosixDirectoryApi | null | undefined; interface NativeDirectorySymbols { openat(directoryDescriptor: number, name: Uint8Array, flags: number, mode: number): number; mkdirat(directoryDescriptor: number, name: Uint8Array, mode: number): number; + linkat( + sourceDirectoryDescriptor: number, + sourceName: Uint8Array, + destinationDirectoryDescriptor: number, + destinationName: Uint8Array, + flags: number, + ): number; renameat( sourceDirectoryDescriptor: number, sourceName: Uint8Array, @@ -386,6 +469,7 @@ function nativeError(operation: string, errno: number): Error & { code?: string; const error = new Error(`${operation} failed with errno ${errno}`) as Error & { code?: string; errno: number }; error.errno = errno; if (errno === 2) error.code = 'ENOENT'; + if (errno === 17) error.code = 'EEXIST'; return error; } @@ -457,6 +541,19 @@ function openPosixDirectoryApi( throw nativeError('mkdirat', errno[0]); } }, + linkAt(sourceDirectoryDescriptor, sourceName, destinationDirectoryDescriptor, destinationName) { + if ( + symbols.linkat( + sourceDirectoryDescriptor, + componentBytes(sourceName), + destinationDirectoryDescriptor, + componentBytes(destinationName), + 0, + ) !== 0 + ) { + throw nativeError('linkat', errno[0]); + } + }, renameAt(sourceDirectoryDescriptor, sourceName, destinationDirectoryDescriptor, destinationName) { if ( symbols.renameat( @@ -630,6 +727,242 @@ export function databaseSyncSnapshotIdentity( return identityFromCanonical(request.mode, canonical, snapshotRoot); } +function requestInputs( + request: ReconciliationRequest, +): readonly { readonly role: ReconciliationInputRole; readonly path: string }[] { + return request.mode === 'bidirectional' + ? [ + { role: 'left', path: request.leftPath }, + { role: 'right', path: request.rightPath }, + ] + : [ + { role: 'source', path: request.sourcePath }, + { role: 'destination', path: request.destinationPath }, + ]; +} + +function physicalRegularFileState(path: string): 'regular' | 'missing' | 'unsafe' { + try { + const stats = lstatSync(path); + return !stats.isSymbolicLink() && stats.isFile() ? 'regular' : 'unsafe'; + } catch (caught) { + return filesystemCode(caught) === 'ENOENT' ? 'missing' : 'unsafe'; + } +} + +function entryIsAbsentAt(directory: BoundDirectory, name: string, options: DatabaseSyncSnapshotOptions): boolean { + let descriptor: number; + try { + descriptor = posixDirectory(options).openAt( + directory.descriptor, + name, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0) | POSIX_CLOSE_ON_EXEC, + ); + } catch (caught) { + return filesystemCode(caught) === 'ENOENT'; + } + closeSync(descriptor); + return false; +} + +interface BootstrapExistingObservation { + readonly canonicalPath: string; + readonly schemaFingerprint: string; + readonly logicalDigest: string; +} + +function inspectBootstrapMainFile(path: string): BootstrapExistingObservation { + const readMainFile = (): { readonly bytes: Uint8Array; readonly identity: FileIdentity } => { + const descriptor = openSync( + path, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0) | POSIX_CLOSE_ON_EXEC, + ); + try { + return readBoundedRegularDescriptor(descriptor, MAX_RECONCILIATION_DATABASE_BYTES); + } finally { + closeSync(descriptor); + } + }; + const sidecars = [`${path}-wal`, `${path}-journal`]; + if (sidecars.some((sidecar) => existsSync(sidecar))) { + throw new ReconciliationError('input-changed', 'Bootstrap source sidecars changed while it was being read.'); + } + const first = readMainFile(); + const second = readMainFile(); + if ( + !sameFileIdentity(first.identity, second.identity) || + sha256(first.bytes) !== sha256(second.bytes) || + sidecars.some((sidecar) => existsSync(sidecar)) + ) { + throw new ReconciliationError('input-changed', 'Bootstrap source changed while it was being read.'); + } + let db: Database; + try { + db = deserializeSnapshotBytes(second.bytes); + } catch { + throw new ReconciliationError('malformed-database', 'Bootstrap source is not a valid SQLite database.'); + } + try { + const image = inspectReconciliationDatabase(db); + const canonicalPath = realpathSync(path); + if (canonicalPath !== path) { + throw new ReconciliationError('input-changed', 'Bootstrap source changed while it was being read.'); + } + return { + canonicalPath, + schemaFingerprint: image.schemaFingerprint, + logicalDigest: image.logicalDigest, + }; + } finally { + db.close(); + } +} + +function inspectBootstrapWalFiles(path: string): BootstrapExistingObservation { + const readPhysicalFile = (physicalPath: string) => { + const descriptor = openSync( + physicalPath, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | (fsConstants.O_NONBLOCK ?? 0) | POSIX_CLOSE_ON_EXEC, + ); + try { + return readBoundedRegularDescriptor(descriptor, MAX_RECONCILIATION_DATABASE_BYTES); + } finally { + closeSync(descriptor); + } + }; + const walPath = `${path}-wal`; + if (existsSync(`${path}-journal`)) { + throw new ReconciliationError('input-changed', 'Bootstrap source journal changed while it was being read.'); + } + const firstMain = readPhysicalFile(path); + const firstWal = readPhysicalFile(walPath); + const secondMain = readPhysicalFile(path); + const secondWal = readPhysicalFile(walPath); + if ( + firstMain.bytes.byteLength + firstWal.bytes.byteLength > MAX_RECONCILIATION_DATABASE_BYTES || + !sameFileIdentity(firstMain.identity, secondMain.identity) || + !sameFileIdentity(firstWal.identity, secondWal.identity) || + sha256(firstMain.bytes) !== sha256(secondMain.bytes) || + sha256(firstWal.bytes) !== sha256(secondWal.bytes) + ) { + throw new ReconciliationError('input-changed', 'Bootstrap source changed while it was being read.'); + } + + const temporaryRoot = mkdtempSync(join(tmpdir(), 'genie-db-bootstrap-plan-')); + const temporaryDatabase = join(temporaryRoot, 'source.db'); + let db: Database | null = null; + try { + writeFileSync(temporaryDatabase, secondMain.bytes, { mode: 0o600 }); + writeFileSync(`${temporaryDatabase}-wal`, secondWal.bytes, { mode: 0o600 }); + db = new Database(temporaryDatabase, { readonly: true, strict: true, safeIntegers: true }); + const image = inspectReconciliationDatabase(db); + const finalMain = readPhysicalFile(path); + const finalWal = readPhysicalFile(walPath); + if ( + !sameFileIdentity(secondMain.identity, finalMain.identity) || + !sameFileIdentity(secondWal.identity, finalWal.identity) || + sha256(secondMain.bytes) !== sha256(finalMain.bytes) || + sha256(secondWal.bytes) !== sha256(finalWal.bytes) || + existsSync(`${path}-journal`) || + realpathSync(path) !== path + ) { + throw new ReconciliationError('input-changed', 'Bootstrap source changed while it was being read.'); + } + return { + canonicalPath: path, + schemaFingerprint: image.schemaFingerprint, + logicalDigest: image.logicalDigest, + }; + } catch (caught) { + if (caught instanceof ReconciliationError) throw caught; + throw new ReconciliationError('malformed-database', 'Bootstrap source WAL image is not a valid database.'); + } finally { + try { + db?.close(); + } finally { + rmSync(temporaryRoot, { recursive: true, force: true }); + } + } +} + +function inspectBootstrapExisting(path: string): BootstrapExistingObservation { + return existsSync(`${path}-wal`) ? inspectBootstrapWalFiles(path) : inspectBootstrapMainFile(path); +} + +/** + * Recognize the one-missing-input bootstrap case without creating anything. + * + * The present side must be a physical exact-current Genie database. The + * absent side must have a stable, user-owned physical parent; + * symlinks and unsafe or missing ancestors are not bootstrap authority. + */ +export function planMissingDatabaseBootstrap( + request: ReconciliationRequest, + options: DatabaseSyncSnapshotOptions = {}, +): MissingDatabaseBootstrapPlan | null { + const inputs = requestInputs(request).map((input) => ({ + ...input, + absolutePath: resolve(input.path), + state: physicalRegularFileState(resolve(input.path)), + })); + const existing = inputs.filter((input) => input.state === 'regular'); + const missing = inputs.filter((input) => input.state === 'missing'); + if (existing.length !== 1 || missing.length !== 1) return null; + + const existingInput = inspectBootstrapExisting(existing[0].absolutePath); + if (existingInput.canonicalPath !== existing[0].absolutePath) return null; + + const missingPath = missing[0].absolutePath; + const missingParentPath = dirname(missingPath); + let missingParent: BoundDirectory; + try { + missingParent = openBoundDirectory(missingParentPath, false, options, false, 'owned-group-writable'); + } catch { + return null; + } + try { + if (!entryIsAbsentAt(missingParent, basename(missingPath), options)) return null; + return Object.freeze({ + request, + existingRole: existing[0].role, + missingRole: missing[0].role, + existingCanonicalPath: existingInput.canonicalPath, + missingPath, + missingParentPath, + missingParentIdentity: missingParent.identity, + schemaFingerprint: existingInput.schemaFingerprint, + logicalDigest: existingInput.logicalDigest, + }); + } finally { + closeBoundDirectory(missingParent); + } +} + +/** + * Select the complete library-owned database-sync plan. + * + * Missing-side recognition is deliberately reached only through the normal + * planner's bounded input-unavailable classification. Callers never need to + * reproduce that fallback or infer a postimage after apply. + */ +export function planDatabaseSyncExecution( + request: ReconciliationRequest, + options: DatabaseSyncSnapshotOptions = {}, +): DatabaseSyncExecutionPlan { + try { + return Object.freeze({ + kind: 'reconciliation', + request, + reconciliation: planDatabaseReconciliation(request), + }); + } catch (caught) { + if (!(caught instanceof ReconciliationError) || caught.code !== 'input-unavailable') throw caught; + const bootstrap = planMissingDatabaseBootstrap(request, options); + if (bootstrap === null) throw caught; + return Object.freeze({ kind: 'bootstrap', request, bootstrap }); + } +} + function requestFromPlan(plan: ReconciliationPlan): ReconciliationRequest { const input = (role: ReconciliationInputRole): string => { const found = plan.inputs.find((candidate) => candidate.role === role); @@ -735,7 +1068,7 @@ function openBoundDirectory( create: boolean, options: DatabaseSyncSnapshotOptions, requirePrivate = false, - allowShared = false, + allowShared: boolean | 'owned-group-writable' = false, ): BoundDirectory { const api = posixDirectory(options); const components = rootComponents(path); @@ -756,12 +1089,22 @@ function openBoundDirectory( descriptor = child.descriptor; } const stats = fstatSync(descriptor); - const currentUid = typeof process.getuid === 'function' ? process.getuid() : stats.uid; + const currentUid = + allowShared === 'owned-group-writable' && options.bootstrapCurrentUid !== undefined + ? options.bootstrapCurrentUid() + : typeof process.getuid === 'function' + ? process.getuid() + : stats.uid; const mode = stats.mode & 0o777; - if ( - !stats.isDirectory() || - (!allowShared && (stats.uid !== currentUid || (requirePrivate ? mode !== 0o700 : (mode & 0o022) !== 0))) - ) { + const ownerIsSafe = allowShared === true || stats.uid === currentUid; + const modeIsSafe = + allowShared === true || + (allowShared === 'owned-group-writable' + ? (mode & 0o002) === 0 + : requirePrivate + ? mode === 0o700 + : (mode & 0o022) === 0); + if (!stats.isDirectory() || !ownerIsSafe || !modeIsSafe) { throw new SnapshotError('manifest-invalid', 'Snapshot root must be a private directory owned by this user.'); } return { path, descriptor, identity: fileIdentity(stats) }; @@ -901,8 +1244,12 @@ function assertBoundDirectory(directory: BoundDirectory): void { } } -function assertBoundRoot(directory: BoundDirectory, options: DatabaseSyncSnapshotOptions): void { - const reopened = openBoundDirectory(directory.path, false, options); +function assertBoundRoot( + directory: BoundDirectory, + options: DatabaseSyncSnapshotOptions, + allowShared: boolean | 'owned-group-writable' = false, +): void { + const reopened = openBoundDirectory(directory.path, false, options, false, allowShared); try { if (!sameFileIdentity(directory.identity, reopened.identity)) { throw new SnapshotError('manifest-invalid', 'Snapshot root identity changed during the operation.'); @@ -971,6 +1318,433 @@ function validateCapturedImage( return { normalizedBytes, sha256: sha256(normalizedBytes) }; } +function writeBytesDescriptor(descriptor: number, bytes: Uint8Array): void { + let offset = 0; + while (offset < bytes.byteLength) { + const written = writeSync(descriptor, bytes, offset, bytes.byteLength - offset, offset); + if (written <= 0) + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap snapshot write was incomplete.'); + offset += written; + } +} + +function sameBootstrapPlan(left: MissingDatabaseBootstrapPlan, right: MissingDatabaseBootstrapPlan): boolean { + return ( + left.existingRole === right.existingRole && + left.missingRole === right.missingRole && + left.existingCanonicalPath === right.existingCanonicalPath && + left.missingPath === right.missingPath && + left.missingParentPath === right.missingParentPath && + sameFileIdentity(left.missingParentIdentity, right.missingParentIdentity) && + left.schemaFingerprint === right.schemaFingerprint && + left.logicalDigest === right.logicalDigest + ); +} + +function publicationError(caught: unknown, cleanupFailures: readonly SnapshotFailureCode[]): SnapshotError { + if (caught instanceof SnapshotError) { + return new SnapshotError(caught.code, caught.message, [...caught.cleanupFailures, ...cleanupFailures]); + } + return new SnapshotError( + 'snapshot-publication-failed', + 'Missing database bootstrap publication failed.', + cleanupFailures, + ); +} + +interface LockedBootstrapTarget { + readonly db: Database; + readonly parentPath: string; + readonly parentIdentity: FileIdentity; + readonly targetName: string; + readonly expectedIdentity: FileIdentity; + transactionOpen: boolean; +} + +function bootstrapHandleMatchesPath(db: Pick): boolean { + const moved = new Int32Array(1); + try { + return db.fileControl(sqliteConstants.SQLITE_FCNTL_HAS_MOVED, moved) === 0 && moved[0] === 0; + } catch { + return false; + } +} + +function requireBootstrapHandleMatchesPath(db: Pick): void { + if (!bootstrapHandleMatchesPath(db)) { + throw new SnapshotError('snapshot-image-mismatch', 'The published bootstrap database changed identity.'); + } +} + +function lockAndValidateBootstrapTarget( + plan: MissingDatabaseBootstrapPlan, + expectedIdentity: FileIdentity, + source: ReconciliationLockedDatabaseInput, + remainingWaitMs: () => number, + options: DatabaseSyncSnapshotOptions, +): LockedBootstrapTarget { + const parent = openBoundDirectory(plan.missingParentPath, false, options, false, 'owned-group-writable'); + let db: Database | null = null; + let transactionOpen = false; + let target: LockedBootstrapTarget | null = null; + let operationFailure: unknown = null; + let parentCloseFailed = false; + const cleanupFailures: SnapshotFailureCode[] = []; + try { + emit(options, { phase: 'bootstrap-target-validate', state: 'before', path: plan.missingPath }); + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, basename(plan.missingPath), expectedIdentity, 'file', options); + const databaseOptions = { + readwrite: true, + create: false, + strict: true, + safeIntegers: true, + } as const; + db = + options.applyOptions?.openDatabase?.(plan.missingPath, databaseOptions) ?? + new Database(plan.missingPath, databaseOptions); + requireBootstrapHandleMatchesPath(db); + db.exec('PRAGMA trusted_schema = OFF'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec(`PRAGMA busy_timeout = ${remainingWaitMs()}`); + db.exec('BEGIN IMMEDIATE'); + transactionOpen = true; + requireBootstrapHandleMatchesPath(db); + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, basename(plan.missingPath), expectedIdentity, 'file', options); + const targetImage = inspectReconciliationDatabase(db); + const sourceImage = source.observe(); + if ( + targetImage.schemaFingerprint !== plan.schemaFingerprint || + targetImage.logicalDigest !== plan.logicalDigest || + sourceImage.schemaFingerprint !== plan.schemaFingerprint || + sourceImage.logicalDigest !== plan.logicalDigest + ) { + throw new SnapshotError( + 'snapshot-image-mismatch', + 'The bootstrap databases did not converge to the expected logical image.', + ); + } + requireBootstrapHandleMatchesPath(db); + assertEntryIdentityAt(parent, basename(plan.missingPath), expectedIdentity, 'file', options); + emit(options, { phase: 'bootstrap-target-validate', state: 'after', path: plan.missingPath }); + target = { + db, + parentPath: plan.missingParentPath, + parentIdentity: parent.identity, + targetName: basename(plan.missingPath), + expectedIdentity, + transactionOpen, + }; + db = null; + transactionOpen = false; + } catch (caught) { + operationFailure = caught; + } + try { + if (options.bootstrapCloseDirectory === undefined) closeBoundDirectory(parent); + else options.bootstrapCloseDirectory(parent.descriptor); + } catch { + parentCloseFailed = true; + cleanupFailures.push('locked-close-failed'); + } + if (operationFailure !== null || parentCloseFailed) { + if (target !== null) { + cleanupFailures.push(...rollbackAndCloseBootstrapTarget(target)); + target = null; + } + if (transactionOpen) { + try { + db?.exec('ROLLBACK'); + } catch { + cleanupFailures.push('locked-rollback-failed'); + } + } + try { + db?.close(); + } catch { + cleanupFailures.push('locked-close-failed'); + } + if (operationFailure !== null && isBusyError(operationFailure)) { + throw new SnapshotError( + 'snapshot-publication-failed', + 'The published bootstrap database could not be locked within the wait bound.', + cleanupFailures, + ); + } + if (operationFailure !== null) throw publicationError(operationFailure, cleanupFailures); + throw new SnapshotError( + 'snapshot-publication-failed', + 'The published bootstrap parent could not be closed safely.', + cleanupFailures, + ); + } + if (target === null) { + throw new SnapshotError('snapshot-publication-failed', 'The published bootstrap target lock was not retained.'); + } + return target; +} + +function requireRetainedBootstrapTarget( + target: LockedBootstrapTarget, + parent: BoundDirectory, + options: DatabaseSyncSnapshotOptions, +): void { + if (!sameFileIdentity(target.parentIdentity, parent.identity)) { + throw new SnapshotError('snapshot-image-mismatch', 'The published bootstrap parent changed identity.'); + } + requireBootstrapHandleMatchesPath(target.db); + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, target.targetName, target.expectedIdentity, 'file', options); +} + +function commitAndCloseBootstrapTarget( + target: LockedBootstrapTarget, + options: DatabaseSyncSnapshotOptions, +): SnapshotFailureCode[] { + const cleanupFailures: SnapshotFailureCode[] = []; + let parent: BoundDirectory | null = null; + let validatedForClose = false; + try { + parent = openBoundDirectory(target.parentPath, false, options, false, 'owned-group-writable'); + requireRetainedBootstrapTarget(target, parent, options); + if (target.transactionOpen) { + target.db.exec('COMMIT'); + target.transactionOpen = false; + } + requireRetainedBootstrapTarget(target, parent, options); + validatedForClose = true; + } catch { + cleanupFailures.push('locked-operation-failed'); + if (target.transactionOpen) { + try { + target.db.exec('ROLLBACK'); + target.transactionOpen = false; + } catch { + cleanupFailures.push('locked-rollback-failed'); + } + } + } + try { + target.db.close(); + } catch { + cleanupFailures.push('locked-close-failed'); + } + if (validatedForClose && parent !== null) { + try { + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, target.targetName, target.expectedIdentity, 'file', options); + } catch { + cleanupFailures.push('locked-operation-failed'); + } + } + if (parent !== null) { + try { + closeBoundDirectory(parent); + } catch { + cleanupFailures.push('locked-close-failed'); + } + } + return cleanupFailures; +} + +function rollbackAndCloseBootstrapTarget(target: LockedBootstrapTarget): SnapshotFailureCode[] { + const cleanupFailures: SnapshotFailureCode[] = []; + if (target.transactionOpen) { + try { + target.db.exec('ROLLBACK'); + target.transactionOpen = false; + } catch { + cleanupFailures.push('locked-rollback-failed'); + } + } + try { + target.db.close(); + } catch { + cleanupFailures.push('locked-close-failed'); + } + return cleanupFailures; +} + +function bootstrapInputReports( + plan: MissingDatabaseBootstrapPlan, + converged: boolean, +): MissingDatabaseBootstrapInputReport[] { + return requestInputs(plan.request).map((input) => ({ + role: input.role, + canonicalPath: input.role === plan.existingRole ? plan.existingCanonicalPath : plan.missingPath, + logicalDigest: converged || input.role === plan.existingRole ? plan.logicalDigest : null, + })); +} + +interface BootstrapStageState { + created: boolean; + descriptor: number | null; + identity: FileIdentity | null; +} + +function prepareBootstrapStage( + stage: BootstrapStageState, + parent: BoundDirectory, + stageName: string, + targetPath: string, + capture: { readonly normalizedBytes: Uint8Array }, + options: DatabaseSyncSnapshotOptions, +): void { + stage.descriptor = openRegularFileAt( + parent, + stageName, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + options, + 0o600, + ); + stage.created = true; + const createdStage = (options.bootstrapStageFstat ?? fstatSync)(stage.descriptor); + stage.identity = fileIdentity(createdStage); + assertEntryIdentityAt(parent, stageName, stage.identity, 'file', options); + emit(options, { phase: 'bootstrap-stage-validate', state: 'before', path: targetPath }); + if (!createdStage.isFile() || (createdStage.mode & 0o777) !== 0o600 || createdStage.nlink !== 1) { + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap staging file is not a private regular file.'); + } + writeBytesDescriptor(stage.descriptor, capture.normalizedBytes); + fsyncSync(stage.descriptor); + const stageStats = (options.bootstrapStageFstat ?? fstatSync)(stage.descriptor); + if ( + !stageStats.isFile() || + (stageStats.mode & 0o777) !== 0o600 || + stageStats.nlink !== 1 || + !sameFileIdentity(stage.identity, stageStats) + ) { + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap staging file is not a private regular file.'); + } + assertEntryIdentityAt(parent, stageName, stage.identity, 'file', options); + emit(options, { phase: 'bootstrap-stage-validate', state: 'after', path: targetPath }); + closeSync(stage.descriptor); + stage.descriptor = null; +} + +function cleanupBootstrapStage( + stage: BootstrapStageState, + parent: BoundDirectory, + stageName: string, + options: DatabaseSyncSnapshotOptions, +): SnapshotFailureCode[] { + const cleanupFailures: SnapshotFailureCode[] = []; + if (stage.created && stage.identity === null && stage.descriptor !== null) { + try { + const cleanupStats = fstatSync(stage.descriptor); + if (cleanupStats.isFile()) stage.identity = fileIdentity(cleanupStats); + else cleanupFailures.push('snapshot-cleanup-failed'); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } + } + if (stage.descriptor !== null) { + try { + closeSync(stage.descriptor); + stage.descriptor = null; + } catch { + cleanupFailures.push('locked-close-failed'); + } + } + if (stage.identity !== null) { + try { + assertEntryIdentityAt(parent, stageName, stage.identity, 'file', options); + posixDirectory(options).unlinkAt(parent.descriptor, stageName, false); + fsyncSync(parent.descriptor); + } catch { + cleanupFailures.push('snapshot-cleanup-failed'); + } + } + if (stage.created && stage.identity === null && !cleanupFailures.includes('snapshot-cleanup-failed')) { + cleanupFailures.push('snapshot-cleanup-failed'); + } + return cleanupFailures; +} + +function publishMissingDatabase( + plan: MissingDatabaseBootstrapPlan, + source: ReconciliationLockedDatabaseInput, + options: DatabaseSyncSnapshotOptions, +): FileIdentity { + const current = planMissingDatabaseBootstrap(plan.request, options); + if (current === null || !sameBootstrapPlan(plan, current)) { + throw new SnapshotError('snapshot-publication-failed', 'Missing database bootstrap inputs changed before locking.'); + } + const observed = source.observe(); + if (observed.schemaFingerprint !== plan.schemaFingerprint || observed.logicalDigest !== plan.logicalDigest) { + throw new SnapshotError('snapshot-image-mismatch', 'Bootstrap source changed before serialization.'); + } + const capture = validateCapturedImage(source.serialize(), plan.schemaFingerprint, plan.logicalDigest); + const parent = openBoundDirectory(plan.missingParentPath, false, options, false, 'owned-group-writable'); + const targetName = basename(plan.missingPath); + const stageName = `.genie-db-bootstrap-${randomUUID()}`; + const stage: BootstrapStageState = { created: false, descriptor: null, identity: null }; + let operationFailure: unknown = null; + const cleanupFailures: SnapshotFailureCode[] = []; + try { + if ( + !sameFileIdentity(plan.missingParentIdentity, parent.identity) || + !entryIsAbsentAt(parent, targetName, options) + ) { + throw new SnapshotError( + 'snapshot-publication-failed', + 'Missing database bootstrap target changed before publication.', + ); + } + prepareBootstrapStage(stage, parent, stageName, plan.missingPath, capture, options); + const stageIdentity = stage.identity; + if (stageIdentity === null) { + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap staging identity was not captured.'); + } + const staged = readBoundedRegularFileAt(parent, stageName, MAX_RECONCILIATION_DATABASE_BYTES, options); + if (!sameFileIdentity(stageIdentity, staged.identity) || sha256(staged.bytes) !== capture.sha256) { + throw new SnapshotError('snapshot-image-mismatch', 'Bootstrap staging image changed before publication.'); + } + + emit(options, { phase: 'bootstrap-publish', state: 'before', path: plan.missingPath }); + assertBoundRoot(parent, options, 'owned-group-writable'); + const api = posixDirectory(options); + if (api.linkAt === undefined) { + throw new SnapshotError( + 'snapshot-publication-failed', + 'Descriptor-relative no-clobber bootstrap publication is unavailable.', + ); + } + api.linkAt(parent.descriptor, stageName, parent.descriptor, targetName); + const published = readBoundedRegularFileAt(parent, targetName, MAX_RECONCILIATION_DATABASE_BYTES, options); + if (!sameFileIdentity(stageIdentity, published.identity) || sha256(published.bytes) !== capture.sha256) { + throw new SnapshotError( + 'snapshot-image-mismatch', + 'Published bootstrap image does not match its staged identity.', + ); + } + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, targetName, stageIdentity, 'file', options); + fsyncSync(parent.descriptor); + assertBoundRoot(parent, options, 'owned-group-writable'); + assertEntryIdentityAt(parent, targetName, stageIdentity, 'file', options); + emit(options, { phase: 'bootstrap-publish', state: 'after', path: plan.missingPath }); + } catch (caught) { + operationFailure = caught; + } finally { + cleanupFailures.push(...cleanupBootstrapStage(stage, parent, stageName, options)); + try { + closeBoundDirectory(parent); + } catch { + cleanupFailures.push('locked-close-failed'); + } + } + if (operationFailure !== null) throw publicationError(operationFailure, cleanupFailures); + if (cleanupFailures.length > 0) { + throw new SnapshotError('snapshot-cleanup-failed', 'Bootstrap staging cleanup failed.', cleanupFailures); + } + if (stage.identity === null) { + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap staging identity was not captured.'); + } + return stage.identity; +} + function makeManifest( identity: SnapshotStoreIdentity, schemaFingerprint: string, @@ -1956,22 +2730,24 @@ function snapshotCleanupFailures(caught: unknown): SnapshotFailureCode[] { return []; } +function snapshotFailureCode(caught: unknown): SnapshotFailureCode { + return caught instanceof SnapshotError + ? caught.code + : caught instanceof ReconciliationLockedOperationError + ? caught.operationCause instanceof SnapshotError + ? caught.operationCause.code + : 'locked-operation-failed' + : 'locked-operation-failed'; +} + function failedRecovery(caught: unknown): SnapshotRecoveryReport { - const failure = - caught instanceof SnapshotError - ? caught.code - : caught instanceof ReconciliationLockedOperationError - ? caught.operationCause instanceof SnapshotError - ? caught.operationCause.code - : 'locked-operation-failed' - : 'locked-operation-failed'; return { reportVersion: SNAPSHOT_REPORT_VERSION, operation: 'recovery', status: 'operational-failure', generationId: null, restoredPaths: [], - failure, + failure: snapshotFailureCode(caught), cleanupFailures: snapshotCleanupFailures(caught), }; } @@ -2099,6 +2875,106 @@ function recoverPrivateGeneration( } } +export function applyMissingDatabaseBootstrap( + plan: MissingDatabaseBootstrapPlan, + options: DatabaseSyncSnapshotOptions = {}, +): MissingDatabaseBootstrapApplyReport { + let target: LockedBootstrapTarget | null = null; + let operationFailure: unknown = null; + const cleanupFailures: SnapshotFailureCode[] = []; + try { + const lockRequest: ReconciliationRequest = { + mode: 'bidirectional', + leftPath: plan.existingCanonicalPath, + rightPath: plan.existingCanonicalPath, + }; + withLockedReconciliationDatabases( + lockRequest, + (inputs, lockContext) => { + const source = inputs[0]; + if (source === undefined) { + throw new SnapshotError('snapshot-publication-failed', 'Bootstrap source lock was not available.'); + } + const publishedIdentity = publishMissingDatabase(plan, source, options); + target = lockAndValidateBootstrapTarget(plan, publishedIdentity, source, lockContext.remainingWaitMs, options); + return { + value: undefined, + afterCommit: () => { + const lockedTarget = target; + if (lockedTarget === null) { + throw new SnapshotError('snapshot-image-mismatch', 'Bootstrap target lock was lost before commit.'); + } + const targetCleanup = commitAndCloseBootstrapTarget(lockedTarget, options); + target = null; + if (targetCleanup.length > 0) { + throw new SnapshotError( + 'snapshot-image-mismatch', + 'Bootstrap target commit or cleanup failed.', + targetCleanup, + ); + } + }, + }; + }, + { + busyTimeoutMs: options.busyTimeoutMs, + onEvent: options.onLockedOperationEvent, + advisoryOnlyPaths: [plan.missingPath], + ...options.applyOptions, + }, + ); + } catch (caught) { + operationFailure = caught; + } finally { + if (target !== null) { + cleanupFailures.push(...rollbackAndCloseBootstrapTarget(target)); + target = null; + } + } + + const inputReports = bootstrapInputReports(plan, operationFailure === null && cleanupFailures.length === 0); + if (operationFailure !== null || cleanupFailures.length > 0) { + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'bootstrap', + status: 'operational-failure', + converged: false, + inputs: inputReports, + failure: operationFailure === null ? 'snapshot-cleanup-failed' : snapshotFailureCode(operationFailure), + cleanupFailures: [ + ...(operationFailure === null ? [] : snapshotCleanupFailures(operationFailure)), + ...cleanupFailures, + ], + }; + } + return { + reportVersion: SNAPSHOT_REPORT_VERSION, + operation: 'bootstrap', + status: 'changed', + converged: true, + inputs: inputReports, + failure: null, + cleanupFailures: [], + }; +} + +export function applyDatabaseSyncExecution( + execution: DatabaseSyncExecutionPlan, + options: DatabaseSyncSnapshotOptions = {}, +): DatabaseSyncExecutionResult { + return execution.kind === 'reconciliation' + ? { + kind: 'reconciliation', + plan: execution.reconciliation, + report: applyDatabaseReconciliationWithSnapshots(execution.reconciliation, options), + } + : { + kind: 'bootstrap', + plan: execution.bootstrap, + report: applyMissingDatabaseBootstrap(execution.bootstrap, options), + }; +} + export function applyDatabaseReconciliationWithSnapshots( plan: ReconciliationPlan, options: DatabaseSyncSnapshotOptions = {}, diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts index c800c5543..39f582a0e 100644 --- a/src/term-commands/v5-db-sync.test.ts +++ b/src/term-commands/v5-db-sync.test.ts @@ -2,7 +2,7 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { openDb, resolveDbPath } from '../lib/v5/genie-db.js'; @@ -27,6 +27,7 @@ interface JsonReport { status: string; schemaFingerprint: string | null; targets: Array<{ role: string; changes: Record }>; + bootstrap?: { sourceRole: string; targetRole: string }; conflicts: { total: number; byTable: Record }; operationalFailure: { code: string; guidance?: string } | null; } | null; @@ -136,7 +137,7 @@ function logicalInventory(path: string): string[] { function sqliteFileInventory(path: string): Record { return Object.fromEntries( - ['', '-wal', '-shm'].map((suffix) => { + ['', '-wal', '-shm', '-journal'].map((suffix) => { const file = `${path}${suffix}`; return [ suffix === '' ? 'database' : suffix.slice(1), @@ -364,6 +365,174 @@ describe('database sync CLI contract', () => { expect(result.stdout).not.toContain('HOSTILE SAME-DATABASE TITLE'); }); + test('bidirectional apply bootstraps either absent side, then repeats as a matching no-op', async () => { + for (const missingFirst of [false, true]) { + const existing = createRepo(`bootstrap-existing-${missingFirst}`); + insertTask(existing.database, `task-${missingFirst}`, `bootstrap-${missingFirst}`); + const missing = join(root, `bootstrap-missing-${missingFirst}.db`); + const args = missingFirst ? [missing, existing.database] : [existing.database, missing]; + const beforeLeft = sqliteFileInventory(args[0]); + const beforeRight = sqliteFileInventory(args[1]); + + const preview = await sync(...args, '--dry-run', '--json'); + expect(preview.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(existsSync(missing)).toBe(false); + expect(sqliteFileInventory(args[0])).toEqual(beforeLeft); + expect(sqliteFileInventory(args[1])).toEqual(beforeRight); + expect(parseJson(preview).plan?.bootstrap).toEqual({ + sourceRole: missingFirst ? 'right' : 'left', + targetRole: missingFirst ? 'left' : 'right', + }); + + const applied = await sync(...args, '--json'); + const appliedReport = parseJson(applied); + expect(applied.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(appliedReport.status).toBe('changed'); + expect(taskTitles(missing)).toEqual([`bootstrap-${missingFirst}`]); + expect(appliedReport.inputs[0].logicalDigest).toBe(appliedReport.inputs[1].logicalDigest); + + const repeated = await sync(...args, '--dry-run', '--json'); + const repeatedReport = parseJson(repeated); + expect(repeated.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(repeatedReport.status).toBe('no-op'); + expect(repeatedReport.inputs[0].logicalDigest).toBe(repeatedReport.inputs[1].logicalDigest); + } + }); + + test('bootstrap accepts a user-owned group-writable repository database directory', async () => { + const existing = createRepo('bootstrap-0775-existing'); + insertTask(existing.database, 'bootstrap-0775-task', 'bootstrap-0775'); + const missingRepo = createRepo('bootstrap-0775-missing', false); + const missingParent = dirname(missingRepo.database); + mkdirSync(missingParent, { mode: 0o775 }); + chmodSync(missingParent, 0o775); + expect(statSync(missingParent).mode & 0o777).toBe(0o775); + const beforeExisting = sqliteFileInventory(existing.database); + const beforeMissing = sqliteFileInventory(missingRepo.database); + + const preview = await sync(existing.database, missingRepo.database, '--dry-run', '--json'); + + expect(preview.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(preview).plan?.bootstrap).toEqual({ sourceRole: 'left', targetRole: 'right' }); + expect(sqliteFileInventory(existing.database)).toEqual(beforeExisting); + expect(sqliteFileInventory(missingRepo.database)).toEqual(beforeMissing); + + const applied = await sync(existing.database, missingRepo.database, '--json'); + + expect(applied.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(applied).status).toBe('changed'); + expect(taskTitles(missingRepo.database)).toEqual(['bootstrap-0775']); + }); + + test('directional bootstrap copies the sole existing side in both role orientations', async () => { + for (const existingRole of ['source', 'destination'] as const) { + const existing = createRepo(`directional-bootstrap-${existingRole}`); + const writer = new Database(existing.database); + writer.exec('PRAGMA journal_mode = WAL'); + writer.exec('PRAGMA wal_autocheckpoint = 0'); + writer + .query('INSERT INTO tasks (id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)') + .run(`task-${existingRole}`, `from-${existingRole}`, 'ready', 1, 1); + expect(existsSync(`${existing.database}-wal`)).toBe(true); + expect(statSync(`${existing.database}-wal`).size).toBeGreaterThan(0); + const missing = join(root, `directional-missing-${existingRole}.db`); + const source = existingRole === 'source' ? existing.database : missing; + const destination = existingRole === 'destination' ? existing.database : missing; + const beforeSource = sqliteFileInventory(source); + const beforeDestination = sqliteFileInventory(destination); + + const preview = await sync('--source', source, '--destination', destination, '--dry-run', '--json'); + const previewReport = parseJson(preview); + expect(preview.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(previewReport.status).toBe('changed'); + expect(previewReport.plan?.bootstrap).toEqual({ + sourceRole: existingRole, + targetRole: existingRole === 'source' ? 'destination' : 'source', + }); + expect(existsSync(missing)).toBe(false); + expect(sqliteFileInventory(source)).toEqual(beforeSource); + expect(sqliteFileInventory(destination)).toEqual(beforeDestination); + + const applied = await sync('--source', source, '--destination', destination, '--json'); + const report = parseJson(applied); + + expect(applied.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(report.plan?.bootstrap).toEqual({ + sourceRole: existingRole, + targetRole: existingRole === 'source' ? 'destination' : 'source', + }); + expect(taskTitles(missing)).toEqual([`from-${existingRole}`]); + const repeated = parseJson(await sync('--source', source, '--destination', destination, '--dry-run', '--json')); + expect(repeated.status).toBe('no-op'); + expect(repeated.inputs[0].logicalDigest).toBe(repeated.inputs[1].logicalDigest); + writer.close(); + } + }); + + test('both absent inputs remain a bounded non-mutating operational failure', async () => { + const left = join(root, 'both-missing-left.db'); + const right = join(root, 'both-missing-right.db'); + + for (const args of [ + [left, right, '--json'], + [left, right, '--dry-run', '--json'], + [right, left, '--dry-run', '--json'], + ['--source', left, '--destination', right, '--json'], + ['--source', left, '--destination', right, '--dry-run', '--json'], + ['--source', right, '--destination', left, '--dry-run', '--json'], + ]) { + const beforeLeft = sqliteFileInventory(left); + const beforeRight = sqliteFileInventory(right); + const result = await sync(...args); + const report = parseJson(result); + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + expect(report.plan?.operationalFailure?.code).toBe('input-unavailable'); + expect(sqliteFileInventory(left)).toEqual(beforeLeft); + expect(sqliteFileInventory(right)).toEqual(beforeRight); + } + }); + + test('safe-parent bootstrap reports stay bounded and omit hostile paths and row content', async () => { + const existing = createRepo('hostile-bootstrap-existing'); + const hostileTitle = 'HOSTILE BOOTSTRAP ROW\nMUST NOT LEAK'; + insertTask(existing.database, 'hostile-bootstrap-task', hostileTitle); + const parent = join(root, 'hostile-bootstrap-parent'); + mkdirSync(parent, { mode: 0o700 }); + const missing = join(parent, 'HOSTILE BOOTSTRAP PATH\nMUST NOT LEAK.db'); + + const result = await sync('--source', missing, '--destination', existing.database, '--json'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(result.stderr).toBe(''); + expect(result.stdout).not.toContain('HOSTILE BOOTSTRAP'); + expect(result.stdout).not.toContain(missing); + expect(result.stdout.length).toBeLessThan(8_000); + expect(parseJson(result)).toMatchObject({ + status: 'changed', + plan: { bootstrap: { sourceRole: 'destination', targetRole: 'source' } }, + }); + expect(taskTitles(missing)).toEqual([hostileTitle]); + }); + + test('missing-side bootstrap preserves committed WAL-only logical content', async () => { + const existing = createRepo('bootstrap-wal-existing'); + const missing = join(root, 'bootstrap-wal-missing.db'); + const writer = new Database(existing.database); + writer.exec('PRAGMA journal_mode = WAL'); + writer.exec('PRAGMA wal_autocheckpoint = 0'); + writer + .query('INSERT INTO tasks (id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)') + .run('wal-task', 'WAL bootstrap', 'ready', 1, 1); + expect(existsSync(`${existing.database}-wal`)).toBe(true); + expect(statSync(`${existing.database}-wal`).size).toBeGreaterThan(0); + + const result = await sync(existing.database, missing, '--json'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(taskTitles(missing)).toEqual(['WAL bootstrap']); + writer.close(); + }); + test('persistent recovery maps converged to exit 6 and hostile uncertain state to exit 5', async () => { const left = createRepo('recovery-left'); const right = createRepo('recovery-right'); @@ -466,7 +635,7 @@ describe('database sync CLI contract', () => { test('operational reports omit hostile input paths and remain bounded', async () => { const destination = createRepo('operational-destination'); - const hostile = join(root, 'HOSTILE PATH\nMUST NOT LEAK.db'); + const hostile = join(root, 'missing-parent', 'HOSTILE PATH\nMUST NOT LEAK.db'); const result = await sync('--source', hostile, '--destination', destination.database, '--dry-run', '--json'); diff --git a/src/term-commands/v5-db-sync.ts b/src/term-commands/v5-db-sync.ts index cd294533d..f4bb69b0f 100644 --- a/src/term-commands/v5-db-sync.ts +++ b/src/term-commands/v5-db-sync.ts @@ -17,16 +17,20 @@ import { type ReconciliationConflict, ReconciliationError, type ReconciliationInputRole, + type ReconciliationPlan, type ReconciliationRequest, type ReconciliationTableName, type ReconciliationTargetReport, - planDatabaseReconciliation, } from '../lib/v5/db-reconciliation.js'; import { + type DatabaseSyncExecutionPlan, + type MissingDatabaseBootstrapApplyReport, + type MissingDatabaseBootstrapPlan, type SnapshotApplyReport, type SnapshotRecoveryReport, type SnapshotRollbackReport, - applyDatabaseReconciliationWithSnapshots, + applyDatabaseSyncExecution, + planDatabaseSyncExecution, rollbackDatabaseReconciliation, } from '../lib/v5/db-sync-snapshots.js'; @@ -74,6 +78,10 @@ interface PlanReport { sameDatabase: boolean; schemaFingerprint: string | null; targets: readonly ReconciliationTargetReport[]; + bootstrap?: { + sourceRole: ReconciliationInputRole; + targetRole: ReconciliationInputRole; + }; conflicts: ConflictCountReport; operationalFailure: { code: string; @@ -259,7 +267,7 @@ function conflictCounts(conflicts: readonly ReconciliationConflict[]): ConflictC return { total: conflicts.length, byTable }; } -function summarizePlan(plan: ReturnType): PlanReport { +function summarizePlan(plan: ReconciliationPlan): PlanReport { return { status: plan.report.status, sameDatabase: plan.report.sameDatabase, @@ -271,6 +279,27 @@ function summarizePlan(plan: ReturnType): Pla }; } +function summarizeBootstrapPlan(plan: MissingDatabaseBootstrapPlan): PlanReport { + return { + status: 'changed', + sameDatabase: false, + schemaFingerprint: plan.schemaFingerprint, + targets: [], + bootstrap: { sourceRole: plan.existingRole, targetRole: plan.missingRole }, + conflicts: { total: 0, byTable: emptyConflictCounts() }, + operationalFailure: null, + historyLimitation: IDENTICAL_HISTORY_ADDITION_LIMITATION, + }; +} + +function bootstrapInputs(plan: MissingDatabaseBootstrapPlan): DatabaseIdentityReport[] { + return requestInputs(plan.request).map((input) => ({ + role: input.role, + databaseIdentity: databaseIdentity(input.path), + logicalDigest: input.role === plan.existingRole ? plan.logicalDigest : null, + })); +} + function planningFailure(caught: unknown): PlanReport { const operationalFailure = caught instanceof ReconciliationError @@ -287,7 +316,7 @@ function planningFailure(caught: unknown): PlanReport { }; } -function resolvedInputs(plan: ReturnType): DatabaseIdentityReport[] { +function resolvedInputs(plan: ReconciliationPlan): DatabaseIdentityReport[] { return plan.inputs.map((input) => ({ role: input.role, databaseIdentity: databaseIdentity(input.canonicalPath), @@ -316,6 +345,31 @@ function summarizeApply(report: SnapshotApplyReport): ApplyReport { }; } +function summarizeBootstrapApply(report: MissingDatabaseBootstrapApplyReport): ApplyReport { + return { + status: report.status, + generationId: null, + recovery: { + status: report.status === 'changed' ? 'none' : 'operational-failure', + generationId: null, + restoredDatabaseIdentities: [], + failure: report.failure, + cleanupFailures: report.cleanupFailures, + }, + apply: null, + failure: report.failure, + cleanupFailures: report.cleanupFailures, + }; +} + +function bootstrapAppliedInputs(report: MissingDatabaseBootstrapApplyReport): DatabaseIdentityReport[] { + return report.inputs.map((input) => ({ + role: input.role, + databaseIdentity: databaseIdentity(input.canonicalPath), + logicalDigest: input.logicalDigest, + })); +} + function summarizeRollback(report: SnapshotRollbackReport): RollbackReport { return { status: report.status, @@ -364,13 +418,16 @@ function execute( }; } - let plan: ReturnType; + let execution: DatabaseSyncExecutionPlan; try { - plan = planDatabaseReconciliation(request); + execution = planDatabaseSyncExecution(request, options); } catch (caught) { return reportForPlanningFailure(request, operation, caught); } - const planReport = summarizePlan(plan); + const planReport = + execution.kind === 'reconciliation' + ? summarizePlan(execution.reconciliation) + : summarizeBootstrapPlan(execution.bootstrap); if (operation === 'dry-run') { return { reportVersion: CLI_REPORT_VERSION, @@ -378,23 +435,26 @@ function execute( operation, mode: request.mode, status: planReport.status, - inputs: resolvedInputs(plan), + inputs: + execution.kind === 'reconciliation' + ? resolvedInputs(execution.reconciliation) + : bootstrapInputs(execution.bootstrap), plan: planReport, apply: null, rollback: null, }; } - const result = applyDatabaseReconciliationWithSnapshots(plan, options); + const result = applyDatabaseSyncExecution(execution, options); return { reportVersion: CLI_REPORT_VERSION, command: 'database-sync', operation, mode: request.mode, - status: result.status, - inputs: resolvedInputs(plan), + status: result.report.status, + inputs: result.kind === 'reconciliation' ? resolvedInputs(result.plan) : bootstrapAppliedInputs(result.report), plan: planReport, - apply: summarizeApply(result), + apply: result.kind === 'reconciliation' ? summarizeApply(result.report) : summarizeBootstrapApply(result.report), rollback: null, }; } @@ -448,6 +508,9 @@ function humanLines(report: DatabaseSyncCliReport): string[] { ]; if (report.plan !== null) { lines.push(`Plan: ${report.plan.status}; conflicts=${report.plan.conflicts.total}`); + if (report.plan.bootstrap !== undefined) { + lines.push(`Bootstrap: ${report.plan.bootstrap.sourceRole} -> ${report.plan.bootstrap.targetRole}`); + } lines.push(...report.plan.targets.map(changesLine)); if (report.plan.operationalFailure !== null) { lines.push(`Failure: ${report.plan.operationalFailure.code}`); From d811e1dee455944ef34c38b6269cbedd447a50f9 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Sun, 2 Aug 2026 18:31:47 +0800 Subject: [PATCH 20/21] fix(db): prefer later versioned records during sync Bidirectional synchronization treated every differing shared key as a conflict, even when tasks and wish groups carried a clear updated_at ordering. Stale sandbox mirrors therefore required manual repair instead of converging. Choose the entire row with the greater bigint updated_at for tasks and wish groups and publish it to both databases. Equal-version differences and unversioned records remain fail-closed. Directional sync and tombstones are unchanged. Verified with the full check suite: 3116 tests passed. --- README.md | 15 ++- src/lib/v5/db-reconciliation.test.ts | 192 +++++++++++++++++++++++++-- src/lib/v5/db-reconciliation.ts | 29 +++- src/term-commands/v5-db-sync.test.ts | 28 +++- 4 files changed, 243 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c6dc0bd18..888bb466e 100644 --- a/README.md +++ b/README.md @@ -183,12 +183,15 @@ genie db sync \ --destination /path/to/destination.db ``` -Bidirectional mode unions additions. It reports a conflict when the same -mutable key differs because the databases do not carry ancestry that could -prove which row is newer. Directional mode makes the source authoritative for -shared mutable task, board, wish-group, hire-roster, and unknown metadata -keys. It writes only the destination. Both modes preserve destination-only rows: -ordinary absence never means deletion. +Bidirectional mode unions additions. When the same task or wish-group key +differs, the row with the higher `updated_at` value wins and is copied exactly +to both databases. Equal `updated_at` values with unequal row content remain a +conflict. Boards, hire-roster entries, and unknown metadata have no trustworthy +update version, so differing rows at the same key also remain conflicts. +Directional mode makes the source authoritative for shared mutable task, +board, wish-group, hire-roster, and unknown metadata keys. It writes only the +destination. Both modes preserve destination-only rows: ordinary absence never +means deletion. When exactly one named database is absent and the other is an exact-current Genie database, sync bootstraps the absent side from the complete logical image diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index 85c681a16..ea871c6cb 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -128,7 +128,7 @@ function insertTask( wish?: string | null; groupName?: string | null; createdAt?: number; - updatedAt?: number; + updatedAt?: number | bigint; lane?: string | null; agentKind?: string | null; heartbeatAt?: number | null; @@ -783,21 +783,190 @@ describe('keyed, edge-set, and history-multiset planning', () => { expect(planDatabaseReconciliation(bidirectional(valid, peer)).status).toBe('changed'); }); - test('bidirectional same-key differences conflict without exposing hostile payloads', () => { + test('bidirectional versioned rows use the later bigint updated_at independent of argument order', () => { + const left = currentDb('version-left'); + const right = currentDb('version-right'); + const earlier = 9_007_199_254_740_992n; + const later = earlier + 1n; + for (const path of [left, right]) { + mutate(path, (db) => { + insertBoard(db, { id: 'board', name: 'shared board' }); + insertTask(db, { id: 'blocker', title: 'shared blocker' }); + }); + } + mutate(left, (db) => { + insertTask(db, { + id: 'left-newer', + boardId: 'board', + title: 'complete left task', + status: 'in_progress', + claimedBy: 'left-worker', + claimedAt: 101, + wish: 'left-wish', + groupName: 'group', + createdAt: 11, + updatedAt: later, + lane: 'active', + agentKind: 'codex', + heartbeatAt: 102, + blockedBy: 'blocker', + blockedReason: 'waiting', + }); + insertTask(db, { id: 'right-newer', title: 'stale left copy', status: 'ready', updatedAt: earlier }); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) + VALUES ('wish', 'group', 'done', '[]', 'left-worker', 201, 202, 12, ?)`, + ).run(later); + }); + mutate(right, (db) => { + insertTask(db, { id: 'left-newer', title: 'stale right copy', status: 'ready', updatedAt: earlier }); + insertTask(db, { + id: 'right-newer', + boardId: 'board', + title: 'complete right task', + status: 'blocked', + claimedBy: 'right-worker', + claimedAt: 301, + wish: 'right-wish', + groupName: 'group', + createdAt: 13, + updatedAt: later, + lane: 'blocked', + agentKind: 'claude', + heartbeatAt: 302, + blockedBy: 'blocker', + blockedReason: 'dependency', + }); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) + VALUES ('wish', 'group', 'ready', '[]', NULL, NULL, NULL, 1, ?)`, + ).run(earlier); + }); + + const forward = planDatabaseReconciliation(bidirectional(left, right)); + const reverse = planDatabaseReconciliation(bidirectional(right, left)); + const leftWinner = { + id: 'left-newer', + boardId: 'board', + title: 'complete left task', + status: 'in_progress', + claimedBy: 'left-worker', + claimedAt: 101n, + wish: 'left-wish', + groupName: 'group', + createdAt: 11n, + updatedAt: later, + lane: 'active', + agentKind: 'codex', + heartbeatAt: 102n, + blockedBy: 'blocker', + blockedReason: 'waiting', + }; + const rightWinner = { + id: 'right-newer', + boardId: 'board', + title: 'complete right task', + status: 'blocked', + claimedBy: 'right-worker', + claimedAt: 301n, + wish: 'right-wish', + groupName: 'group', + createdAt: 13n, + updatedAt: later, + lane: 'blocked', + agentKind: 'claude', + heartbeatAt: 302n, + blockedBy: 'blocker', + blockedReason: 'dependency', + }; + + expect(forward.status).toBe('changed'); + expect(reverse.status).toBe('changed'); + expect(target(forward, 'left').changes.tasks).toEqual([rightWinner]); + expect(target(forward, 'right').changes.tasks).toEqual([leftWinner]); + expect(target(reverse, 'left').changes.tasks).toEqual([leftWinner]); + expect(target(reverse, 'right').changes.tasks).toEqual([rightWinner]); + expect(target(forward, 'right').changes.wishGroups).toEqual([ + { + wish: 'wish', + name: 'group', + status: 'done', + dependsOn: '[]', + assignee: 'left-worker', + startedAt: 201n, + completedAt: 202n, + createdAt: 12n, + updatedAt: later, + }, + ]); + expect(target(reverse, 'left').changes.wishGroups).toEqual(target(forward, 'right').changes.wishGroups); + expect(new Set(forward.targets.map((item) => item.postimageDigest)).size).toBe(1); + expect(new Set(reverse.targets.map((item) => item.postimageDigest)).size).toBe(1); + expect(target(forward, 'left').postimageDigest).toBe(target(reverse, 'right').postimageDigest); + }); + + test('bidirectional equal-version row differences remain conflicts', () => { + const left = currentDb('equal-version-left'); + const right = currentDb('equal-version-right'); + for (const [path, side] of [ + [left, 'left'], + [right, 'right'], + ] as const) { + mutate(path, (db) => { + insertTask(db, { id: 'task', title: `${side} task`, updatedAt: 7 }); + db.query( + `INSERT INTO wish_groups + (wish, name, status, depends_on, assignee, created_at, updated_at) + VALUES ('wish', 'group', 'ready', '[]', ?, 1, 9)`, + ).run(side); + }); + } + + const plan = planDatabaseReconciliation(bidirectional(left, right)); + + expect(plan.status).toBe('conflict'); + expect(plan.conflicts).toEqual([ + expect.objectContaining({ table: 'tasks', reason: 'same-key-difference' }), + expect.objectContaining({ table: 'wish_groups', reason: 'same-key-difference' }), + ]); + }); + + test('bidirectional unversioned same-key differences conflict without exposing hostile payloads', () => { const left = currentDb('left'); const right = currentDb('right'); - mutate(left, (db) => insertBoard(db, { id: 'same', name: '' })); - mutate(right, (db) => insertBoard(db, { id: 'same', name: ' right' })); + mutate(left, (db) => { + insertBoard(db, { id: 'same', name: '' }); + db.query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('wish', 'agent', NULL, '/left', 1, 'hired')`, + ).run(); + db.query("INSERT INTO meta (key, value) VALUES ('unknown', '')").run(); + }); + mutate(right, (db) => { + insertBoard(db, { id: 'same', name: ' right' }); + db.query( + `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) + VALUES ('wish', 'agent', NULL, '/right', 1, 'hired')`, + ).run(); + db.query("INSERT INTO meta (key, value) VALUES ('unknown', '')").run(); + }); const plan = planDatabaseReconciliation(bidirectional(left, right)); const reportJson = JSON.stringify(plan.report); expect(plan.status).toBe('conflict'); - expect(plan.conflicts).toHaveLength(1); - expect(plan.conflicts[0]).toMatchObject({ table: 'boards', reason: 'same-key-difference' }); + expect(plan.conflicts).toEqual([ + expect.objectContaining({ table: 'boards', reason: 'same-key-difference' }), + expect.objectContaining({ table: 'hire_roster', reason: 'same-key-difference' }), + expect.objectContaining({ table: 'meta', reason: 'same-key-difference' }), + ]); expect(plan.targets.every((item) => item.postimageDigest === null)).toBe(true); expect(reportJson).not.toContain('hostile-title'); expect(reportJson).not.toContain('left-title'); + expect(reportJson).not.toContain('hostile-meta'); + expect(reportJson).not.toContain('left-meta'); }); test('directional source wins shared keyed rows and unknown meta while destination-only rows are preserved', () => { @@ -809,12 +978,17 @@ describe('keyed, edge-set, and history-multiset planning', () => { ] as const) { mutate(path, (db) => { insertBoard(db, { id: 'shared-board', name: `board-${suffix}` }); - insertTask(db, { id: 'shared-task', boardId: 'shared-board', title: `task-${suffix}` }); + insertTask(db, { + id: 'shared-task', + boardId: 'shared-board', + title: `task-${suffix}`, + updatedAt: suffix === 'source' ? 1 : 999, + }); db.query( `INSERT INTO wish_groups (wish, name, status, depends_on, assignee, started_at, completed_at, created_at, updated_at) - VALUES ('w', 'g', 'ready', '[]', ?, NULL, NULL, 1, 1)`, - ).run(suffix); + VALUES ('w', 'g', 'ready', '[]', ?, NULL, NULL, 1, ?)`, + ).run(suffix, suffix === 'source' ? 1 : 999); db.query( `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index 0e70920c5..da2f44d19 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -1914,6 +1914,7 @@ function reconcileBidirectionalKeyed( values: (row: T) => readonly (string | bigint | null)[], conflicts: ReconciliationConflict[], excludedKey?: string, + version?: (row: T) => bigint, ): void { const keys = new Set([...left.keys(), ...right.keys()]); for (const key of [...keys].sort(compareCanonical)) { @@ -1923,7 +1924,13 @@ function reconcileBidirectionalKeyed( if (leftRow === undefined && rightRow !== undefined) left.set(key, rightRow); else if (rightRow === undefined && leftRow !== undefined) right.set(key, leftRow); else if (leftRow !== undefined && rightRow !== undefined && !rowEqual(leftRow, rightRow, values)) { - addConflict(conflicts, table, 'same-key-difference', key); + if (version === undefined || version(leftRow) === version(rightRow)) { + addConflict(conflicts, table, 'same-key-difference', key); + } else { + const winner = version(leftRow) > version(rightRow) ? leftRow : rightRow; + left.set(key, winner); + right.set(key, winner); + } } } } @@ -2165,8 +2172,24 @@ function reconcileStates( if (mode === 'bidirectional') { reconcileBidirectionalKeyed('boards', left.boards, right.boards, boardValues, conflicts); - reconcileBidirectionalKeyed('tasks', left.tasks, right.tasks, taskValues, conflicts); - reconcileBidirectionalKeyed('wish_groups', left.wishGroups, right.wishGroups, wishGroupValues, conflicts); + reconcileBidirectionalKeyed( + 'tasks', + left.tasks, + right.tasks, + taskValues, + conflicts, + undefined, + (row) => row.updatedAt, + ); + reconcileBidirectionalKeyed( + 'wish_groups', + left.wishGroups, + right.wishGroups, + wishGroupValues, + conflicts, + undefined, + (row) => row.updatedAt, + ); reconcileBidirectionalKeyed('hire_roster', left.hireRoster, right.hireRoster, hireRosterValues, conflicts); reconcileBidirectionalKeyed('meta', left.meta, right.meta, metaValues, conflicts, markerKey); reconcileBidirectionalSet(left.taskDependencies, right.taskDependencies); diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts index 39f582a0e..0597c55ad 100644 --- a/src/term-commands/v5-db-sync.test.ts +++ b/src/term-commands/v5-db-sync.test.ts @@ -96,14 +96,14 @@ function parseJson(result: CliResult): JsonReport { return JSON.parse(result.stdout) as JsonReport; } -function insertTask(path: string, id: string, title: string): void { +function insertTask(path: string, id: string, title: string, updatedAt: number | bigint = 1): void { const db = openDb({ path }); db.query('INSERT INTO tasks (id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)').run( id, title, 'ready', 1, - 1, + updatedAt, ); db.close(); } @@ -670,6 +670,28 @@ describe('database sync CLI contract', () => { expect(parseJson(noOp).status).toBe('no-op'); }); + test('bidirectional apply converges both left-newer and right-newer tasks independent of argument order', async () => { + const left = createRepo('version-left'); + const right = createRepo('version-right'); + const earlier = 9_007_199_254_740_992n; + const later = earlier + 1n; + insertTask(left.database, 'left-newer', 'left wins', later); + insertTask(right.database, 'left-newer', 'stale right', earlier); + insertTask(left.database, 'right-newer', 'stale left', earlier); + insertTask(right.database, 'right-newer', 'right wins', later); + + const result = await sync(left.database, right.database, '--json'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(result).status).toBe('changed'); + expect(taskTitles(left.database)).toEqual(['left wins', 'right wins']); + expect(taskTitles(right.database)).toEqual(['left wins', 'right wins']); + + const reversed = await sync(right.database, left.database, '--json'); + expect(reversed.code).toBe(DATABASE_SYNC_EXIT_CODES.success); + expect(parseJson(reversed).status).toBe('no-op'); + }); + test('bidirectional apply propagates an explicit roster tombstone without a schema migration', async () => { const host = createRepo('tombstone-host'); const guest = createRepo('tombstone-guest'); @@ -696,7 +718,7 @@ describe('database sync CLI contract', () => { } }); - test('bidirectional mutable conflicts use exit 3, bounded digests/counts, and zero mutation', async () => { + test('bidirectional equal-version mutable conflicts use exit 3, bounded digests/counts, and zero mutation', async () => { const left = createRepo('left'); const right = createRepo('right'); insertTask(left.database, 't_shared', 'LEFT SECRET'); From e800563dc49bafdd70d8cdae3c4b653fb4607fa9 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Sun, 2 Aug 2026 19:16:53 +0800 Subject: [PATCH 21/21] fix(db): address reconciliation review findings Make deletion markers versioned so a later rehire or deletion converges safely. Use immediate task-state transactions to preserve ordering under concurrent writers. Treat empty WAL and SHM hardlinks as inert and retain native errno details. Fail closed on unknown CLI outcomes and count recovery cleanup failures. Make CLI and filesystem tests portable and isolated. Verified with bun run check: 3113 tests passed. --- README.md | 13 ++-- scripts/release-docs.test.ts | 6 +- src/genie.ts | 6 +- src/lib/interactivity.ts | 2 + src/lib/v5/db-reconciliation.test.ts | 60 ++++++++++++-- src/lib/v5/db-reconciliation.ts | 69 ++++++++++++---- src/lib/v5/db-sync-snapshots.test.ts | 46 ++++++++--- src/lib/v5/db-sync-snapshots.ts | 2 +- src/lib/v5/reconciliation-tombstone.ts | 21 +++-- src/lib/v5/task-state.test.ts | 15 ++-- src/lib/v5/task-state.ts | 49 +++++++++--- src/term-commands/v5-db-sync.test.ts | 104 +++++++++++++++++-------- src/term-commands/v5-db-sync.ts | 68 ++++++++++------ tests/support/codex-dogfood-harness.ts | 2 +- 14 files changed, 339 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 888bb466e..195cce2a9 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,10 @@ genie db sync \ Bidirectional mode unions additions. When the same task or wish-group key differs, the row with the higher `updated_at` value wins and is copied exactly to both databases. Equal `updated_at` values with unequal row content remain a -conflict. Boards, hire-roster entries, and unknown metadata have no trustworthy -update version, so differing rows at the same key also remain conflicts. +conflict. Hire-roster rows and their deletion tombstones resolve by their +`hired_at` or deletion timestamp, so a later explicit re-hire can supersede a +replicated deletion. Boards and unknown metadata have no trustworthy update +version, so differing rows at the same key remain conflicts. Directional mode makes the source authoritative for shared mutable task, board, wish-group, hire-roster, and unknown metadata keys. It writes only the destination. Both modes preserve destination-only rows: ordinary absence never @@ -203,10 +205,9 @@ An explicit roster unhire (the `roster_unhire` UI-bridge operation) is the exception. It records a versioned tombstone in the existing metadata table, so a later sync removes the matching hire-roster row instead of resurrecting it. This changes neither the SQLite schema nor its `user_version`; reconciliation -reports the explicit removal in its deletion count. Tombstones are durable and -deletion wins while any replica retains one. Re-hiring the same agent clears -the local tombstone, but an offline replica with the old tombstone can reassert -the deletion when it returns. +reports the explicit removal in its deletion count. Tombstones are durable, but +their timestamp participates in reconciliation: a later re-hire wins over an +older deletion, and a later unhire wins over the replicated live row. Dependency edges are unioned. Task events and legacy stage-log entries preserve the maximum occurrence count observed on either applicable side. Independent diff --git a/scripts/release-docs.test.ts b/scripts/release-docs.test.ts index 3699af15f..5cfcda79b 100644 --- a/scripts/release-docs.test.ts +++ b/scripts/release-docs.test.ts @@ -750,11 +750,11 @@ describe('Group E release and documentation contracts', () => { }); test('database reconciliation docs describe explicit live paths and both bounded lock layers', () => { - const readme = read('README.md'); - expect(readme).toContain('An explicit\npath may name a database that active Genie processes also use.'); + const readme = read('README.md').replace(/\s+/g, ' '); + expect(readme).toContain('An explicit path may name a database that active Genie processes also use.'); expect(readme).toContain('canonical-path advisory locks'); expect(readme).toContain('SQLite write locks'); - expect(readme).toContain('bounds the combined\nwait for both lock layers'); + expect(readme).toContain('bounds the combined wait for both lock layers'); expect(readme).toContain('repeating one is an actionable usage error'); expect(readme).not.toContain('It never discovers or connects a live shared database.'); }); diff --git a/src/genie.ts b/src/genie.ts index 7c2dfd8f7..40408d3cf 100644 --- a/src/genie.ts +++ b/src/genie.ts @@ -200,10 +200,6 @@ registerOmniCommands(program); // Universal workspace check — ensures workspace exists before commands that need it // ============================================================================ -// `db` takes explicit database paths and must validate them before any legacy -// workspace prompt can create state. It is therefore standalone like task and -// board, without broadening the legacy middleware's shared exemption list. -const requestedRootCommand = process.argv.slice(2).find((argument) => !argument.startsWith('-')); -if (requestedRootCommand !== 'db') installWorkspaceCheck(program); +installWorkspaceCheck(program); await program.parseAsync(process.argv); diff --git a/src/lib/interactivity.ts b/src/lib/interactivity.ts index 86137e435..268e3e455 100644 --- a/src/lib/interactivity.ts +++ b/src/lib/interactivity.ts @@ -67,6 +67,8 @@ const WORKSPACE_EXEMPT = new Set([ // behavior. 'task', 'board', + // `db` operates only on explicit database paths and owns their validation. + 'db', // `idea` is the one-verb quick-capture (roadmap board's Idea lane). Same v5 // sqlite-backed self-resolving DB as `task`/`board`; it must work in a fresh // repo with no workspace.json (QA: `genie idea` on a fresh repo). diff --git a/src/lib/v5/db-reconciliation.test.ts b/src/lib/v5/db-reconciliation.test.ts index ea871c6cb..ebd82e111 100644 --- a/src/lib/v5/db-reconciliation.test.ts +++ b/src/lib/v5/db-reconciliation.test.ts @@ -33,7 +33,7 @@ import { } from './db-reconciliation.js'; import { openDb } from './genie-db.js'; import { RECONCILIATION_TOMBSTONE_PREFIX, reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; -import { unhireAgent } from './task-state.js'; +import { hireAgent, unhireAgent } from './task-state.js'; let fixtureRoot: string; @@ -90,7 +90,9 @@ async function spawnFlockHolder(lockPath: string, holdMs: number): Promise { expect(plan.report.targets[0].changes.deletions).toBe(0); }); - test('a roster tombstone propagates a guest deletion instead of resurrecting the live host row', () => { + test('versioned roster tombstones support delete, later re-hire, and later delete', () => { const host = currentDb('host'); const guest = currentDb('guest'); for (const path of [host, guest]) { @@ -1034,7 +1036,7 @@ describe('keyed, edge-set, and history-multiset planning', () => { expect(plan.status).toBe('changed'); expect(target(plan, 'left').changes.deletions).toEqual([ - { table: 'hire_roster', wish: 'w', agentAdapterId: 'agent' }, + expect.objectContaining({ table: 'hire_roster', wish: 'w', agentAdapterId: 'agent' }), ]); expect(target(plan, 'right').changes.deletions).toEqual([]); expect(plan.report.targets.map(({ changes }) => changes.deletions)).toEqual([1, 0]); @@ -1048,6 +1050,30 @@ describe('keyed, edge-set, and history-multiset planning', () => { }); } expect(planDatabaseReconciliation(bidirectional(host, guest)).status).toBe('no-op'); + + mutate(host, (db) => { + const rehired = hireAgent(db, { wish: 'w', agentAdapterId: 'agent', worktree: '/wt/re-hired' }); + expect(rehired.hiredAt).toBeGreaterThan(1); + }); + const resurrection = planDatabaseReconciliation(bidirectional(host, guest)); + expect(resurrection.status).toBe('changed'); + expect(resurrection.targets.every((item) => item.changes.deletions.length === 0)).toBe(true); + expect(applyDatabaseReconciliation(resurrection)).toMatchObject({ status: 'changed', converged: true }); + for (const path of [host, guest]) { + mutate(path, (db) => { + expect(db.query('SELECT worktree FROM hire_roster').all()).toEqual([{ worktree: '/wt/re-hired' }]); + }); + } + expect(planDatabaseReconciliation(bidirectional(host, guest)).status).toBe('no-op'); + + mutate(guest, (db) => expect(unhireAgent(db, 'w', 'agent')).toBe(true)); + const secondDeletion = planDatabaseReconciliation(bidirectional(host, guest)); + expect(secondDeletion.status).toBe('changed'); + expect(target(secondDeletion, 'left').changes.deletions).toHaveLength(1); + expect(applyDatabaseReconciliation(secondDeletion)).toMatchObject({ status: 'changed', converged: true }); + for (const path of [host, guest]) { + mutate(path, (db) => expect(db.query('SELECT * FROM hire_roster').all()).toEqual([])); + } }); test('malformed tombstones and a live-row contradiction fail closed before planning', () => { @@ -1068,7 +1094,12 @@ describe('keyed, edge-set, and history-multiset planning', () => { (wish, agent_adapter_id, profile, worktree, hired_at, state) VALUES ('w', 'agent', NULL, '/wt/agent', 1, 'hired')`, ).run(); - const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish: 'w', agentAdapterId: 'agent' }); + const tombstone = reconciliationTombstoneMeta({ + table: 'hire_roster', + wish: 'w', + agentAdapterId: 'agent', + deletedAt: 1, + }); db.query('INSERT INTO meta (key, value) VALUES (?, ?)').run(tombstone.key, tombstone.value); }); expect(dryRunDatabaseReconciliation(bidirectional(contradictory, peer)).operationalFailure?.code).toBe( @@ -1381,6 +1412,25 @@ describe('same-file, idempotency, and bounded failures', () => { expect(plan.targets.every((item) => !Object.values(item.changes).some((rows) => rows.length > 0))).toBe(true); }); + test('hardlink aliases remain same-database no-ops with inert SQLite sidecar pathnames', () => { + const path = currentDb('inert-sidecar-alias'); + const alias = join(fixtureRoot, 'inert-sidecar-hardlink.db'); + linkSync(path, alias); + writeFileSync(`${path}-wal`, ''); + writeFileSync(`${path}-shm`, ''); + + for (const request of [ + bidirectional(path, alias), + bidirectional(alias, path), + { mode: 'directional' as const, sourcePath: path, destinationPath: alias }, + { mode: 'directional' as const, sourcePath: alias, destinationPath: path }, + ]) { + const plan = planDatabaseReconciliation(request); + expect(plan.status).toBe('same-database'); + expect(plan.sameDatabase).toBe(true); + } + }); + test('hardlink aliases with a committed path-specific WAL are rejected independent of order and mode', () => { const path = currentDb('wal-alias'); const alias = join(fixtureRoot, 'wal-hardlink.db'); diff --git a/src/lib/v5/db-reconciliation.ts b/src/lib/v5/db-reconciliation.ts index da2f44d19..d441bc02c 100644 --- a/src/lib/v5/db-reconciliation.ts +++ b/src/lib/v5/db-reconciliation.ts @@ -440,7 +440,7 @@ interface PhysicalInput { readonly canonicalPath: string; readonly device: string; readonly inode: string; - readonly hasSidecars: boolean; + readonly hasWalContent: boolean; } interface ColumnFingerprint { @@ -794,7 +794,9 @@ function resolvePhysicalInput(path: string): PhysicalInput { canonicalPath, device: String(stats.dev), inode: String(stats.ino), - hasSidecars: hasWal || hasShm, + // SQLite and Bun may leave empty WAL/SHM pathnames after all handles + // close. Only WAL bytes can carry path-specific database state. + hasWalContent: walBytes > 0, }; } catch (caught) { if (caught instanceof ReconciliationError) throw caught; @@ -821,7 +823,7 @@ function revalidatePhysicalInput(input: PhysicalInput): void { function samePhysicalInput(left: PhysicalInput, right: PhysicalInput): boolean { if (left.canonicalPath === right.canonicalPath) return true; if (left.device !== right.device || left.inode !== right.inode) return false; - if (left.hasSidecars || right.hasSidecars) { + if (left.hasWalContent || right.hasWalContent) { throw error('invalid-data', 'Hardlink aliases with path-specific SQLite sidecars are ambiguous.'); } return true; @@ -1752,7 +1754,12 @@ function reconciliationTombstones(state: LogicalState): ReconciliationTombstone[ } catch { throw error('invalid-data', 'A reconciliation input contains invalid tombstone metadata.'); } - return tombstones.sort((left, right) => compareCanonical(JSON.stringify(left), JSON.stringify(right))); + return tombstones.sort((left, right) => + compareCanonical( + canonicalTuple([left.wish, left.agentAdapterId, left.deletedAt]), + canonicalTuple([right.wish, right.agentAdapterId, right.deletedAt]), + ), + ); } function tombstoneKey(tombstone: ReconciliationTombstone): string { @@ -1764,17 +1771,32 @@ function tombstoneHasLiveRow(state: LogicalState, tombstone: ReconciliationTombs } function validateInputTombstones(state: LogicalState): void { - if (reconciliationTombstones(state).some((tombstone) => tombstoneHasLiveRow(state, tombstone))) { - throw error('invalid-data', 'A reconciliation input contains both a tombstone and its live row.'); + for (const tombstone of reconciliationTombstones(state)) { + const live = state.hireRoster.get(tombstoneKey(tombstone)); + if (live !== undefined && live.hiredAt <= tombstone.deletedAt) { + throw error('invalid-data', 'A reconciliation input contains a live row no newer than its tombstone.'); + } } } -function applyTombstonesToState(state: LogicalState): void { +function applyTombstonesToState(state: LogicalState, conflicts?: ReconciliationConflict[]): void { for (const tombstone of reconciliationTombstones(state)) { - state.hireRoster.delete(tombstoneKey(tombstone)); + const key = tombstoneKey(tombstone); + const live = state.hireRoster.get(key); + if (live === undefined) continue; + if (live.hiredAt > tombstone.deletedAt) continue; + if (live.hiredAt === tombstone.deletedAt) { + if (conflicts !== undefined) addConflict(conflicts, 'hire_roster', 'same-key-difference', key); + continue; + } + state.hireRoster.delete(key); } } +function metaVersion(row: MetaReconciliationRow): bigint | null { + return parseReconciliationTombstoneMeta(row.key, row.value)?.deletedAt ?? null; +} + function boardValues(row: BoardReconciliationRow): readonly (string | bigint | null)[] { return [row.id, row.name, row.createdAt, row.lanes]; } @@ -1914,7 +1936,7 @@ function reconcileBidirectionalKeyed( values: (row: T) => readonly (string | bigint | null)[], conflicts: ReconciliationConflict[], excludedKey?: string, - version?: (row: T) => bigint, + version?: (row: T) => bigint | null, ): void { const keys = new Set([...left.keys(), ...right.keys()]); for (const key of [...keys].sort(compareCanonical)) { @@ -1924,10 +1946,12 @@ function reconcileBidirectionalKeyed( if (leftRow === undefined && rightRow !== undefined) left.set(key, rightRow); else if (rightRow === undefined && leftRow !== undefined) right.set(key, leftRow); else if (leftRow !== undefined && rightRow !== undefined && !rowEqual(leftRow, rightRow, values)) { - if (version === undefined || version(leftRow) === version(rightRow)) { + const leftVersion = version?.(leftRow) ?? null; + const rightVersion = version?.(rightRow) ?? null; + if (leftVersion === null || rightVersion === null || leftVersion === rightVersion) { addConflict(conflicts, table, 'same-key-difference', key); } else { - const winner = version(leftRow) > version(rightRow) ? leftRow : rightRow; + const winner = leftVersion > rightVersion ? leftRow : rightRow; left.set(key, winner); right.set(key, winner); } @@ -2190,8 +2214,16 @@ function reconcileStates( undefined, (row) => row.updatedAt, ); - reconcileBidirectionalKeyed('hire_roster', left.hireRoster, right.hireRoster, hireRosterValues, conflicts); - reconcileBidirectionalKeyed('meta', left.meta, right.meta, metaValues, conflicts, markerKey); + reconcileBidirectionalKeyed( + 'hire_roster', + left.hireRoster, + right.hireRoster, + hireRosterValues, + conflicts, + undefined, + (row) => row.hiredAt, + ); + reconcileBidirectionalKeyed('meta', left.meta, right.meta, metaValues, conflicts, markerKey, metaVersion); reconcileBidirectionalSet(left.taskDependencies, right.taskDependencies); reconcileBidirectionalCounts(left.stageLog, right.stageLog); reconcileBidirectionalCounts(left.taskEvents, right.taskEvents); @@ -2209,8 +2241,15 @@ function reconcileStates( } if (conflicts.length === 0) { - applyTombstonesToState(left); - applyTombstonesToState(right); + if (mode === 'bidirectional') { + applyTombstonesToState(left, conflicts); + applyTombstonesToState(right); + } else { + applyTombstonesToState(left); + applyTombstonesToState(right, conflicts); + } + } + if (conflicts.length === 0) { if (mode === 'bidirectional') validatePlannedTargetIntegrity(left, 'left', conflicts); validatePlannedTargetIntegrity(right, mode === 'bidirectional' ? 'right' : 'destination', conflicts); } diff --git a/src/lib/v5/db-sync-snapshots.test.ts b/src/lib/v5/db-sync-snapshots.test.ts index 663ebee31..829084105 100644 --- a/src/lib/v5/db-sync-snapshots.test.ts +++ b/src/lib/v5/db-sync-snapshots.test.ts @@ -9,6 +9,7 @@ import { linkSync, mkdirSync, mkdtempSync, + openSync, readFileSync, readdirSync, readlinkSync, @@ -160,8 +161,8 @@ function descriptorCount(): number { return readdirSync(directory).length; } -function privateSnapshotDirectories(): string[] { - return readdirSync(tmpdir()).filter((name) => name.startsWith('genie-db-sync-private-')); +function privateSnapshotDirectories(directory: string): string[] { + return readdirSync(directory).filter((name) => name.startsWith('genie-db-sync-private-')); } function leaveCompleteGeneration(left: string, right: string): { root: string; directory: string } { @@ -1994,15 +1995,18 @@ describe('database sync snapshots', () => { expect(existsSync(join(root, '.staging-darwin'))).toBe(false); expect(generationDirectories(root)).toHaveLength(1); - const privateBefore = privateSnapshotDirectories(); + const privateTemporaryDirectory = join(fixtureRoot, 'darwin-private-temp'); + mkdirSync(privateTemporaryDirectory, { mode: 0o700 }); + const privateBefore = privateSnapshotDirectories(privateTemporaryDirectory); insertBoard(right, 'private'); expect( applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { keepSnapshots: 0, posixDirectory: darwin, + privateRoot: { temporaryDirectory: privateTemporaryDirectory }, }).status, ).toBe('changed'); - expect(privateSnapshotDirectories()).toEqual(privateBefore); + expect(privateSnapshotDirectories(privateTemporaryDirectory)).toEqual(privateBefore); expect(directoryRemovals).toBeGreaterThanOrEqual(3); expect(candidates).toContain('darwin:/usr/lib/libSystem.B.dylib'); }); @@ -2013,16 +2017,19 @@ describe('database sync snapshots', () => { insertBoard(right, 'private'); const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; const identity = databaseSyncSnapshotIdentity(request); + const privateTemporaryDirectory = join(fixtureRoot, 'private-success-temp'); + mkdirSync(privateTemporaryDirectory, { mode: 0o700 }); let privateRoot: string | undefined; let privateMode: number | undefined; const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { keepSnapshots: 0, + privateRoot: { temporaryDirectory: privateTemporaryDirectory }, onEvent: (event) => { if (event.phase === 'payload-write' && event.path === undefined && privateRoot === undefined) { // The generation ID is intentionally not sufficient to discover the OS-private root. - privateRoot = readdirSync(tmpdir()) + privateRoot = readdirSync(privateTemporaryDirectory) .filter((name) => name.startsWith('genie-db-sync-private-')) - .map((name) => join(tmpdir(), name)) + .map((name) => join(privateTemporaryDirectory, name)) .find((path) => statSync(path).isDirectory()); privateMode = privateRoot === undefined ? undefined : statSync(privateRoot).mode & 0o777; } @@ -2089,11 +2096,14 @@ describe('database sync snapshots', () => { keepSnapshots: 3, }).status, ).toBe('changed'); - const privateBefore = privateSnapshotDirectories(); + const privateTemporaryDirectory = join(fixtureRoot, 'private-prune-failure-temp'); + mkdirSync(privateTemporaryDirectory, { mode: 0o700 }); + const privateBefore = privateSnapshotDirectories(privateTemporaryDirectory); insertBoard(right, 'private'); let injected = false; const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { keepSnapshots: 0, + privateRoot: { temporaryDirectory: privateTemporaryDirectory }, onEvent: (event) => { if (!injected && event.phase === 'prune' && event.state === 'before') { injected = true; @@ -2106,7 +2116,7 @@ describe('database sync snapshots', () => { status: 'changed', cleanupFailures: ['snapshot-cleanup-failed'], }); - expect(privateSnapshotDirectories()).toEqual(privateBefore); + expect(privateSnapshotDirectories(privateTemporaryDirectory)).toEqual(privateBefore); expect(generationDirectories(databaseSyncSnapshotIdentity(request).root)).toHaveLength(1); }); @@ -2115,14 +2125,17 @@ describe('database sync snapshots', () => { const right = currentDb('private-failure-right'); insertBoard(right, 'private'); const request = { mode: 'bidirectional' as const, leftPath: left, rightPath: right }; + const privateTemporaryDirectory = join(fixtureRoot, 'private-cleanup-failure-temp'); + mkdirSync(privateTemporaryDirectory, { mode: 0o700 }); let leakedRoot: string | undefined; const report = applyDatabaseReconciliationWithSnapshots(planDatabaseReconciliation(request), { keepSnapshots: 0, + privateRoot: { temporaryDirectory: privateTemporaryDirectory }, onEvent: (event) => { if (event.phase === 'payload-write' && leakedRoot === undefined) { - leakedRoot = readdirSync(tmpdir()) + leakedRoot = readdirSync(privateTemporaryDirectory) .filter((name) => name.startsWith('genie-db-sync-private-')) - .map((name) => join(tmpdir(), name)) + .map((name) => join(privateTemporaryDirectory, name)) .find((path) => statSync(path).isDirectory()); } }, @@ -2329,6 +2342,19 @@ describe('database sync snapshots', () => { expect(resolveSnapshotPosixDirectory({ platform: 'win32', architecture: 'x64' })).toBeNull(); }); + test('native POSIX directory listing reads names from the dirent name offset', () => { + const directory = join(fixtureRoot, 'dirent-name-offset'); + mkdirSync(directory); + writeFileSync(join(directory, 'alpha-generation'), 'alpha'); + writeFileSync(join(directory, 'beta-generation'), 'beta'); + const descriptor = openSync(directory, 'r'); + try { + expect([...nativePosixDirectory().list(descriptor)].sort()).toEqual(['alpha-generation', 'beta-generation']); + } finally { + closeSync(descriptor); + } + }); + test('root and child descriptors close on invalid options, staging faults, and reacquisition', () => { const native = nativePosixDirectory(); const left = currentDb('descriptor-lifetime-left'); diff --git a/src/lib/v5/db-sync-snapshots.ts b/src/lib/v5/db-sync-snapshots.ts index 248d35081..befa375c4 100644 --- a/src/lib/v5/db-sync-snapshots.ts +++ b/src/lib/v5/db-sync-snapshots.ts @@ -563,7 +563,7 @@ function openPosixDirectoryApi( componentBytes(destinationName), ) !== 0 ) { - throw new Error('renameat failed'); + throw nativeError('renameat', errno[0]); } }, unlinkAt(directoryDescriptor, name, directory) { diff --git a/src/lib/v5/reconciliation-tombstone.ts b/src/lib/v5/reconciliation-tombstone.ts index 0e718bc77..0e17223cd 100644 --- a/src/lib/v5/reconciliation-tombstone.ts +++ b/src/lib/v5/reconciliation-tombstone.ts @@ -1,22 +1,23 @@ import { Buffer } from 'node:buffer'; export const RECONCILIATION_TOMBSTONE_PREFIX = 'genie:reconciliation-tombstone:v1:'; -export const RECONCILIATION_TOMBSTONE_VALUE = 'deleted'; const MAX_ENCODED_KEY_BYTES = 4_096; +const MAX_SQLITE_INTEGER = 9_223_372_036_854_775_807n; export interface ReconciliationTombstone { readonly table: 'hire_roster'; readonly wish: string; readonly agentAdapterId: string; + readonly deletedAt: bigint; } export interface ReconciliationTombstoneMeta { readonly key: string; - readonly value: typeof RECONCILIATION_TOMBSTONE_VALUE; + readonly value: string; } -function tombstoneParts(tombstone: ReconciliationTombstone): readonly string[] { +function tombstoneParts(tombstone: Pick): readonly string[] { return [tombstone.wish, tombstone.agentAdapterId]; } @@ -26,10 +27,14 @@ function encodedParts(parts: readonly string[]): string { return encoded; } -export function reconciliationTombstoneMeta(tombstone: ReconciliationTombstone): ReconciliationTombstoneMeta { +export function reconciliationTombstoneMeta( + tombstone: Omit & { readonly deletedAt: number | bigint }, +): ReconciliationTombstoneMeta { + const deletedAt = BigInt(tombstone.deletedAt); + if (deletedAt < 0n || deletedAt > MAX_SQLITE_INTEGER) invalidTombstone(); return { key: `${RECONCILIATION_TOMBSTONE_PREFIX}${tombstone.table}:${encodedParts(tombstoneParts(tombstone))}`, - value: RECONCILIATION_TOMBSTONE_VALUE, + value: deletedAt.toString(), }; } @@ -55,12 +60,14 @@ function decodedParts(encoded: string): readonly [string, string] { export function parseReconciliationTombstoneMeta(key: string, value: string): ReconciliationTombstone | null { if (!key.startsWith(RECONCILIATION_TOMBSTONE_PREFIX)) return null; - if (value !== RECONCILIATION_TOMBSTONE_VALUE) invalidTombstone(); + if (!/^(0|[1-9][0-9]*)$/.test(value)) invalidTombstone(); + const deletedAt = BigInt(value); + if (deletedAt > MAX_SQLITE_INTEGER) invalidTombstone(); const suffix = key.slice(RECONCILIATION_TOMBSTONE_PREFIX.length); const separator = suffix.indexOf(':'); if (separator < 1) invalidTombstone(); const table = suffix.slice(0, separator); const [first, second] = decodedParts(suffix.slice(separator + 1)); - if (table === 'hire_roster') return { table, wish: first, agentAdapterId: second }; + if (table === 'hire_roster') return { table, wish: first, agentAdapterId: second, deletedAt }; return invalidTombstone(); } diff --git a/src/lib/v5/task-state.test.ts b/src/lib/v5/task-state.test.ts index f24325c8b..407cdb69f 100644 --- a/src/lib/v5/task-state.test.ts +++ b/src/lib/v5/task-state.test.ts @@ -661,12 +661,16 @@ describe('hire roster (single-row upsert / delete)', () => { .query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name") .all() as Array<{ name: string }> ).map(({ name }) => name); - const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish: 'w', agentAdapterId: 'a' }); + const tombstone = reconciliationTombstoneMeta({ + table: 'hire_roster', + wish: 'w', + agentAdapterId: 'a', + deletedAt: 0, + }); expect(unhireAgent(db, 'w', 'a')).toBe(false); - expect(db.query('SELECT value FROM meta WHERE key = ?').get(tombstone.key)).toEqual({ - value: tombstone.value, - }); + const stored = db.query('SELECT value FROM meta WHERE key = ?').get(tombstone.key) as { value: string }; + expect(stored.value).toMatch(/^[1-9][0-9]*$/); expect(db.query('PRAGMA user_version').get()).toEqual(schemaVersion); expect( ( @@ -676,9 +680,10 @@ describe('hire roster (single-row upsert / delete)', () => { ).map(({ name }) => name), ).toEqual(tableNames); - hireAgent(db, { wish: 'w', agentAdapterId: 'a', worktree: '/wt/re-hired' }); + const rehired = hireAgent(db, { wish: 'w', agentAdapterId: 'a', worktree: '/wt/re-hired' }); expect(db.query('SELECT value FROM meta WHERE key = ?').get(tombstone.key)).toBeNull(); expect(getHire(db, 'w', 'a')?.worktree).toBe('/wt/re-hired'); + expect(BigInt(rehired.hiredAt)).toBeGreaterThan(BigInt(stored.value)); }); test('listHires scopes by wish and orders stably', () => { diff --git a/src/lib/v5/task-state.ts b/src/lib/v5/task-state.ts index 526a5f635..24079d83b 100644 --- a/src/lib/v5/task-state.ts +++ b/src/lib/v5/task-state.ts @@ -13,7 +13,7 @@ import type { Database } from 'bun:sqlite'; import { createHash, randomBytes } from 'node:crypto'; -import { reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; +import { parseReconciliationTombstoneMeta, reconciliationTombstoneMeta } from './reconciliation-tombstone.js'; // ============================================================================ // Type boundaries @@ -1290,6 +1290,10 @@ function mapHire(row: RawHire): HireRosterRow { }; } +function maxBigInt(left: bigint, right: bigint): bigint { + return left > right ? left : right; +} + /** * Hire an agent adapter into a wish. Idempotent single-row upsert keyed on * `(wish, agent_adapter_id)`: a re-hire refreshes profile/worktree/state but @@ -1299,15 +1303,22 @@ function mapHire(row: RawHire): HireRosterRow { * The transaction also clears this replica's reconciliation tombstone. */ export function hireAgent(db: Database, input: HireAgentInput): HireRosterRow { - const now = Date.now(); const state = input.state ?? 'hired'; - const tombstone = reconciliationTombstoneMeta({ + const tombstoneKey = reconciliationTombstoneMeta({ table: 'hire_roster', wish: input.wish, agentAdapterId: input.agentAdapterId, - }); + deletedAt: 0, + }).key; db.transaction(() => { - db.query('DELETE FROM meta WHERE key = ?').run(tombstone.key); + const priorMarker = db.query('SELECT value FROM meta WHERE key = ?').get(tombstoneKey) as { value: string } | null; + const priorDeletion = + priorMarker === null ? null : parseReconciliationTombstoneMeta(tombstoneKey, priorMarker.value); + const now = Number( + priorDeletion === null ? BigInt(Date.now()) : maxBigInt(BigInt(Date.now()), priorDeletion.deletedAt + 1n), + ); + if (!Number.isSafeInteger(now)) throw new Error('Roster reconciliation version exceeds the safe timestamp range.'); + db.query('DELETE FROM meta WHERE key = ?').run(tombstoneKey); db.query( `INSERT INTO hire_roster (wish, agent_adapter_id, profile, worktree, hired_at, state) VALUES (?, ?, ?, ?, ?, ?) @@ -1316,7 +1327,7 @@ export function hireAgent(db: Database, input: HireAgentInput): HireRosterRow { worktree = excluded.worktree, state = excluded.state`, ).run(input.wish, input.agentAdapterId, input.profile ?? null, input.worktree, now, state); - })(); + }).immediate(); return getHire(db, input.wish, input.agentAdapterId) as HireRosterRow; } @@ -1326,12 +1337,26 @@ export function hireAgent(db: Database, input: HireAgentInput): HireRosterRow { * boolean reports whether this call removed a live local row. */ export function unhireAgent(db: Database, wish: string, agentAdapterId: string): boolean { - const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish, agentAdapterId }); - return db.transaction(() => { - const res = db.query('DELETE FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?').run(wish, agentAdapterId); - db.query('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)').run(tombstone.key, tombstone.value); - return res.changes > 0; - })(); + const tombstoneKey = reconciliationTombstoneMeta({ table: 'hire_roster', wish, agentAdapterId, deletedAt: 0 }).key; + return db + .transaction(() => { + const live = db + .query('SELECT hired_at FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?') + .get(wish, agentAdapterId) as { hired_at: number } | null; + const priorMarker = db.query('SELECT value FROM meta WHERE key = ?').get(tombstoneKey) as { + value: string; + } | null; + const priorDeletion = + priorMarker === null ? null : parseReconciliationTombstoneMeta(tombstoneKey, priorMarker.value); + let deletedAt = BigInt(Date.now()); + if (live !== null) deletedAt = maxBigInt(deletedAt, BigInt(live.hired_at) + 1n); + if (priorDeletion !== null) deletedAt = maxBigInt(deletedAt, priorDeletion.deletedAt + 1n); + const tombstone = reconciliationTombstoneMeta({ table: 'hire_roster', wish, agentAdapterId, deletedAt }); + const res = db.query('DELETE FROM hire_roster WHERE wish = ? AND agent_adapter_id = ?').run(wish, agentAdapterId); + db.query('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)').run(tombstone.key, tombstone.value); + return res.changes > 0; + }) + .immediate(); } export function getHire(db: Database, wish: string, agentAdapterId: string): HireRosterRow | null { diff --git a/src/term-commands/v5-db-sync.test.ts b/src/term-commands/v5-db-sync.test.ts index 0597c55ad..5e37141ff 100644 --- a/src/term-commands/v5-db-sync.test.ts +++ b/src/term-commands/v5-db-sync.test.ts @@ -80,6 +80,7 @@ async function cli(cwd: string, ...args: string[]): Promise { env: { ...process.env, NO_COLOR: '1', + GENIE_HOME: join(root, 'genie-home'), GENIE_TEST_SKIP_PGSERVE: '1', }, }); @@ -176,7 +177,7 @@ async function syncWithInjectedReport(report: unknown, ...args: string[]): Promi cwd: root, stdout: 'pipe', stderr: 'pipe', - env: { ...process.env, NO_COLOR: '1' }, + env: { ...process.env, NO_COLOR: '1', GENIE_HOME: join(root, 'runner-genie-home') }, }); const stdout = await new Response(proc.stdout).text(); const stderr = await new Response(proc.stderr).text(); @@ -302,40 +303,61 @@ describe('database sync CLI contract', () => { const otherSnapshotRoot = join(root, 'other-snapshots'); const generationA = `${'a'.repeat(64)}--0000000000000001--00000000-0000-4000-8000-000000000001`; const generationB = `${'b'.repeat(64)}--0000000000000002--00000000-0000-4000-8000-000000000002`; - const cases = [ - [left.database, '--snapshot-root', snapshotRoot], - ['--source', left.database, '--snapshot-root', snapshotRoot], - [ - left.database, - right.database, - '--source', - left.database, - '--destination', - right.database, - '--snapshot-root', - snapshotRoot, - ], - [left.database, right.database, '--dry-run', '--keep-snapshots', '3'], - [left.database, right.database, '--keep-snapshots', '-1', '--snapshot-root', snapshotRoot], - [left.database, right.database, '--busy-timeout-ms', '2147483648', '--snapshot-root', snapshotRoot], - [left.database, right.database, '--snapshot-root', 'relative'], - [left.database, right.database, '--rollback', 'not-a-generation', '--snapshot-root', snapshotRoot], - ['--source', left.database, '--source', right.database, '--destination', right.database], - ['--source', left.database, '--destination', right.database, '--destination', left.database], - [left.database, right.database, '--snapshot-root', snapshotRoot, '--snapshot-root', otherSnapshotRoot], - [left.database, right.database, '--rollback', generationA, '--rollback', generationB], - [left.database, right.database, '--keep-snapshots', '1', '--keep-snapshots', '2'], - [left.database, right.database, '--busy-timeout-ms', '0', '--busy-timeout-ms', '1'], - [left.database, right.database, '--dry-run', '--dry-run'], - [left.database, right.database, '--json', '--json'], + const duplicate = 'may be specified only once'; + const cases: Array<{ args: string[]; expectedStderr?: string }> = [ + { args: [left.database, '--snapshot-root', snapshotRoot] }, + { args: ['--source', left.database, '--snapshot-root', snapshotRoot] }, + { + args: [ + left.database, + right.database, + '--source', + left.database, + '--destination', + right.database, + '--snapshot-root', + snapshotRoot, + ], + }, + { args: [left.database, right.database, '--dry-run', '--keep-snapshots', '3'] }, + { args: [left.database, right.database, '--keep-snapshots', '-1', '--snapshot-root', snapshotRoot] }, + { args: [left.database, right.database, '--busy-timeout-ms', '2147483648', '--snapshot-root', snapshotRoot] }, + { args: [left.database, right.database, '--snapshot-root', 'relative'] }, + { args: [left.database, right.database, '--rollback', 'not-a-generation', '--snapshot-root', snapshotRoot] }, + { + args: ['--source', left.database, '--source', right.database, '--destination', right.database], + expectedStderr: duplicate, + }, + { + args: ['--source', left.database, '--destination', right.database, '--destination', left.database], + expectedStderr: duplicate, + }, + { + args: [left.database, right.database, '--snapshot-root', snapshotRoot, '--snapshot-root', otherSnapshotRoot], + expectedStderr: duplicate, + }, + { + args: [left.database, right.database, '--rollback', generationA, '--rollback', generationB], + expectedStderr: duplicate, + }, + { + args: [left.database, right.database, '--keep-snapshots', '1', '--keep-snapshots', '2'], + expectedStderr: duplicate, + }, + { + args: [left.database, right.database, '--busy-timeout-ms', '0', '--busy-timeout-ms', '1'], + expectedStderr: duplicate, + }, + { args: [left.database, right.database, '--dry-run', '--dry-run'], expectedStderr: duplicate }, + { args: [left.database, right.database, '--json', '--json'], expectedStderr: duplicate }, ]; - for (const [index, args] of cases.entries()) { + for (const { args, expectedStderr } of cases) { const result = await sync(...args); expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.usage); expect(result.stdout).toBe(''); expect(result.stderr).toContain('genie db sync --help'); - if (index >= 8) expect(result.stderr).toContain('may be specified only once'); + if (expectedStderr !== undefined) expect(result.stderr).toContain(expectedStderr); } expect(databaseInventory(left.database)).toEqual(beforeLeft); expect(databaseInventory(right.database)).toEqual(beforeRight); @@ -564,7 +586,7 @@ describe('database sync CLI contract', () => { expect(databaseInventory(right.database)).toEqual(beforeRight); }); - test('cleanup failures map an otherwise successful subprocess report to exit 4', async () => { + test('recovery cleanup failures map an otherwise successful subprocess report to exit 4', async () => { const report = { reportVersion: 1, command: 'database-sync', @@ -581,11 +603,11 @@ describe('database sync CLI contract', () => { generationId: null, restoredDatabaseIdentities: [], failure: null, - cleanupFailures: [], + cleanupFailures: ['snapshot-cleanup-failed'], }, apply: null, failure: null, - cleanupFailures: ['snapshot-cleanup-failed'], + cleanupFailures: [], }, rollback: null, }; @@ -598,6 +620,24 @@ describe('database sync CLI contract', () => { expect(result.stdout).toContain('Cleanup failures: 1'); }); + test('unknown runtime report statuses fail closed with exit 4', async () => { + const report = { + reportVersion: 1, + command: 'database-sync', + operation: 'apply', + mode: 'bidirectional', + status: 'future-status', + inputs: [], + plan: null, + apply: null, + rollback: null, + }; + + const result = await syncWithInjectedReport(report, '/unused/left.db', '/unused/right.db'); + + expect(result.code).toBe(DATABASE_SYNC_EXIT_CODES.operationalFailure); + }); + test('directional dry-run reports bounded identities and counts without row content or writes', async () => { const source = createRepo('source'); const destination = createRepo('destination'); diff --git a/src/term-commands/v5-db-sync.ts b/src/term-commands/v5-db-sync.ts index f4bb69b0f..98a7bdfcc 100644 --- a/src/term-commands/v5-db-sync.ts +++ b/src/term-commands/v5-db-sync.ts @@ -115,12 +115,14 @@ interface RollbackReport { cleanupFailures: SnapshotRollbackReport['cleanupFailures']; } +type DatabaseSyncCliStatus = PlanReport['status'] | ApplyReport['status'] | RollbackReport['status']; + interface DatabaseSyncCliReport { reportVersion: typeof CLI_REPORT_VERSION; command: 'database-sync'; operation: DatabaseSyncOperation; mode: ReconciliationRequest['mode']; - status: string; + status: DatabaseSyncCliStatus; inputs: readonly DatabaseIdentityReport[]; plan: PlanReport | null; apply: ApplyReport | null; @@ -460,29 +462,40 @@ function execute( } function cleanupFailureCount(report: DatabaseSyncCliReport): number { - if (report.apply !== null) return report.apply.cleanupFailures.length; + if (report.apply !== null) { + return report.apply.cleanupFailures.length + report.apply.recovery.cleanupFailures.length; + } return report.rollback?.cleanupFailures.length ?? 0; } function exitCode(report: DatabaseSyncCliReport): DatabaseSyncExitCode { - if (report.status === 'conflict') return DATABASE_SYNC_EXIT_CODES.conflict; - if (report.status === 'uncertain') return DATABASE_SYNC_EXIT_CODES.uncertain; - if (report.status === 'converged' || report.status === 'recovered') { - return DATABASE_SYNC_EXIT_CODES.recoveryHandled; - } - if ( - report.status === 'operational-failure' || - report.status === 'lock-timeout' || - report.status === 'preimage-changed' || - (report.status === 'rolled-back' && report.operation !== 'rollback') || - report.status === 'expected-postimage' || - report.status === 'partial-commit' || - report.status === 'unexpected-intervening-write' || - cleanupFailureCount(report) > 0 - ) { - return DATABASE_SYNC_EXIT_CODES.operationalFailure; + if (cleanupFailureCount(report) > 0) return DATABASE_SYNC_EXIT_CODES.operationalFailure; + switch (report.status) { + case 'no-op': + case 'changed': + case 'same-database': + return DATABASE_SYNC_EXIT_CODES.success; + case 'rolled-back': + return report.operation === 'rollback' + ? DATABASE_SYNC_EXIT_CODES.success + : DATABASE_SYNC_EXIT_CODES.operationalFailure; + case 'conflict': + return DATABASE_SYNC_EXIT_CODES.conflict; + case 'uncertain': + return DATABASE_SYNC_EXIT_CODES.uncertain; + case 'converged': + case 'recovered': + return DATABASE_SYNC_EXIT_CODES.recoveryHandled; + case 'operational-failure': + case 'lock-timeout': + case 'preimage-changed': + case 'expected-postimage': + case 'partial-commit': + case 'unexpected-intervening-write': + return DATABASE_SYNC_EXIT_CODES.operationalFailure; + default: + return DATABASE_SYNC_EXIT_CODES.operationalFailure; } - return DATABASE_SYNC_EXIT_CODES.success; } function shortDigest(value: string | null): string { @@ -549,10 +562,11 @@ function handleSync( databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions, + rawArguments: readonly string[], dependencies: DatabaseSyncCommandDependencies, ): void { try { - rejectRepeatedOptions(process.argv.slice(2)); + rejectRepeatedOptions(rawArguments); const request = parseRequest(databaseA, databaseB, options); const operation = parseOperation(options); const snapshotRoot = parseSnapshotRoot(options.snapshotRoot); @@ -607,7 +621,17 @@ export function registerV5DatabaseSyncCommand( 'Exit codes: 0 success, 1 parser error, 2 usage, 3 conflict, ' + '4 operational failure, 5 uncertain/manual, 6 recovery handled (rerun).\n', ) - .action((databaseA: string | undefined, databaseB: string | undefined, options: DatabaseSyncOptions) => - handleSync(databaseA, databaseB, options, dependencies), + .action( + ( + databaseA: string | undefined, + databaseB: string | undefined, + options: DatabaseSyncOptions, + command: Command, + ) => { + let root = command; + while (root.parent !== null) root = root.parent; + const rawArguments = (root as Command & { readonly rawArgs?: readonly string[] }).rawArgs ?? []; + handleSync(databaseA, databaseB, options, rawArguments.slice(2), dependencies); + }, ); } diff --git a/tests/support/codex-dogfood-harness.ts b/tests/support/codex-dogfood-harness.ts index 694cbd706..733e05442 100644 --- a/tests/support/codex-dogfood-harness.ts +++ b/tests/support/codex-dogfood-harness.ts @@ -268,7 +268,7 @@ export async function runDogfoodEntry( const input = normalizeInput(rawInput); const ownsRoot = dependencies.root === undefined; const root = dependencies.root ?? mkdtempSync(join(tmpdir(), 'genie-dogfood-entry-')); - mkdirSync(root, { recursive: true, mode: 0o755 }); + if (!ownsRoot) mkdirSync(root, { recursive: true, mode: 0o755 }); try { const previous = verifyGeneration( 'previous',