diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 77e9f65f11..f9291bcc70 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -27,6 +27,13 @@ import { waitForWindowsHandleRelease, type LbugConnectionHandle, } from './lbug-config.js'; +import { + finalizeLbugSidecarsAfterClose, + isMissingShadowSidecarError, + preflightLbugSidecars, + quarantineWalForMissingShadow, + shadowSidecarRecoveryMessage, +} from './sidecar-recovery.js'; import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js'; import { logger } from '../logger.js'; @@ -437,6 +444,157 @@ const queryAndDrain = async (targetConn: lbug.Connection, cypher: string): Promi await drainQueryResult(queryResult); }; +const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1'; + +const isReadOnlyShadowReplayError = (err: unknown): boolean => { + const msg = err instanceof Error ? err.message : String(err); + return /replay shadow pages under read-only mode/i.test(msg); +}; + +const reopenReadOnlyAfterMissingShadow = async ( + dbPath: string, + err: unknown, +): Promise => { + try { + await quarantineWalForMissingShadow(dbPath, { + logger, + level: 'warn', + reason: 'read-only recovery', + }); + } catch { + throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + } + + const reopened = await openLbugConnection(lbug, dbPath, { readOnly: true }); + try { + await queryAndDrain(reopened.conn, READ_ONLY_SHADOW_REPLAY_PROBE); + return reopened; + } catch (retryErr) { + await closeLbugConnection(reopened); + if (isMissingShadowSidecarError(retryErr) || isReadOnlyShadowReplayError(retryErr)) { + throw new Error(shadowSidecarRecoveryMessage(dbPath, retryErr)); + } + throw retryErr; + } +}; + +const reopenWritableAfterMissingShadow = async ( + dbPath: string, + err: unknown, +): Promise => { + try { + await quarantineWalForMissingShadow(dbPath, { + logger, + level: 'warn', + reason: 'writable recovery', + }); + } catch { + throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + } + + return await openLbugConnection(lbug, dbPath); +}; + +const ensureReadOnlyConnectionUsable = async ( + dbPath: string, + handle: LbugConnectionHandle, +): Promise => { + try { + await queryAndDrain(handle.conn, READ_ONLY_SHADOW_REPLAY_PROBE); + return handle; + } catch (err) { + if (isMissingShadowSidecarError(err)) { + await closeLbugConnection(handle); + return await reopenReadOnlyAfterMissingShadow(dbPath, err); + } + if (!isReadOnlyShadowReplayError(err)) { + await closeLbugConnection(handle); + throw err; + } + } + + await closeLbugConnection(handle); + + const writable = await openLbugConnection(lbug, dbPath); + let missingShadowError: unknown; + try { + await queryAndDrain(writable.conn, READ_ONLY_SHADOW_REPLAY_PROBE); + } catch (err) { + if (isMissingShadowSidecarError(err)) { + missingShadowError = err; + } else { + throw err; + } + } finally { + await closeLbugConnection(writable); + } + if (missingShadowError) { + return await reopenReadOnlyAfterMissingShadow(dbPath, missingShadowError); + } + + const reopened = await openLbugConnection(lbug, dbPath, { readOnly: true }); + try { + await queryAndDrain(reopened.conn, READ_ONLY_SHADOW_REPLAY_PROBE); + return reopened; + } catch (err) { + await closeLbugConnection(reopened); + if (isMissingShadowSidecarError(err)) { + throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + } + throw err; + } +}; + +const resetOpenConnectionState = (): void => { + currentDbPath = null; + ftsLoaded = false; + vectorExtensionLoaded = false; + ensuredFTSIndexes.clear(); +}; + +const runSchemaCreationQueries = async (dbPath: string): Promise => { + for (const schemaQuery of SCHEMA_QUERIES) { + try { + await queryAndDrain(conn, schemaQuery); + } catch (err) { + if (isMissingShadowSidecarError(err)) { + return err; + } + + const msg = err instanceof Error ? err.message : String(err); + // Suppression list: + // - "already exists": expected idempotent re-create on existing DBs + // - "could not set lock on file": LadybugDB v0.16.1 emits this on + // Windows when CREATE NODE TABLE runs against a path that was + // just opened (the WAL handle from a fresh Database briefly + // contests the table's first-write lock). The table is created + // anyway and any genuine cross-process lock contention surfaces + // on the next operation via withLbugDb's retry. Logging it here + // would just be noise in CI. + // + // WAL corruption: the first DDL write after DB open triggers WAL + // replay — if the WAL file was left in a corrupt state by an + // interrupted previous run, the native engine throws here. Rather + // than logging a WARN and continuing in a broken state, close the + // DB cleanly and surface an actionable error so the caller (serve, + // MCP, analyze) can exit with a clear recovery message. + if (isWalCorruptionError(err)) { + await safeClose(); + resetOpenConnectionState(); + throw new Error( + `LadybugDB WAL corruption detected at ${dbPath}. ${WAL_RECOVERY_SUGGESTION}\n` + + ` Original error: ${msg.slice(0, 200)}`, + ); + } + if (!msg.includes('already exists') && !isDbBusyError(err) && !isReadOnlyDbError(err)) { + logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`); + } + } + } + + return null; +}; + export const initLbug = async (dbPath: string) => { return runWithSessionLock(() => ensureLbugInitialized(dbPath)); }; @@ -580,58 +738,46 @@ const doInitLbug = async (dbPath: string, readOnly: boolean = false) => { // Ensure parent directory exists const parentDir = path.dirname(dbPath); await fs.mkdir(parentDir, { recursive: true }); + await preflightLbugSidecars(dbPath, { + mode: readOnly ? 'read-only' : 'write', + logger, + allowQuarantine: true, + }); const opened = readOnly ? await openLbugConnection(lbug, dbPath, { readOnly: true }) : await openLbugConnection(lbug, dbPath); - db = opened.db; - conn = opened.conn; + const usable = readOnly ? await ensureReadOnlyConnectionUsable(dbPath, opened) : opened; + db = usable.db; + conn = usable.conn; currentDbReadOnly = readOnly; } finally { await releaseInitLock(); } - for (const schemaQuery of SCHEMA_QUERIES) { - try { - await queryAndDrain(conn, schemaQuery); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - // Suppression list: - // - "already exists": expected idempotent re-create on existing DBs - // - "could not set lock on file": LadybugDB v0.16.1 emits this on - // Windows when CREATE NODE TABLE runs against a path that was - // just opened (the WAL handle from a fresh Database briefly - // contests the table's first-write lock). The table is created - // anyway and any genuine cross-process lock contention surfaces - // on the next operation via withLbugDb's retry. Logging it here - // would just be noise in CI. - // - // WAL corruption: the first DDL write after DB open triggers WAL - // replay — if the WAL file was left in a corrupt state by an - // interrupted previous run, the native engine throws here. Rather - // than logging a WARN and continuing in a broken state, close the - // DB cleanly and surface an actionable error so the caller (serve, - // MCP, analyze) can exit with a clear recovery message. - if (isWalCorruptionError(err)) { + if (!readOnly) { + const missingShadowError = await runSchemaCreationQueries(dbPath); + if (missingShadowError) { + await safeClose(); + resetOpenConnectionState(); + const reopened = await reopenWritableAfterMissingShadow(dbPath, missingShadowError); + db = reopened.db; + conn = reopened.conn; + currentDbReadOnly = false; + + const retryMissingShadowError = await runSchemaCreationQueries(dbPath); + if (retryMissingShadowError) { await safeClose(); - currentDbPath = null; - ftsLoaded = false; - vectorExtensionLoaded = false; - ensuredFTSIndexes.clear(); - throw new Error( - `LadybugDB WAL corruption detected at ${dbPath}. ${WAL_RECOVERY_SUGGESTION}\n` + - ` Original error: ${msg.slice(0, 200)}`, - ); - } - if (!msg.includes('already exists') && !isDbBusyError(err) && !isReadOnlyDbError(err)) { - logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`); + resetOpenConnectionState(); + throw new Error(shadowSidecarRecoveryMessage(dbPath, retryMissingShadowError)); } } } - // FTS powers baseline search, so initialize it with the core DB. VECTOR is - // only required for semantic embeddings and is probed lazily there. - await loadFTSExtension(); + // FTS powers baseline search, so initialize it with the core DB. Read-only + // serve/MCP paths must never run DDL or trigger network INSTALL; analyze owns + // schema/index creation and extension installation. + await loadFTSExtension(undefined, readOnly ? { policy: 'load-only' } : {}); currentDbPath = dbPath; return { db, conn }; @@ -1348,8 +1494,10 @@ export const flushWAL = async (): Promise => { try { const checkpointResult = await conn.query('CHECKPOINT'); await drainQueryResult(checkpointResult); - } catch { - /* ignore — older LadybugDB or schemaless DB may not accept it */ + } catch (err) { + logger.debug( + `GitNexus: LadybugDB CHECKPOINT skipped/failed during WAL flush: ${summarizeError(err)}`, + ); } }; @@ -1404,6 +1552,9 @@ export const safeClose = async (): Promise => { ); } } + if (closingDbPath) { + await finalizeLbugSidecarsAfterClose(closingDbPath, { logger }); + } }; export const closeLbug = async (): Promise => { diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index f551c65f81..e0dfdab7a4 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -25,6 +25,12 @@ import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION, } from './lbug-config.js'; +import { + isMissingShadowSidecarError, + preflightLbugSidecars, + quarantineWalForMissingShadow, + shadowSidecarRecoveryMessage, +} from './sidecar-recovery.js'; /** * Probe whether a Windows FTS extension binary is locally installed under @@ -304,16 +310,115 @@ const WAITER_TIMEOUT_MS = 15_000; const LOCK_RETRY_ATTEMPTS = 3; const LOCK_RETRY_DELAY_MS = 2000; +const SHADOW_REPLAY_PROBE_QUERY = 'MATCH (n) RETURN n LIMIT 1'; + +const isReadOnlyShadowReplayError = (err: unknown): boolean => { + const msg = err instanceof Error ? err.message : String(err); + return /replay shadow pages under read-only mode/i.test(msg); +}; + +const poolSidecarLogger = { + warn: (message: string): void => { + realStderrWrite(`${message}\n`); + }, + debug: (_message: string): void => {}, + info: (message: string): void => { + realStderrWrite(`${message}\n`); + }, +}; + +async function probeDatabaseForShadowReplay(db: lbug.Database): Promise { + const conn = createConnection(db); + try { + const queryResult = await conn.query(SHADOW_REPLAY_PROBE_QUERY); + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + await result.getAll(); + result.close?.(); + } finally { + await conn.close().catch(() => {}); + } +} + +async function replayShadowPagesWithWritableOpen(dbPath: string): Promise { + let db: lbug.Database | undefined; + try { + db = createLbugDatabase(lbug, dbPath, { throwOnWalReplayFailure: false }); + await db.init(); + await probeDatabaseForShadowReplay(db); + } catch (err) { + if (isMissingShadowSidecarError(err)) { + try { + await quarantineWalForMissingShadow(dbPath, { + logger: poolSidecarLogger, + level: 'warn', + reason: 'pool writable replay recovery', + }); + return; + } catch { + throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + } + } + throw err; + } finally { + if (db) await db.close().catch(() => {}); + } +} async function openReadOnlyDatabase(dbPath: string): Promise { let db: lbug.Database | undefined; silenceStdout(); try { + await preflightLbugSidecars(dbPath, { + mode: 'read-only', + logger: poolSidecarLogger, + allowQuarantine: true, + }); db = createLbugDatabase(lbug, dbPath, { readOnly: true, throwOnWalReplayFailure: false, }); await db.init(); + try { + await probeDatabaseForShadowReplay(db); + } catch (err) { + if (isMissingShadowSidecarError(err)) { + await db.close().catch(() => {}); + db = undefined; + try { + await quarantineWalForMissingShadow(dbPath, { + logger: poolSidecarLogger, + level: 'warn', + reason: 'pool read-only recovery', + }); + } catch { + throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + } + await preflightLbugSidecars(dbPath, { + mode: 'read-only', + logger: poolSidecarLogger, + allowQuarantine: true, + }); + db = createLbugDatabase(lbug, dbPath, { + readOnly: true, + throwOnWalReplayFailure: false, + }); + await db.init(); + await probeDatabaseForShadowReplay(db); + return db; + } + if (!isReadOnlyShadowReplayError(err)) { + throw err; + } + await db.close().catch(() => {}); + db = undefined; + await replayShadowPagesWithWritableOpen(dbPath); + db = createLbugDatabase(lbug, dbPath, { + readOnly: true, + throwOnWalReplayFailure: false, + }); + await db.init(); + await probeDatabaseForShadowReplay(db); + } return db; } catch (err) { if (db) await db.close().catch(() => {}); @@ -423,6 +528,13 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { } } + if ( + lastError.message.startsWith('LadybugDB checkpoint sidecar is missing') || + isMissingShadowSidecarError(lastError) + ) { + throw lastError; + } + const isLockError = lastError.message.includes('Could not set lock') || lastError.message.includes('lock'); if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break; diff --git a/gitnexus/src/core/lbug/sidecar-recovery.ts b/gitnexus/src/core/lbug/sidecar-recovery.ts new file mode 100644 index 0000000000..08cc76f9b1 --- /dev/null +++ b/gitnexus/src/core/lbug/sidecar-recovery.ts @@ -0,0 +1,254 @@ +import fs from 'fs/promises'; +import path from 'path'; + +export type LbugSidecarState = + | { kind: 'clean'; dbPath: string } + | { kind: 'wal-with-shadow'; dbPath: string; walBytes: number; shadowBytes: number } + | { kind: 'tiny-orphan-wal'; dbPath: string; walBytes: number } + | { kind: 'orphan-wal'; dbPath: string; walBytes: number } + | { kind: 'orphan-shadow'; dbPath: string; shadowBytes: number }; + +export interface SidecarRecoveryLogger { + warn: (message: string) => void; + info?: (message: string) => void; + debug?: (message: string) => void; +} + +export const TINY_ORPHAN_WAL_BYTES = 4 * 1024; + +const warnedKeys = new Set(); + +const missing = (err: unknown): boolean => + (err as NodeJS.ErrnoException | undefined)?.code === 'ENOENT'; + +const sidecarPreflightDisabled = (): boolean => + /^(1|true|yes|on)$/i.test(process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT ?? ''); + +const statIfExists = async (filePath: string): Promise<{ size: number } | null> => { + try { + const statFn = (fs as typeof fs & { stat?: typeof fs.stat }).stat; + if (typeof statFn === 'function') { + const stat = await statFn(filePath); + return { size: stat.size }; + } + // Some focused unit tests provide a deliberately tiny fs mock. Treat a + // path as present only when access succeeds, with an unknown/zero size. + await fs.access(filePath); + return { size: 0 }; + } catch (err) { + if (missing(err)) return null; + throw err; + } +}; + +const logDebug = (logger: SidecarRecoveryLogger, message: string): void => { + if (logger.debug) logger.debug(message); +}; + +const logInfo = (logger: SidecarRecoveryLogger, message: string): void => { + if (logger.info) logger.info(message); + else logDebug(logger, message); +}; + +const warnOnce = (logger: SidecarRecoveryLogger, key: string, message: string): void => { + if (warnedKeys.has(key)) { + logDebug(logger, message); + return; + } + warnedKeys.add(key); + logger.warn(message); +}; + +export const isMissingShadowSidecarError = (err: unknown): boolean => { + const msg = err instanceof Error ? err.message : String(err); + return /Cannot open file .*\.shadow: No such file or directory/i.test(msg); +}; + +export const shadowSidecarRecoveryMessage = (dbPath: string, err: unknown): string => { + const msg = err instanceof Error ? err.message : String(err); + return ( + `LadybugDB checkpoint sidecar is missing for ${dbPath}. ` + + 'Rebuild the index with `gitnexus analyze --force --index-only` and restart `gitnexus serve`.' + + `\n Original error: ${msg.slice(0, 200)}` + ); +}; + +export async function inspectLbugSidecars(dbPath: string): Promise { + const wal = await statIfExists(`${dbPath}.wal`); + const shadow = await statIfExists(`${dbPath}.shadow`); + + if (wal && shadow) { + return { kind: 'wal-with-shadow', dbPath, walBytes: wal.size, shadowBytes: shadow.size }; + } + if (wal) { + if (wal.size <= TINY_ORPHAN_WAL_BYTES) { + return { kind: 'tiny-orphan-wal', dbPath, walBytes: wal.size }; + } + return { kind: 'orphan-wal', dbPath, walBytes: wal.size }; + } + if (shadow) { + return { kind: 'orphan-shadow', dbPath, shadowBytes: shadow.size }; + } + return { kind: 'clean', dbPath }; +} + +export async function quarantineWalForMissingShadow( + dbPath: string, + options: { + logger: SidecarRecoveryLogger; + level?: 'debug' | 'info' | 'warn'; + reason?: string; + }, +): Promise { + const walPath = `${dbPath}.wal`; + const quarantinePath = `${walPath}.missing-shadow.${Date.now()}-${Math.random() + .toString(36) + .slice(2)}`; + await fs.rename(walPath, quarantinePath); + + const message = + `GitNexus: quarantined WAL ${path.basename(quarantinePath)} because LadybugDB shadow sidecar was missing; ` + + `continuing from last checkpoint${options.reason ? ` (${options.reason})` : ''}`; + + if (options.level === 'warn') { + warnOnce(options.logger, `${dbPath}:missing-shadow-quarantine`, message); + } else if (options.level === 'info') { + logInfo(options.logger, message); + } else { + logDebug(options.logger, message); + } + + return quarantinePath; +} + +export async function preflightLbugSidecars( + dbPath: string, + options: { + mode: 'read-only' | 'write'; + logger: SidecarRecoveryLogger; + allowQuarantine: boolean; + }, +): Promise { + let state: LbugSidecarState; + try { + state = await inspectLbugSidecars(dbPath); + } catch (err) { + logDebug( + options.logger, + `GitNexus: unable to inspect LadybugDB sidecars before ${options.mode} open; continuing without preflight repair: ${(err as Error).message}`, + ); + return { kind: 'clean', dbPath }; + } + if (sidecarPreflightDisabled() || !options.allowQuarantine) return state; + + if (state.kind === 'tiny-orphan-wal') { + await quarantineWalForMissingShadow(dbPath, { + logger: options.logger, + level: 'debug', + reason: `${options.mode} preflight tiny orphan WAL (${state.walBytes} bytes)`, + }); + return inspectLbugSidecars(dbPath); + } + + if (state.kind === 'orphan-wal') { + warnOnce( + options.logger, + `${dbPath}:orphan-wal-preflight:${options.mode}`, + `GitNexus: found ${state.walBytes} byte lbug.wal without lbug.shadow before ${options.mode} open; ` + + 'will rely on LadybugDB replay/recovery instead of deleting pending WAL data.', + ); + } + + return state; +} + +export async function finalizeLbugSidecarsAfterClose( + dbPath: string, + options: { logger: SidecarRecoveryLogger }, +): Promise { + if (sidecarPreflightDisabled()) return; + + let state: LbugSidecarState; + try { + state = await inspectLbugSidecars(dbPath); + } catch (err) { + logDebug( + options.logger, + `GitNexus: unable to inspect LadybugDB sidecars after close; skipping post-close repair: ${(err as Error).message}`, + ); + return; + } + if (state.kind === 'clean' || state.kind === 'wal-with-shadow') return; + + for (const delayMs of [25, 50, 100]) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + try { + state = await inspectLbugSidecars(dbPath); + } catch (err) { + logDebug( + options.logger, + `GitNexus: unable to inspect LadybugDB sidecars after close; skipping post-close repair: ${(err as Error).message}`, + ); + return; + } + if (state.kind === 'clean' || state.kind === 'wal-with-shadow') return; + } + + if (state.kind === 'tiny-orphan-wal') { + try { + await quarantineWalForMissingShadow(dbPath, { + logger: options.logger, + level: 'debug', + reason: `post-close tiny orphan WAL (${state.walBytes} bytes)`, + }); + } catch (err) { + if (!missing(err)) { + warnOnce( + options.logger, + `${dbPath}:post-close-tiny-quarantine-failed`, + `GitNexus: failed to quarantine tiny orphan WAL after close (${(err as Error).message}); next read may recover reactively.`, + ); + } + } + return; + } + + if (state.kind === 'orphan-wal') { + warnOnce( + options.logger, + `${dbPath}:post-close-orphan-wal`, + `GitNexus: lbug.wal (${state.walBytes} bytes) remains without lbug.shadow after close; ` + + 'keeping it for recovery. If this repeats, run `gitnexus analyze --force --index-only` or the sidecar repair command.', + ); + } +} + +export async function listQuarantinedMissingShadowWals(dbPath: string): Promise { + const dir = path.dirname(dbPath); + const base = path.basename(dbPath); + let entries: string[]; + try { + entries = await fs.readdir(dir); + } catch (err) { + if (missing(err)) return []; + throw err; + } + return entries + .filter((entry) => entry.startsWith(`${base}.wal.missing-shadow.`)) + .map((entry) => path.join(dir, entry)) + .sort(); +} + +export async function cleanQuarantinedMissingShadowWals(dbPath: string): Promise { + const files = await listQuarantinedMissingShadowWals(dbPath); + const deleted: string[] = []; + for (const file of files) { + await fs.unlink(file); + deleted.push(file); + } + return deleted; +} + +export const _resetSidecarRecoveryWarningsForTest = (): void => { + warnedKeys.clear(); +}; diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 796fa04e17..87053ffda1 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -130,6 +130,7 @@ export const isIgnorableGraphQueryError = (err: unknown): boolean => { }; export const SPA_FALLBACK_REGEX = /^(?!\/api(?:\/|$))(?!.*\.\w{1,10}$).*/; +export const PNA_PREFLIGHT_PATH_REGEX = /^\/.*$/; export const resolveWebDistDir = async ( primaryDir: string, @@ -718,7 +719,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // on OPTIONS requests and expects the allow header in the response. // Note: the actual Allow-Private-Network header is already set by the global // middleware above, so we just need to call next() here. - app.options('*', (_req, res, next) => { + app.options(PNA_PREFLIGHT_PATH_REGEX, (_req, res, next) => { next(); }); diff --git a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts index c5f4b67730..8c29b203b3 100644 --- a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts +++ b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts @@ -41,6 +41,7 @@ function makeFsMock(dbPath: string) { throw ENOENT; }), unlink: vi.fn(async () => {}), + rename: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), open: makeOpenMock(), }, @@ -78,8 +79,8 @@ describe('doInitLbug WAL corruption guard — structural', () => { expect(schemaLoopBody).toMatch(/await safeClose\(\)/); }); - it('WAL guard resets currentDbPath to null', () => { - expect(schemaLoopBody).toMatch(/currentDbPath = null/); + it('WAL guard resets open connection state', () => { + expect(schemaLoopBody).toMatch(/resetOpenConnectionState\(\)/); }); it('WAL guard throws with WAL_RECOVERY_SUGGESTION in the message', () => { @@ -211,6 +212,284 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { await adapter.closeLbug(); }); + it('quarantines the WAL and retries writable schema creation when shadow sidecar is missing', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-writable-shadow-missing/lbug'; + const missingShadowError = new Error( + `IO exception: Cannot open file ${dbPath}.shadow: No such file or directory`, + ); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const firstConn = { + query: vi.fn().mockRejectedValueOnce(missingShadowError).mockResolvedValue(queryResult), + close: vi.fn(async () => {}), + }; + const firstDb = { close: vi.fn(async () => {}) }; + const recoveredConn = { + query: vi.fn(async () => queryResult), + close: vi.fn(async () => {}), + }; + const recoveredDb = { close: vi.fn(async () => {}) }; + const openLbugConnectionMock = vi + .fn() + .mockResolvedValueOnce({ db: firstDb, conn: firstConn }) + .mockResolvedValueOnce({ db: recoveredDb, conn: recoveredConn }); + const fsMock = makeFsMock(dbPath); + const ensureMock = vi.fn(async () => false); + const warnMock = vi.fn(); + + vi.doMock('fs/promises', () => fsMock); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: openLbugConnectionMock, + closeLbugConnection: async (handle: { conn: typeof firstConn; db: typeof firstDb }) => { + await handle.conn.close(); + await handle.db.close(); + }, + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: ensureMock, + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + await expect(adapter.initLbug(dbPath)).resolves.toBeDefined(); + + expect(openLbugConnectionMock).toHaveBeenCalledTimes(2); + expect(fsMock.default.rename).toHaveBeenCalledWith( + `${dbPath}.wal`, + expect.stringContaining(`${dbPath}.wal.missing-shadow.`), + ); + expect(recoveredConn.query).toHaveBeenCalledWith(SCHEMA_MOCK.SCHEMA_QUERIES[0]); + expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning')); + + await adapter.closeLbug(); + }); + + it('skips schema DDL and uses load-only FTS policy for read-only opens', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-readonly-schema-skip/lbug'; + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const conn = { + query: vi.fn(async () => queryResult), + close: vi.fn(async () => {}), + }; + const db = { close: vi.fn(async () => {}) }; + const openLbugConnectionMock = vi.fn(async () => ({ db, conn })); + const ensureMock = vi.fn(async () => false); + const warnMock = vi.fn(); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: openLbugConnectionMock, + closeLbugConnection: vi.fn(async () => {}), + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: ensureMock, + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + await expect(adapter.withLbugDb(dbPath, async () => 'ok', { readOnly: true })).resolves.toBe( + 'ok', + ); + + expect(openLbugConnectionMock).toHaveBeenCalledWith(expect.anything(), dbPath, { + readOnly: true, + }); + expect(conn.query).not.toHaveBeenCalledWith(SCHEMA_MOCK.SCHEMA_QUERIES[0]); + expect(ensureMock).toHaveBeenCalledWith(expect.any(Function), 'fts', 'FTS', { + policy: 'load-only', + }); + expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning')); + + await adapter.closeLbug(); + }); + + it('replays dirty shadow pages with a temporary writable open before read-only serving', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-readonly-shadow-replay/lbug'; + const shadowReplayError = new Error( + "Runtime exception: Couldn't replay shadow pages under read-only mode. Please re-open the database with read-write mode to replay shadow pages.", + ); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const readOnlyConn1 = { + query: vi.fn().mockRejectedValueOnce(shadowReplayError), + close: vi.fn(async () => {}), + }; + const readOnlyDb1 = { close: vi.fn(async () => {}) }; + const writableConn = { + query: vi.fn(async () => queryResult), + close: vi.fn(async () => {}), + }; + const writableDb = { close: vi.fn(async () => {}) }; + const readOnlyConn2 = { + query: vi.fn(async () => queryResult), + close: vi.fn(async () => {}), + }; + const readOnlyDb2 = { close: vi.fn(async () => {}) }; + const openLbugConnectionMock = vi + .fn() + .mockResolvedValueOnce({ db: readOnlyDb1, conn: readOnlyConn1 }) + .mockResolvedValueOnce({ db: writableDb, conn: writableConn }) + .mockResolvedValueOnce({ db: readOnlyDb2, conn: readOnlyConn2 }); + const ensureMock = vi.fn(async () => false); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: openLbugConnectionMock, + closeLbugConnection: async (handle: { + conn: typeof readOnlyConn1; + db: typeof readOnlyDb1; + }) => { + await handle.conn.close(); + await handle.db.close(); + }, + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: ensureMock, + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + await expect(adapter.withLbugDb(dbPath, async () => 'ok', { readOnly: true })).resolves.toBe( + 'ok', + ); + + expect(openLbugConnectionMock).toHaveBeenNthCalledWith(1, expect.anything(), dbPath, { + readOnly: true, + }); + expect(openLbugConnectionMock).toHaveBeenNthCalledWith(2, expect.anything(), dbPath); + expect(openLbugConnectionMock).toHaveBeenNthCalledWith(3, expect.anything(), dbPath, { + readOnly: true, + }); + expect(readOnlyConn1.close).toHaveBeenCalled(); + expect(readOnlyDb1.close).toHaveBeenCalled(); + expect(writableConn.query).toHaveBeenCalledWith('MATCH (n) RETURN n LIMIT 1'); + expect(writableConn.close).toHaveBeenCalled(); + expect(writableDb.close).toHaveBeenCalled(); + expect(readOnlyConn2.query).toHaveBeenCalledWith('MATCH (n) RETURN n LIMIT 1'); + expect(ensureMock).toHaveBeenCalledWith(expect.any(Function), 'fts', 'FTS', { + policy: 'load-only', + }); + + await adapter.closeLbug(); + }); + + it('quarantines the WAL and reopens read-only when the shadow sidecar is missing', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-readonly-shadow-missing/lbug'; + const missingShadowError = new Error( + `IO exception: Cannot open file ${dbPath}.shadow: No such file or directory`, + ); + const readOnlyConn = { + query: vi.fn().mockRejectedValueOnce(missingShadowError), + close: vi.fn(async () => {}), + }; + const readOnlyDb = { close: vi.fn(async () => {}) }; + const recoveredConn = { + query: vi.fn(async () => ({ getAll: vi.fn(async () => []), close: vi.fn() })), + close: vi.fn(async () => {}), + }; + const recoveredDb = { close: vi.fn(async () => {}) }; + const openLbugConnectionMock = vi + .fn() + .mockResolvedValueOnce({ + db: readOnlyDb, + conn: readOnlyConn, + }) + .mockResolvedValueOnce({ + db: recoveredDb, + conn: recoveredConn, + }); + const fsMock = makeFsMock(dbPath); + + vi.doMock('fs/promises', () => fsMock); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: openLbugConnectionMock, + closeLbugConnection: async (handle: { conn: typeof readOnlyConn; db: typeof readOnlyDb }) => { + await handle.conn.close(); + await handle.db.close(); + }, + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => false), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + await expect(adapter.withLbugDb(dbPath, async () => 'ok', { readOnly: true })).resolves.toBe( + 'ok', + ); + expect(openLbugConnectionMock).toHaveBeenCalledTimes(2); + expect(readOnlyConn.close).toHaveBeenCalled(); + expect(readOnlyDb.close).toHaveBeenCalled(); + expect(fsMock.default.rename).toHaveBeenCalledWith( + `${dbPath}.wal`, + expect.stringContaining(`${dbPath}.wal.missing-shadow.`), + ); + + await adapter.closeLbug(); + }); + it('calls safeClose() (db.close) when WAL corruption is detected mid-schema', async () => { vi.resetModules(); diff --git a/gitnexus/test/unit/pool-wal-recovery.test.ts b/gitnexus/test/unit/pool-wal-recovery.test.ts index cda42806ab..0ccf8362a9 100644 --- a/gitnexus/test/unit/pool-wal-recovery.test.ts +++ b/gitnexus/test/unit/pool-wal-recovery.test.ts @@ -6,7 +6,8 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -const { stderrWriteMock } = vi.hoisted(() => ({ +const { connectionQueryMock, stderrWriteMock } = vi.hoisted(() => ({ + connectionQueryMock: vi.fn(), stderrWriteMock: vi.fn(), })); @@ -23,6 +24,7 @@ vi.mock('@ladybugdb/core', () => ({ Database: vi.fn(), Connection: vi.fn(function (this: any) { this.close = vi.fn().mockResolvedValue(undefined); + this.query = connectionQueryMock; }), }, })); @@ -68,6 +70,11 @@ describe('WAL corruption recovery in doInitLbug (#1402)', () => { (fs.rename as any).mockReset(); mockInit.mockReset(); mockClose.mockReset(); + connectionQueryMock.mockReset(); + connectionQueryMock.mockResolvedValue({ + getAll: vi.fn().mockResolvedValue([]), + close: vi.fn(), + }); mockInit.mockResolvedValue(undefined); mockClose.mockResolvedValue(undefined); (fs.stat as any).mockResolvedValue({}); @@ -110,6 +117,79 @@ describe('WAL corruption recovery in doInitLbug (#1402)', () => { ); }); + it('replays shadow pages with a temporary writable open before pooling read-only DBs', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-shadow-replay/lbug'; + + const readOnlyDb1 = makeMockDb(); + const writableDb = makeMockDb(); + const readOnlyDb2 = makeMockDb(); + connectionQueryMock + .mockRejectedValueOnce( + new Error( + "Runtime exception: Couldn't replay shadow pages under read-only mode. Please re-open the database with read-write mode to replay shadow pages.", + ), + ) + .mockResolvedValue({ + getAll: vi.fn().mockResolvedValue([]), + close: vi.fn(), + }); + (createLbugDatabase as any) + .mockReturnValueOnce(readOnlyDb1) + .mockReturnValueOnce(writableDb) + .mockReturnValueOnce(readOnlyDb2); + + await initLbug('test-repo-shadow-replay', dbPath); + + expect(createLbugDatabase).toHaveBeenNthCalledWith( + 1, + expect.anything(), + dbPath, + expect.objectContaining({ readOnly: true, throwOnWalReplayFailure: false }), + ); + expect(createLbugDatabase).toHaveBeenNthCalledWith( + 2, + expect.anything(), + dbPath, + expect.objectContaining({ throwOnWalReplayFailure: false }), + ); + expect(createLbugDatabase).toHaveBeenNthCalledWith( + 3, + expect.anything(), + dbPath, + expect.objectContaining({ readOnly: true, throwOnWalReplayFailure: false }), + ); + expect(readOnlyDb1.close).toHaveBeenCalled(); + expect(writableDb.close).toHaveBeenCalled(); + expect(fs.rename).not.toHaveBeenCalled(); + }); + + it('quarantines WAL and reopens read-only when the Ladybug shadow sidecar is missing', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-shadow-missing/lbug'; + + const readOnlyDb1 = makeMockDb(); + const readOnlyDb2 = makeMockDb(); + connectionQueryMock + .mockRejectedValueOnce( + new Error(`IO exception: Cannot open file ${dbPath}.shadow: No such file or directory`), + ) + .mockResolvedValue({ + getAll: vi.fn().mockResolvedValue([]), + close: vi.fn(), + }); + (createLbugDatabase as any).mockReturnValueOnce(readOnlyDb1).mockReturnValueOnce(readOnlyDb2); + + await initLbug('test-repo-shadow-missing', dbPath); + + expect(createLbugDatabase).toHaveBeenCalledTimes(2); + expect(readOnlyDb1.close).toHaveBeenCalled(); + expect(fs.rename).toHaveBeenCalledWith( + dbPath + '.wal', + expect.stringContaining('.wal.missing-shadow.'), + ); + }); + it('does not quarantine on lock error (preserves existing lock retry)', async () => { const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation((callback: any) => { diff --git a/gitnexus/test/unit/sidecar-recovery.test.ts b/gitnexus/test/unit/sidecar-recovery.test.ts new file mode 100644 index 0000000000..994f6e4e2a --- /dev/null +++ b/gitnexus/test/unit/sidecar-recovery.test.ts @@ -0,0 +1,123 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + _resetSidecarRecoveryWarningsForTest, + finalizeLbugSidecarsAfterClose, + inspectLbugSidecars, + listQuarantinedMissingShadowWals, + preflightLbugSidecars, + TINY_ORPHAN_WAL_BYTES, +} from '../../src/core/lbug/sidecar-recovery.js'; + +const logger = () => ({ warn: vi.fn(), info: vi.fn(), debug: vi.fn() }); + +describe('LadybugDB sidecar recovery', () => { + let dir: string; + let dbPath: string; + + beforeEach(async () => { + _resetSidecarRecoveryWarningsForTest(); + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-sidecar-recovery-')); + dbPath = path.join(dir, 'lbug'); + await fs.writeFile(dbPath, 'db'); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('classifies clean sidecars', async () => { + await expect(inspectLbugSidecars(dbPath)).resolves.toEqual({ kind: 'clean', dbPath }); + }); + + it('classifies WAL with shadow as replayable by LadybugDB', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(128)); + await fs.writeFile(`${dbPath}.shadow`, Buffer.alloc(64)); + + await expect(inspectLbugSidecars(dbPath)).resolves.toEqual({ + kind: 'wal-with-shadow', + dbPath, + walBytes: 128, + shadowBytes: 64, + }); + }); + + it('preflight quarantines tiny orphan WAL without WARN noise', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(34)); + const log = logger(); + + const state = await preflightLbugSidecars(dbPath, { + mode: 'read-only', + logger: log, + allowQuarantine: true, + }); + + expect(state.kind).toBe('clean'); + await expect(fs.stat(`${dbPath}.wal`)).rejects.toMatchObject({ code: 'ENOENT' }); + const files = await fs.readdir(dir); + expect(files.some((file) => file.startsWith('lbug.wal.missing-shadow.'))).toBe(true); + expect(log.warn).not.toHaveBeenCalled(); + expect(log.debug).toHaveBeenCalledWith(expect.stringContaining('preflight tiny orphan WAL')); + }); + + it('does not silently quarantine large orphan WAL during preflight', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1)); + const log = logger(); + + const state = await preflightLbugSidecars(dbPath, { + mode: 'read-only', + logger: log, + allowQuarantine: true, + }); + + expect(state).toEqual({ + kind: 'orphan-wal', + dbPath, + walBytes: TINY_ORPHAN_WAL_BYTES + 1, + }); + await expect(fs.stat(`${dbPath}.wal`)).resolves.toBeDefined(); + expect(log.warn).toHaveBeenCalledTimes(1); + }); + + it('finalize quarantines tiny orphan WAL after close', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(34)); + const log = logger(); + + await finalizeLbugSidecarsAfterClose(dbPath, { logger: log }); + + await expect(fs.stat(`${dbPath}.wal`)).rejects.toMatchObject({ code: 'ENOENT' }); + const files = await fs.readdir(dir); + expect(files.some((file) => file.startsWith('lbug.wal.missing-shadow.'))).toBe(true); + expect(log.warn).not.toHaveBeenCalled(); + }); + + it('can be disabled through GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT', async () => { + vi.stubEnv('GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT', '1'); + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(34)); + const log = logger(); + + const state = await preflightLbugSidecars(dbPath, { + mode: 'read-only', + logger: log, + allowQuarantine: true, + }); + + expect(state.kind).toBe('tiny-orphan-wal'); + await expect(fs.stat(`${dbPath}.wal`)).resolves.toBeDefined(); + }); + + it('lists only missing-shadow WAL quarantine files for cleanup', async () => { + await fs.writeFile(`${dbPath}.wal.missing-shadow.1-a`, ''); + await fs.writeFile(`${dbPath}.wal.missing-shadow.2-b`, ''); + await fs.writeFile(`${dbPath}.wal.corrupt.3-c`, ''); + await fs.writeFile(path.join(dir, 'other.wal.missing-shadow.4-d'), ''); + + await expect(listQuarantinedMissingShadowWals(dbPath)).resolves.toEqual([ + `${dbPath}.wal.missing-shadow.1-a`, + `${dbPath}.wal.missing-shadow.2-b`, + ]); + }); +}); diff --git a/gitnexus/test/unit/web-ui-serving.test.ts b/gitnexus/test/unit/web-ui-serving.test.ts index bd1c7faaf2..4099bebfa6 100644 --- a/gitnexus/test/unit/web-ui-serving.test.ts +++ b/gitnexus/test/unit/web-ui-serving.test.ts @@ -17,6 +17,7 @@ import { registerWebUI, resolveWebDistDir, landingPageHtml, + PNA_PREFLIGHT_PATH_REGEX, SPA_FALLBACK_REGEX, staticCacheControlSetHeaders, } from '../../src/server/api.js'; @@ -323,4 +324,11 @@ describe('Real Express dispatch — API and asset isolation', () => { const status = await makeRequest(app, 'GET', '/'); expect(status).toBe(200); }); + + it('registers the global OPTIONS preflight matcher without Express 5 wildcard errors', async () => { + const app = express(); + expect(() => app.options(PNA_PREFLIGHT_PATH_REGEX, (_req, _res, next) => next())).not.toThrow(); + const status = await makeRequest(app, 'OPTIONS', '/api/health'); + expect(status).toBe(404); + }); });