diff --git a/.env.example b/.env.example index c809f7653c05..d193fa35a04a 100644 --- a/.env.example +++ b/.env.example @@ -1033,6 +1033,10 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # Used by: src/lib/db/core.ts::getDbHealthCheckIntervalMs(). #OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS=21600000 +# WAL truncate cadence override (ms). Set to 0 to disable. Default: 21600000 (6h). +# Used by: src/lib/db/core.ts::getWalTruncateIntervalMs(). +#OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS=21600000 + # Skip the Redis-backed auth cache used by API key lookups (forces DB reads). # Used by: src/lib/db/apiKeys.ts. Set to 1 to disable. Default: enabled. #OMNIROUTE_DISABLE_REDIS_AUTH_CACHE=0 diff --git a/changelog.d/fixes/10781-wal-truncate-scheduler.md b/changelog.d/fixes/10781-wal-truncate-scheduler.md new file mode 100644 index 000000000000..4eb13a271bbb --- /dev/null +++ b/changelog.d/fixes/10781-wal-truncate-scheduler.md @@ -0,0 +1 @@ +- fix(db): periodically run `wal_checkpoint(TRUNCATE)` so the SQLite WAL file shrinks on long-running servers (default 6h, override with `OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS`, `0` disables) (#10781) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index ccb6a6981fa9..981cdc473b92 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -95,6 +95,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_CRYPT_KEY` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** for `STORAGE_ENCRYPTION_KEY`. Accepted as a fallback when the primary variable is absent. | | `OMNIROUTE_API_KEY_BASE64` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** (Base64-encoded form) accepted as a fallback. Decoded automatically before use. | | `OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS` | _(unset)_ | `src/lib/db/core.ts` | Override the periodic SQLite healthcheck interval (ms). When unset, defaults are derived from `NODE_ENV`. | +| `OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS` | `21600000` (6h) | `src/lib/db/core.ts` | Override the periodic `wal_checkpoint(TRUNCATE)` interval (ms). Auto-checkpoint never shrinks the WAL file itself, and a long-running server never closes its DB. `0` disables. | | `OMNIROUTE_SKIP_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts`, `src/lib/db/healthCheck.ts` | Set to `1` to skip the DB healthcheck entirely on startup. Useful for short-lived tasks and integration tests. | | `OMNIROUTE_FORCE_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts` | Set to `1` to force the DB healthcheck loop on, even when it would normally be skipped (e.g., short-lived tasks). | | `OMNIROUTE_SKIP_POSTINSTALL` | `0` | `scripts/postinstall.mjs` | Set to `1` to skip the native-runtime warm-up during `npm install`. Useful in CI/headless installs where sqlite is already built. | diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 202d76be9f9e..b7021be80e29 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -962,6 +962,51 @@ function startDbHealthCheckScheduler(db: SqliteDatabase) { dbHealthCheckTimer.unref?.(); } +let walTruncateTimer: NodeJS.Timeout | null = null; + +function getWalTruncateIntervalMs(): number { + const rawValue = process.env.OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS; + if (typeof rawValue === "string" && rawValue.trim().length > 0) { + const parsed = Number(rawValue); + if (Number.isFinite(parsed) && parsed >= 0) { + return parsed; + } + } + return 6 * 60 * 60 * 1000; +} + +function clearWalTruncateScheduler() { + if (walTruncateTimer) { + clearInterval(walTruncateTimer); + walTruncateTimer = null; + } +} + +// Auto-checkpoint moves WAL pages back into the main DB file but never shrinks the WAL +// file itself; only wal_checkpoint(TRUNCATE) does, and a long-running server never closes its DB. +function startWalTruncateScheduler(db: SqliteDatabase) { + clearWalTruncateScheduler(); + if (isCloud || isBuildPhase || isAutomatedTestProcess()) return; + + const intervalMs = getWalTruncateIntervalMs(); + if (intervalMs <= 0) return; + + walTruncateTimer = setInterval(() => { + try { + if (!db.open) return; + // TRUNCATE waits for readers; under concurrent write load it can no-op without + // shrinking the file. That is expected — it retries on the next tick. + if (checkpointDb(db, "TRUNCATE")) { + console.log("[DB] Periodic SQLite WAL checkpoint completed (TRUNCATE)."); + } + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn("[DB] Periodic WAL truncate failed:", message); + } + }, intervalMs); + walTruncateTimer.unref?.(); +} + export function runManagedDbHealthCheck(options?: { autoRepair?: boolean }) { const db = getDbInstance(); return runDbHealthCheck(db, { @@ -1308,6 +1353,7 @@ export function getDbInstance(): SqliteDatabase { } startDbHealthCheckScheduler(db); + startWalTruncateScheduler(db); // Log the resolved absolute DATA_DIR + SQLITE_FILE once at init so a // multi-replica / Docker volume-topology mismatch (each replica opening a // different on-disk DB → "phantom"/missing combos & connections) is @@ -1335,6 +1381,7 @@ export function pingDb(): boolean { export function closeDbInstance(options?: { checkpointMode?: CheckpointMode | null }): boolean { clearDbHealthCheckScheduler(); + clearWalTruncateScheduler(); const db = getDb(); if (!db) return false; diff --git a/tests/unit/db-wal-truncate-scheduler.test.ts b/tests/unit/db-wal-truncate-scheduler.test.ts new file mode 100644 index 000000000000..cbd93cde7e96 --- /dev/null +++ b/tests/unit/db-wal-truncate-scheduler.test.ts @@ -0,0 +1,82 @@ +/** + * A WAL never shrinks on its own: only `wal_checkpoint(TRUNCATE)` reclaims the file, and a + * long-running server never closes its DB (observed locally: a 154 MB WAL on a 143 MB base). + * + * The scheduler cannot be exercised directly in a unit test: it gates itself off under + * `isAutomatedTestProcess()`, same as the pre-existing DB health-check scheduler it is + * modeled on (see tests/unit/lib/jobRegistry/boot-wiring.test.ts for the same constraint). + * This reads the wiring instead. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +function readSource(relativePath: string): string { + return fs.readFileSync(path.join(process.cwd(), relativePath), "utf8"); +} + +const CORE_PATH = "src/lib/db/core.ts"; + +test("a periodic WAL truncate scheduler is started when the DB instance boots", () => { + const source = readSource(CORE_PATH); + assert.match( + source, + /startWalTruncateScheduler\(db\)/, + "getDbInstance() must start the WAL truncate scheduler alongside the DB health-check scheduler" + ); +}); + +test("the WAL truncate scheduler runs wal_checkpoint(TRUNCATE), not a lighter mode", () => { + const source = readSource(CORE_PATH); + const fnStart = source.indexOf("function startWalTruncateScheduler"); + assert.notEqual(fnStart, -1, "startWalTruncateScheduler must exist"); + const fnBody = source.slice(fnStart, fnStart + 1200); + assert.match( + fnBody, + /checkpointDb\(db, "TRUNCATE"\)/, + "the scheduled checkpoint must request TRUNCATE mode — a lighter mode would not shrink the WAL file" + ); +}); + +test("the WAL truncate scheduler is cleared on close, like the health-check scheduler", () => { + const source = readSource(CORE_PATH); + const fnStart = source.indexOf("export function closeDbInstance"); + assert.notEqual(fnStart, -1, "closeDbInstance must exist"); + const fnBody = source.slice(fnStart, fnStart + 300); + assert.match(fnBody, /clearDbHealthCheckScheduler\(\)/); + assert.match( + fnBody, + /clearWalTruncateScheduler\(\)/, + "closeDbInstance() must clear the WAL truncate timer so it does not outlive the DB handle" + ); +}); + +test("the truncate interval is overridable via OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS", () => { + const source = readSource(CORE_PATH); + assert.match( + source, + /OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS/, + "the interval must be operator-configurable, matching OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS" + ); +}); + +test("the scheduler self-gates the same way the DB health-check scheduler does", () => { + const source = readSource(CORE_PATH); + const fnStart = source.indexOf("function startWalTruncateScheduler"); + const fnBody = source.slice(fnStart, fnStart + 300); + assert.match( + fnBody, + /isCloud \|\| isBuildPhase \|\| isAutomatedTestProcess\(\)/, + "must not run during cloud/build/test contexts, same as startDbHealthCheckScheduler" + ); +}); + +test("the new env var is documented", () => { + const docs = readSource("docs/reference/ENVIRONMENT.md"); + assert.match( + docs, + /OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS/, + "docs/reference/ENVIRONMENT.md must document the new env var (check:env-doc-sync)" + ); +});