From 5ffb1a65eaa8981f673951392fb0b4da6ce8b449 Mon Sep 17 00:00:00 2001 From: oyi77 Date: Thu, 24 Sep 2026 22:34:31 +0700 Subject: [PATCH] fix(backend): index conversation_turn_nodes(last_seen_at) for retention sweep The 6h cleanup deletes in batches via WHERE last_seen_at < ?, which full-scanned every batch without an index. Covers the remaining open item of #13973 (TRUNCATE removal landed in #14005). --- .../fixes/13973-turn-nodes-last-seen-index.md | 1 + .../186_turn_nodes_last_seen_index.sql | 9 ++ ...migration-186-turn-nodes-last-seen.test.ts | 94 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 changelog.d/fixes/13973-turn-nodes-last-seen-index.md create mode 100644 src/lib/db/migrations/186_turn_nodes_last_seen_index.sql create mode 100644 tests/unit/db/migration-186-turn-nodes-last-seen.test.ts diff --git a/changelog.d/fixes/13973-turn-nodes-last-seen-index.md b/changelog.d/fixes/13973-turn-nodes-last-seen-index.md new file mode 100644 index 00000000000..24b57999eab --- /dev/null +++ b/changelog.d/fixes/13973-turn-nodes-last-seen-index.md @@ -0,0 +1 @@ +- **fix(backend):** index `conversation_turn_nodes(last_seen_at)` so the 6h retention sweep range-scans instead of full-scanning every batch ([#13973](https://github.com/diegosouzapw/OmniRoute/issues/13973)) diff --git a/src/lib/db/migrations/186_turn_nodes_last_seen_index.sql b/src/lib/db/migrations/186_turn_nodes_last_seen_index.sql new file mode 100644 index 00000000000..e382c2092b2 --- /dev/null +++ b/src/lib/db/migrations/186_turn_nodes_last_seen_index.sql @@ -0,0 +1,9 @@ +-- Index the 6h retention sweep's filter column (#13973). +-- cleanupConversationTurnNodes() deletes in batches via +-- `WHERE last_seen_at < ?` (see deleteFromTableBeforeInBatches in +-- src/lib/db/cleanup/usagePurge.ts). Without an index the inner SELECT +-- full-scans conversation_turn_nodes on every batch; with it the sweep is a +-- range scan. The table's other indexes (conversation_id, parent_id, +-- conversation_id+content_hash) don't cover this predicate. +CREATE INDEX IF NOT EXISTS idx_turn_nodes_last_seen + ON conversation_turn_nodes(last_seen_at); diff --git a/tests/unit/db/migration-186-turn-nodes-last-seen.test.ts b/tests/unit/db/migration-186-turn-nodes-last-seen.test.ts new file mode 100644 index 00000000000..e441cf89f23 --- /dev/null +++ b/tests/unit/db/migration-186-turn-nodes-last-seen.test.ts @@ -0,0 +1,94 @@ +// #13973: the 6h retention sweep filters conversation_turn_nodes on +// last_seen_at, which had no index — every cleanup batch full-scanned the +// table. Migration 186 adds idx_turn_nodes_last_seen. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import Database from "better-sqlite3"; + +const repoMigrations = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "../../../src/lib/db/migrations" +); +const migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-migration-186-")); +fs.copyFileSync( + path.join(repoMigrations, "186_turn_nodes_last_seen_index.sql"), + path.join(migrationsDir, "186_turn_nodes_last_seen_index.sql") +); +const originalMigrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR; +process.env.OMNIROUTE_MIGRATIONS_DIR = migrationsDir; + +const { runMigrations } = await import("../../../src/lib/db/migrationRunner.ts"); + +test.after(() => { + fs.rmSync(migrationsDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (originalMigrationsDir === undefined) delete process.env.OMNIROUTE_MIGRATIONS_DIR; + else process.env.OMNIROUTE_MIGRATIONS_DIR = originalMigrationsDir; +}); + +function openDb(): Database.Database { + const db = new Database(":memory:"); + db.exec( + `CREATE TABLE conversation_turn_nodes ( + id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL, + parent_id TEXT, + role TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '', + last_correlation_id TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL + );` + ); + return db; +} + +function indexes(db: Database.Database): string[] { + return ( + db + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'conversation_turn_nodes'") + .all() as Array<{ name: string }> + ).map((row) => row.name); +} + +test("migration 186 creates idx_turn_nodes_last_seen and a second run is a no-op", () => { + const db = openDb(); + try { + assert.equal(runMigrations(db, { isNewDb: true }), 1); + assert.ok(indexes(db).includes("idx_turn_nodes_last_seen")); + assert.equal(runMigrations(db, { isNewDb: true }), 0); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "186", name: "turn_nodes_last_seen_index" }, + ]); + } finally { + db.close(); + } +}); + +test("the retention predicate uses the index (no full scan)", () => { + const db = openDb(); + try { + assert.equal(runMigrations(db, { isNewDb: true }), 1); + // Seed enough rows that the planner prefers the index over a scan. + const insert = db.prepare( + "INSERT INTO conversation_turn_nodes (id, conversation_id, role, content_hash, first_seen_at, last_seen_at) VALUES (?, ?, 'user', 'h', '2026-01-01T00:00:00.000Z', ?)" + ); + for (let i = 0; i < 500; i++) { + insert.run(`n${i}`, `c${i % 10}`, `2026-09-${String((i % 28) + 1).padStart(2, "0")}T00:00:00.000Z`); + } + const plan = db + .prepare( + "EXPLAIN QUERY PLAN SELECT rowid FROM conversation_turn_nodes WHERE last_seen_at < ? LIMIT 100" + ) + .all("2026-06-01T00:00:00.000Z") as Array<{ detail: string }>; + assert.ok( + plan.some((row) => row.detail.includes("idx_turn_nodes_last_seen")), + `expected index use, got: ${JSON.stringify(plan)}` + ); + } finally { + db.close(); + } +});