fix(backend): index conversation_turn_nodes(last_seen_at) for retention sweep

changelog.d/fixes/13973-turn-nodes-last-seen-index.md

@@ -0,0 +1 @@
+- **fix(backend):** index `conversation_turn_nodes(last_seen_at)` so the 6h retention sweep range-scans instead of full-scanning every batch ([#13973](https://github.com/diegosouzapw/OmniRoute/issues/13973))

src/lib/db/migrations/186_turn_nodes_last_seen_index.sql

@@ -0,0 +1,9 @@
+-- Index the 6h retention sweep's filter column (#13973).
+-- cleanupConversationTurnNodes() deletes in batches via
+-- `WHERE last_seen_at < ?` (see deleteFromTableBeforeInBatches in
+-- src/lib/db/cleanup/usagePurge.ts). Without an index the inner SELECT
+-- full-scans conversation_turn_nodes on every batch; with it the sweep is a
+-- range scan. The table's other indexes (conversation_id, parent_id,
+-- conversation_id+content_hash) don't cover this predicate.
+CREATE INDEX IF NOT EXISTS idx_turn_nodes_last_seen
+  ON conversation_turn_nodes(last_seen_at);

tests/unit/db/migration-186-turn-nodes-last-seen.test.ts

@@ -0,0 +1,94 @@
+// #13973: the 6h retention sweep filters conversation_turn_nodes on
+// last_seen_at, which had no index — every cleanup batch full-scanned the
+// table. Migration 186 adds idx_turn_nodes_last_seen.
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import Database from "better-sqlite3";
+
+const repoMigrations = path.join(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "../../../src/lib/db/migrations"
+);
+const migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-migration-186-"));
+fs.copyFileSync(
+  path.join(repoMigrations, "186_turn_nodes_last_seen_index.sql"),
+  path.join(migrationsDir, "186_turn_nodes_last_seen_index.sql")
+);
+const originalMigrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR;
+process.env.OMNIROUTE_MIGRATIONS_DIR = migrationsDir;
+
+const { runMigrations } = await import("../../../src/lib/db/migrationRunner.ts");
+
+test.after(() => {
+  fs.rmSync(migrationsDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  if (originalMigrationsDir === undefined) delete process.env.OMNIROUTE_MIGRATIONS_DIR;
+  else process.env.OMNIROUTE_MIGRATIONS_DIR = originalMigrationsDir;
+});
+
+function openDb(): Database.Database {
+  const db = new Database(":memory:");
+  db.exec(
+    `CREATE TABLE conversation_turn_nodes (
+       id TEXT PRIMARY KEY,
+       conversation_id TEXT NOT NULL,
+       parent_id TEXT,
+       role TEXT NOT NULL,
+       content_hash TEXT NOT NULL DEFAULT '',
+       last_correlation_id TEXT,
+       first_seen_at TEXT NOT NULL,
+       last_seen_at TEXT NOT NULL
+     );`
+  );
+  return db;
+}
+
+function indexes(db: Database.Database): string[] {
+  return (
+    db
+      .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'conversation_turn_nodes'")
+      .all() as Array<{ name: string }>
+  ).map((row) => row.name);
+}
+
+test("migration 186 creates idx_turn_nodes_last_seen and a second run is a no-op", () => {
+  const db = openDb();
+  try {
+    assert.equal(runMigrations(db, { isNewDb: true }), 1);
+    assert.ok(indexes(db).includes("idx_turn_nodes_last_seen"));
+    assert.equal(runMigrations(db, { isNewDb: true }), 0);
+    assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [
+      { version: "186", name: "turn_nodes_last_seen_index" },
+    ]);
+  } finally {
+    db.close();
+  }
+});
+
+test("the retention predicate uses the index (no full scan)", () => {
+  const db = openDb();
+  try {
+    assert.equal(runMigrations(db, { isNewDb: true }), 1);
+    // Seed enough rows that the planner prefers the index over a scan.
+    const insert = db.prepare(
+      "INSERT INTO conversation_turn_nodes (id, conversation_id, role, content_hash, first_seen_at, last_seen_at) VALUES (?, ?, 'user', 'h', '2026-01-01T00:00:00.000Z', ?)"
+    );
+    for (let i = 0; i < 500; i++) {
+      insert.run(`n${i}`, `c${i % 10}`, `2026-09-${String((i % 28) + 1).padStart(2, "0")}T00:00:00.000Z`);
+    }
+    const plan = db
+      .prepare(
+        "EXPLAIN QUERY PLAN SELECT rowid FROM conversation_turn_nodes WHERE last_seen_at < ? LIMIT 100"
+      )
+      .all("2026-06-01T00:00:00.000Z") as Array<{ detail: string }>;
+    assert.ok(
+      plan.some((row) => row.detail.includes("idx_turn_nodes_last_seen")),
+      `expected index use, got: ${JSON.stringify(plan)}`
+    );
+  } finally {
+    db.close();
+  }
+});