Backport call log storage refactor to release/v3.6.6

README.md

@@ -387,10 +387,10 @@ When a call fails, the dev doesn't know if it was a rate limit, expired token, w
 
 - **Unified Logs Dashboard** — 4 tabs: Request Logs, Proxy Logs, Audit Logs, Console
 - **Console Log Viewer** — Real-time terminal-style viewer with color-coded levels, auto-scroll, search, filter
-- **SQLite Proxy Logs** — Persistent logs that survive server restarts
+- **SQLite Summary Logs** — Request and proxy log indexes stay queryable across restarts without loading large payload blobs into SQLite
 - **Translator Playground** — 4 debugging modes: Playground (format translation), Chat Tester (round-trip), Test Bench (batch), Live Monitor (real-time)
 - **Request Telemetry** — p50/p95/p99 latency + X-Request-Id tracing
-- **File-Based Logging with Rotation** — App logs rotate by size, retention days, and archive count; call log artifacts rotate by retention days and file count
+- **File-Based Detail Artifacts** — App logs rotate by size, retention days, and archive count; detailed request/response payloads live in `DATA_DIR/call_logs/` and rotate independently of SQLite summaries
 - **System Info Report** — `npm run system-info` generates `system-info.txt` with your full environment (Node version, OmniRoute version, OS, CLI tools, Docker/PM2 status). Attach it when reporting issues for instant triage.
 
 </details>
@@ -2023,8 +2023,10 @@ opencode
 
 **No request logs**
 
-- Request artifacts are written to `DATA_DIR/call_logs/` as one JSON file per request
+- `call_logs` in SQLite stores summary metadata for the Request Logs table and analytics views
+- Detailed request/response payloads are written to `DATA_DIR/call_logs/` as one JSON artifact per request
 - Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads
+- `Export Logs` reads the artifact files on demand, while `Export All` includes the `call_logs/` directory alongside `storage.sqlite`
 - Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log`
 - Adjust `APP_LOG_MAX_FILE_SIZE`, `APP_LOG_RETENTION_DAYS`, `APP_LOG_MAX_FILES`, and `CALL_LOG_MAX_ENTRIES` as needed
 

open-sse/utils/streamPayloadCollector.ts

@@ -344,6 +344,10 @@ function buildClaudeSummary(events: StructuredSSEEvent[], fallbackModel?: string
         input: unknown;
         inputJson: string;
       };
+  type ClaudeContentBlock =
+    | { type: "text"; text: string }
+    | { type: "thinking"; thinking: string; signature?: string }
+    | { type: "tool_use"; id: string; name: string; input: unknown };
 
   const blocks = new Map<number, ClaudeBlock>();
   const usage: JsonRecord = {};
@@ -456,7 +460,7 @@ function buildClaudeSummary(events: StructuredSSEEvent[], fallbackModel?: string
 
   const content = [...blocks.values()]
     .sort((a, b) => a.index - b.index)
-    .flatMap((block) => {
+    .flatMap<ClaudeContentBlock>((block) => {
       if (block.type === "text") {
         return block.text
           ? [

src/app/api/db-backups/exportAll/route.ts

@@ -1,8 +1,10 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getDbInstance, SQLITE_FILE } from "@/lib/db/core";
+import { CALL_LOGS_DIR } from "@/lib/usage/callLogArtifacts";
 import fs from "fs";
 import path from "path";
 import os from "os";
+import { execSync } from "node:child_process";
 import { isAuthenticated } from "@/shared/utils/apiAuth";
 
 /**
@@ -34,7 +36,7 @@ export async function GET(request: NextRequest) {
 
       // 1. Export database using native backup API
       const dbBackupPath = path.join(tempDir, "storage.sqlite");
-      db.backup(dbBackupPath);
+      await db.backup(dbBackupPath);
 
       // 2. Export settings as JSON
       const settings: Record<string, string> = {};
@@ -89,7 +91,12 @@ export async function GET(request: NextRequest) {
       }
       fs.writeFileSync(path.join(tempDir, "api-keys.json"), JSON.stringify(apiKeys, null, 2));
 
-      // 6. Export metadata
+      // 6. Export call log artifacts directory
+      if (CALL_LOGS_DIR && fs.existsSync(CALL_LOGS_DIR)) {
+        fs.cpSync(CALL_LOGS_DIR, path.join(tempDir, "call_logs"), { recursive: true });
+      }
+
+      // 7. Export metadata
       const metadata = {
         exportedAt: new Date().toISOString(),
         version: process.env.npm_package_version || "unknown",
@@ -100,13 +107,13 @@ export async function GET(request: NextRequest) {
           "combos.json - Combo configurations",
           "providers.json - Provider connections (no credentials)",
           "api-keys.json - API key metadata (masked)",
+          "call_logs/ - Detailed call log artifacts",
         ],
       };
       fs.writeFileSync(path.join(tempDir, "metadata.json"), JSON.stringify(metadata, null, 2));
 
       // Create ZIP using tar (available on all Linux/macOS, and the archiver npm package is not installed)
       // We'll use Node.js built-in zlib to create a simple tar.gz instead
-      const { execSync } = require("node:child_process");
       const tarPath = zipPath.replace(".zip", ".tar.gz");
       execSync(`tar -czf "${tarPath}" -C "${path.dirname(tempDir)}" "${path.basename(tempDir)}"`, {
         timeout: 30000,

src/app/api/logs/export/route.ts

@@ -1,4 +1,5 @@
 import { getDbInstance } from "@/lib/db/core";
+import { exportCallLogsSince } from "@/lib/usage/callLogs";
 
 /**
  * GET /api/logs/export — export logs as JSON
@@ -19,10 +20,7 @@ export async function GET(request: Request) {
 
     if (logType === "call-logs" || logType === "request-logs") {
       tableName = "call_logs";
-      const stmt = db.prepare(
-        "SELECT * FROM call_logs WHERE timestamp >= @since ORDER BY timestamp DESC"
-      );
-      rows = stmt.all({ since });
+      rows = await exportCallLogsSince(since);
     } else if (logType === "proxy-logs") {
       tableName = "proxy_logs";
       const stmt = db.prepare(

src/app/api/search/stats/route.ts

@@ -42,7 +42,7 @@ export async function GET(request: Request) {
     const recentRows = db
       .prepare(
         `
-        SELECT request_body, provider, timestamp
+        SELECT request_summary, provider, timestamp
         FROM call_logs
         WHERE request_type = 'search'
         ORDER BY timestamp DESC
@@ -55,12 +55,11 @@ export async function GET(request: Request) {
       let query = "";
       let filters = {};
       try {
-        const body = JSON.parse(row.request_body);
-        query = body.query || "";
-        const { query: _q, provider: _p, ...rest } = body;
-        filters = rest;
+        const summary = JSON.parse(row.request_summary);
+        query = summary.query || "";
+        filters = summary.filters || {};
       } catch {
-        // Unparseable request_body
+        // Unparseable request_summary
       }
       return {
         query,

src/app/api/settings/purge-logs/route.ts

@@ -1,18 +1,20 @@
 import { NextResponse } from "next/server";
-import { getDbInstance } from "@/lib/db/core";
 import { getCallLogRetentionDays } from "@/lib/logEnv";
+import { deleteCallLogsBefore } from "@/lib/usage/callLogs";
 import { isAuthenticated } from "@/shared/utils/apiAuth";
 
 export async function POST(request: Request) {
   if (!(await isAuthenticated(request))) {
     return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
   }
   try {
-    const db = getDbInstance();
     const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000;
     const cutoff = new Date(Date.now() - retentionMs).toISOString();
-    const result = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(cutoff);
-    return NextResponse.json({ deleted: result.changes });
+    const result = deleteCallLogsBefore(cutoff);
+    return NextResponse.json({
+      deleted: result.deletedRows,
+      deletedArtifacts: result.deletedArtifacts,
+    });
   } catch (err: unknown) {
     const error = err instanceof Error ? err.message : String(err);
     return NextResponse.json({ error }, { status: 500 });

src/app/api/v1/search/analytics/route.ts

@@ -33,7 +33,7 @@ export async function GET(req: Request) {
         `SELECT
           COUNT(*) as total,
           COALESCE(SUM(CASE WHEN timestamp >= ? THEN 1 ELSE 0 END), 0) as today,
-          COALESCE(SUM(CASE WHEN status >= 400 OR error IS NOT NULL THEN 1 ELSE 0 END), 0) as errors,
+          COALESCE(SUM(CASE WHEN status >= 400 OR error_summary IS NOT NULL THEN 1 ELSE 0 END), 0) as errors,
           AVG(CASE WHEN duration > 0 THEN duration END) as avg_duration,
           COALESCE(SUM(CASE WHEN duration > 0 AND duration < 5 THEN 1 ELSE 0 END), 0) as cached
          FROM call_logs

src/lib/compliance/index.ts

@@ -18,6 +18,7 @@ import {
   getProxyLogsTableMaxRows,
 } from "../logEnv";
 import { generateRequestId, getRequestId } from "@/shared/utils/requestId";
+import { deleteCallLogsBefore, trimCallLogsToMaxRows } from "../usage/callLogs";
 
 /** @returns {import("better-sqlite3").Database | null} */
 function getDb() {
@@ -539,8 +540,8 @@ export function cleanupExpiredLogs() {
   }
 
   try {
-    const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(callCutoff);
-    deletedCallLogs = r2.changes;
+    const r2 = deleteCallLogsBefore(callCutoff);
+    deletedCallLogs = r2.deletedRows;
   } catch {
     /* table may not exist */
   }
@@ -577,22 +578,8 @@ export function cleanupExpiredLogs() {
   const BATCH_SIZE = 5000;
   if (callLogsMaxRows > 0) {
     try {
-      let currentCount = db.prepare("SELECT COUNT(*) as cnt FROM call_logs").get() as {
-        cnt: number;
-      };
-      while (currentCount.cnt > callLogsMaxRows) {
-        const toDelete = Math.min(currentCount.cnt - callLogsMaxRows, BATCH_SIZE);
-        const trimmed = db
-          .prepare(
-            `DELETE FROM call_logs WHERE id IN (
-              SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ?
-            )`
-          )
-          .run(toDelete);
-        trimmedCallLogs += trimmed.changes;
-        currentCount.cnt -= trimmed.changes;
-        if (trimmed.changes === 0) break;
-      }
+      const trimmed = trimCallLogsToMaxRows(callLogsMaxRows);
+      trimmedCallLogs = trimmed.deletedRows;
     } catch {
       /* best effort */
     }