From 39bbc410069fb46c2ed7b0f28e6e997945422485 Mon Sep 17 00:00:00 2001 From: "R.D." Date: Wed, 15 Apr 2026 22:33:56 -0400 Subject: [PATCH] Backport call log storage refactor to release/v3.6.6 --- README.md | 8 +- open-sse/utils/streamPayloadCollector.ts | 6 +- src/app/api/db-backups/exportAll/route.ts | 13 +- src/app/api/logs/export/route.ts | 6 +- src/app/api/search/stats/route.ts | 11 +- src/app/api/settings/purge-logs/route.ts | 10 +- src/app/api/v1/search/analytics/route.ts | 2 +- src/lib/compliance/index.ts | 23 +- src/lib/db/core.ts | 220 +++++- src/lib/db/migrationRunner.ts | 72 ++ src/lib/db/migrations/001_initial_schema.sql | 26 +- src/lib/db/migrations/022_add_memory_fts5.sql | 36 +- .../025_call_logs_summary_storage.sql | 142 ++++ src/lib/usage/callLogArtifacts.ts | 181 +++++ src/lib/usage/callLogs.ts | 730 +++++++++++------- src/lib/usage/migrations.ts | 128 ++- src/shared/components/RequestLoggerDetail.tsx | 15 + tests/unit/call-log-cap.test.ts | 544 ++++++------- tests/unit/call-log-file-rotation.test.ts | 137 +++- tests/unit/db-migration-runner.test.ts | 191 +++++ tests/unit/log-export-routes.test.mjs | 86 +++ tests/unit/usage-migrations.test.ts | 32 +- 22 files changed, 1900 insertions(+), 719 deletions(-) create mode 100644 src/lib/db/migrations/025_call_logs_summary_storage.sql create mode 100644 src/lib/usage/callLogArtifacts.ts create mode 100644 tests/unit/log-export-routes.test.mjs diff --git a/README.md b/README.md index c443e86dce48..723606151638 100644 --- a/README.md +++ b/README.md @@ -387,10 +387,10 @@ When a call fails, the dev doesn't know if it was a rate limit, expired token, w - **Unified Logs Dashboard** — 4 tabs: Request Logs, Proxy Logs, Audit Logs, Console - **Console Log Viewer** — Real-time terminal-style viewer with color-coded levels, auto-scroll, search, filter -- **SQLite Proxy Logs** — Persistent logs that survive server restarts +- **SQLite Summary Logs** — Request and proxy log indexes stay queryable across restarts without loading large payload blobs into SQLite - **Translator Playground** — 4 debugging modes: Playground (format translation), Chat Tester (round-trip), Test Bench (batch), Live Monitor (real-time) - **Request Telemetry** — p50/p95/p99 latency + X-Request-Id tracing -- **File-Based Logging with Rotation** — App logs rotate by size, retention days, and archive count; call log artifacts rotate by retention days and file count +- **File-Based Detail Artifacts** — App logs rotate by size, retention days, and archive count; detailed request/response payloads live in `DATA_DIR/call_logs/` and rotate independently of SQLite summaries - **System Info Report** — `npm run system-info` generates `system-info.txt` with your full environment (Node version, OmniRoute version, OS, CLI tools, Docker/PM2 status). Attach it when reporting issues for instant triage. @@ -2023,8 +2023,10 @@ opencode **No request logs** -- Request artifacts are written to `DATA_DIR/call_logs/` as one JSON file per request +- `call_logs` in SQLite stores summary metadata for the Request Logs table and analytics views +- Detailed request/response payloads are written to `DATA_DIR/call_logs/` as one JSON artifact per request - Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads +- `Export Logs` reads the artifact files on demand, while `Export All` includes the `call_logs/` directory alongside `storage.sqlite` - Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log` - Adjust `APP_LOG_MAX_FILE_SIZE`, `APP_LOG_RETENTION_DAYS`, `APP_LOG_MAX_FILES`, and `CALL_LOG_MAX_ENTRIES` as needed diff --git a/open-sse/utils/streamPayloadCollector.ts b/open-sse/utils/streamPayloadCollector.ts index 96395bee99ed..0dc409aeb16c 100644 --- a/open-sse/utils/streamPayloadCollector.ts +++ b/open-sse/utils/streamPayloadCollector.ts @@ -344,6 +344,10 @@ function buildClaudeSummary(events: StructuredSSEEvent[], fallbackModel?: string input: unknown; inputJson: string; }; + type ClaudeContentBlock = + | { type: "text"; text: string } + | { type: "thinking"; thinking: string; signature?: string } + | { type: "tool_use"; id: string; name: string; input: unknown }; const blocks = new Map(); const usage: JsonRecord = {}; @@ -456,7 +460,7 @@ function buildClaudeSummary(events: StructuredSSEEvent[], fallbackModel?: string const content = [...blocks.values()] .sort((a, b) => a.index - b.index) - .flatMap((block) => { + .flatMap((block) => { if (block.type === "text") { return block.text ? [ diff --git a/src/app/api/db-backups/exportAll/route.ts b/src/app/api/db-backups/exportAll/route.ts index 3a786ae6a97f..4cb2e75144a3 100644 --- a/src/app/api/db-backups/exportAll/route.ts +++ b/src/app/api/db-backups/exportAll/route.ts @@ -1,8 +1,10 @@ import { NextRequest, NextResponse } from "next/server"; import { getDbInstance, SQLITE_FILE } from "@/lib/db/core"; +import { CALL_LOGS_DIR } from "@/lib/usage/callLogArtifacts"; import fs from "fs"; import path from "path"; import os from "os"; +import { execSync } from "node:child_process"; import { isAuthenticated } from "@/shared/utils/apiAuth"; /** @@ -34,7 +36,7 @@ export async function GET(request: NextRequest) { // 1. Export database using native backup API const dbBackupPath = path.join(tempDir, "storage.sqlite"); - db.backup(dbBackupPath); + await db.backup(dbBackupPath); // 2. Export settings as JSON const settings: Record = {}; @@ -89,7 +91,12 @@ export async function GET(request: NextRequest) { } fs.writeFileSync(path.join(tempDir, "api-keys.json"), JSON.stringify(apiKeys, null, 2)); - // 6. Export metadata + // 6. Export call log artifacts directory + if (CALL_LOGS_DIR && fs.existsSync(CALL_LOGS_DIR)) { + fs.cpSync(CALL_LOGS_DIR, path.join(tempDir, "call_logs"), { recursive: true }); + } + + // 7. Export metadata const metadata = { exportedAt: new Date().toISOString(), version: process.env.npm_package_version || "unknown", @@ -100,13 +107,13 @@ export async function GET(request: NextRequest) { "combos.json - Combo configurations", "providers.json - Provider connections (no credentials)", "api-keys.json - API key metadata (masked)", + "call_logs/ - Detailed call log artifacts", ], }; fs.writeFileSync(path.join(tempDir, "metadata.json"), JSON.stringify(metadata, null, 2)); // Create ZIP using tar (available on all Linux/macOS, and the archiver npm package is not installed) // We'll use Node.js built-in zlib to create a simple tar.gz instead - const { execSync } = require("node:child_process"); const tarPath = zipPath.replace(".zip", ".tar.gz"); execSync(`tar -czf "${tarPath}" -C "${path.dirname(tempDir)}" "${path.basename(tempDir)}"`, { timeout: 30000, diff --git a/src/app/api/logs/export/route.ts b/src/app/api/logs/export/route.ts index 3ee0390ff92e..3e25814332b8 100644 --- a/src/app/api/logs/export/route.ts +++ b/src/app/api/logs/export/route.ts @@ -1,4 +1,5 @@ import { getDbInstance } from "@/lib/db/core"; +import { exportCallLogsSince } from "@/lib/usage/callLogs"; /** * GET /api/logs/export — export logs as JSON @@ -19,10 +20,7 @@ export async function GET(request: Request) { if (logType === "call-logs" || logType === "request-logs") { tableName = "call_logs"; - const stmt = db.prepare( - "SELECT * FROM call_logs WHERE timestamp >= @since ORDER BY timestamp DESC" - ); - rows = stmt.all({ since }); + rows = await exportCallLogsSince(since); } else if (logType === "proxy-logs") { tableName = "proxy_logs"; const stmt = db.prepare( diff --git a/src/app/api/search/stats/route.ts b/src/app/api/search/stats/route.ts index a7dfc78b585d..361f1f05c328 100644 --- a/src/app/api/search/stats/route.ts +++ b/src/app/api/search/stats/route.ts @@ -42,7 +42,7 @@ export async function GET(request: Request) { const recentRows = db .prepare( ` - SELECT request_body, provider, timestamp + SELECT request_summary, provider, timestamp FROM call_logs WHERE request_type = 'search' ORDER BY timestamp DESC @@ -55,12 +55,11 @@ export async function GET(request: Request) { let query = ""; let filters = {}; try { - const body = JSON.parse(row.request_body); - query = body.query || ""; - const { query: _q, provider: _p, ...rest } = body; - filters = rest; + const summary = JSON.parse(row.request_summary); + query = summary.query || ""; + filters = summary.filters || {}; } catch { - // Unparseable request_body + // Unparseable request_summary } return { query, diff --git a/src/app/api/settings/purge-logs/route.ts b/src/app/api/settings/purge-logs/route.ts index 66cdb178ed65..b752b7dbab90 100644 --- a/src/app/api/settings/purge-logs/route.ts +++ b/src/app/api/settings/purge-logs/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; -import { getDbInstance } from "@/lib/db/core"; import { getCallLogRetentionDays } from "@/lib/logEnv"; +import { deleteCallLogsBefore } from "@/lib/usage/callLogs"; import { isAuthenticated } from "@/shared/utils/apiAuth"; export async function POST(request: Request) { @@ -8,11 +8,13 @@ export async function POST(request: Request) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } try { - const db = getDbInstance(); const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000; const cutoff = new Date(Date.now() - retentionMs).toISOString(); - const result = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(cutoff); - return NextResponse.json({ deleted: result.changes }); + const result = deleteCallLogsBefore(cutoff); + return NextResponse.json({ + deleted: result.deletedRows, + deletedArtifacts: result.deletedArtifacts, + }); } catch (err: unknown) { const error = err instanceof Error ? err.message : String(err); return NextResponse.json({ error }, { status: 500 }); diff --git a/src/app/api/v1/search/analytics/route.ts b/src/app/api/v1/search/analytics/route.ts index 2e1168550460..f11545ef782a 100644 --- a/src/app/api/v1/search/analytics/route.ts +++ b/src/app/api/v1/search/analytics/route.ts @@ -33,7 +33,7 @@ export async function GET(req: Request) { `SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN timestamp >= ? THEN 1 ELSE 0 END), 0) as today, - COALESCE(SUM(CASE WHEN status >= 400 OR error IS NOT NULL THEN 1 ELSE 0 END), 0) as errors, + COALESCE(SUM(CASE WHEN status >= 400 OR error_summary IS NOT NULL THEN 1 ELSE 0 END), 0) as errors, AVG(CASE WHEN duration > 0 THEN duration END) as avg_duration, COALESCE(SUM(CASE WHEN duration > 0 AND duration < 5 THEN 1 ELSE 0 END), 0) as cached FROM call_logs diff --git a/src/lib/compliance/index.ts b/src/lib/compliance/index.ts index e52685c1ed7e..28a248e41bf3 100644 --- a/src/lib/compliance/index.ts +++ b/src/lib/compliance/index.ts @@ -18,6 +18,7 @@ import { getProxyLogsTableMaxRows, } from "../logEnv"; import { generateRequestId, getRequestId } from "@/shared/utils/requestId"; +import { deleteCallLogsBefore, trimCallLogsToMaxRows } from "../usage/callLogs"; /** @returns {import("better-sqlite3").Database | null} */ function getDb() { @@ -539,8 +540,8 @@ export function cleanupExpiredLogs() { } try { - const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(callCutoff); - deletedCallLogs = r2.changes; + const r2 = deleteCallLogsBefore(callCutoff); + deletedCallLogs = r2.deletedRows; } catch { /* table may not exist */ } @@ -577,22 +578,8 @@ export function cleanupExpiredLogs() { const BATCH_SIZE = 5000; if (callLogsMaxRows > 0) { try { - let currentCount = db.prepare("SELECT COUNT(*) as cnt FROM call_logs").get() as { - cnt: number; - }; - while (currentCount.cnt > callLogsMaxRows) { - const toDelete = Math.min(currentCount.cnt - callLogsMaxRows, BATCH_SIZE); - const trimmed = db - .prepare( - `DELETE FROM call_logs WHERE id IN ( - SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ? - )` - ) - .run(toDelete); - trimmedCallLogs += trimmed.changes; - currentCount.cnt -= trimmed.changes; - if (trimmed.changes === 0) break; - } + const trimmed = trimCallLogsToMaxRows(callLogsMaxRows); + trimmedCallLogs = trimmed.deletedRows; } catch { /* best effort */ } diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index ee090c61415b..d0fbc3659c23 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -10,6 +10,12 @@ import fs from "fs"; import { resolveDataDir, getLegacyDotDataDir } from "../dataPaths"; import { runMigrations } from "./migrationRunner"; import { runDbHealthCheck } from "./healthCheck"; +import { parseStoredPayload } from "../logPayloads"; +import { + buildArtifactRelativePath, + writeCallArtifact, + type CallLogArtifact, +} from "../usage/callLogArtifacts"; type SqliteDatabase = import("better-sqlite3").Database; type JsonRecord = Record; @@ -181,11 +187,15 @@ const SCHEMA_SQL = ` combo_name TEXT, combo_step_id TEXT, combo_execution_key TEXT, - request_body TEXT, - response_body TEXT, - error TEXT, + error_summary TEXT, + detail_state TEXT DEFAULT 'none', artifact_relpath TEXT, - has_pipeline_details INTEGER DEFAULT 0 + artifact_size_bytes INTEGER DEFAULT NULL, + artifact_sha256 TEXT DEFAULT NULL, + has_request_body INTEGER DEFAULT 0, + has_response_body INTEGER DEFAULT 0, + has_pipeline_details INTEGER DEFAULT 0, + request_summary TEXT ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); @@ -437,6 +447,34 @@ function ensureCallLogsColumns(db: SqliteDatabase) { db.exec("ALTER TABLE call_logs ADD COLUMN combo_execution_key TEXT DEFAULT NULL"); console.log("[DB] Added call_logs.combo_execution_key column"); } + if (!columnNames.has("error_summary")) { + db.exec("ALTER TABLE call_logs ADD COLUMN error_summary TEXT DEFAULT NULL"); + console.log("[DB] Added call_logs.error_summary column"); + } + if (!columnNames.has("detail_state")) { + db.exec("ALTER TABLE call_logs ADD COLUMN detail_state TEXT DEFAULT 'none'"); + console.log("[DB] Added call_logs.detail_state column"); + } + if (!columnNames.has("artifact_size_bytes")) { + db.exec("ALTER TABLE call_logs ADD COLUMN artifact_size_bytes INTEGER DEFAULT NULL"); + console.log("[DB] Added call_logs.artifact_size_bytes column"); + } + if (!columnNames.has("artifact_sha256")) { + db.exec("ALTER TABLE call_logs ADD COLUMN artifact_sha256 TEXT DEFAULT NULL"); + console.log("[DB] Added call_logs.artifact_sha256 column"); + } + if (!columnNames.has("has_request_body")) { + db.exec("ALTER TABLE call_logs ADD COLUMN has_request_body INTEGER DEFAULT 0"); + console.log("[DB] Added call_logs.has_request_body column"); + } + if (!columnNames.has("has_response_body")) { + db.exec("ALTER TABLE call_logs ADD COLUMN has_response_body INTEGER DEFAULT 0"); + console.log("[DB] Added call_logs.has_response_body column"); + } + if (!columnNames.has("request_summary")) { + db.exec("ALTER TABLE call_logs ADD COLUMN request_summary TEXT DEFAULT NULL"); + console.log("[DB] Added call_logs.request_summary column"); + } db.exec( "CREATE INDEX IF NOT EXISTS idx_call_logs_requested_model ON call_logs(requested_model)" @@ -456,6 +494,165 @@ function hasColumn(db: SqliteDatabase, tableName: string, columnName: string): b return rows.some((row) => row.name === columnName); } +function hasTable(db: SqliteDatabase, tableName: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName) + ); +} + +function parseLegacyError(value: unknown): unknown { + if (typeof value !== "string" || value.trim().length === 0) return null; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +function offloadLegacyCallLogDetails(db: SqliteDatabase) { + if (!hasTable(db, "call_logs_v1_legacy")) return; + + type LegacyCallLogRow = { + id: string; + timestamp: string | null; + method: string | null; + path: string | null; + status: number | null; + model: string | null; + requested_model: string | null; + provider: string | null; + account: string | null; + connection_id: string | null; + duration: number | null; + tokens_in: number | null; + tokens_out: number | null; + tokens_cache_read: number | null; + tokens_cache_creation: number | null; + tokens_reasoning: number | null; + request_type: string | null; + source_format: string | null; + target_format: string | null; + api_key_id: string | null; + api_key_name: string | null; + combo_name: string | null; + combo_step_id: string | null; + combo_execution_key: string | null; + request_body: string | null; + response_body: string | null; + error: string | null; + }; + + const pendingRows = db + .prepare( + ` + SELECT legacy.* + FROM call_logs_v1_legacy AS legacy + JOIN call_logs AS current ON current.id = legacy.id + WHERE current.detail_state = 'legacy-inline' + ORDER BY legacy.timestamp ASC + ` + ) + .all() as LegacyCallLogRow[]; + + if (pendingRows.length === 0) { + db.exec("DROP TABLE IF EXISTS call_logs_v1_legacy"); + return; + } + + const updateStmt = db.prepare(` + UPDATE call_logs + SET artifact_relpath = @artifactRelPath, + artifact_size_bytes = @artifactSizeBytes, + artifact_sha256 = @artifactSha256, + detail_state = 'ready' + WHERE id = @id + `); + const markMissingStmt = db.prepare(` + UPDATE call_logs + SET detail_state = 'missing', + artifact_relpath = NULL, + artifact_size_bytes = NULL, + artifact_sha256 = NULL + WHERE id = ? + `); + + let failed = 0; + const tx = db.transaction(() => { + for (const row of pendingRows) { + const artifact: CallLogArtifact = { + schemaVersion: 4, + summary: { + id: row.id, + timestamp: row.timestamp || new Date().toISOString(), + method: row.method || "POST", + path: row.path || "/v1/chat/completions", + status: row.status || 0, + model: row.model || "-", + requestedModel: row.requested_model || null, + provider: row.provider || "-", + account: row.account || "-", + connectionId: row.connection_id || null, + duration: row.duration || 0, + tokens: { + in: row.tokens_in || 0, + out: row.tokens_out || 0, + cacheRead: row.tokens_cache_read ?? null, + cacheWrite: row.tokens_cache_creation ?? null, + reasoning: row.tokens_reasoning ?? null, + }, + requestType: row.request_type || null, + sourceFormat: row.source_format || null, + targetFormat: row.target_format || null, + apiKeyId: row.api_key_id || null, + apiKeyName: row.api_key_name || null, + comboName: row.combo_name || null, + comboStepId: row.combo_step_id || null, + comboExecutionKey: row.combo_execution_key || null, + }, + requestBody: parseStoredPayload(row.request_body), + responseBody: parseStoredPayload(row.response_body), + error: parseLegacyError(row.error), + }; + + const artifactResult = writeCallArtifact( + artifact, + buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id) + ); + if (!artifactResult) { + failed++; + markMissingStmt.run(row.id); + continue; + } + + updateStmt.run({ + id: row.id, + artifactRelPath: artifactResult.relPath, + artifactSizeBytes: artifactResult.sizeBytes, + artifactSha256: artifactResult.sha256, + }); + } + }); + + tx(); + + if (failed > 0) { + console.warn( + `[DB] Kept call_logs_v1_legacy after partial call log offload (${failed} failed row(s)).` + ); + return; + } + + db.exec("DROP TABLE IF EXISTS call_logs_v1_legacy"); + try { + db.pragma("wal_checkpoint(TRUNCATE)"); + db.exec("VACUUM"); + console.log(`[DB] Offloaded ${pendingRows.length} legacy call log detail row(s) to artifacts.`); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn("[DB] Legacy call log compaction finished without VACUUM:", message); + } +} + function isAutomatedTestProcess(): boolean { return ( typeof process !== "undefined" && @@ -694,7 +891,22 @@ export function getDbInstance(): SqliteDatabase { "combo_call_log_targets" ); } + if ( + hasColumn(db, "call_logs", "detail_state") && + hasColumn(db, "call_logs", "request_summary") && + hasColumn(db, "call_logs", "has_request_body") && + hasColumn(db, "call_logs", "has_response_body") && + !hasColumn(db, "call_logs", "request_body") && + !hasColumn(db, "call_logs", "response_body") && + !hasColumn(db, "call_logs", "error") + ) { + db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( + "025", + "call_logs_summary_storage" + ); + } runMigrations(db); + offloadLegacyCallLogDetails(db); // Auto-migrate from db.json if exists if (jsonDbFile && fs.existsSync(jsonDbFile)) { diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index d7478477dbd8..6a21b13be0f7 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -50,6 +50,15 @@ const MIGRATIONS_DIR = resolveMigrationsDir(); */ const MAX_PENDING_MIGRATIONS_ON_EXISTING_DB = 5; +const RENAMED_MIGRATION_COMPATIBILITY = [ + { + fromVersion: "022", + fromName: "call_logs_summary_storage", + toVersion: "025", + toName: "call_logs_summary_storage", + }, +] as const; + /** * Ensure the schema_migrations tracking table exists. */ @@ -133,6 +142,68 @@ function detectNameMismatches( return mismatches; } +function reconcileRenumberedMigrations( + db: Database.Database, + files: Array<{ version: string; name: string; path: string }> +): boolean { + let repaired = false; + + for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { + const hasTargetFile = files.some( + (file) => + file.version === compatibility.toVersion && file.name === compatibility.toName + ); + const hasSourceFile = files.some( + (file) => + file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ); + + if (!hasTargetFile || !hasSourceFile) { + continue; + } + + const legacyRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(compatibility.fromVersion, compatibility.fromName) as + | { version: string; name: string } + | undefined; + if (!legacyRow) { + continue; + } + + const targetRow = db + .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") + .get(compatibility.toVersion) as { version: string } | undefined; + + const applyRepair = db.transaction(() => { + if (targetRow) { + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( + compatibility.fromVersion, + compatibility.fromName + ); + } else { + db.prepare( + "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" + ).run( + compatibility.toVersion, + compatibility.toName, + compatibility.fromVersion, + compatibility.fromName + ); + } + }); + + applyRepair(); + repaired = true; + console.warn( + `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` + + `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.` + ); + } + + return repaired; +} + /** * Create a pre-migration backup of the SQLite database using VACUUM INTO. * Returns the backup path on success, null on failure. @@ -174,6 +245,7 @@ export function runMigrations(db: Database.Database): number { ensureMigrationsTable(db); const files = getMigrationFiles(); + reconcileRenumberedMigrations(db, files); const applied = getAppliedVersions(db); const appliedRecords = getAppliedRecords(db); diff --git a/src/lib/db/migrations/001_initial_schema.sql b/src/lib/db/migrations/001_initial_schema.sql index 42d3fe5aba6b..6423a2f104e0 100644 --- a/src/lib/db/migrations/001_initial_schema.sql +++ b/src/lib/db/migrations/001_initial_schema.sql @@ -109,8 +109,8 @@ CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); CREATE TABLE IF NOT EXISTS call_logs ( - id TEXT PRIMARY KEY, - timestamp TEXT NOT NULL, + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, method TEXT, path TEXT, status INTEGER, @@ -129,15 +129,19 @@ CREATE TABLE IF NOT EXISTS call_logs ( source_format TEXT, target_format TEXT, api_key_id TEXT, - api_key_name TEXT, - combo_name TEXT, - combo_step_id TEXT, - combo_execution_key TEXT, - request_body TEXT, - response_body TEXT, - error TEXT, - artifact_relpath TEXT, - has_pipeline_details INTEGER DEFAULT 0 + api_key_name TEXT, + combo_name TEXT, + combo_step_id TEXT, + combo_execution_key TEXT, + error_summary TEXT, + detail_state TEXT DEFAULT 'none', + artifact_relpath TEXT, + artifact_size_bytes INTEGER DEFAULT NULL, + artifact_sha256 TEXT DEFAULT NULL, + has_request_body INTEGER DEFAULT 0, + has_response_body INTEGER DEFAULT 0, + has_pipeline_details INTEGER DEFAULT 0, + request_summary TEXT ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); diff --git a/src/lib/db/migrations/022_add_memory_fts5.sql b/src/lib/db/migrations/022_add_memory_fts5.sql index 32887cc561b1..8042a130805c 100644 --- a/src/lib/db/migrations/022_add_memory_fts5.sql +++ b/src/lib/db/migrations/022_add_memory_fts5.sql @@ -22,38 +22,12 @@ CREATE INDEX IF NOT EXISTS idx_memories_session ON memories(session_id); CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type); CREATE INDEX IF NOT EXISTS idx_memories_expires ON memories(expires_at); --- Create FTS5 virtual table for full-text search on memories +-- Transitional setup only: create the FTS table shell without backfilling data or +-- attaching UUID-based triggers. The follow-up migration 023_fix_memory_fts_uuid.sql +-- converts memories to a stable INTEGER-backed join key, recreates the FTS table, +-- and performs the real backfill safely. CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( content, key, - content='memories', - content_rowid='id' + content='memories' ); - --- KNOWN LIMITATION: content_rowid='id' requires an INTEGER rowid, but memories.id is TEXT (UUID). --- SQLite FTS5 silently accepts the TEXT UUID as rowid during INSERT (line 15), but the internal rowid --- won't match memories.id, causing FTS5 content-based inserts to fail lookup by actual rowid. --- Production code (src/lib/memory/retrieval.ts) gracefully falls back to keyword scoring via getRelevanceScore() --- when FTS5 matching fails. This is a pre-existing known limitation; fixing it requires: --- (a) Adding an INTEGER primary key column to memories table, or --- (b) Using external content FTS5 (content=) with proper INTEGER sync. --- For now, keyword-based fallback is acceptable and maintains backward compatibility. - --- Sync trigger for INSERT — keep FTS5 in sync when new memories are added -CREATE TRIGGER IF NOT EXISTS memory_fts_ai AFTER INSERT ON memories BEGIN - INSERT INTO memory_fts(rowid, content, key) VALUES (new.id, new.content, new.key); -END; - --- Sync trigger for DELETE — keep FTS5 in sync when memories are removed -CREATE TRIGGER IF NOT EXISTS memory_fts_ad AFTER DELETE ON memories BEGIN - INSERT INTO memory_fts(memory_fts, rowid, content, key) VALUES('delete', old.id, old.content, old.key); -END; - --- Sync trigger for UPDATE — keep FTS5 in sync when memories are modified -CREATE TRIGGER IF NOT EXISTS memory_fts_au AFTER UPDATE ON memories BEGIN - INSERT INTO memory_fts(memory_fts, rowid, content, key) VALUES('delete', old.id, old.content, old.key); - INSERT INTO memory_fts(rowid, content, key) VALUES (new.id, new.content, new.key); -END; - --- Populate FTS5 table with existing memory data -INSERT INTO memory_fts(rowid, content, key) SELECT id, content, key FROM memories; diff --git a/src/lib/db/migrations/025_call_logs_summary_storage.sql b/src/lib/db/migrations/025_call_logs_summary_storage.sql new file mode 100644 index 000000000000..182144616696 --- /dev/null +++ b/src/lib/db/migrations/025_call_logs_summary_storage.sql @@ -0,0 +1,142 @@ +-- 025_call_logs_summary_storage.sql +-- Rebuild call_logs so SQLite stores summary metadata only. + +DROP INDEX IF EXISTS idx_cl_combo_target; +DROP INDEX IF EXISTS idx_call_logs_request_type; +DROP INDEX IF EXISTS idx_call_logs_requested_model; +DROP INDEX IF EXISTS idx_cl_status; +DROP INDEX IF EXISTS idx_cl_timestamp; + +ALTER TABLE call_logs RENAME TO call_logs_v1_legacy; + +CREATE TABLE call_logs ( + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, + method TEXT, + path TEXT, + status INTEGER, + model TEXT, + requested_model TEXT, + provider TEXT, + account TEXT, + connection_id TEXT, + duration INTEGER DEFAULT 0, + tokens_in INTEGER DEFAULT 0, + tokens_out INTEGER DEFAULT 0, + tokens_cache_read INTEGER DEFAULT NULL, + tokens_cache_creation INTEGER DEFAULT NULL, + tokens_reasoning INTEGER DEFAULT NULL, + request_type TEXT, + source_format TEXT, + target_format TEXT, + api_key_id TEXT, + api_key_name TEXT, + combo_name TEXT, + combo_step_id TEXT, + combo_execution_key TEXT, + error_summary TEXT, + detail_state TEXT DEFAULT 'none', + artifact_relpath TEXT, + artifact_size_bytes INTEGER DEFAULT NULL, + artifact_sha256 TEXT DEFAULT NULL, + has_request_body INTEGER DEFAULT 0, + has_response_body INTEGER DEFAULT 0, + has_pipeline_details INTEGER DEFAULT 0, + request_summary TEXT +); + +INSERT OR REPLACE INTO call_logs ( + id, + timestamp, + method, + path, + status, + model, + requested_model, + provider, + account, + connection_id, + duration, + tokens_in, + tokens_out, + tokens_cache_read, + tokens_cache_creation, + tokens_reasoning, + request_type, + source_format, + target_format, + api_key_id, + api_key_name, + combo_name, + combo_step_id, + combo_execution_key, + error_summary, + detail_state, + artifact_relpath, + artifact_size_bytes, + artifact_sha256, + has_request_body, + has_response_body, + has_pipeline_details, + request_summary +) +SELECT + id, + timestamp, + method, + path, + status, + model, + requested_model, + provider, + account, + connection_id, + COALESCE(duration, 0), + COALESCE(tokens_in, 0), + COALESCE(tokens_out, 0), + tokens_cache_read, + tokens_cache_creation, + tokens_reasoning, + request_type, + source_format, + target_format, + api_key_id, + api_key_name, + combo_name, + combo_step_id, + combo_execution_key, + CASE + WHEN error IS NULL OR TRIM(CAST(error AS TEXT)) = '' THEN NULL + WHEN LENGTH(CAST(error AS TEXT)) > 4000 THEN SUBSTR(CAST(error AS TEXT), 1, 4000) + ELSE CAST(error AS TEXT) + END AS error_summary, + CASE + WHEN artifact_relpath IS NOT NULL AND TRIM(artifact_relpath) != '' THEN 'ready' + WHEN COALESCE(request_body, '') != '' OR COALESCE(response_body, '') != '' OR COALESCE(error, '') != '' + THEN 'legacy-inline' + ELSE 'none' + END AS detail_state, + NULLIF(TRIM(artifact_relpath), '') AS artifact_relpath, + NULL AS artifact_size_bytes, + NULL AS artifact_sha256, + CASE WHEN request_body IS NOT NULL AND TRIM(request_body) != '' THEN 1 ELSE 0 END AS has_request_body, + CASE WHEN response_body IS NOT NULL AND TRIM(response_body) != '' THEN 1 ELSE 0 END AS has_response_body, + COALESCE(has_pipeline_details, 0) AS has_pipeline_details, + CASE + WHEN request_type = 'search' AND request_body IS NOT NULL AND json_valid(request_body) + THEN json_object( + 'query', + COALESCE(json_extract(request_body, '$.query'), ''), + 'filters', + json(COALESCE(json_remove(request_body, '$.query', '$.provider'), '{}')) + ) + ELSE NULL + END AS request_summary +FROM call_logs_v1_legacy; + +CREATE INDEX idx_cl_timestamp ON call_logs(timestamp); +CREATE INDEX idx_cl_status ON call_logs(status); +CREATE INDEX idx_call_logs_requested_model ON call_logs(requested_model); +CREATE INDEX idx_call_logs_request_type ON call_logs(request_type); +CREATE INDEX idx_cl_combo_target + ON call_logs(combo_name, combo_execution_key, timestamp); diff --git a/src/lib/usage/callLogArtifacts.ts b/src/lib/usage/callLogArtifacts.ts new file mode 100644 index 000000000000..daa2d6c8b66e --- /dev/null +++ b/src/lib/usage/callLogArtifacts.ts @@ -0,0 +1,181 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; +import { resolveDataDir } from "../dataPaths"; + +const isCloud = typeof globalThis.caches === "object" && globalThis.caches !== null; +const isBuildPhase = process.env.NEXT_PHASE === "phase-production-build"; +const DATA_DIR = resolveDataDir({ isCloud }); + +export const CALL_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "call_logs"); + +export type CallLogDetailState = "none" | "ready" | "missing" | "corrupt" | "legacy-inline"; + +export type CallLogArtifact = { + schemaVersion: 4; + summary: { + id: string; + timestamp: string; + method: string; + path: string; + status: number; + model: string; + requestedModel: string | null; + provider: string; + account: string; + connectionId: string | null; + duration: number; + tokens: { + in: number; + out: number; + cacheRead: number | null; + cacheWrite: number | null; + reasoning: number | null; + }; + requestType: string | null; + sourceFormat: string | null; + targetFormat: string | null; + apiKeyId: string | null; + apiKeyName: string | null; + comboName: string | null; + comboStepId: string | null; + comboExecutionKey: string | null; + }; + requestBody: unknown; + responseBody: unknown; + error: unknown; + pipeline?: RequestPipelinePayloads; +}; + +export type CallLogArtifactWriteResult = { + relPath: string; + sizeBytes: number; + sha256: string; +}; + +export function buildArtifactRelativePath(timestamp: string, id: string) { + const parsed = new Date(timestamp); + const safeTimestamp = ( + Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString() + ).replace(/[:]/g, "-"); + const dateFolder = safeTimestamp.slice(0, 10); + return path.posix.join(dateFolder, `${safeTimestamp}_${id}.json`); +} + +export function writeCallArtifact( + artifact: CallLogArtifact, + relativePath = buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id) +): CallLogArtifactWriteResult | null { + if (!CALL_LOGS_DIR || isBuildPhase) return null; + + const absPath = path.join(CALL_LOGS_DIR, relativePath); + const tmpPath = `${absPath}.${process.pid}.${Date.now()}.tmp`; + + try { + const serialized = JSON.stringify(artifact, null, 2); + const sizeBytes = Buffer.byteLength(serialized); + const sha256 = crypto.createHash("sha256").update(serialized).digest("hex"); + + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(tmpPath, serialized); + fs.renameSync(tmpPath, absPath); + + return { + relPath: relativePath, + sizeBytes, + sha256, + }; + } catch (error) { + try { + fs.rmSync(tmpPath, { force: true }); + } catch { + // Best effort cleanup only. + } + console.error("[callLogs] Failed to write request artifact:", (error as Error).message); + return null; + } +} + +export function readCallArtifact(relativePath: string | null): { + artifact: CallLogArtifact | null; + state: "ready" | "missing" | "corrupt"; +} { + if (!CALL_LOGS_DIR || !relativePath) { + return { artifact: null, state: "missing" }; + } + + try { + const absPath = path.join(CALL_LOGS_DIR, relativePath); + if (!fs.existsSync(absPath)) { + return { artifact: null, state: "missing" }; + } + return { + artifact: JSON.parse(fs.readFileSync(absPath, "utf8")) as CallLogArtifact, + state: "ready", + }; + } catch (error) { + console.error("[callLogs] Failed to read request artifact:", (error as Error).message); + return { artifact: null, state: "corrupt" }; + } +} + +export function deleteCallArtifact(relativePath: string | null): boolean { + if (!CALL_LOGS_DIR || !relativePath) return false; + + try { + const absPath = path.join(CALL_LOGS_DIR, relativePath); + if (!fs.existsSync(absPath)) return false; + fs.rmSync(absPath, { force: true }); + return true; + } catch { + return false; + } +} + +export function cleanupEmptyCallLogDirs(baseDir = CALL_LOGS_DIR) { + if (!baseDir || !fs.existsSync(baseDir)) return; + + try { + for (const entry of fs.readdirSync(baseDir)) { + const entryPath = path.join(baseDir, entry); + const stat = fs.statSync(entryPath); + if (!stat.isDirectory()) continue; + if (fs.readdirSync(entryPath).length === 0) { + fs.rmSync(entryPath, { recursive: true, force: true }); + } + } + } catch { + // Best effort only. + } +} + +export function listCallLogArtifactFiles(baseDir = CALL_LOGS_DIR) { + if (!baseDir || !fs.existsSync(baseDir)) return []; + + return fs + .readdirSync(baseDir) + .flatMap((entry) => { + const entryPath = path.join(baseDir, entry); + try { + const stat = fs.statSync(entryPath); + if (!stat.isDirectory()) return []; + + return fs + .readdirSync(entryPath) + .filter((file) => file.endsWith(".json")) + .map((file) => { + const absPath = path.join(entryPath, file); + const fileStat = fs.statSync(absPath); + return { + relativePath: path.posix.join(entry, file), + absPath, + mtimeMs: fileStat.mtimeMs, + }; + }); + } catch { + return []; + } + }) + .sort((a, b) => b.mtimeMs - a.mtimeMs); +} diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index fa57df9a53a3..6c79cc3d49e8 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -1,10 +1,8 @@ /** - * Call Logs — extracted from usageDb.js (T-15) + * Structured call log management. * - * Structured call log management: save, query, rotate, and - * unified single-artifact disk storage for the Logger UI. - * - * @module lib/usage/callLogs + * SQLite stores only summary metadata. Detailed request/response payloads live in + * filesystem artifacts and are loaded only for explicit detail/export flows. */ import fs from "fs"; @@ -12,7 +10,7 @@ import path from "path"; import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { getDbInstance } from "../db/core"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; -import { shouldPersistToDisk, CALL_LOGS_DIR } from "./migrations"; +import { shouldPersistToDisk } from "./migrations"; import { getLoggedInputTokens, getLoggedOutputTokens, @@ -22,53 +20,70 @@ import { } from "./tokenAccounting"; import { isNoLog } from "../compliance"; import { sanitizePII } from "../piiSanitizer"; -import { - protectPayloadForLog, - parseStoredPayload, - serializePayloadForStorage, -} from "../logPayloads"; -import { getCallLogMaxEntries, getCallLogRetentionDays } from "../logEnv"; +import { protectPayloadForLog, parseStoredPayload } from "../logPayloads"; +import { getCallLogMaxEntries, getCallLogRetentionDays, getCallLogsTableMaxRows } from "../logEnv"; import { pickMaskedDisplayValue } from "@/shared/utils/maskEmail"; +import { + CALL_LOGS_DIR, + cleanupEmptyCallLogDirs, + deleteCallArtifact, + listCallLogArtifactFiles, + readCallArtifact, + writeCallArtifact, + type CallLogArtifact, + type CallLogDetailState, +} from "./callLogArtifacts"; type JsonRecord = Record; -type CallLogArtifact = { - schemaVersion: 3; - summary: { - id: string; - timestamp: string; - method: string; - path: string; - status: number; - model: string; - requestedModel: string | null; - provider: string; - account: string; - connectionId: string | null; - duration: number; - tokens: { - in: number; - out: number; - cacheRead: number | null; - cacheWrite: number | null; - reasoning: number | null; - }; - requestType: string | null; - sourceFormat: string | null; - targetFormat: string | null; - apiKeyId: string | null; - apiKeyName: string | null; - comboName: string | null; - comboStepId: string | null; - comboExecutionKey: string | null; - }; - requestBody: unknown; - responseBody: unknown; - error: unknown; - pipeline?: RequestPipelinePayloads; +type CallLogSummaryRow = { + id: string; + timestamp: string | null; + method: string | null; + path: string | null; + status: number | null; + model: string | null; + requested_model: string | null; + provider: string | null; + account: string | null; + connection_id: string | null; + duration: number | null; + tokens_in: number | null; + tokens_out: number | null; + tokens_cache_read: number | null; + tokens_cache_creation: number | null; + tokens_reasoning: number | null; + request_type: string | null; + source_format: string | null; + target_format: string | null; + api_key_id: string | null; + api_key_name: string | null; + combo_name: string | null; + combo_step_id: string | null; + combo_execution_key: string | null; + error_summary: string | null; + detail_state: string | null; + artifact_relpath: string | null; + artifact_size_bytes: number | null; + artifact_sha256: string | null; + has_request_body: number | null; + has_response_body: number | null; + has_pipeline_details: number | null; + request_summary: string | null; +}; + +type LegacyInlineRow = { + request_body: string | null; + response_body: string | null; + error: string | null; }; -const CALL_LOG_INLINE_BODY_LIMIT = 256 * 1024; +type DeleteResult = { + deletedRows: number; + deletedArtifacts: number; +}; + +let logIdCounter = 0; function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; @@ -87,9 +102,29 @@ function toStringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } -function hasTruncatedFlag(value: unknown): boolean { - if (!value || typeof value !== "object" || Array.isArray(value)) return false; - return (value as Record)._truncated === true; +function truncateText(value: string, maxLength: number) { + return value.length > maxLength ? value.slice(0, maxLength) : value; +} + +function parseInlineError(value: unknown): unknown { + if (typeof value !== "string" || value.trim().length === 0) return null; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +function normalizeDetailState(value: unknown): CallLogDetailState { + if ( + value === "ready" || + value === "missing" || + value === "corrupt" || + value === "legacy-inline" + ) { + return value; + } + return "none"; } function sanitizeErrorForLog(error: unknown): unknown { @@ -105,14 +140,18 @@ function sanitizeErrorForLog(error: unknown): unknown { return protectPayloadForLog(error); } -function toStoredErrorString(error: unknown): string | null { +function toStoredErrorSummary(error: unknown): string | null { const sanitized = sanitizeErrorForLog(error); if (sanitized === null || sanitized === undefined) return null; - if (typeof sanitized === "string") return sanitized; + + if (typeof sanitized === "string") { + return truncateText(sanitized, 4000); + } + try { - return JSON.stringify(sanitized); + return truncateText(JSON.stringify(sanitized), 4000); } catch { - return String(sanitized); + return truncateText(String(sanitized), 4000); } } @@ -121,9 +160,7 @@ function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | n const protectedPayloads: RequestPipelinePayloads = {}; for (const [key, value] of Object.entries(payloads as JsonRecord)) { - if (value === null || value === undefined) { - continue; - } + if (value === null || value === undefined) continue; if (key === "streamChunks" && value && typeof value === "object") { const chunks = value as Record; @@ -146,7 +183,28 @@ function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | n return Object.keys(protectedPayloads).length > 0 ? protectedPayloads : null; } -let logIdCounter = 0; +function buildRequestSummary(requestType: string | null, requestBody: unknown): string | null { + if (requestType !== "search") return null; + + const body = asRecord(requestBody); + if (Object.keys(body).length === 0) return null; + + const summary: JsonRecord = {}; + if (typeof body.query === "string" && body.query.trim().length > 0) { + summary.query = sanitizePII(body.query).text; + } + + const filters = Object.fromEntries( + Object.entries(body).filter(([key]) => key !== "query" && key !== "provider") + ); + if (Object.keys(filters).length > 0) { + summary.filters = filters; + } + + if (Object.keys(summary).length === 0) return null; + return JSON.stringify(summary); +} + function generateLogId() { logIdCounter++; return `${Date.now()}-${logIdCounter}`; @@ -164,7 +222,10 @@ async function resolveAccountName(connectionId: string | null | undefined) { const connections = await getProviderConnections(); const conn = connections.find((item) => item.id === connectionId); if (conn) { - account = pickMaskedDisplayValue([conn.name, conn.email], account); + account = pickMaskedDisplayValue( + [toStringOrNull(conn.name), toStringOrNull(conn.email)], + account + ); } } catch { // Best-effort lookup only. @@ -173,15 +234,6 @@ async function resolveAccountName(connectionId: string | null | undefined) { return account; } -function buildArtifactRelativePath(timestamp: string, id: string) { - const parsed = new Date(timestamp); - const safeTimestamp = ( - Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString() - ).replace(/[:]/g, "-"); - const dateFolder = safeTimestamp.slice(0, 10); - return path.posix.join(dateFolder, `${safeTimestamp}_${id}.json`); -} - function buildArtifact( logEntry: { id: string; @@ -215,7 +267,7 @@ function buildArtifact( pipelinePayloads: RequestPipelinePayloads | null ): CallLogArtifact { return { - schemaVersion: 3, + schemaVersion: 4, summary: { id: logEntry.id, timestamp: logEntry.timestamp, @@ -251,34 +303,11 @@ function buildArtifact( }; } -function writeCallArtifact(artifact: CallLogArtifact): string | null { - if (!CALL_LOGS_DIR) return null; - - const relPath = buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id); - const absPath = path.join(CALL_LOGS_DIR, relPath); - - try { - fs.mkdirSync(path.dirname(absPath), { recursive: true }); - fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2)); - rotateCallLogs(); - return relPath; - } catch (error) { - console.error("[callLogs] Failed to write request artifact:", (error as Error).message); - return null; - } -} - -function readArtifactFromDisk(relativePath: string | null) { - if (!CALL_LOGS_DIR || !relativePath) return null; - - try { - const absPath = path.join(CALL_LOGS_DIR, relativePath); - if (!fs.existsSync(absPath)) return null; - return JSON.parse(fs.readFileSync(absPath, "utf8")) as CallLogArtifact; - } catch (error) { - console.error("[callLogs] Failed to read request artifact:", (error as Error).message); - return null; - } +function hasTable(tableName: string): boolean { + const db = getDbInstance(); + return Boolean( + db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName) + ); } function readLegacyLogFromDisk(entry: { @@ -324,67 +353,210 @@ function readLegacyLogFromDisk(entry: { return null; } -function cleanupEmptyCallLogDirs() { - if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return; +function clearArtifactReference(relativePath: string, nextState: CallLogDetailState) { + const db = getDbInstance(); + db.prepare( + ` + UPDATE call_logs + SET detail_state = ?, + artifact_relpath = NULL, + artifact_size_bytes = NULL, + artifact_sha256 = NULL + WHERE artifact_relpath = ? + ` + ).run(nextState, relativePath); +} + +function listReferencedArtifacts() { + const db = getDbInstance(); + const rows = db + .prepare("SELECT artifact_relpath FROM call_logs WHERE artifact_relpath IS NOT NULL") + .all() as Array<{ artifact_relpath: string | null }>; + + return new Set( + rows.map((row) => row.artifact_relpath).filter((value): value is string => Boolean(value)) + ); +} + +function deleteCallLogRowsByIds(ids: string[]): DeleteResult { + if (ids.length === 0) { + return { deletedRows: 0, deletedArtifacts: 0 }; + } + + const db = getDbInstance(); + const placeholders = ids.map(() => "?").join(", "); + const rows = db + .prepare(`SELECT artifact_relpath FROM call_logs WHERE id IN (${placeholders})`) + .all(...ids) as Array<{ artifact_relpath: string | null }>; + + const result = db.prepare(`DELETE FROM call_logs WHERE id IN (${placeholders})`).run(...ids); + let deletedArtifacts = 0; + for (const row of rows) { + if (deleteCallArtifact(row.artifact_relpath)) { + deletedArtifacts++; + } + } + cleanupEmptyCallLogDirs(); + + return { + deletedRows: result.changes, + deletedArtifacts, + }; +} + +export function cleanupOrphanCallLogFiles(baseDir = CALL_LOGS_DIR) { + if (!baseDir || !fs.existsSync(baseDir)) return 0; try { - for (const entry of fs.readdirSync(CALL_LOGS_DIR)) { - const entryPath = path.join(CALL_LOGS_DIR, entry); - const stat = fs.statSync(entryPath); - if (!stat.isDirectory()) continue; - if (fs.readdirSync(entryPath).length === 0) { - fs.rmSync(entryPath, { recursive: true, force: true }); + const referenced = listReferencedArtifacts(); + let deleted = 0; + for (const file of listCallLogArtifactFiles(baseDir)) { + if (referenced.has(file.relativePath)) continue; + if (deleteCallArtifact(file.relativePath)) { + deleted++; } } - } catch { - // Best effort only. + cleanupEmptyCallLogDirs(baseDir); + return deleted; + } catch (error) { + console.error("[callLogs] Failed to prune orphan request artifacts:", (error as Error).message); + return 0; } } export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?: number) { - if (!baseDir || !fs.existsSync(baseDir)) return; + if (!baseDir || !fs.existsSync(baseDir)) return 0; const limit = maxEntries ?? getCallLogMaxEntries(); - if (!Number.isInteger(limit) || limit < 1) return; + if (!Number.isInteger(limit) || limit < 1) return 0; try { - const files = fs - .readdirSync(baseDir) - .flatMap((entry) => { - const entryPath = path.join(baseDir, entry); - try { - const stat = fs.statSync(entryPath); - if (!stat.isDirectory()) return []; - - return fs - .readdirSync(entryPath) - .filter((file) => file.endsWith(".json")) - .map((file) => { - const filePath = path.join(entryPath, file); - const fileStat = fs.statSync(filePath); - return { filePath, mtimeMs: fileStat.mtimeMs }; - }); - } catch { - return []; - } - }) - .sort((a, b) => b.mtimeMs - a.mtimeMs); - + let deleted = 0; + const files = listCallLogArtifactFiles(baseDir); for (const file of files.slice(limit)) { - try { - fs.rmSync(file.filePath, { force: true }); - } catch { - // Best effort only. + if (deleteCallArtifact(file.relativePath)) { + clearArtifactReference(file.relativePath, "missing"); + deleted++; } } - - cleanupEmptyCallLogDirs(); + cleanupEmptyCallLogDirs(baseDir); + return deleted; } catch (error) { console.error( "[callLogs] Failed to prune overflow request artifacts:", (error as Error).message ); + return 0; + } +} + +export function deleteCallLogsBefore(cutoff: string): DeleteResult { + const db = getDbInstance(); + const ids = db + .prepare("SELECT id FROM call_logs WHERE timestamp < ? ORDER BY timestamp ASC") + .all(cutoff) + .map((row) => String((row as { id: string }).id)); + + return deleteCallLogRowsByIds(ids); +} + +export function trimCallLogsToMaxRows(maxRows = getCallLogsTableMaxRows()) { + if (!Number.isInteger(maxRows) || maxRows < 1) { + return { deletedRows: 0, deletedArtifacts: 0 }; } + + const db = getDbInstance(); + let deletedRows = 0; + let deletedArtifacts = 0; + const batchSize = 5000; + + while (true) { + const currentCount = db.prepare("SELECT COUNT(*) AS cnt FROM call_logs").get() as { + cnt: number; + }; + if (currentCount.cnt <= maxRows) break; + + const toDelete = Math.min(currentCount.cnt - maxRows, batchSize); + const ids = db + .prepare("SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ?") + .all(toDelete) + .map((row) => String((row as { id: string }).id)); + const result = deleteCallLogRowsByIds(ids); + deletedRows += result.deletedRows; + deletedArtifacts += result.deletedArtifacts; + if (result.deletedRows === 0) break; + } + + return { deletedRows, deletedArtifacts }; +} + +function mapSummaryRow(row: CallLogSummaryRow) { + const detailState = normalizeDetailState(row.detail_state); + return { + id: row.id, + timestamp: row.timestamp, + method: row.method, + path: row.path, + status: toNumber(row.status), + model: row.model, + requestedModel: row.requested_model, + provider: row.provider, + account: row.account, + connectionId: row.connection_id, + duration: toNumber(row.duration), + tokens: { + in: toNumber(row.tokens_in), + out: toNumber(row.tokens_out), + cacheRead: row.tokens_cache_read != null ? toNumber(row.tokens_cache_read) : null, + cacheWrite: row.tokens_cache_creation != null ? toNumber(row.tokens_cache_creation) : null, + reasoning: row.tokens_reasoning != null ? toNumber(row.tokens_reasoning) : null, + }, + requestType: row.request_type, + sourceFormat: row.source_format, + targetFormat: row.target_format, + apiKeyId: row.api_key_id, + apiKeyName: row.api_key_name, + comboName: row.combo_name, + comboStepId: row.combo_step_id, + comboExecutionKey: row.combo_execution_key, + error: row.error_summary, + detailState, + artifactRelPath: row.artifact_relpath, + artifactSizeBytes: row.artifact_size_bytes, + artifactSha256: row.artifact_sha256, + requestSummary: row.request_summary ? parseStoredPayload(row.request_summary) : null, + hasRequestBody: toNumber(row.has_request_body) === 1, + hasResponseBody: toNumber(row.has_response_body) === 1, + hasPipelineDetails: toNumber(row.has_pipeline_details) === 1, + }; +} + +function buildLegacyPipelinePayloads(id: string) { + const detailed = getRequestDetailLogByCallLogId(id); + if (!detailed) return null; + + return { + clientRequest: detailed.client_request ?? null, + providerRequest: detailed.translated_request ?? null, + providerResponse: detailed.provider_response ?? null, + clientResponse: detailed.client_response ?? null, + }; +} + +function getLegacyInlineDetail(id: string) { + if (!hasTable("call_logs_v1_legacy")) return null; + + const db = getDbInstance(); + const row = db + .prepare("SELECT request_body, response_body, error FROM call_logs_v1_legacy WHERE id = ?") + .get(id) as LegacyInlineRow | undefined; + if (!row) return null; + + return { + requestBody: parseStoredPayload(row.request_body), + responseBody: parseStoredPayload(row.response_body), + error: parseInlineError(row.error), + }; } export async function saveCallLog(entry: any) { @@ -402,7 +574,6 @@ export async function saveCallLog(entry: any) { const protectedError = sanitizeErrorForLog(entry.error); const account = await resolveAccountName(entry.connectionId || null); - const logEntry = { id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(), timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(), @@ -429,11 +600,42 @@ export async function saveCallLog(entry: any) { comboStepId: toStringOrNull(entry.comboStepId), comboExecutionKey: toStringOrNull(entry.comboExecutionKey) || toStringOrNull(entry.comboStepId), - requestBody: serializePayloadForStorage(protectedRequestBody, CALL_LOG_INLINE_BODY_LIMIT), - responseBody: serializePayloadForStorage(protectedResponseBody, CALL_LOG_INLINE_BODY_LIMIT), - error: toStoredErrorString(protectedError), }; + const requestSummary = noLogEnabled + ? null + : buildRequestSummary(logEntry.requestType, protectedRequestBody); + const detailExpected = + !noLogEnabled && + (protectedRequestBody !== null || + protectedResponseBody !== null || + protectedError !== null || + protectedPipelinePayloads !== null); + + let detailState: CallLogDetailState = "none"; + let artifactRelPath: string | null = null; + let artifactSizeBytes: number | null = null; + let artifactSha256: string | null = null; + + if (detailExpected) { + const artifact = buildArtifact( + logEntry, + protectedRequestBody, + protectedResponseBody, + protectedError, + protectedPipelinePayloads + ); + const artifactResult = writeCallArtifact(artifact); + if (artifactResult) { + detailState = "ready"; + artifactRelPath = artifactResult.relPath; + artifactSizeBytes = artifactResult.sizeBytes; + artifactSha256 = artifactResult.sha256; + } else { + detailState = "missing"; + } + } + const db = getDbInstance(); db.prepare( ` @@ -441,42 +643,35 @@ export async function saveCallLog(entry: any) { id, timestamp, method, path, status, model, requested_model, provider, account, connection_id, duration, tokens_in, tokens_out, tokens_cache_read, tokens_cache_creation, tokens_reasoning, - request_type, source_format, - target_format, api_key_id, api_key_name, combo_name, combo_step_id, - combo_execution_key, request_body, response_body, error, artifact_relpath, - has_pipeline_details + request_type, source_format, target_format, api_key_id, api_key_name, + combo_name, combo_step_id, combo_execution_key, error_summary, detail_state, + artifact_relpath, artifact_size_bytes, artifact_sha256, + has_request_body, has_response_body, has_pipeline_details, request_summary ) VALUES ( @id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, @account, @connectionId, @duration, @tokensIn, @tokensOut, @tokensCacheRead, @tokensCacheCreation, @tokensReasoning, - @requestType, @sourceFormat, - @targetFormat, @apiKeyId, @apiKeyName, @comboName, @comboStepId, - @comboExecutionKey, @requestBody, @responseBody, @error, NULL, 0 + @requestType, @sourceFormat, @targetFormat, @apiKeyId, @apiKeyName, + @comboName, @comboStepId, @comboExecutionKey, @errorSummary, @detailState, + @artifactRelPath, @artifactSizeBytes, @artifactSha256, + @hasRequestBody, @hasResponseBody, @hasPipelineDetails, @requestSummary ) ` - ).run(logEntry); + ).run({ + ...logEntry, + errorSummary: toStoredErrorSummary(protectedError), + detailState, + artifactRelPath, + artifactSizeBytes, + artifactSha256, + hasRequestBody: protectedRequestBody !== null ? 1 : 0, + hasResponseBody: protectedResponseBody !== null ? 1 : 0, + hasPipelineDetails: protectedPipelinePayloads ? 1 : 0, + requestSummary, + }); - if (!noLogEnabled) { - const artifact = buildArtifact( - logEntry, - protectedRequestBody, - protectedResponseBody, - protectedError, - protectedPipelinePayloads - ); - const artifactRelPath = writeCallArtifact(artifact); - - if (artifactRelPath) { - db.prepare( - ` - UPDATE call_logs - SET artifact_relpath = ?, has_pipeline_details = ? - WHERE id = ? - ` - ).run(artifactRelPath, protectedPipelinePayloads ? 1 : 0, logEntry.id); - } - } + rotateCallLogs(); } catch (error) { console.error("[callLogs] Failed to save call log:", (error as Error).message); } @@ -486,18 +681,13 @@ export function rotateCallLogs() { if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return; try { - const entries = fs.readdirSync(CALL_LOGS_DIR); - const now = Date.now(); const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000; + const cutoff = new Date(Date.now() - retentionMs).toISOString(); - for (const entry of entries) { - const entryPath = path.join(CALL_LOGS_DIR, entry); - const stat = fs.statSync(entryPath); - if (stat.isDirectory() && now - stat.mtimeMs > retentionMs) { - fs.rmSync(entryPath, { recursive: true, force: true }); - } - } + deleteCallLogsBefore(cutoff); + trimCallLogsToMaxRows(getCallLogsTableMaxRows()); cleanupOverflowCallLogFiles(CALL_LOGS_DIR, getCallLogMaxEntries()); + cleanupOrphanCallLogFiles(CALL_LOGS_DIR); } catch (error) { console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message); } @@ -519,7 +709,7 @@ export async function getCallLogs(filter: any = {}) { if (filter.status) { if (filter.status === "error") { - conditions.push("(status >= 400 OR error IS NOT NULL)"); + conditions.push("(status >= 400 OR error_summary IS NOT NULL)"); } else if (filter.status === "ok") { conditions.push("status >= 200 AND status < 300"); } else { @@ -555,8 +745,9 @@ export async function getCallLogs(filter: any = {}) { model LIKE @searchQ OR path LIKE @searchQ OR account LIKE @searchQ OR requested_model LIKE @searchQ OR provider LIKE @searchQ OR api_key_name LIKE @searchQ OR api_key_id LIKE @searchQ OR - combo_name LIKE @searchQ OR CAST(status AS TEXT) LIKE @searchQ - OR combo_step_id LIKE @searchQ OR combo_execution_key LIKE @searchQ + combo_name LIKE @searchQ OR CAST(status AS TEXT) LIKE @searchQ OR + combo_step_id LIKE @searchQ OR combo_execution_key LIKE @searchQ OR + error_summary LIKE @searchQ )`); params.searchQ = `%${filter.search}%`; } @@ -568,133 +759,98 @@ export async function getCallLogs(filter: any = {}) { const limit = filter.limit || 200; sql += ` ORDER BY timestamp DESC LIMIT ${limit}`; - const rows = db.prepare(sql).all(params); - - return rows.map((row) => { - const l = asRecord(row); - return { - id: toStringOrNull(l.id), - timestamp: toStringOrNull(l.timestamp), - method: toStringOrNull(l.method), - path: toStringOrNull(l.path), - status: toNumber(l.status), - model: toStringOrNull(l.model), - requestedModel: toStringOrNull(l.requested_model), - provider: toStringOrNull(l.provider), - account: toStringOrNull(l.account), - duration: toNumber(l.duration), - tokens: { - in: toNumber(l.tokens_in), - out: toNumber(l.tokens_out), - cacheRead: l.tokens_cache_read != null ? toNumber(l.tokens_cache_read) : null, - cacheWrite: l.tokens_cache_creation != null ? toNumber(l.tokens_cache_creation) : null, - reasoning: l.tokens_reasoning != null ? toNumber(l.tokens_reasoning) : null, - }, - sourceFormat: toStringOrNull(l.source_format), - targetFormat: toStringOrNull(l.target_format), - error: toStringOrNull(l.error), - comboName: toStringOrNull(l.combo_name), - comboStepId: toStringOrNull(l.combo_step_id), - comboExecutionKey: toStringOrNull(l.combo_execution_key), - apiKeyId: toStringOrNull(l.api_key_id), - apiKeyName: toStringOrNull(l.api_key_name), - hasRequestBody: typeof l.request_body === "string" && l.request_body.length > 0, - hasResponseBody: typeof l.response_body === "string" && l.response_body.length > 0, - hasPipelineDetails: toNumber(l.has_pipeline_details) === 1, - }; - }); -} - -function buildLegacyPipelinePayloads(id: string) { - const detailed = getRequestDetailLogByCallLogId(id); - if (!detailed) return null; - - return { - clientRequest: detailed.client_request ?? null, - providerRequest: detailed.translated_request ?? null, - providerResponse: detailed.provider_response ?? null, - clientResponse: detailed.client_response ?? null, - }; + const rows = db.prepare(sql).all(params) as CallLogSummaryRow[]; + return rows.map(mapSummaryRow); } export async function getCallLogById(id: string) { const db = getDbInstance(); - const row = db.prepare("SELECT * FROM call_logs WHERE id = ?").get(id); + const row = db.prepare("SELECT * FROM call_logs WHERE id = ?").get(id) as + | CallLogSummaryRow + | undefined; if (!row) return null; - const entryRow = asRecord(row); - const artifactRelPath = toStringOrNull(entryRow.artifact_relpath); - const entry = { - id: toStringOrNull(entryRow.id), - timestamp: toStringOrNull(entryRow.timestamp), - method: toStringOrNull(entryRow.method), - path: toStringOrNull(entryRow.path), - status: toNumber(entryRow.status), - model: toStringOrNull(entryRow.model), - requestedModel: toStringOrNull(entryRow.requested_model), - provider: toStringOrNull(entryRow.provider), - account: toStringOrNull(entryRow.account), - connectionId: toStringOrNull(entryRow.connection_id), - duration: toNumber(entryRow.duration), - tokens: { - in: toNumber(entryRow.tokens_in), - out: toNumber(entryRow.tokens_out), - cacheRead: entryRow.tokens_cache_read != null ? toNumber(entryRow.tokens_cache_read) : null, - cacheWrite: - entryRow.tokens_cache_creation != null ? toNumber(entryRow.tokens_cache_creation) : null, - reasoning: entryRow.tokens_reasoning != null ? toNumber(entryRow.tokens_reasoning) : null, - }, - sourceFormat: toStringOrNull(entryRow.source_format), - targetFormat: toStringOrNull(entryRow.target_format), - apiKeyId: toStringOrNull(entryRow.api_key_id), - apiKeyName: toStringOrNull(entryRow.api_key_name), - comboName: toStringOrNull(entryRow.combo_name), - comboStepId: toStringOrNull(entryRow.combo_step_id), - comboExecutionKey: toStringOrNull(entryRow.combo_execution_key), - requestBody: parseStoredPayload(entryRow.request_body), - responseBody: parseStoredPayload(entryRow.response_body), - error: toStringOrNull(entryRow.error), - artifactRelPath, - hasPipelineDetails: toNumber(entryRow.has_pipeline_details) === 1, - }; + const entry = mapSummaryRow(row); + let detailState = entry.detailState; + let artifactRelPath = entry.artifactRelPath; - const artifact = readArtifactFromDisk(artifactRelPath); - if (artifact) { - return { - ...entry, - requestBody: artifact.requestBody ?? entry.requestBody, - responseBody: artifact.responseBody ?? entry.responseBody, - error: artifact.error ?? entry.error, - pipelinePayloads: artifact.pipeline ?? null, - hasPipelineDetails: Boolean(artifact.pipeline) || entry.hasPipelineDetails, - }; + if (artifactRelPath) { + const artifactResult = readCallArtifact(artifactRelPath); + if (artifactResult.state === "ready" && artifactResult.artifact) { + return { + ...entry, + detailState: "ready" as const, + requestBody: artifactResult.artifact.requestBody ?? null, + responseBody: artifactResult.artifact.responseBody ?? null, + error: artifactResult.artifact.error ?? entry.error, + pipelinePayloads: artifactResult.artifact.pipeline ?? buildLegacyPipelinePayloads(id), + hasPipelineDetails: Boolean(artifactResult.artifact.pipeline) || entry.hasPipelineDetails, + }; + } + + detailState = artifactResult.state; + if (artifactResult.state === "missing") { + clearArtifactReference(artifactRelPath, "missing"); + artifactRelPath = null; + } else { + db.prepare("UPDATE call_logs SET detail_state = ? WHERE id = ?").run("corrupt", id); + } } - const needsLegacyDisk = - hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody) || !artifactRelPath; - if (needsLegacyDisk) { - const legacyEntry = readLegacyLogFromDisk(entry); - if (legacyEntry) { + if (detailState === "legacy-inline") { + const legacyInline = getLegacyInlineDetail(id); + if (legacyInline) { const legacyPipeline = buildLegacyPipelinePayloads(id); return { ...entry, - requestBody: legacyEntry.requestBody ?? entry.requestBody, - responseBody: legacyEntry.responseBody ?? entry.responseBody, - error: legacyEntry.error ?? entry.error, + detailState, + artifactRelPath, + ...legacyInline, pipelinePayloads: legacyPipeline, - hasPipelineDetails: Boolean(legacyPipeline), + hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails, }; } } - const legacyPipeline = buildLegacyPipelinePayloads(id); - if (legacyPipeline) { + const legacyDisk = readLegacyLogFromDisk(entry); + if (legacyDisk) { + const legacyPipeline = buildLegacyPipelinePayloads(id); return { ...entry, + detailState, + artifactRelPath, + requestBody: legacyDisk.requestBody ?? null, + responseBody: legacyDisk.responseBody ?? null, + error: legacyDisk.error ?? entry.error, pipelinePayloads: legacyPipeline, - hasPipelineDetails: true, + hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails, }; } - return entry; + const legacyPipeline = buildLegacyPipelinePayloads(id); + return { + ...entry, + detailState, + artifactRelPath, + requestBody: null, + responseBody: null, + error: entry.error, + pipelinePayloads: legacyPipeline, + hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails, + }; +} + +export async function exportCallLogsSince(since: string) { + const db = getDbInstance(); + const ids = db + .prepare("SELECT id FROM call_logs WHERE timestamp >= ? ORDER BY timestamp DESC") + .all(since) + .map((row) => String((row as { id: string }).id)); + + const logs: unknown[] = []; + for (const id of ids) { + const log = await getCallLogById(id); + if (log) logs.push(log); + } + return logs; } diff --git a/src/lib/usage/migrations.ts b/src/lib/usage/migrations.ts index 2cd0275aef8b..0780ff938161 100644 --- a/src/lib/usage/migrations.ts +++ b/src/lib/usage/migrations.ts @@ -12,6 +12,9 @@ import path from "path"; import { ZipFile } from "yazl"; import { getDbInstance, isCloud, isBuildPhase, DATA_DIR } from "../db/core"; import { getLegacyDotDataDir, isSamePath } from "../dataPaths"; +import { protectPayloadForLog } from "../logPayloads"; +import { sanitizePII } from "../piiSanitizer"; +import { writeCallArtifact, type CallLogArtifact } from "./callLogArtifacts"; export const shouldPersistToDisk = !isCloud && !isBuildPhase; @@ -41,6 +44,25 @@ type ArchiveTarget = { deleteAfterArchive: boolean; }; +function buildLegacyRequestSummary(requestType: unknown, requestBody: unknown) { + if (requestType !== "search" || !requestBody || typeof requestBody !== "object") return null; + + const record = requestBody as Record; + const summary: Record = {}; + if (typeof record.query === "string" && record.query.trim().length > 0) { + summary.query = sanitizePII(record.query).text; + } + + const filters = Object.fromEntries( + Object.entries(record).filter(([key]) => key !== "query" && key !== "provider") + ); + if (Object.keys(filters).length > 0) { + summary.filters = filters; + } + + return Object.keys(summary).length > 0 ? JSON.stringify(summary) : null; +} + function copyIfMissing(fromPath: string | null, toPath: string | null, label: string) { if (!fromPath || !toPath) return; if (!fs.existsSync(fromPath) || fs.existsSync(toPath)) return; @@ -297,25 +319,96 @@ export function migrateUsageJsonToSqlite() { console.log(`[usageDb] Migrating ${logs.length} call log entries from JSON → SQLite...`); const insert = db.prepare(` - INSERT OR IGNORE INTO call_logs (id, timestamp, method, path, status, model, provider, + INSERT OR IGNORE INTO call_logs (id, timestamp, method, path, status, model, requested_model, provider, account, connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error, - artifact_relpath, has_pipeline_details) - VALUES (@id, @timestamp, @method, @path, @status, @model, @provider, + api_key_id, api_key_name, combo_name, combo_step_id, combo_execution_key, error_summary, + detail_state, artifact_relpath, artifact_size_bytes, artifact_sha256, + has_request_body, has_response_body, has_pipeline_details, request_summary) + VALUES (@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, @account, @connectionId, @duration, @tokensIn, @tokensOut, @sourceFormat, @targetFormat, - @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, - NULL, 0) + @apiKeyId, @apiKeyName, @comboName, @comboStepId, @comboExecutionKey, @errorSummary, + @detailState, @artifactRelPath, @artifactSizeBytes, @artifactSha256, + @hasRequestBody, @hasResponseBody, @hasPipelineDetails, @requestSummary) `); const tx = db.transaction(() => { for (const log of logs) { + const id = log.id || `${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; + const timestamp = log.timestamp || new Date().toISOString(); + const protectedRequestBody = log.requestBody + ? protectPayloadForLog(log.requestBody) + : null; + const protectedResponseBody = log.responseBody + ? protectPayloadForLog(log.responseBody) + : null; + const protectedError = + log.error && typeof log.error === "object" + ? protectPayloadForLog(log.error) + : log.error || null; + const detailExpected = + protectedRequestBody !== null || + protectedResponseBody !== null || + protectedError !== null; + + let detailState: "none" | "ready" | "missing" = "none"; + let artifactRelPath: string | null = null; + let artifactSizeBytes: number | null = null; + let artifactSha256: string | null = null; + + if (detailExpected) { + const artifact: CallLogArtifact = { + schemaVersion: 4, + summary: { + id, + timestamp, + method: log.method || "POST", + path: log.path || "/v1/chat/completions", + status: log.status || 0, + model: log.model || "-", + requestedModel: log.requestedModel || null, + provider: log.provider || "-", + account: log.account || "-", + connectionId: log.connectionId || null, + duration: log.duration || 0, + tokens: { + in: log.tokens?.in ?? 0, + out: log.tokens?.out ?? 0, + cacheRead: null, + cacheWrite: null, + reasoning: null, + }, + requestType: log.requestType || null, + sourceFormat: log.sourceFormat || null, + targetFormat: log.targetFormat || null, + apiKeyId: log.apiKeyId || null, + apiKeyName: log.apiKeyName || null, + comboName: log.comboName || null, + comboStepId: log.comboStepId || null, + comboExecutionKey: log.comboExecutionKey || null, + }, + requestBody: protectedRequestBody, + responseBody: protectedResponseBody, + error: protectedError, + }; + const artifactResult = writeCallArtifact(artifact); + if (artifactResult) { + detailState = "ready"; + artifactRelPath = artifactResult.relPath; + artifactSizeBytes = artifactResult.sizeBytes; + artifactSha256 = artifactResult.sha256; + } else { + detailState = "missing"; + } + } + insert.run({ - id: log.id || `${Date.now()}-${Math.random().toString(36).slice(2, 6)}`, - timestamp: log.timestamp || new Date().toISOString(), + id, + timestamp, method: log.method || "POST", path: log.path || null, status: log.status || 0, model: log.model || null, + requestedModel: log.requestedModel || null, provider: log.provider || null, account: log.account || null, connectionId: log.connectionId || null, @@ -327,9 +420,22 @@ export function migrateUsageJsonToSqlite() { apiKeyId: log.apiKeyId || null, apiKeyName: log.apiKeyName || null, comboName: log.comboName || null, - requestBody: log.requestBody ? JSON.stringify(log.requestBody) : null, - responseBody: log.responseBody ? JSON.stringify(log.responseBody) : null, - error: log.error || null, + comboStepId: log.comboStepId || null, + comboExecutionKey: log.comboExecutionKey || log.comboStepId || null, + errorSummary: + typeof protectedError === "string" + ? protectedError.slice(0, 4000) + : protectedError + ? JSON.stringify(protectedError).slice(0, 4000) + : null, + detailState, + artifactRelPath, + artifactSizeBytes, + artifactSha256, + hasRequestBody: protectedRequestBody ? 1 : 0, + hasResponseBody: protectedResponseBody ? 1 : 0, + hasPipelineDetails: 0, + requestSummary: buildLegacyRequestSummary(log.requestType, protectedRequestBody), }); } }); diff --git a/src/shared/components/RequestLoggerDetail.tsx b/src/shared/components/RequestLoggerDetail.tsx index 2cd4171c29bf..2e6bfef6ccde 100644 --- a/src/shared/components/RequestLoggerDetail.tsx +++ b/src/shared/components/RequestLoggerDetail.tsx @@ -109,6 +109,12 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC : []; const requestJson = detail?.requestBody ? toPrettyJson(detail.requestBody) : null; const responseJson = detail?.responseBody ? toPrettyJson(detail.responseBody) : null; + const detailIssue = + detail?.detailState === "missing" + ? "Detailed payload artifact is no longer available for this log entry." + : detail?.detailState === "corrupt" + ? "Detailed payload artifact could not be parsed." + : null; const tokenStats = { totalIn: detail?.tokens?.in ?? log.tokens?.in ?? 0, totalOut: detail?.tokens?.out ?? log.tokens?.out ?? 0, @@ -289,6 +295,15 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC )} + {detailIssue && ( +
+
+ Detail Status +
+
{detailIssue}
+
+ )} + {loading ? (
Loading request details... diff --git a/tests/unit/call-log-cap.test.ts b/tests/unit/call-log-cap.test.ts index 80ef62f9cb0b..2aee31a542b8 100644 --- a/tests/unit/call-log-cap.test.ts +++ b/tests/unit/call-log-cap.test.ts @@ -6,7 +6,8 @@ import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-artifacts-")); process.env.DATA_DIR = TEST_DATA_DIR; -process.env.CALL_LOG_RETENTION_DAYS = "1"; +process.env.CALL_LOG_RETENTION_DAYS = "3650"; +process.env.CALL_LOG_MAX_ENTRIES = "100"; const core = await import("../../src/lib/db/core.ts"); const callLogs = await import("../../src/lib/usage/callLogs.ts"); @@ -18,7 +19,53 @@ async function resetStorage() { fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); } +function insertCallLog(row) { + const db = core.getDbInstance(); + db.prepare( + ` + INSERT INTO call_logs ( + id, timestamp, method, path, status, model, requested_model, provider, account, + connection_id, duration, tokens_in, tokens_out, source_format, target_format, + api_key_id, api_key_name, combo_name, combo_step_id, combo_execution_key, + error_summary, detail_state, artifact_relpath, artifact_size_bytes, artifact_sha256, + has_request_body, has_response_body, has_pipeline_details, request_summary + ) + VALUES ( + @id, @timestamp, @method, @path, @status, @model, @requested_model, @provider, @account, + @connection_id, @duration, @tokens_in, @tokens_out, @source_format, @target_format, + @api_key_id, @api_key_name, @combo_name, @combo_step_id, @combo_execution_key, + @error_summary, @detail_state, @artifact_relpath, @artifact_size_bytes, @artifact_sha256, + @has_request_body, @has_response_body, @has_pipeline_details, @request_summary + ) + ` + ).run({ + requested_model: null, + connection_id: null, + duration: 0, + tokens_in: 0, + tokens_out: 0, + source_format: null, + target_format: null, + api_key_id: null, + api_key_name: null, + combo_name: null, + combo_step_id: null, + combo_execution_key: null, + error_summary: null, + detail_state: "none", + artifact_relpath: null, + artifact_size_bytes: null, + artifact_sha256: null, + has_request_body: 0, + has_response_body: 0, + has_pipeline_details: 0, + request_summary: null, + ...row, + }); +} + test.beforeEach(async () => { + process.env.CALL_LOG_RETENTION_DAYS = "3650"; await resetStorage(); }); @@ -27,7 +74,7 @@ test.after(() => { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); -test("call logs store a single per-request artifact with pipeline details", async () => { +test("saveCallLog stores only summary metadata in SQLite and writes detailed artifact", async () => { const timestamp = "2026-03-30T12:34:56.789Z"; const logId = "req_artifact_1"; @@ -56,7 +103,10 @@ test("call logs store a single per-request artifact with pipeline details", asyn const logs = await callLogs.getCallLogs({ limit: 5 }); assert.equal(logs.length, 1); + assert.equal(logs[0].hasRequestBody, true); + assert.equal(logs[0].hasResponseBody, true); assert.equal(logs[0].hasPipelineDetails, true); + assert.equal(logs[0].detailState, "ready"); const detail = await callLogs.getCallLogById(logId); assert.equal(detail?.requestedModel, "openai/gpt-5"); @@ -72,56 +122,106 @@ test("call logs store a single per-request artifact with pipeline details", asyn /^2026-03-30\/2026-03-30T12-34-56\.789Z_req_artifact_1\.json$/ ); + const db = core.getDbInstance(); + const columns = db + .prepare("SELECT name FROM pragma_table_info('call_logs') ORDER BY cid") + .all() + .map((row) => row.name); + assert.equal(columns.includes("request_body"), false); + assert.equal(columns.includes("response_body"), false); + assert.equal(columns.includes("error"), false); + + const summaryRow = db + .prepare( + ` + SELECT detail_state, artifact_relpath, has_request_body, has_response_body, has_pipeline_details + FROM call_logs WHERE id = ? + ` + ) + .get(logId); + assert.equal(summaryRow.detail_state, "ready"); + assert.equal(summaryRow.has_request_body, 1); + assert.equal(summaryRow.has_response_body, 1); + assert.equal(summaryRow.has_pipeline_details, 1); + assert.equal(typeof summaryRow.artifact_relpath, "string"); + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", detail.artifactRelPath); const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); assert.equal(artifact.summary.id, logId); assert.equal(artifact.summary.requestedModel, "openai/gpt-5"); - assert.equal(artifact.summary.comboName, "combo-a"); - assert.equal(artifact.summary.comboStepId, "step-openai-a"); assert.equal(artifact.summary.comboExecutionKey, "combo-a:0:step-openai-a"); - assert.equal(artifact.pipeline.clientRawRequest.body.raw, true); - assert.equal("sourceRequest" in artifact.pipeline, false); }); -test("call log artifact rotation removes directories older than CALL_LOG_RETENTION_DAYS", async () => { - const oldDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-10"); - const freshDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-30"); - fs.mkdirSync(oldDir, { recursive: true }); - fs.mkdirSync(freshDir, { recursive: true }); - fs.writeFileSync(path.join(oldDir, "old.json"), "{}"); - fs.writeFileSync(path.join(freshDir, "fresh.json"), "{}"); +test("rotateCallLogs removes expired rows and orphaned artifacts but keeps fresh referenced artifacts", async () => { + process.env.CALL_LOG_RETENTION_DAYS = "1"; + const oldRelPath = "2026-03-10/2026-03-10T00-00-00.000Z_old.json"; + const oldAbsPath = path.join(TEST_DATA_DIR, "call_logs", oldRelPath); + fs.mkdirSync(path.dirname(oldAbsPath), { recursive: true }); + fs.writeFileSync(oldAbsPath, JSON.stringify({ schemaVersion: 4 }, null, 2)); - const oldTime = new Date("2026-03-10T00:00:00.000Z"); - const freshTime = new Date(); - fs.utimesSync(oldDir, oldTime, oldTime); - fs.utimesSync(freshDir, freshTime, freshTime); + insertCallLog({ + id: "expired-log", + timestamp: "2026-03-10T00:00:00.000Z", + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + account: "Expired", + detail_state: "ready", + artifact_relpath: oldRelPath, + has_request_body: 1, + }); - callLogs.rotateCallLogs(); + await callLogs.saveCallLog({ + id: "fresh-log", + timestamp: new Date().toISOString(), + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + requestBody: { ok: true }, + }); - assert.equal(fs.existsSync(oldDir), false); - assert.equal(fs.existsSync(freshDir), true); -}); + const freshRow = core + .getDbInstance() + .prepare("SELECT artifact_relpath FROM call_logs WHERE id = ?") + .get("fresh-log"); + const freshAbsPath = path.join(TEST_DATA_DIR, "call_logs", freshRow.artifact_relpath); + assert.equal( + core + .getDbInstance() + .prepare("SELECT COUNT(*) AS cnt FROM call_logs WHERE id = ?") + .get("expired-log").cnt, + 0 + ); + assert.equal(fs.existsSync(oldAbsPath), false); + assert.equal(fs.existsSync(freshAbsPath), true); + + callLogs.rotateCallLogs(); -test("getCallLogs applies provider, account, apiKey, combo, and search filters together", async () => { const db = core.getDbInstance(); - const insert = db.prepare(` - INSERT INTO call_logs ( - id, timestamp, method, path, status, model, requested_model, provider, account, - connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error, - artifact_relpath, has_pipeline_details - ) - VALUES ( - @id, @timestamp, @method, @path, @status, @model, @requested_model, @provider, @account, - @connection_id, @duration, @tokens_in, @tokens_out, @source_format, @target_format, - @api_key_id, @api_key_name, @combo_name, @request_body, @response_body, @error, - @artifact_relpath, @has_pipeline_details - ) - `); + assert.equal( + db.prepare("SELECT COUNT(*) AS cnt FROM call_logs WHERE id = ?").get("fresh-log").cnt, + 1 + ); + assert.equal(fs.existsSync(freshAbsPath), true); + + const orphanDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-31"); + const orphanFile = path.join(orphanDir, "orphan.json"); + fs.mkdirSync(orphanDir, { recursive: true }); + fs.writeFileSync(orphanFile, "{}"); + callLogs.cleanupOrphanCallLogFiles(); + assert.equal(fs.existsSync(orphanFile), false); + + process.env.CALL_LOG_RETENTION_DAYS = "3650"; +}); - insert.run({ +test("getCallLogs applies provider, account, apiKey, combo, search, and status filters with summary fields", async () => { + insertCallLog({ id: "filter-hit", - timestamp: "2026-03-30T12:34:56", + timestamp: "2026-03-30T12:34:56.000Z", method: "POST", path: "/v1/chat/completions", status: 200, @@ -138,20 +238,17 @@ test("getCallLogs applies provider, account, apiKey, combo, and search filters t api_key_id: "key-1", api_key_name: "Primary Key", combo_name: "combo-a", - request_body: JSON.stringify({ ok: true }), - response_body: JSON.stringify({ ok: true }), - error: null, - artifact_relpath: null, - has_pipeline_details: 0, + combo_step_id: "step-a", + combo_execution_key: "combo-a:0:step-a", + has_request_body: 1, }); - insert.run({ + insertCallLog({ id: "filter-miss", timestamp: "2026-03-30T12:35:56.000Z", method: "POST", path: "/v1/embeddings", - status: 500, + status: 200, model: "gemini/text-embedding", - requested_model: null, provider: "gemini", account: "Backup Account", connection_id: "conn-2", @@ -162,214 +259,93 @@ test("getCallLogs applies provider, account, apiKey, combo, and search filters t target_format: "gemini", api_key_id: "key-2", api_key_name: "Backup Key", - combo_name: null, - request_body: null, - response_body: null, - error: "upstream failed", - artifact_relpath: null, - has_pipeline_details: 0, }); - - const providerFiltered = await callLogs.getCallLogs({ provider: "openai" }); - assert.deepEqual( - providerFiltered.map((row) => row.id), - ["filter-hit"] - ); - - const accountFiltered = await callLogs.getCallLogs({ account: "primary" }); - assert.deepEqual( - accountFiltered.map((row) => row.id), - ["filter-hit"] - ); - - const apiKeyFiltered = await callLogs.getCallLogs({ apiKey: "Primary" }); - assert.deepEqual( - apiKeyFiltered.map((row) => row.id), - ["filter-hit"] - ); - - const comboFiltered = await callLogs.getCallLogs({ combo: true }); - assert.deepEqual( - comboFiltered.map((row) => row.id), - ["filter-hit"] - ); - - const searchFiltered = await callLogs.getCallLogs({ search: "gpt-5" }); - assert.deepEqual( - searchFiltered.map((row) => row.id), - ["filter-hit"] - ); -}); - -test("getCallLogs applies status filters for error, ok, and explicit status codes", async () => { - const db = core.getDbInstance(); - const insert = db.prepare(` - INSERT INTO call_logs ( - id, timestamp, method, path, status, model, requested_model, provider, account, - connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error, - artifact_relpath, has_pipeline_details - ) - VALUES ( - @id, @timestamp, @method, @path, @status, @model, @requested_model, @provider, @account, - @connection_id, @duration, @tokens_in, @tokens_out, @source_format, @target_format, - @api_key_id, @api_key_name, @combo_name, @request_body, @response_body, @error, - @artifact_relpath, @has_pipeline_details - ) - `); - - insert.run({ - id: "status-ok", - timestamp: "2026-03-30T12:34:56", + insertCallLog({ + id: "status-error-by-message", + timestamp: "2026-03-30T12:36:56.000Z", method: "POST", path: "/v1/chat/completions", - status: 201, + status: 200, model: "openai/gpt-4.1", - requested_model: null, provider: "openai", account: "Primary Account", - connection_id: "conn-1", - duration: 12, - tokens_in: 1, - tokens_out: 2, - source_format: "openai", - target_format: "openai", - api_key_id: null, - api_key_name: null, - combo_name: null, - request_body: null, - response_body: null, - error: null, - artifact_relpath: null, - has_pipeline_details: 0, + error_summary: "synthetic failure", }); - insert.run({ + insertCallLog({ id: "status-error", - timestamp: "2026-03-30T12:35:56", + timestamp: "2026-03-30T12:37:56.000Z", method: "POST", path: "/v1/chat/completions", status: 500, model: "openai/gpt-4.1", - requested_model: null, provider: "openai", account: "Primary Account", - connection_id: "conn-2", - duration: 13, - tokens_in: 1, - tokens_out: 2, - source_format: "openai", - target_format: "openai", - api_key_id: null, - api_key_name: null, - combo_name: null, - request_body: null, - response_body: null, - error: null, - artifact_relpath: null, - has_pipeline_details: 0, - }); - insert.run({ - id: "status-error-by-message", - timestamp: "2026-03-30T12:36:56", - method: "POST", - path: "/v1/chat/completions", - status: 200, - model: "openai/gpt-4.1", - requested_model: null, - provider: "openai", - account: "Primary Account", - connection_id: "conn-3", - duration: 14, - tokens_in: 1, - tokens_out: 2, - source_format: "openai", - target_format: "openai", - api_key_id: null, - api_key_name: null, - combo_name: null, - request_body: null, - response_body: null, - error: "synthetic failure", - artifact_relpath: null, - has_pipeline_details: 0, }); - const errorFiltered = await callLogs.getCallLogs({ status: "error" }); assert.deepEqual( - errorFiltered.map((row) => row.id), - ["status-error-by-message", "status-error"] + (await callLogs.getCallLogs({ provider: "openai" })).map((row) => row.id), + ["status-error", "status-error-by-message", "filter-hit"] ); - - const okFiltered = await callLogs.getCallLogs({ status: "ok" }); assert.deepEqual( - okFiltered.map((row) => row.id), - ["status-error-by-message", "status-ok"] + (await callLogs.getCallLogs({ account: "primary" })).map((row) => row.id), + ["status-error", "status-error-by-message", "filter-hit"] ); - - const numericFiltered = await callLogs.getCallLogs({ status: "500" }); assert.deepEqual( - numericFiltered.map((row) => row.id), - ["status-error"] + (await callLogs.getCallLogs({ apiKey: "Primary" })).map((row) => row.id), + ["filter-hit"] + ); + assert.deepEqual( + (await callLogs.getCallLogs({ combo: true })).map((row) => row.id), + ["filter-hit"] + ); + assert.deepEqual( + (await callLogs.getCallLogs({ search: "gpt-5" })).map((row) => row.id), + ["filter-hit"] + ); + assert.deepEqual( + (await callLogs.getCallLogs({ status: "error" })).map((row) => row.id), + ["status-error", "status-error-by-message"] ); - - const invalidNumericFiltered = await callLogs.getCallLogs({ status: "not-a-number" }); assert.deepEqual( - invalidNumericFiltered.map((row) => row.id), - ["status-error-by-message", "status-error", "status-ok"] + (await callLogs.getCallLogs({ status: "ok" })).map((row) => row.id), + ["status-error-by-message", "filter-miss", "filter-hit"] ); }); -test("getCallLogById falls back to legacy disk payloads and request_detail_logs when artifacts are absent", async () => { +test("getCallLogById falls back to legacy inline rows and request_detail_logs", async () => { const db = core.getDbInstance(); + db.exec(` + CREATE TABLE call_logs_v1_legacy ( + id TEXT PRIMARY KEY, + request_body TEXT, + response_body TEXT, + error TEXT + ); + `); + + insertCallLog({ + id: "legacy-read", + timestamp: "2026-03-30T12:34:56.000Z", + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + account: "Primary Account", + detail_state: "legacy-inline", + has_request_body: 1, + has_response_body: 1, + }); db.prepare( ` - INSERT INTO call_logs ( - id, timestamp, method, path, status, model, requested_model, provider, account, - connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error, - artifact_relpath, has_pipeline_details - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO call_logs_v1_legacy (id, request_body, response_body, error) + VALUES (?, ?, ?, ?) ` ).run( "legacy-read", - "2026-03-30T12:34:56", - "POST", - "/v1/chat/completions", - 200, - "openai/gpt-4.1", - null, - "openai", - "Primary Account", - "conn-1", - 42, - 11, - 7, - "openai", - "openai", - null, - null, - null, - JSON.stringify({ _truncated: true }), - JSON.stringify({ _truncated: true }), - "db-error", - null, - 0 - ); - - const legacyDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-30"); - fs.mkdirSync(legacyDir, { recursive: true }); - fs.writeFileSync( - path.join(legacyDir, "123456_alt_200.json"), - JSON.stringify( - { - requestBody: { recovered: "request" }, - responseBody: { recovered: "response" }, - error: "legacy-error", - }, - null, - 2 - ) + JSON.stringify({ recovered: "request" }), + JSON.stringify({ recovered: "response" }), + JSON.stringify({ message: "legacy-error" }) ); detailedLogs.saveRequestDetailLog({ @@ -381,10 +357,9 @@ test("getCallLogById falls back to legacy disk payloads and request_detail_logs }); const detail = await callLogs.getCallLogById("legacy-read"); - assert.deepEqual(detail?.requestBody, { recovered: "request" }); assert.deepEqual(detail?.responseBody, { recovered: "response" }); - assert.equal(detail?.error, "legacy-error"); + assert.deepEqual(detail?.error, { message: "legacy-error" }); assert.equal(detail?.pipelinePayloads?.clientRequest?.body?.from, "detail-client"); assert.equal(detail?.pipelinePayloads?.providerRequest?.body?.from, "detail-provider-request"); assert.equal(detail?.pipelinePayloads?.providerResponse?.body?.from, "detail-provider-response"); @@ -392,94 +367,38 @@ test("getCallLogById falls back to legacy disk payloads and request_detail_logs assert.equal(detail?.hasPipelineDetails, true); }); -test("getCallLogById returns legacy pipeline details even when no legacy disk artifact exists", async () => { - const db = core.getDbInstance(); - db.prepare( - ` - INSERT INTO call_logs ( - id, timestamp, method, path, status, model, requested_model, provider, account, - connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error, - artifact_relpath, has_pipeline_details - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` - ).run( - "legacy-pipeline-only", - "2026-03-30T12:34:57", - "POST", - "/v1/chat/completions", - 200, - "openai/gpt-4.1", - null, - "openai", - "Primary Account", - "conn-1", - 1, - 1, - 1, - "openai", - "openai", - null, - null, - null, - JSON.stringify({ stored: "request" }), - JSON.stringify({ stored: "response" }), - null, - null, - 0 - ); - - detailedLogs.saveRequestDetailLog({ - call_log_id: "legacy-pipeline-only", - client_request: { fallback: "pipeline-only" }, - }); - - const detail = await callLogs.getCallLogById("legacy-pipeline-only"); - - assert.deepEqual(detail?.requestBody, { stored: "request" }); - assert.deepEqual(detail?.responseBody, { stored: "response" }); - assert.deepEqual(detail?.pipelinePayloads?.clientRequest, { fallback: "pipeline-only" }); - assert.equal(detail?.hasPipelineDetails, true); -}); - -test("saveCallLog keeps payloads below 256KB inline in sqlite", async () => { - const requestBody = { payload: "x".repeat(64 * 1024) }; - const responseBody = { payload: "y".repeat(96 * 1024) }; - - await callLogs.saveCallLog({ - id: "inline-payload-limit", - timestamp: "2026-03-31T09:00:00.000Z", +test("getCallLogById marks missing artifacts explicitly and clears stale DB pointers", async () => { + insertCallLog({ + id: "missing-artifact", + timestamp: "2026-03-30T12:34:56.000Z", method: "POST", path: "/v1/chat/completions", status: 200, model: "openai/gpt-4.1", provider: "openai", - duration: 5, - requestBody, - responseBody, + account: "Primary Account", + detail_state: "ready", + artifact_relpath: "2026-03-30/missing.json", + has_request_body: 1, }); + const detail = await callLogs.getCallLogById("missing-artifact"); + assert.equal(detail?.detailState, "missing"); + assert.equal(detail?.requestBody, null); + const db = core.getDbInstance(); const row = db - .prepare("SELECT request_body, response_body FROM call_logs WHERE id = ?") - .get("inline-payload-limit"); - const storedRequest = JSON.parse(row.request_body); - const storedResponse = JSON.parse(row.response_body); - - assert.equal(storedRequest._truncated, undefined); - assert.equal(storedResponse._truncated, undefined); - - const detail = await callLogs.getCallLogById("inline-payload-limit"); - assert.equal(detail?.requestBody?.payload?.length, requestBody.payload.length); - assert.equal(detail?.responseBody?.payload?.length, responseBody.payload.length); + .prepare("SELECT artifact_relpath, detail_state FROM call_logs WHERE id = ?") + .get("missing-artifact"); + assert.equal(row.artifact_relpath, null); + assert.equal(row.detail_state, "missing"); }); -test("saveCallLog still truncates oversized inline sqlite payloads above 256KB", async () => { +test("saveCallLog keeps large payloads out of SQLite while preserving explicit detail export", async () => { const requestBody = { payload: "x".repeat(320 * 1024) }; await callLogs.saveCallLog({ - id: "truncated-inline-payload-limit", + id: "artifact-only-large-payload", timestamp: "2026-03-31T09:05:00.000Z", method: "POST", path: "/v1/chat/completions", @@ -487,18 +406,35 @@ test("saveCallLog still truncates oversized inline sqlite payloads above 256KB", model: "openai/gpt-4.1", provider: "openai", duration: 7, + requestType: "search", requestBody, + error: "upstream unavailable", }); const db = core.getDbInstance(); const row = db - .prepare("SELECT request_body FROM call_logs WHERE id = ?") - .get("truncated-inline-payload-limit"); - const storedRequest = JSON.parse(row.request_body); + .prepare( + ` + SELECT detail_state, has_request_body, artifact_relpath, error_summary, request_summary + FROM call_logs WHERE id = ? + ` + ) + .get("artifact-only-large-payload"); + assert.equal(row.detail_state, "ready"); + assert.equal(row.has_request_body, 1); + assert.equal(typeof row.artifact_relpath, "string"); + assert.equal(row.error_summary, "upstream unavailable"); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", row.artifact_relpath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.equal(artifact.requestBody.payload.length, requestBody.payload.length); + + const detail = await callLogs.getCallLogById("artifact-only-large-payload"); + assert.equal(detail?.requestBody?.payload.length, requestBody.payload.length); - assert.equal(storedRequest._truncated, true); - assert.equal(storedRequest._preview.length, 256 * 1024); - assert.ok(storedRequest._originalSize > storedRequest._preview.length); + const exported = await callLogs.exportCallLogsSince("2026-03-31T00:00:00.000Z"); + assert.equal(exported.length, 1); + assert.equal(exported[0].requestBody.payload.length, requestBody.payload.length); }); test("saveCallLog logs and returns when sqlite persistence throws unexpectedly", async () => { diff --git a/tests/unit/call-log-file-rotation.test.ts b/tests/unit/call-log-file-rotation.test.ts index 56f20ee32b6f..9d3d51c8941f 100644 --- a/tests/unit/call-log-file-rotation.test.ts +++ b/tests/unit/call-log-file-rotation.test.ts @@ -13,11 +13,44 @@ process.env.DATA_DIR = TEST_DATA_DIR; process.env.CALL_LOG_RETENTION_DAYS = "7"; process.env.CALL_LOG_MAX_ENTRIES = "2"; +const core = await import("../../src/lib/db/core.ts"); const { rotateCallLogs, cleanupOverflowCallLogFiles } = await import("../../src/lib/usage/callLogs.ts"); -const { CALL_LOGS_DIR } = await import("../../src/lib/usage/migrations.ts"); +const { CALL_LOGS_DIR } = await import("../../src/lib/usage/callLogArtifacts.ts"); + +function insertCallLog(row) { + const db = core.getDbInstance(); + db.prepare( + ` + INSERT INTO call_logs ( + id, timestamp, method, path, status, model, provider, account, detail_state, artifact_relpath, + has_request_body + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ).run( + row.id, + row.timestamp, + row.method || "POST", + row.path || "/v1/chat/completions", + row.status || 200, + row.model || "openai/gpt-4.1", + row.provider || "openai", + row.account || "acct", + row.detail_state || "ready", + row.artifact_relpath || null, + row.has_request_body || 1 + ); +} + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); test.after(() => { + core.resetDbInstance(); if (ORIGINAL_DATA_DIR === undefined) { delete process.env.DATA_DIR; } else { @@ -44,34 +77,79 @@ test("call log file rotation honors both retention days and file count", () => { fs.rmSync(CALL_LOGS_DIR, { recursive: true, force: true }); fs.mkdirSync(CALL_LOGS_DIR, { recursive: true }); - const oldDir = path.join(CALL_LOGS_DIR, "2026-03-01"); - const activeDir = path.join(CALL_LOGS_DIR, "2026-03-31"); - fs.mkdirSync(oldDir, { recursive: true }); - fs.mkdirSync(activeDir, { recursive: true }); - - const oldFile = path.join(oldDir, "080000_old_200.json"); - const keepA = path.join(activeDir, "090000_keep-a_200.json"); - const keepB = path.join(activeDir, "091000_keep-b_200.json"); - const keepC = path.join(activeDir, "092000_keep-c_200.json"); + const oldRelPath = "2026-03-01/080000_old_200.json"; + const keepARelPath = "2026-04-12/090000_keep-a_200.json"; + const keepBRelPath = "2026-04-13/091000_keep-b_200.json"; + const keepCRelPath = "2026-04-14/092000_keep-c_200.json"; - for (const file of [oldFile, keepA, keepB, keepC]) { - fs.writeFileSync(file, JSON.stringify({ file }), "utf8"); + for (const relativePath of [oldRelPath, keepARelPath, keepBRelPath, keepCRelPath]) { + const absolutePath = path.join(CALL_LOGS_DIR, relativePath); + fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); + fs.writeFileSync(absolutePath, JSON.stringify({ relativePath }), "utf8"); } + insertCallLog({ + id: "old-log", + timestamp: "2026-03-01T08:00:00.000Z", + artifact_relpath: oldRelPath, + }); + insertCallLog({ + id: "keep-a", + timestamp: "2026-04-12T09:00:00.000Z", + artifact_relpath: keepARelPath, + }); + insertCallLog({ + id: "keep-b", + timestamp: "2026-04-13T09:10:00.000Z", + artifact_relpath: keepBRelPath, + }); + insertCallLog({ + id: "keep-c", + timestamp: "2026-04-14T09:20:00.000Z", + artifact_relpath: keepCRelPath, + }); + const now = Date.now(); const oneDay = 24 * 60 * 60 * 1000; - fs.utimesSync(oldFile, new Date(now - 10 * oneDay), new Date(now - 10 * oneDay)); - fs.utimesSync(oldDir, new Date(now - 10 * oneDay), new Date(now - 10 * oneDay)); - fs.utimesSync(keepA, new Date(now - 3 * oneDay), new Date(now - 3 * oneDay)); - fs.utimesSync(keepB, new Date(now - 2 * oneDay), new Date(now - 2 * oneDay)); - fs.utimesSync(keepC, new Date(now - oneDay), new Date(now - oneDay)); + fs.utimesSync( + path.join(CALL_LOGS_DIR, oldRelPath), + new Date(now - 10 * oneDay), + new Date(now - 10 * oneDay) + ); + fs.utimesSync( + path.join(CALL_LOGS_DIR, keepARelPath), + new Date(now - 3 * oneDay), + new Date(now - 3 * oneDay) + ); + fs.utimesSync( + path.join(CALL_LOGS_DIR, keepBRelPath), + new Date(now - 2 * oneDay), + new Date(now - 2 * oneDay) + ); + fs.utimesSync( + path.join(CALL_LOGS_DIR, keepCRelPath), + new Date(now - oneDay), + new Date(now - oneDay) + ); rotateCallLogs(); - assert.equal(fs.existsSync(oldDir), false); - assert.equal(fs.existsSync(keepA), false); - assert.equal(fs.existsSync(keepB), true); - assert.equal(fs.existsSync(keepC), true); + const db = core.getDbInstance(); + assert.equal( + db.prepare("SELECT COUNT(*) AS cnt FROM call_logs WHERE id = ?").get("old-log").cnt, + 0 + ); + assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, oldRelPath)), false); + + const keepARow = db + .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?") + .get("keep-a"); + assert.equal(keepARow.detail_state, "missing"); + assert.equal(keepARow.artifact_relpath, null); + assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, keepARelPath)), false); + + assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, keepBRelPath)), true); + assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, keepCRelPath)), true); }); test("rotateCallLogs swallows filesystem errors during cleanup", () => { @@ -166,11 +244,24 @@ test("cleanupOverflowCallLogFiles ignores rmSync failures for old artifacts", () const dayDir = path.join(CALL_LOGS_DIR, "2026-04-02"); fs.mkdirSync(dayDir, { recursive: true }); - const newerFile = path.join(dayDir, "110000_keep.json"); - const olderFile = path.join(dayDir, "100000_remove.json"); + const newerRelPath = "2026-04-02/110000_keep.json"; + const olderRelPath = "2026-04-02/100000_remove.json"; + const newerFile = path.join(CALL_LOGS_DIR, newerRelPath); + const olderFile = path.join(CALL_LOGS_DIR, olderRelPath); fs.writeFileSync(newerFile, "{}"); fs.writeFileSync(olderFile, "{}"); + insertCallLog({ + id: "keep", + timestamp: "2026-04-02T11:00:00.000Z", + artifact_relpath: newerRelPath, + }); + insertCallLog({ + id: "remove", + timestamp: "2026-04-02T10:00:00.000Z", + artifact_relpath: olderRelPath, + }); + const now = Date.now(); fs.utimesSync(newerFile, new Date(now), new Date(now)); fs.utimesSync(olderFile, new Date(now - 5_000), new Date(now - 5_000)); diff --git a/tests/unit/db-migration-runner.test.ts b/tests/unit/db-migration-runner.test.ts index d31160c3be98..2445598ebfcf 100644 --- a/tests/unit/db-migration-runner.test.ts +++ b/tests/unit/db-migration-runner.test.ts @@ -61,6 +61,15 @@ function createDb() { return new Database(":memory:"); } +const REAL_022_ADD_MEMORY_FTS5_SQL = fs.readFileSync( + path.resolve("src/lib/db/migrations/022_add_memory_fts5.sql"), + "utf8" +); +const REAL_023_FIX_MEMORY_FTS_UUID_SQL = fs.readFileSync( + path.resolve("src/lib/db/migrations/023_fix_memory_fts_uuid.sql"), + "utf8" +); + test("runMigrations applies pending files sequentially in version order", serial, async () => { const runner = await importFresh("src/lib/db/migrationRunner.ts"); const db = createDb(); @@ -346,3 +355,185 @@ test( } } ); + +test( + "runMigrations rehomes legacy call_logs_summary_storage tracking so 022_add_memory_fts5 can still apply", + serial, + async () => { + const runner = await importFresh("src/lib/db/migrationRunner.ts"); + const db = createDb(); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `); + db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( + "022", + "call_logs_summary_storage" + ); + + const count = withMockedMigrationFs( + { + "022_add_memory_fts5.sql": "CREATE TABLE memory_fts_shadow (id INTEGER);", + "025_call_logs_summary_storage.sql": "CREATE TABLE call_log_summary_shadow (id INTEGER);", + }, + () => runner.runMigrations(db) + ); + + assert.equal(count, 1); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "022", name: "add_memory_fts5" }, + { version: "025", name: "call_logs_summary_storage" }, + ] + ); + assert.ok( + db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("memory_fts_shadow") + ); + assert.equal( + db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("call_log_summary_shadow"), + undefined + ); + } finally { + db.close(); + } + } +); + +test( + "runMigrations drops stale 022 call_logs_summary_storage rows when 025 is already tracked", + serial, + async () => { + const runner = await importFresh("src/lib/db/migrationRunner.ts"); + const db = createDb(); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `); + db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( + "022", + "call_logs_summary_storage" + ); + db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( + "025", + "call_logs_summary_storage" + ); + + const count = withMockedMigrationFs( + { + "022_add_memory_fts5.sql": "CREATE TABLE memory_fts_shadow_dupe (id INTEGER);", + "025_call_logs_summary_storage.sql": + "CREATE TABLE call_log_summary_shadow_dupe (id INTEGER);", + }, + () => runner.runMigrations(db) + ); + + assert.equal(count, 1); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "022", name: "add_memory_fts5" }, + { version: "025", name: "call_logs_summary_storage" }, + ] + ); + assert.ok( + db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("memory_fts_shadow_dupe") + ); + assert.equal( + db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("call_log_summary_shadow_dupe"), + undefined + ); + } finally { + db.close(); + } + } +); + +test( + "memory FTS migrations upgrade existing UUID memories without datatype mismatches", + serial, + async () => { + const runner = await importFresh("src/lib/db/migrationRunner.ts"); + const db = createDb(); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE memories ( + id TEXT PRIMARY KEY, + api_key_id TEXT NOT NULL, + session_id TEXT, + type TEXT NOT NULL, + key TEXT, + content TEXT NOT NULL, + metadata TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + expires_at TEXT + ); + `); + db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( + "021", + "combo_call_log_targets" + ); + db.prepare( + "INSERT INTO memories (id, api_key_id, session_id, type, key, content, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)" + ).run( + "550e8400-e29b-41d4-a716-446655440000", + "key-1", + "session-1", + "factual", + "topic", + "memory content", + "{}" + ); + + const count = withMockedMigrationFs( + { + "022_add_memory_fts5.sql": REAL_022_ADD_MEMORY_FTS5_SQL, + "023_fix_memory_fts_uuid.sql": REAL_023_FIX_MEMORY_FTS_UUID_SQL, + }, + () => runner.runMigrations(db) + ); + + assert.equal(count, 2); + assert.deepEqual( + db.prepare("SELECT version FROM _omniroute_migrations ORDER BY version").all(), + [{ version: "021" }, { version: "022" }, { version: "023" }] + ); + assert.deepEqual( + db.prepare("SELECT memory_id, content FROM memories").get(), + { memory_id: 1, content: "memory content" } + ); + assert.deepEqual( + db.prepare("SELECT rowid, content, key FROM memory_fts").get(), + { rowid: 1, content: "memory content", key: "topic" } + ); + } finally { + db.close(); + } + } +); diff --git a/tests/unit/log-export-routes.test.mjs b/tests/unit/log-export-routes.test.mjs new file mode 100644 index 000000000000..2ab38d445377 --- /dev/null +++ b/tests/unit/log-export-routes.test.mjs @@ -0,0 +1,86 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-export-routes-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.CALL_LOG_RETENTION_DAYS = "3650"; + +const core = await import("../../src/lib/db/core.ts"); +const callLogs = await import("../../src/lib/usage/callLogs.ts"); +const exportRoute = await import("../../src/app/api/logs/export/route.ts"); +const exportAllRoute = await import("../../src/app/api/db-backups/exportAll/route.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("GET /api/logs/export returns explicit detailed payloads from artifact storage", async () => { + await callLogs.saveCallLog({ + id: "export-route-log", + timestamp: new Date().toISOString(), + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + requestBody: { messages: [{ role: "user", content: "hello export" }] }, + responseBody: { id: "resp-export", ok: true }, + }); + + const response = await exportRoute.GET( + new Request("http://localhost/api/logs/export?hours=24&type=call-logs") + ); + const body = await response.json(); + + assert.equal(response.status, 200); + assert.equal(body.count, 1); + assert.equal(body.logs[0].id, "export-route-log"); + assert.deepEqual(body.logs[0].requestBody, { + messages: [{ role: "user", content: "hello export" }], + }); + assert.deepEqual(body.logs[0].responseBody, { id: "resp-export", ok: true }); + assert.equal(body.logs[0].detailState, "ready"); +}); + +test("GET /api/db-backups/exportAll includes call_logs artifacts in the archive", async () => { + await callLogs.saveCallLog({ + id: "backup-route-log", + timestamp: new Date().toISOString(), + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + requestBody: { hello: "backup" }, + }); + + const response = await exportAllRoute.GET( + new Request("http://localhost/api/db-backups/exportAll") + ); + assert.equal(response.status, 200); + + const archiveBuffer = Buffer.from(await response.arrayBuffer()); + const archivePath = path.join(TEST_DATA_DIR, "backup-export.tar.gz"); + fs.writeFileSync(archivePath, archiveBuffer); + + const listing = execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" }); + assert.match(listing, /call_logs\//); + assert.match(listing, /call_logs\/.+\.json/); + assert.match(listing, /metadata\.json/); + assert.match(listing, /storage\.sqlite/); +}); diff --git a/tests/unit/usage-migrations.test.ts b/tests/unit/usage-migrations.test.ts index 48d577da417f..1fc4667cfe7f 100644 --- a/tests/unit/usage-migrations.test.ts +++ b/tests/unit/usage-migrations.test.ts @@ -225,7 +225,7 @@ test("migrateUsageJsonToSqlite migrates usage history aliases and TTFT fallbacks ]); }); -test("migrateUsageJsonToSqlite migrates call logs, serializes payloads, and ignores duplicate ids", () => { +test("migrateUsageJsonToSqlite migrates call logs to summary rows and ignores duplicate ids", () => { writeJson(CALL_LOGS_JSON_FILE, { logs: [ { @@ -271,7 +271,7 @@ test("migrateUsageJsonToSqlite migrates call logs, serializes payloads, and igno .prepare( ` SELECT id, method, path, status, provider, account, connection_id, - request_body, response_body, error + detail_state, artifact_relpath, has_request_body, has_response_body, error_summary FROM call_logs ORDER BY timestamp ASC ` @@ -287,10 +287,13 @@ test("migrateUsageJsonToSqlite migrates call logs, serializes payloads, and igno provider: "openai", account: "acct-a", connection_id: "conn-a", - request_body: JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), - response_body: JSON.stringify({ id: "resp-1" }), - error: "bad upstream", + detail_state: "ready", + artifact_relpath: rows[0].artifact_relpath, + has_request_body: 1, + has_response_body: 1, + error_summary: "bad upstream", }); + assert.equal(typeof rows[0].artifact_relpath, "string"); assert.equal(rows[1].id.length > 0, true); assert.equal(rows[1].method, "POST"); assert.equal(rows[1].path, null); @@ -298,9 +301,22 @@ test("migrateUsageJsonToSqlite migrates call logs, serializes payloads, and igno assert.equal(rows[1].provider, null); assert.equal(rows[1].account, null); assert.equal(rows[1].connection_id, null); - assert.deepEqual(JSON.parse(rows[1].request_body), { foo: "bar" }); - assert.equal(rows[1].response_body, null); - assert.equal(rows[1].error, null); + assert.equal(rows[1].detail_state, "ready"); + assert.equal(rows[1].has_request_body, 1); + assert.equal(rows[1].has_response_body, 0); + assert.equal(rows[1].error_summary, null); + + const firstArtifact = JSON.parse( + fs.readFileSync(path.join(TEST_DATA_DIR, "call_logs", rows[0].artifact_relpath), "utf8") + ); + assert.deepEqual(firstArtifact.requestBody, { messages: [{ role: "user", content: "hi" }] }); + assert.deepEqual(firstArtifact.responseBody, { id: "resp-1" }); + + const secondArtifact = JSON.parse( + fs.readFileSync(path.join(TEST_DATA_DIR, "call_logs", rows[1].artifact_relpath), "utf8") + ); + assert.deepEqual(secondArtifact.requestBody, { foo: "bar" }); + assert.equal(secondArtifact.responseBody, null); }); test("migrateUsageJsonToSqlite renames empty JSON payloads without inserting rows", () => {