From a653d86ac3203551e9d038c1767315d1f7cc11c1 Mon Sep 17 00:00:00 2001 From: dat Date: Mon, 1 Jun 2026 04:03:49 +0000 Subject: [PATCH] fix(usage): [AI-000] backfill usageDaily project maps so project cost reconciles Legacy usageDaily rows written before the per-project feature shipped carry day.cost / byModel / byApiKey but omit byProject / byApiKeyProject. In daily-summary periods (7d/30d/60d/all), /dashboard/projects summed day.cost for the Est. Cost card while Usage-by-Project summed byProject, so the project table and the By-Project API key chart undercounted. Today/24h was correct because it recomputes live from usageHistory. usageHistory is the complete authoritative record (no pruning), and sum(usageHistory.cost) per day == day.cost, so re-aggregating is lossless. - extract getLocalDateKey/aggregateEntryToDay into helpers/aggregate.js (leaf module) to break the migration->usageRepo->driver->migrate import cycle - add migration v2 rebuilding usageDaily from usageHistory; passes stored cost through (no pricing recompute) and parsed tokens JSON; leaves history-less daily rows untouched; idempotent - v2 is column-defensive: selects NULL for project when the column predates the feature, since versioned migrations run before the additive column sync - regression test reconciles byProject/byApiKeyProject/byModel/byApiKey with day.cost, plus the pre-project-column upgrade path Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/db/helpers/aggregate.js | 63 +++++++++ .../002-backfill-usage-daily-projects.js | 52 +++++++ src/lib/db/migrations/index.js | 3 +- src/lib/db/repos/usageRepo.js | 61 +------- src/lib/db/schema.js | 2 +- tests/unit/usage-daily-backfill.test.js | 131 ++++++++++++++++++ 6 files changed, 250 insertions(+), 62 deletions(-) create mode 100644 src/lib/db/helpers/aggregate.js create mode 100644 src/lib/db/migrations/002-backfill-usage-daily-projects.js create mode 100644 tests/unit/usage-daily-backfill.test.js diff --git a/src/lib/db/helpers/aggregate.js b/src/lib/db/helpers/aggregate.js new file mode 100644 index 00000000000..295b6a733f5 --- /dev/null +++ b/src/lib/db/helpers/aggregate.js @@ -0,0 +1,63 @@ +// Canonical daily-usage aggregation. Pure and synchronous so both the live +// writer (usageRepo.saveRequestUsage) and the backfill migration can share one +// source of truth without pulling in the driver/migrate import cycle. + +export function getLocalDateKey(timestamp) { + const d = timestamp ? new Date(timestamp) : new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; +} + +function addToCounter(target, key, values) { + if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 }; + target[key].requests += values.requests || 1; + target[key].promptTokens += values.promptTokens || 0; + target[key].completionTokens += values.completionTokens || 0; + target[key].cost += values.cost || 0; + if (values.meta) Object.assign(target[key], values.meta); +} + +export function aggregateEntryToDay(day, entry) { + const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0; + const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0; + const cost = entry.cost || 0; + const vals = { promptTokens, completionTokens, cost }; + + day.requests = (day.requests || 0) + 1; + day.promptTokens = (day.promptTokens || 0) + promptTokens; + day.completionTokens = (day.completionTokens || 0) + completionTokens; + day.cost = (day.cost || 0) + cost; + + day.byProvider ||= {}; + day.byModel ||= {}; + day.byAccount ||= {}; + day.byApiKey ||= {}; + day.byEndpoint ||= {}; + day.byProject ||= {}; + day.byApiKeyProject ||= {}; + + if (entry.provider) addToCounter(day.byProvider, entry.provider, vals); + + const modelKey = entry.provider ? `${entry.model}|${entry.provider}` : entry.model; + addToCounter(day.byModel, modelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } }); + + if (entry.connectionId) { + addToCounter(day.byAccount, entry.connectionId, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } }); + } + + const apiKeyVal = entry.apiKey && typeof entry.apiKey === "string" ? entry.apiKey : "local-no-key"; + const akModelKey = `${apiKeyVal}|${entry.model}|${entry.provider || "unknown"}`; + addToCounter(day.byApiKey, akModelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider, apiKey: entry.apiKey || null } }); + + const endpoint = entry.endpoint || "Unknown"; + const epKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`; + addToCounter(day.byEndpoint, epKey, { ...vals, meta: { endpoint, rawModel: entry.model, provider: entry.provider } }); + + const projectValue = entry.project && typeof entry.project === "string" ? entry.project : null; + const projectKeySegment = projectValue || "__untagged__"; + const projectKey = `${projectKeySegment}|${entry.model}|${entry.provider || "unknown"}`; + addToCounter(day.byProject, projectKey, { ...vals, meta: { project: projectValue, rawModel: entry.model, provider: entry.provider } }); + + // API key × project, for the per-API-key usage-by-project chart. + const akProjectKey = `${apiKeyVal}|${projectKeySegment}`; + addToCounter(day.byApiKeyProject, akProjectKey, { ...vals, meta: { apiKey: entry.apiKey || null, project: projectValue } }); +} diff --git a/src/lib/db/migrations/002-backfill-usage-daily-projects.js b/src/lib/db/migrations/002-backfill-usage-daily-projects.js new file mode 100644 index 00000000000..cf18d18d8f2 --- /dev/null +++ b/src/lib/db/migrations/002-backfill-usage-daily-projects.js @@ -0,0 +1,52 @@ +// Rebuild usageDaily from usageHistory so legacy daily rows gain byProject / +// byApiKeyProject maps. Daily rows written before the per-project feature +// shipped carry day.cost / byModel / byApiKey but omit the project maps, so the +// 7d/30d/60d/all stats undercounted Usage-by-Project vs the Est. Cost card. +// +// usageHistory is the complete authoritative record (no pruning anywhere), and +// sum(usageHistory.cost) per day == day.cost, so re-aggregating is lossless. +import { getLocalDateKey, aggregateEntryToDay } from "../helpers/aggregate.js"; +import { parseJson, stringifyJson } from "../helpers/jsonCol.js"; + +export default { + version: 2, + name: "backfill-usage-daily-projects", + up(db) { + // Versioned migrations run BEFORE the additive schema sync that adds missing + // columns, so on an upgrade from a build predating the `project` column the + // column may not exist yet. Select NULL for it in that case — those rows are + // genuinely untagged (the feature didn't exist when they were written). + const columns = new Set(db.all(`PRAGMA table_info(usageHistory)`).map((column) => column.name)); + const projectSelect = columns.has("project") ? "project" : "NULL AS project"; + + const rows = db.all( + `SELECT timestamp, provider, model, connectionId, apiKey, endpoint, ${projectSelect}, cost, tokens FROM usageHistory` + ); + if (!rows.length) return; // no history → leave existing daily rows untouched + + const byDay = {}; + for (const row of rows) { + const dateKey = getLocalDateKey(row.timestamp); + byDay[dateKey] ||= { + requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, + byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, + byEndpoint: {}, byProject: {}, byApiKeyProject: {}, + }; + aggregateEntryToDay(byDay[dateKey], { + provider: row.provider, model: row.model, connectionId: row.connectionId, + apiKey: row.apiKey, endpoint: row.endpoint, project: row.project, + cost: row.cost, // stored column — never recompute via pricing + tokens: parseJson(row.tokens, {}), // nested shape aggregateEntryToDay expects + }); + } + + // Only days that have history rows are rebuilt; daily-only dates (e.g. an + // imported summary with no underlying history) are left untouched. + for (const [dateKey, day] of Object.entries(byDay)) { + db.run( + `INSERT INTO usageDaily(dateKey, data) VALUES(?, ?) ON CONFLICT(dateKey) DO UPDATE SET data = excluded.data`, + [dateKey, stringifyJson(day)] + ); + } + }, +}; diff --git a/src/lib/db/migrations/index.js b/src/lib/db/migrations/index.js index 13fb4bfe339..150a0d0e309 100644 --- a/src/lib/db/migrations/index.js +++ b/src/lib/db/migrations/index.js @@ -2,8 +2,9 @@ // Each migration: { version: number, name: string, up(db): void } // Versions MUST be unique and monotonically increasing. import m001 from "./001-initial.js"; +import m002 from "./002-backfill-usage-daily-projects.js"; -export const MIGRATIONS = [m001].sort((a, b) => a.version - b.version); +export const MIGRATIONS = [m001, m002].sort((a, b) => a.version - b.version); export function latestVersion() { return MIGRATIONS.length ? MIGRATIONS[MIGRATIONS.length - 1].version : 0; diff --git a/src/lib/db/repos/usageRepo.js b/src/lib/db/repos/usageRepo.js index fad12befab0..78f826fb483 100644 --- a/src/lib/db/repos/usageRepo.js +++ b/src/lib/db/repos/usageRepo.js @@ -2,6 +2,7 @@ import { EventEmitter } from "events"; import { getAdapter } from "../driver.js"; import { parseJson, stringifyJson } from "../helpers/jsonCol.js"; import { getMeta, setMeta } from "../helpers/metaStore.js"; +import { getLocalDateKey, aggregateEntryToDay } from "../helpers/aggregate.js"; const PENDING_TIMEOUT_MS = 60 * 1000; const RING_CAP = 50; @@ -27,66 +28,6 @@ const connCache = global._connectionMapCache; export const statsEmitter = global._statsEmitter; -function getLocalDateKey(timestamp) { - const d = timestamp ? new Date(timestamp) : new Date(); - return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; -} - -function addToCounter(target, key, values) { - if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 }; - target[key].requests += values.requests || 1; - target[key].promptTokens += values.promptTokens || 0; - target[key].completionTokens += values.completionTokens || 0; - target[key].cost += values.cost || 0; - if (values.meta) Object.assign(target[key], values.meta); -} - -function aggregateEntryToDay(day, entry) { - const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0; - const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0; - const cost = entry.cost || 0; - const vals = { promptTokens, completionTokens, cost }; - - day.requests = (day.requests || 0) + 1; - day.promptTokens = (day.promptTokens || 0) + promptTokens; - day.completionTokens = (day.completionTokens || 0) + completionTokens; - day.cost = (day.cost || 0) + cost; - - day.byProvider ||= {}; - day.byModel ||= {}; - day.byAccount ||= {}; - day.byApiKey ||= {}; - day.byEndpoint ||= {}; - day.byProject ||= {}; - day.byApiKeyProject ||= {}; - - if (entry.provider) addToCounter(day.byProvider, entry.provider, vals); - - const modelKey = entry.provider ? `${entry.model}|${entry.provider}` : entry.model; - addToCounter(day.byModel, modelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } }); - - if (entry.connectionId) { - addToCounter(day.byAccount, entry.connectionId, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } }); - } - - const apiKeyVal = entry.apiKey && typeof entry.apiKey === "string" ? entry.apiKey : "local-no-key"; - const akModelKey = `${apiKeyVal}|${entry.model}|${entry.provider || "unknown"}`; - addToCounter(day.byApiKey, akModelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider, apiKey: entry.apiKey || null } }); - - const endpoint = entry.endpoint || "Unknown"; - const epKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`; - addToCounter(day.byEndpoint, epKey, { ...vals, meta: { endpoint, rawModel: entry.model, provider: entry.provider } }); - - const projectValue = entry.project && typeof entry.project === "string" ? entry.project : null; - const projectKeySegment = projectValue || "__untagged__"; - const projectKey = `${projectKeySegment}|${entry.model}|${entry.provider || "unknown"}`; - addToCounter(day.byProject, projectKey, { ...vals, meta: { project: projectValue, rawModel: entry.model, provider: entry.provider } }); - - // API key × project, for the per-API-key usage-by-project chart. - const akProjectKey = `${apiKeyVal}|${projectKeySegment}`; - addToCounter(day.byApiKeyProject, akProjectKey, { ...vals, meta: { apiKey: entry.apiKey || null, project: projectValue } }); -} - function pushToRing(entry) { recentRing.items.push(entry); if (recentRing.items.length > RING_CAP) { diff --git a/src/lib/db/schema.js b/src/lib/db/schema.js index 34581e3d689..6d405a1c704 100644 --- a/src/lib/db/schema.js +++ b/src/lib/db/schema.js @@ -1,5 +1,5 @@ // Latest schema version — bumped when a migration is added in ./migrations/ -export const SCHEMA_VERSION = 1; +export const SCHEMA_VERSION = 2; export const PRAGMA_SQL = ` PRAGMA journal_mode = WAL; diff --git a/tests/unit/usage-daily-backfill.test.js b/tests/unit/usage-daily-backfill.test.js new file mode 100644 index 00000000000..c96864728be --- /dev/null +++ b/tests/unit/usage-daily-backfill.test.js @@ -0,0 +1,131 @@ +// Regression guard for the /dashboard/projects cost mismatch: legacy usageDaily +// rows written before the per-project feature shipped carry day.cost / byModel / +// byApiKey but omit byProject / byApiKeyProject. In daily-summary periods +// (7d/30d/60d/all) the Est. Cost card summed day.cost while Usage-by-Project +// summed byProject → undercount. Migration v2 rebuilds usageDaily from the +// authoritative usageHistory so every day regains the project maps. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +let tempDir; +const originalDataDir = process.env.DATA_DIR; + +beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-backfill-")); + process.env.DATA_DIR = tempDir; + delete global._dbAdapter; + vi.resetModules(); +}); + +afterEach(() => { + try { global._dbAdapter?.instance?.close?.(); } catch {} + delete global._dbAdapter; + if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; +}); + +const DATE_KEY = "2026-05-23"; +const insertHistory = (db, { project, model, provider, prompt, completion, cost, apiKey }) => + db.run( + `INSERT INTO usageHistory(timestamp, provider, model, connectionId, apiKey, endpoint, project, promptTokens, completionTokens, cost, status, tokens, meta) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + `${DATE_KEY}T10:00:00.000Z`, provider, model, "conn-1", apiKey, "/v1/messages", + project, prompt, completion, cost, "ok", + JSON.stringify({ prompt_tokens: prompt, completion_tokens: completion }), "{}", + ] + ); + +const sumCost = (mapObject) => + Object.values(mapObject || {}).reduce((total, entry) => total + (entry.cost || 0), 0); + +describe("usageDaily project backfill (migration v2)", () => { + it("rebuilds byProject / byApiKeyProject so sums reconcile with day.cost", async () => { + // Boot 1 — fresh DB applies the full migration chain (no history → no-op). + const { getAdapter } = await import("@/lib/db/driver.js"); + const db = await getAdapter(); + + // Seed history rows directly (NOT saveRequestUsage, which would write a + // correct daily row and defeat the legacy simulation). + insertHistory(db, { project: "alpha", model: "claude-opus-4-8", provider: "anthropic", prompt: 1000, completion: 200, cost: 1.5, apiKey: "sk-a" }); + insertHistory(db, { project: "beta", model: "claude-opus-4-8", provider: "anthropic", prompt: 500, completion: 100, cost: 0.75, apiKey: "sk-b" }); + insertHistory(db, { project: null, model: "gpt-5", provider: "openai", prompt: 300, completion: 50, cost: 0.25, apiKey: null }); + const historyTotal = 1.5 + 0.75 + 0.25; + + // Overwrite that date's daily row with a legacy-shaped blob: has cost + + // byModel + byApiKey, but NO byProject / byApiKeyProject. + const legacyDay = { + requests: 3, promptTokens: 1800, completionTokens: 350, cost: historyTotal, + byProvider: { anthropic: { requests: 2, cost: 2.25 }, openai: { requests: 1, cost: 0.25 } }, + byModel: { "claude-opus-4-8|anthropic": { requests: 2, cost: 2.25 }, "gpt-5|openai": { requests: 1, cost: 0.25 } }, + byApiKey: { "sk-a|claude-opus-4-8|anthropic": { requests: 1, cost: 1.5 } }, + byEndpoint: {}, + }; + db.run( + `INSERT INTO usageDaily(dateKey, data) VALUES(?, ?) ON CONFLICT(dateKey) DO UPDATE SET data = excluded.data`, + [DATE_KEY, JSON.stringify(legacyDay)] + ); + + // A daily-only date with no underlying history — must survive untouched. + const orphanDay = { requests: 9, cost: 9.99, byProject: {} }; + db.run(`INSERT INTO usageDaily(dateKey, data) VALUES(?, ?)`, ["2020-01-01", JSON.stringify(orphanDay)]); + + // Roll schemaVersion back so the next boot re-applies v2. + db.run(`UPDATE _meta SET value = '1' WHERE key = 'schemaVersion'`); + db.close?.(); + + // Boot 2 — simulate restart; v2 re-runs and backfills. + delete global._dbAdapter; + vi.resetModules(); + const { getAdapter: getAdapter2 } = await import("@/lib/db/driver.js"); + const db2 = await getAdapter2(); + + const rebuilt = JSON.parse(db2.get(`SELECT data FROM usageDaily WHERE dateKey = ?`, [DATE_KEY]).data); + + expect(Object.keys(rebuilt.byProject).length).toBeGreaterThan(0); + expect(Object.keys(rebuilt.byApiKeyProject).length).toBeGreaterThan(0); + + // The whole point: every dimension reconciles with the same day total. + expect(rebuilt.cost).toBeCloseTo(historyTotal, 9); + expect(sumCost(rebuilt.byProject)).toBeCloseTo(historyTotal, 9); + expect(sumCost(rebuilt.byApiKeyProject)).toBeCloseTo(historyTotal, 9); + expect(sumCost(rebuilt.byModel)).toBeCloseTo(historyTotal, 9); + expect(sumCost(rebuilt.byApiKey)).toBeCloseTo(historyTotal, 9); + + // Project meta is carried onto the keys. + const projectNames = Object.values(rebuilt.byProject).map((entry) => entry.project); + expect(projectNames).toEqual(expect.arrayContaining(["alpha", "beta"])); + // Untagged request lands under the null project bucket. + expect(projectNames).toContain(null); + + // Orphan daily-only date untouched. + const orphan = JSON.parse(db2.get(`SELECT data FROM usageDaily WHERE dateKey = ?`, ["2020-01-01"]).data); + expect(orphan).toEqual(orphanDay); + }); + + it("does not crash when usageHistory predates the project column", async () => { + // Versioned migrations run before the additive column sync, so a DB upgrading + // from a build without the `project` column must not throw in v2's SELECT. + const { getAdapter } = await import("@/lib/db/driver.js"); + const m2 = (await import("@/lib/db/migrations/002-backfill-usage-daily-projects.js")).default; + + const db = await getAdapter(); + // Drop the project column to simulate a pre-feature schema (SQLite ≥ 3.35). + db.exec(`ALTER TABLE usageHistory DROP COLUMN project`); + expect(db.all(`PRAGMA table_info(usageHistory)`).map((column) => column.name)).not.toContain("project"); + + db.run( + `INSERT INTO usageHistory(timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, status, tokens, meta) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [`${DATE_KEY}T10:00:00.000Z`, "anthropic", "claude-opus-4-8", "conn-1", "sk-a", "/v1/messages", 1000, 200, 1.5, "ok", JSON.stringify({ prompt_tokens: 1000, completion_tokens: 200 }), "{}"] + ); + + expect(() => m2.up(db)).not.toThrow(); + + const rebuilt = JSON.parse(db.get(`SELECT data FROM usageDaily WHERE dateKey = ?`, [DATE_KEY]).data); + // Rows from before the feature are correctly bucketed as untagged. + expect(sumCost(rebuilt.byProject)).toBeCloseTo(1.5, 9); + expect(Object.values(rebuilt.byProject).map((entry) => entry.project)).toEqual([null]); + }); +});