Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions open-sse/mcp-server/analyticsShape.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/**
* Readers for the /api/usage/analytics response.
*
* The route returns totals under `summary` and per-model rows under `byModel`.
* It has no top-level `totalCost`, `requestCount`, `successRate` or
* `avgLatencyMs`, and it only honours a `range` query param (1d/7d/30d/...),
* never `period`. The MCP handlers used to read the missing top-level fields
* and pass `period=session`, so every cost, request count and provider metric
* they reported was zero. These readers are the single place that knows the
* real shape.
*/

type JsonRecord = Record<string, unknown>;

function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}

function asArray(value: unknown): JsonRecord[] {
return Array.isArray(value)
? value.filter((item): item is JsonRecord => asRecord(item) === item)
: [];
}

function asNumber(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
}

const PERIOD_TO_RANGE: Record<string, string> = {
session: "1d",
day: "1d",
week: "7d",
month: "30d",
};

export function analyticsRangeForPeriod(period: string | undefined): string {
return (period && PERIOD_TO_RANGE[period]) || "30d";
}

export interface AnalyticsTotals {
totalCost: number;
requestCount: number;
promptTokens: number;
completionTokens: number;
}

export function readAnalyticsTotals(raw: unknown): AnalyticsTotals {
const summary = asRecord(asRecord(raw).summary);
return {
totalCost: asNumber(summary.totalCost),
requestCount: asNumber(summary.totalRequests),
promptTokens: asNumber(summary.promptTokens),
completionTokens: asNumber(summary.completionTokens),
};
}

export interface ProviderMetrics {
requestCount: number;
avgLatencyMs: number;
successRate: number;
}

// byModel rows carry per-model request counts, latency and a success percentage.
// A provider's numbers are the request-weighted aggregate of its rows.
export function readProviderMetrics(raw: unknown, provider: string): ProviderMetrics {
const rows = asArray(asRecord(raw).byModel).filter((row) => row.provider === provider);
const requestCount = rows.reduce((sum, row) => sum + asNumber(row.requests), 0);
if (requestCount === 0) return { requestCount: 0, avgLatencyMs: 0, successRate: 0 };

const latencyWeighted = rows.reduce(
(sum, row) => sum + asNumber(row.avgLatencyMs) * asNumber(row.requests),
0
);
const successful = rows.reduce((sum, row) => {
const pct = typeof row.successRatePct === "string" ? Number(row.successRatePct) : 0;
return sum + (Number.isFinite(pct) ? pct : 0) * asNumber(row.requests);
}, 0);

return {
requestCount,
avgLatencyMs: Math.round(latencyWeighted / requestCount),
successRate: Math.round((successful / requestCount / 100) * 10000) / 10000,
};
}
19 changes: 7 additions & 12 deletions open-sse/mcp-server/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import { startMcpHeartbeat } from "./runtimeHeartbeat.ts";
import { countUniqueMcpTools } from "./toolCount.ts";
import { z } from "zod";
import { closeAuditDb, logToolCall } from "./audit.ts";
import { analyticsRangeForPeriod, readAnalyticsTotals } from "./analyticsShape.ts";
import {
evaluateToolScopes,
resolveCallerScopeContext,
Expand Down Expand Up @@ -572,26 +573,20 @@ async function handleCostReport(args: { period?: string }) {
const start = Date.now();
try {
const period = args.period || "session";
const rangeMap: Record<string, string> = {
session: "1d",
day: "1d",
week: "7d",
month: "30d",
};
const range = rangeMap[period] || "30d";
const range = analyticsRangeForPeriod(period);
const raw = toRecord(
await omniRouteFetch(`/api/usage/analytics?range=${encodeURIComponent(range)}`)
);
const tokenCount = toRecord(raw.tokenCount);
const totals = readAnalyticsTotals(raw);
const budget = toRecord(raw.budget);

const result = {
period,
totalCost: toNumber(raw.totalCost, 0),
requestCount: toNumber(raw.requestCount, 0),
totalCost: totals.totalCost,
requestCount: totals.requestCount,
tokenCount: {
prompt: toNumber(tokenCount.prompt, 0),
completion: toNumber(tokenCount.completion, 0),
prompt: totals.promptTokens,
completion: totals.completionTokens,
},
byProvider: toArray(raw.byProvider),
byModel: toArray(raw.byModel),
Expand Down
29 changes: 15 additions & 14 deletions open-sse/mcp-server/tools/advancedTools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/

import { logToolCall } from "../audit.ts";
import { readAnalyticsTotals, readProviderMetrics } from "../analyticsShape.ts";
import { toSafeMcpErrorMessage } from "../errorMessage.ts";
import { getMcpHttpAuthHeadersForInternalFetch } from "../httpAuthContext.ts";
import { getInternalServiceAuthHeaders } from "../../../src/lib/api/internalServiceAuth.ts";
Expand Down Expand Up @@ -340,8 +341,8 @@ export async function handleSetBudgetGuard(args: {
// Get current session cost
let spent = 0;
try {
const analytics = toRecord(await apiFetch("/api/usage/analytics?period=session"));
spent = toNumber(analytics.totalCost, 0);
const analytics = toRecord(await apiFetch("/api/usage/analytics?range=1d"));
spent = readAnalyticsTotals(analytics).totalCost;
} catch {
/* ignore if analytics not available */
}
Expand Down Expand Up @@ -626,7 +627,7 @@ export async function handleGetProviderMetrics(args: { provider: string }) {
const [healthRaw, quotaRaw, analyticsRaw] = await Promise.allSettled([
apiFetch("/api/monitoring/health"),
apiFetch(`/api/usage/quota?provider=${encodeURIComponent(args.provider)}`),
apiFetch(`/api/usage/analytics?period=session&provider=${encodeURIComponent(args.provider)}`),
apiFetch(`/api/usage/analytics?range=1d&provider=${encodeURIComponent(args.provider)}`),
]);

const health = healthRaw.status === "fulfilled" ? toRecord(healthRaw.value) : {};
Expand All @@ -641,11 +642,13 @@ export async function handleGetProviderMetrics(args: { provider: string }) {
);
const providerQuota = quota.providers.find((p) => p.provider === args.provider) || null;

const providerMetrics = readProviderMetrics(analytics, args.provider);

const result = {
provider: args.provider,
successRate: toNumber(analytics.successRate, 1.0),
requestCount: toNumber(analytics.requestCount, 0),
avgLatencyMs: toNumber(analytics.avgLatencyMs, 0),
successRate: providerMetrics.successRate,
requestCount: providerMetrics.requestCount,
avgLatencyMs: providerMetrics.avgLatencyMs,
p50LatencyMs: toNumber(analytics.p50LatencyMs, 0),
p95LatencyMs: toNumber(analytics.p95LatencyMs, 0),
p99LatencyMs: toNumber(analytics.p99LatencyMs, 0),
Expand Down Expand Up @@ -841,21 +844,19 @@ export async function handleSyncPricing(args: { sources?: string[]; dryRun?: boo
export async function handleGetSessionSnapshot() {
const start = Date.now();
try {
const analytics = toRecord(
await apiFetch("/api/usage/analytics?period=session").catch(() => ({}))
);
const tokenCount = toRecord(analytics.tokenCount);
const analytics = toRecord(await apiFetch("/api/usage/analytics?range=1d").catch(() => ({})));
const totals = readAnalyticsTotals(analytics);
const byModel = toArrayOfRecords(analytics.byModel);
const byProvider = toArrayOfRecords(analytics.byProvider);

const result = {
sessionStart: toString(analytics.sessionStart, new Date().toISOString()),
duration: toString(analytics.duration, "unknown"),
requestCount: toNumber(analytics.requestCount, 0),
costTotal: toNumber(analytics.totalCost, 0),
requestCount: totals.requestCount,
costTotal: totals.totalCost,
tokenCount: {
prompt: toNumber(tokenCount.prompt, 0),
completion: toNumber(tokenCount.completion, 0),
prompt: totals.promptTokens,
completion: totals.completionTokens,
},
topModels: byModel.slice(0, 5).map((model) => ({
model: toString(model.model, "unknown"),
Expand Down
2 changes: 1 addition & 1 deletion open-sse/mcp-server/tools/pickFastestModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ async function fetchTelemetrySources(): Promise<TelemetrySources> {
apiFetch("/api/combos"),
apiFetch("/api/monitoring/health"),
apiFetch("/api/usage/quota"),
apiFetch("/api/usage/analytics?period=session"),
apiFetch("/api/usage/analytics?range=1d"),
]);

const analytics = toRecord(settledValue(analyticsRaw));
Expand Down
86 changes: 86 additions & 0 deletions tests/unit/mcp-analytics-shape.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";

import {
analyticsRangeForPeriod,
readAnalyticsTotals,
readProviderMetrics,
} from "../../open-sse/mcp-server/analyticsShape.ts";

// The shape /api/usage/analytics actually returns: totals live under `summary`,
// and there is no top-level totalCost, requestCount, successRate or avgLatencyMs.
const ANALYTICS = {
summary: {
totalCost: 7.5,
totalRequests: 1200,
promptTokens: 400000,
completionTokens: 90000,
},
byProvider: [
{ provider: "antigravity", requests: 1160, cost: 7.3 },
{ provider: "claude", requests: 40, cost: 0.2 },
],
byModel: [
{
model: "gemini-3.6-flash",
provider: "antigravity",
requests: 1000,
avgLatencyMs: 800,
successRatePct: "99.00",
},
{
model: "gemini-3.1-pro",
provider: "antigravity",
requests: 160,
avgLatencyMs: 2000,
successRatePct: "95.00",
},
{
model: "claude-sonnet",
provider: "claude",
requests: 40,
avgLatencyMs: 1500,
successRatePct: "100.00",
},
],
};

describe("analytics shape readers", () => {
it("reads the totals from summary, not the top level", () => {
const totals = readAnalyticsTotals(ANALYTICS);
assert.equal(totals.totalCost, 7.5);
assert.equal(totals.requestCount, 1200);
assert.equal(totals.promptTokens, 400000);
assert.equal(totals.completionTokens, 90000);
});

it("returns zeros when summary is missing", () => {
const totals = readAnalyticsTotals({ byProvider: [] });
assert.equal(totals.totalCost, 0);
assert.equal(totals.requestCount, 0);
});

it("aggregates one provider from the byModel rows", () => {
const metrics = readProviderMetrics(ANALYTICS, "antigravity");
assert.equal(metrics.requestCount, 1160);
// Weighted by requests: (1000*800 + 160*2000) / 1160 = 965.517...
assert.equal(metrics.avgLatencyMs, 966);
// Successful requests: 1000*0.99 + 160*0.95 = 1142, over 1160 = 0.98448
assert.ok(Math.abs(metrics.successRate - 0.9845) < 0.001);
});

it("returns zeros for a provider with no rows", () => {
const metrics = readProviderMetrics(ANALYTICS, "openai");
assert.equal(metrics.requestCount, 0);
assert.equal(metrics.avgLatencyMs, 0);
assert.equal(metrics.successRate, 0);
});

it("maps a report period onto the range the analytics route accepts", () => {
assert.equal(analyticsRangeForPeriod("session"), "1d");
assert.equal(analyticsRangeForPeriod("day"), "1d");
assert.equal(analyticsRangeForPeriod("week"), "7d");
assert.equal(analyticsRangeForPeriod("month"), "30d");
assert.equal(analyticsRangeForPeriod(undefined), "30d");
});
});
Loading