feat(admin): routing election + exclusion telemetry

apps/api/src/routes/admin-routing-analytics.spec.ts

@@ -62,11 +62,13 @@ describe("admin routing analytics endpoint", () => {
 		} else {
 			process.env.ADMIN_EMAILS = originalAdminEmails;
 		}
-		// Neither table hangs off a cascade root that deleteAll() clears, so the
-		// fixtures inserted here have to be removed explicitly or the next run
+		// None of these tables hang off a cascade root that deleteAll() clears, so
+		// the fixtures inserted here have to be removed explicitly or the next run
 		// collides on their fixed ids. Discounts go through the cached client so
 		// the cached lookup is dropped with them.
 		await db.delete(tables.modelProviderMappingHistoryHourly);
+		await db.delete(tables.routingElectionHourly);
+		await db.delete(tables.routingExclusionHourly);
 		await cdb.delete(tables.discount);
 		await deleteAll();
 	});
@@ -179,6 +181,153 @@ describe("admin routing analytics endpoint", () => {
 		expect(summaryA.uptime).toBe(80);
 	});
 
+	it("reports election paths, eligibility and service-tier coverage", async () => {
+		const hour = currentHourStart();
+		await db.insert(tables.modelProviderMappingHistoryHourly).values([
+			{
+				id: "routing-telemetry-a",
+				modelId: testModel.id,
+				providerId: providerA,
+				modelProviderMappingId: `${testModel.id}-${providerA}`,
+				hourTimestamp: hour,
+				logsCount: 10,
+				serviceTierExplicitCount: 2,
+				serviceTierImplicitCount: 6,
+				serviceTierServedCount: 5,
+				serviceTierUnconfirmedCount: 3,
+			},
+		]);
+		await db.insert(tables.routingElectionHourly).values([
+			{
+				id: "routing-election-scored",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerA,
+				selectionReason: "weighted-score",
+				requestCount: 2,
+				candidateCount: 6,
+			},
+			{
+				id: "routing-election-pinned",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerA,
+				selectionReason: "direct-provider-specified",
+				requestCount: 8,
+				candidateCount: 8,
+			},
+		]);
+		await db.insert(tables.routingExclusionHourly).values([
+			{
+				id: "routing-exclusion-tier",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerB,
+				reason: "service_tier",
+				excludedCount: 6,
+				candidateCount: 10,
+			},
+			{
+				id: "routing-exclusion-vision",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerB,
+				reason: "vision",
+				excludedCount: 1,
+				candidateCount: 10,
+			},
+		]);
+
+		const res = await get(`?modelId=${testModel.id}&window=24h`, cookie);
+		expect(res.status).toBe(200);
+		const body = await res.json();
+
+		expect(body.elections.requestCount).toBe(10);
+		expect(body.elections.scoredCount).toBe(2);
+		// (6 + 8) candidates over 10 requests
+		expect(body.elections.averageCandidateCount).toBe(1.4);
+		expect(body.elections.byKind).toEqual([
+			{ kind: "pinned", requestCount: 8 },
+			{ kind: "scored", requestCount: 2 },
+		]);
+
+		const telemetryHour = body.hourly.find(
+			(h: { hour: string }) => h.hour === hour.toISOString(),
+		);
+		expect(telemetryHour.elections).toEqual([
+			{ kind: "pinned", requestCount: 8 },
+			{ kind: "scored", requestCount: 2 },
+		]);
+
+		const eligibilityB = body.eligibility.find(
+			(e: { providerId: string }) => e.providerId === providerB,
+		);
+		// 7 exclusions across 2 reasons against a candidate count of 10
+		expect(eligibilityB.excludedCount).toBe(7);
+		expect(eligibilityB.candidateCount).toBe(10);
+		expect(eligibilityB.exclusionRate).toBe(0.7);
+		expect(eligibilityB.topReason).toBe("service_tier");
+
+		// A mapping that was never excluded reports no rate rather than 0%, so the
+		// dashboard can tell "always eligible" apart from "no data".
+		const eligibilityA = body.eligibility.find(
+			(e: { providerId: string }) => e.providerId === providerA,
+		);
+		expect(eligibilityA.exclusionRate).toBeNull();
+		expect(eligibilityA.serviceTier).toEqual({
+			requestCount: 10,
+			explicit: 2,
+			implicit: 6,
+			served: 5,
+			unconfirmed: 3,
+		});
+
+		expect(body.exclusions).toEqual([
+			{ reason: "service_tier", excludedCount: 6 },
+			{ reason: "vision", excludedCount: 1 },
+		]);
+		expect(body.serviceTier).toEqual({
+			requestCount: 10,
+			explicit: 2,
+			implicit: 6,
+			served: 5,
+			unconfirmed: 3,
+		});
+	});
+
+	it("caps the exclusion rate when one request fires several reasons", async () => {
+		const hour = currentHourStart();
+		// Two reasons on every one of the 4 requests the mapping was a candidate
+		// in: 8 exclusions against 4 candidates would read as 200% ineligible.
+		await db.insert(tables.routingExclusionHourly).values([
+			{
+				id: "routing-exclusion-multi-a",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerB,
+				reason: "service_tier",
+				excludedCount: 4,
+				candidateCount: 4,
+			},
+			{
+				id: "routing-exclusion-multi-b",
+				hourTimestamp: hour,
+				modelId: testModel.id,
+				providerId: providerB,
+				reason: "vision",
+				excludedCount: 4,
+				candidateCount: 4,
+			},
+		]);
+
+		const res = await get(`?modelId=${testModel.id}&window=24h`, cookie);
+		const body = await res.json();
+		const eligibilityB = body.eligibility.find(
+			(e: { providerId: string }) => e.providerId === providerB,
+		);
+		expect(eligibilityB.exclusionRate).toBe(1);
+	});
+
 	it("scores the discounted selection price", async () => {
 		const before = await get(`?modelId=${testModel.id}&window=24h`, cookie);
 		const baseline = await before.json();