From b6cb573507f9c83b8787798d081dd773ac52b44b Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Sun, 26 Jul 2026 20:30:27 -0300 Subject: [PATCH 1/6] feat(web): add public leaderboard provider race endpoint Weekly token volume per model lab for the kilo.ai/leaderboard/race visualization. Grouped at week x model_provider_company x is_open_weights so one payload drives both the per-lab race and an open-weight vs proprietary toggle. Lab mapping comes from the dbt model_dim seed rather than being re-derived in SQL. Depends on the kilocode-dbt usage_daily backfill that adds is_open_weights. --- .../public/leaderboard-provider-race/route.ts | 68 +++++++++++++++++++ apps/web/src/lib/redis-keys.ts | 1 + 2 files changed, 69 insertions(+) create mode 100644 apps/web/src/app/api/public/leaderboard-provider-race/route.ts diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts new file mode 100644 index 0000000000..5b9c2ff3c2 --- /dev/null +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts @@ -0,0 +1,68 @@ +import { z } from 'zod'; + +import { + createPublicSnowflakeReport, + publicSnowflakeReportOptions, +} from '@/lib/public-snowflake-report'; +import { LEADERBOARD_PROVIDER_RACE_REDIS_KEY } from '@/lib/redis-keys'; + +// Weekly token volume per model lab, from a fixed start date through the most +// recent complete day. Grouped at week x model_provider_company x +// is_open_weights so a single payload drives both the per-lab "race" view and +// an open-weight vs proprietary toggle. model_provider_company and +// is_open_weights are maintained in the dbt model_dim seed (kilocode-dbt), so +// the lab mapping lives in one place rather than being re-derived here. +const LEADERBOARD_PROVIDER_RACE_QUERY = ` +select + to_char(date_trunc('week', ud.usage_date), 'YYYY-MM-DD') as week_start + , ud.model_provider_company as provider + , ud.is_open_weights + , sum(ud.total_tokens) as tokens +from kilo_dw.dbt_prod.usage_daily as ud +where + ud.usage_date >= '2025-07-01' + and ud.usage_date < current_date() + and ud.total_tokens > 0 +group by 1, 2, 3 +order by 1, 4 desc; +`; + +const providerRaceSchema = z.array( + z.object({ + weekStart: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + provider: z.string().min(1), + isOpenWeights: z.boolean(), + tokens: z.number(), + }) +); + +type ProviderRace = z.infer; + +function parseProviderRace(rows: string[][]): ProviderRace { + return rows.map(row => { + const [weekStart, provider, rawOpenWeights, rawTokens] = row; + const tokens = Number(rawTokens); + + if (!weekStart || !provider || !Number.isFinite(tokens)) { + throw new Error('Snowflake returned an invalid provider race row'); + } + + return providerRaceSchema.element.parse({ + weekStart, + provider, + isOpenWeights: rawOpenWeights === 'true' || rawOpenWeights === true, + tokens, + }); + }); +} + +export const GET = createPublicSnowflakeReport({ + cacheKey: LEADERBOARD_PROVIDER_RACE_REDIS_KEY, + errorMessage: 'Failed to fetch leaderboard provider race', + parseRows: parseProviderRace, + query: LEADERBOARD_PROVIDER_RACE_QUERY, + schema: providerRaceSchema, + source: 'public-leaderboard-provider-race-api', +}); + +export const OPTIONS = publicSnowflakeReportOptions; diff --git a/apps/web/src/lib/redis-keys.ts b/apps/web/src/lib/redis-keys.ts index a692943155..442443edfe 100644 --- a/apps/web/src/lib/redis-keys.ts +++ b/apps/web/src/lib/redis-keys.ts @@ -52,6 +52,7 @@ export const LEADERBOARD_MODEL_PROVIDER_USAGE_REDIS_KEY = redisKey( 'public-api:leaderboard-model-provider-usage' ); export const LEADERBOARD_MODEL_USAGE_REDIS_KEY = redisKey('public-api:leaderboard-model-usage'); +export const LEADERBOARD_PROVIDER_RACE_REDIS_KEY = redisKey('public-api:leaderboard-provider-race'); export const REQUEST_LOGGING_OPT_INS_REDIS_KEY = redisKey('ai-gateway:request-logging-opt-ins'); From 3d41bacf2dc6f464cd7dbd145bb4457c8cfc4564 Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Sun, 26 Jul 2026 20:42:24 -0300 Subject: [PATCH 2/6] fix(web): exclude partial current week from provider race Match the prior '52 complete weeks' behavior so the latest returned week is always complete (keeps the insights 'latest complete week' stat correct). --- .../src/app/api/public/leaderboard-provider-race/route.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts index 5b9c2ff3c2..5586327717 100644 --- a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts @@ -7,11 +7,12 @@ import { import { LEADERBOARD_PROVIDER_RACE_REDIS_KEY } from '@/lib/redis-keys'; // Weekly token volume per model lab, from a fixed start date through the most -// recent complete day. Grouped at week x model_provider_company x +// recent complete week. Grouped at week x model_provider_company x // is_open_weights so a single payload drives both the per-lab "race" view and // an open-weight vs proprietary toggle. model_provider_company and // is_open_weights are maintained in the dbt model_dim seed (kilocode-dbt), so // the lab mapping lives in one place rather than being re-derived here. +// The partial current week is excluded so every returned week is complete. const LEADERBOARD_PROVIDER_RACE_QUERY = ` select to_char(date_trunc('week', ud.usage_date), 'YYYY-MM-DD') as week_start @@ -21,7 +22,7 @@ select from kilo_dw.dbt_prod.usage_daily as ud where ud.usage_date >= '2025-07-01' - and ud.usage_date < current_date() + and date_trunc('week', ud.usage_date) < date_trunc('week', current_date()) and ud.total_tokens > 0 group by 1, 2, 3 order by 1, 4 desc; From 4ac7c58b07837b313193f109424b78590c902ca0 Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Mon, 27 Jul 2026 10:03:22 -0300 Subject: [PATCH 3/6] fix(web): surface unknown is_open_weights as null in provider race Map only explicit 'true'/'false' strings to a boolean and return null for NULL/unmapped models so the open/closed split isn't silently corrupted by model_dim mapping gaps. Also removes the string===boolean comparison that broke the typecheck (TS2367). --- .../api/public/leaderboard-provider-race/route.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts index 5586327717..e5c80a2958 100644 --- a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts @@ -32,13 +32,24 @@ const providerRaceSchema = z.array( z.object({ weekStart: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), provider: z.string().min(1), - isOpenWeights: z.boolean(), + isOpenWeights: z.boolean().nullable(), tokens: z.number(), }) ); type ProviderRace = z.infer; +// is_open_weights is maintained in the dbt model_dim seed and is NULL for +// unmapped models (the ones bucketed as provider = 'other'). Only the explicit +// 'true'/'false' strings map to a boolean; anything else (NULL from the SQL API, +// or a future mapping gap) becomes null so the client can distinguish "unknown" +// from a confirmed closed-weight lab instead of silently bucketing gaps as closed. +function parseOpenWeights(raw: string): boolean | null { + if (raw === 'true') return true; + if (raw === 'false') return false; + return null; +} + function parseProviderRace(rows: string[][]): ProviderRace { return rows.map(row => { const [weekStart, provider, rawOpenWeights, rawTokens] = row; @@ -51,7 +62,7 @@ function parseProviderRace(rows: string[][]): ProviderRace { return providerRaceSchema.element.parse({ weekStart, provider, - isOpenWeights: rawOpenWeights === 'true' || rawOpenWeights === true, + isOpenWeights: parseOpenWeights(rawOpenWeights), tokens, }); }); From 03df4abd9778753c34ea4375cefb7f3219a0d3a9 Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Mon, 27 Jul 2026 10:37:37 -0300 Subject: [PATCH 4/6] test(web): cover leaderboard provider race parsing and error paths Mocks the Snowflake statement executor and Redis to exercise the route without a warehouse: 503 when unconfigured, 502 on query failure or a malformed row, CORS/OPTIONS, and the is_open_weights mapping where only 'true'/'false' become booleans and NULL/unknown become null. --- .../leaderboard-provider-race/route.test.ts | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts new file mode 100644 index 0000000000..541a6d8ec7 --- /dev/null +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts @@ -0,0 +1,109 @@ +import { describe, test, expect, jest } from '@jest/globals'; +import type * as SnowflakeModule from '@/lib/snowflake'; +import type { GET as RouteGet } from './route'; + +jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() })); + +jest.mock('@/lib/redis', () => ({ + redisClient: { + get: jest.fn<() => Promise>().mockResolvedValue(null), + set: jest.fn().mockResolvedValue('OK'), + }, +})); + +jest.mock('@/lib/snowflake', () => ({ + resolveSnowflakeConfig: jest.fn(), + executeSnowflakeStatement: jest.fn(), +})); + +const FAKE_CONFIG = { accountHost: 'test.snowflakecomputing.com' }; + +// The route builds a module-level in-process cache (1h TTL) at import time, so +// each test loads it in an isolated module registry to keep that cache from +// leaking between cases. +async function loadGet(opts: { + config?: unknown; + rows?: string[][]; + statementError?: Error; +}): Promise { + let GET!: typeof RouteGet; + await jest.isolateModulesAsync(async () => { + const snowflake = (await import('@/lib/snowflake')) as SnowflakeModule; + const config = opts.config === undefined ? FAKE_CONFIG : opts.config; + jest + .mocked(snowflake.resolveSnowflakeConfig) + .mockReturnValue(config as ReturnType); + if (opts.statementError) { + jest.mocked(snowflake.executeSnowflakeStatement).mockRejectedValue(opts.statementError); + } else { + jest.mocked(snowflake.executeSnowflakeStatement).mockResolvedValue(opts.rows ?? []); + } + ({ GET } = await import('./route')); + }); + return GET; +} + +describe('GET /api/public/leaderboard-provider-race', () => { + test('returns 503 when Snowflake is not configured', async () => { + const GET = await loadGet({ config: null }); + + const response = await GET(); + + expect(response.status).toBe(503); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + }); + + test('returns 502 when the Snowflake query fails (e.g. is_open_weights backfill not landed)', async () => { + const GET = await loadGet({ statementError: new Error('invalid identifier IS_OPEN_WEIGHTS') }); + + const response = await GET(); + + expect(response.status).toBe(502); + }); + + test('maps explicit is_open_weights to booleans and NULL/unknown to null', async () => { + const GET = await loadGet({ + rows: [ + ['2026-07-14', 'Anthropic', 'false', '12345'], + ['2026-07-14', 'Alibaba', 'true', '6789'], + // The SQL API returns NULL cells as null at runtime despite the + // string[] row type; unmapped models land here. + ['2026-07-14', 'other', null, '999'] as unknown as string[], + // A future model_dim mapping gap must not be silently bucketed as closed. + ['2026-07-14', 'Mystery', 'unexpected', '111'], + ], + }); + + const response = await GET(); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual([ + { weekStart: '2026-07-14', provider: 'Anthropic', isOpenWeights: false, tokens: 12345 }, + { weekStart: '2026-07-14', provider: 'Alibaba', isOpenWeights: true, tokens: 6789 }, + { weekStart: '2026-07-14', provider: 'other', isOpenWeights: null, tokens: 999 }, + { weekStart: '2026-07-14', provider: 'Mystery', isOpenWeights: null, tokens: 111 }, + ]); + expect(response.headers.get('Cache-Control')).toContain('public'); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + }); + + test('returns 502 when a row is malformed', async () => { + const GET = await loadGet({ rows: [['2026-07-14', 'Anthropic', 'true', 'not-a-number']] }); + + const response = await GET(); + + expect(response.status).toBe(502); + }); +}); + +describe('OPTIONS /api/public/leaderboard-provider-race', () => { + test('returns 204 with CORS headers', async () => { + const { OPTIONS } = await import('./route'); + + const response = OPTIONS(); + + expect(response.status).toBe(204); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + expect(response.headers.get('Access-Control-Allow-Methods')).toBe('GET'); + }); +}); From 8447a0a8ac826eba7a0bee94c8c3af984809308c Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Mon, 27 Jul 2026 10:50:30 -0300 Subject: [PATCH 5/6] fix(web): correct type errors in provider race route test Type the redis set mock so mockResolvedValue isn't inferred as never, and replace the namespace-style SnowflakeModule type import (TS2709) with a ReturnType alias over a type-only function import. Verified with tsgo. --- .../leaderboard-provider-race/route.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts index 541a6d8ec7..a35c35a24f 100644 --- a/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, jest } from '@jest/globals'; -import type * as SnowflakeModule from '@/lib/snowflake'; +import type { resolveSnowflakeConfig } from '@/lib/snowflake'; import type { GET as RouteGet } from './route'; jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() })); @@ -7,7 +7,7 @@ jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() })); jest.mock('@/lib/redis', () => ({ redisClient: { get: jest.fn<() => Promise>().mockResolvedValue(null), - set: jest.fn().mockResolvedValue('OK'), + set: jest.fn<() => Promise>().mockResolvedValue('OK'), }, })); @@ -16,23 +16,23 @@ jest.mock('@/lib/snowflake', () => ({ executeSnowflakeStatement: jest.fn(), })); -const FAKE_CONFIG = { accountHost: 'test.snowflakecomputing.com' }; +type SnowflakeConfig = ReturnType; + +const FAKE_CONFIG = { accountHost: 'test.snowflakecomputing.com' } as SnowflakeConfig; // The route builds a module-level in-process cache (1h TTL) at import time, so // each test loads it in an isolated module registry to keep that cache from // leaking between cases. async function loadGet(opts: { - config?: unknown; + config?: SnowflakeConfig; rows?: string[][]; statementError?: Error; }): Promise { let GET!: typeof RouteGet; await jest.isolateModulesAsync(async () => { - const snowflake = (await import('@/lib/snowflake')) as SnowflakeModule; + const snowflake = await import('@/lib/snowflake'); const config = opts.config === undefined ? FAKE_CONFIG : opts.config; - jest - .mocked(snowflake.resolveSnowflakeConfig) - .mockReturnValue(config as ReturnType); + jest.mocked(snowflake.resolveSnowflakeConfig).mockReturnValue(config); if (opts.statementError) { jest.mocked(snowflake.executeSnowflakeStatement).mockRejectedValue(opts.statementError); } else { From 44337739016a424e6e0e792ebf64411405f78d3c Mon Sep 17 00:00:00 2001 From: Pedro Heyerdahl Date: Mon, 27 Jul 2026 12:58:44 -0300 Subject: [PATCH 6/6] fix(web): exclude infra-artifact BYOK models from provider race query Filter URI/path ids (s3://, gs://...), ckpt: checkpoint refs, and HuggingFace class names (AtlasForCausalLM) out of the race aggregation. These are single-org self-hosted/eval traffic that can't be attributed to a lab and would dominate the 'other' bucket. Applied in the race query only - the dbt usage_daily base model stays complete. --- .../app/api/public/leaderboard-provider-race/route.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts index e5c80a2958..89c1fdd025 100644 --- a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts @@ -13,6 +13,12 @@ import { LEADERBOARD_PROVIDER_RACE_REDIS_KEY } from '@/lib/redis-keys'; // is_open_weights are maintained in the dbt model_dim seed (kilocode-dbt), so // the lab mapping lives in one place rather than being re-derived here. // The partial current week is excluded so every returned week is complete. +// Infrastructure-artifact BYOK model ids are excluded here (not in the base +// dbt model, which stays complete): URI/path ids (s3://, gs://, ...), ckpt: +// checkpoint refs, and HuggingFace class names (e.g. AtlasForCausalLM). These +// are customers pointing Kilo at their own checkpoints/endpoints - single-org +// noise that cannot be attributed to a model lab and would otherwise dominate +// the "other" bucket. const LEADERBOARD_PROVIDER_RACE_QUERY = ` select to_char(date_trunc('week', ud.usage_date), 'YYYY-MM-DD') as week_start @@ -24,6 +30,11 @@ where ud.usage_date >= '2025-07-01' and date_trunc('week', ud.usage_date) < date_trunc('week', current_date()) and ud.total_tokens > 0 + and not ( + ud.model like '%://%' + or ud.model ilike 'ckpt:%' + or ud.model ilike '%ForCausalLM' + ) group by 1, 2, 3 order by 1, 4 desc; `;