diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts new file mode 100644 index 0000000000..a35c35a24f --- /dev/null +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts @@ -0,0 +1,109 @@ +import { describe, test, expect, jest } from '@jest/globals'; +import type { resolveSnowflakeConfig } from '@/lib/snowflake'; +import type { GET as RouteGet } from './route'; + +jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() })); + +jest.mock('@/lib/redis', () => ({ + redisClient: { + get: jest.fn<() => Promise>().mockResolvedValue(null), + set: jest.fn<() => Promise>().mockResolvedValue('OK'), + }, +})); + +jest.mock('@/lib/snowflake', () => ({ + resolveSnowflakeConfig: jest.fn(), + executeSnowflakeStatement: jest.fn(), +})); + +type SnowflakeConfig = ReturnType; + +const FAKE_CONFIG = { accountHost: 'test.snowflakecomputing.com' } as SnowflakeConfig; + +// The route builds a module-level in-process cache (1h TTL) at import time, so +// each test loads it in an isolated module registry to keep that cache from +// leaking between cases. +async function loadGet(opts: { + config?: SnowflakeConfig; + rows?: string[][]; + statementError?: Error; +}): Promise { + let GET!: typeof RouteGet; + await jest.isolateModulesAsync(async () => { + const snowflake = await import('@/lib/snowflake'); + const config = opts.config === undefined ? FAKE_CONFIG : opts.config; + jest.mocked(snowflake.resolveSnowflakeConfig).mockReturnValue(config); + if (opts.statementError) { + jest.mocked(snowflake.executeSnowflakeStatement).mockRejectedValue(opts.statementError); + } else { + jest.mocked(snowflake.executeSnowflakeStatement).mockResolvedValue(opts.rows ?? []); + } + ({ GET } = await import('./route')); + }); + return GET; +} + +describe('GET /api/public/leaderboard-provider-race', () => { + test('returns 503 when Snowflake is not configured', async () => { + const GET = await loadGet({ config: null }); + + const response = await GET(); + + expect(response.status).toBe(503); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + }); + + test('returns 502 when the Snowflake query fails (e.g. is_open_weights backfill not landed)', async () => { + const GET = await loadGet({ statementError: new Error('invalid identifier IS_OPEN_WEIGHTS') }); + + const response = await GET(); + + expect(response.status).toBe(502); + }); + + test('maps explicit is_open_weights to booleans and NULL/unknown to null', async () => { + const GET = await loadGet({ + rows: [ + ['2026-07-14', 'Anthropic', 'false', '12345'], + ['2026-07-14', 'Alibaba', 'true', '6789'], + // The SQL API returns NULL cells as null at runtime despite the + // string[] row type; unmapped models land here. + ['2026-07-14', 'other', null, '999'] as unknown as string[], + // A future model_dim mapping gap must not be silently bucketed as closed. + ['2026-07-14', 'Mystery', 'unexpected', '111'], + ], + }); + + const response = await GET(); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual([ + { weekStart: '2026-07-14', provider: 'Anthropic', isOpenWeights: false, tokens: 12345 }, + { weekStart: '2026-07-14', provider: 'Alibaba', isOpenWeights: true, tokens: 6789 }, + { weekStart: '2026-07-14', provider: 'other', isOpenWeights: null, tokens: 999 }, + { weekStart: '2026-07-14', provider: 'Mystery', isOpenWeights: null, tokens: 111 }, + ]); + expect(response.headers.get('Cache-Control')).toContain('public'); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + }); + + test('returns 502 when a row is malformed', async () => { + const GET = await loadGet({ rows: [['2026-07-14', 'Anthropic', 'true', 'not-a-number']] }); + + const response = await GET(); + + expect(response.status).toBe(502); + }); +}); + +describe('OPTIONS /api/public/leaderboard-provider-race', () => { + test('returns 204 with CORS headers', async () => { + const { OPTIONS } = await import('./route'); + + const response = OPTIONS(); + + expect(response.status).toBe(204); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + expect(response.headers.get('Access-Control-Allow-Methods')).toBe('GET'); + }); +}); diff --git a/apps/web/src/app/api/public/leaderboard-provider-race/route.ts b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts new file mode 100644 index 0000000000..89c1fdd025 --- /dev/null +++ b/apps/web/src/app/api/public/leaderboard-provider-race/route.ts @@ -0,0 +1,91 @@ +import { z } from 'zod'; + +import { + createPublicSnowflakeReport, + publicSnowflakeReportOptions, +} from '@/lib/public-snowflake-report'; +import { LEADERBOARD_PROVIDER_RACE_REDIS_KEY } from '@/lib/redis-keys'; + +// Weekly token volume per model lab, from a fixed start date through the most +// recent complete week. Grouped at week x model_provider_company x +// is_open_weights so a single payload drives both the per-lab "race" view and +// an open-weight vs proprietary toggle. model_provider_company and +// is_open_weights are maintained in the dbt model_dim seed (kilocode-dbt), so +// the lab mapping lives in one place rather than being re-derived here. +// The partial current week is excluded so every returned week is complete. +// Infrastructure-artifact BYOK model ids are excluded here (not in the base +// dbt model, which stays complete): URI/path ids (s3://, gs://, ...), ckpt: +// checkpoint refs, and HuggingFace class names (e.g. AtlasForCausalLM). These +// are customers pointing Kilo at their own checkpoints/endpoints - single-org +// noise that cannot be attributed to a model lab and would otherwise dominate +// the "other" bucket. +const LEADERBOARD_PROVIDER_RACE_QUERY = ` +select + to_char(date_trunc('week', ud.usage_date), 'YYYY-MM-DD') as week_start + , ud.model_provider_company as provider + , ud.is_open_weights + , sum(ud.total_tokens) as tokens +from kilo_dw.dbt_prod.usage_daily as ud +where + ud.usage_date >= '2025-07-01' + and date_trunc('week', ud.usage_date) < date_trunc('week', current_date()) + and ud.total_tokens > 0 + and not ( + ud.model like '%://%' + or ud.model ilike 'ckpt:%' + or ud.model ilike '%ForCausalLM' + ) +group by 1, 2, 3 +order by 1, 4 desc; +`; + +const providerRaceSchema = z.array( + z.object({ + weekStart: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + provider: z.string().min(1), + isOpenWeights: z.boolean().nullable(), + tokens: z.number(), + }) +); + +type ProviderRace = z.infer; + +// is_open_weights is maintained in the dbt model_dim seed and is NULL for +// unmapped models (the ones bucketed as provider = 'other'). Only the explicit +// 'true'/'false' strings map to a boolean; anything else (NULL from the SQL API, +// or a future mapping gap) becomes null so the client can distinguish "unknown" +// from a confirmed closed-weight lab instead of silently bucketing gaps as closed. +function parseOpenWeights(raw: string): boolean | null { + if (raw === 'true') return true; + if (raw === 'false') return false; + return null; +} + +function parseProviderRace(rows: string[][]): ProviderRace { + return rows.map(row => { + const [weekStart, provider, rawOpenWeights, rawTokens] = row; + const tokens = Number(rawTokens); + + if (!weekStart || !provider || !Number.isFinite(tokens)) { + throw new Error('Snowflake returned an invalid provider race row'); + } + + return providerRaceSchema.element.parse({ + weekStart, + provider, + isOpenWeights: parseOpenWeights(rawOpenWeights), + tokens, + }); + }); +} + +export const GET = createPublicSnowflakeReport({ + cacheKey: LEADERBOARD_PROVIDER_RACE_REDIS_KEY, + errorMessage: 'Failed to fetch leaderboard provider race', + parseRows: parseProviderRace, + query: LEADERBOARD_PROVIDER_RACE_QUERY, + schema: providerRaceSchema, + source: 'public-leaderboard-provider-race-api', +}); + +export const OPTIONS = publicSnowflakeReportOptions; diff --git a/apps/web/src/lib/redis-keys.ts b/apps/web/src/lib/redis-keys.ts index a692943155..442443edfe 100644 --- a/apps/web/src/lib/redis-keys.ts +++ b/apps/web/src/lib/redis-keys.ts @@ -52,6 +52,7 @@ export const LEADERBOARD_MODEL_PROVIDER_USAGE_REDIS_KEY = redisKey( 'public-api:leaderboard-model-provider-usage' ); export const LEADERBOARD_MODEL_USAGE_REDIS_KEY = redisKey('public-api:leaderboard-model-usage'); +export const LEADERBOARD_PROVIDER_RACE_REDIS_KEY = redisKey('public-api:leaderboard-provider-race'); export const REQUEST_LOGGING_OPT_INS_REDIS_KEY = redisKey('ai-gateway:request-logging-opt-ins');