diff --git a/changelog.d/fixes/15396-unify-free-provider-lists.md b/changelog.d/fixes/15396-unify-free-provider-lists.md new file mode 100644 index 000000000000..18ae8d01b062 --- /dev/null +++ b/changelog.d/fixes/15396-unify-free-provider-lists.md @@ -0,0 +1 @@ +- **fix(rankings):** free provider ranking now follows the documented free-model catalog ([#15396](https://github.com/diegosouzapw/OmniRoute/pull/15396)) — thanks @maxmad64bis diff --git a/src/lib/freeProviderRankings.ts b/src/lib/freeProviderRankings.ts index 2fb000987eda..e762110ddf25 100644 --- a/src/lib/freeProviderRankings.ts +++ b/src/lib/freeProviderRankings.ts @@ -9,6 +9,7 @@ */ import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/constants/providers"; +import { providerHasFreeModels } from "@/shared/utils/freeModels"; import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry"; import { listModelIntelligence } from "./db/modelIntelligence"; import { getProviderConnections } from "./db/providers"; @@ -82,29 +83,35 @@ function getFreeProviders() { }); } - // OAuth providers with free tier + // OAuth providers with a documented free tier for (const [id, p] of Object.entries(OAUTH_PROVIDERS)) { - if ("hasFree" in p && p.hasFree) { + if (providerHasFreeModels(id)) { providers.push({ id, name: p.name, icon: p.icon, color: p.color, - textIcon: "textIcon" in p ? (p as any).textIcon : undefined, + textIcon: + typeof p === "object" && p !== null && "textIcon" in p + ? ((p as { textIcon?: unknown }).textIcon as string | undefined) + : undefined, category: "oauth", }); } } - // API key providers with free tier + // API key providers with a documented free tier for (const [id, p] of Object.entries(APIKEY_PROVIDERS)) { - if ("hasFree" in p && p.hasFree) { + if (providerHasFreeModels(id)) { providers.push({ id, name: p.name, icon: p.icon, color: p.color, - textIcon: "textIcon" in p ? (p as any).textIcon : undefined, + textIcon: + typeof p === "object" && p !== null && "textIcon" in p + ? ((p as { textIcon?: unknown }).textIcon as string | undefined) + : undefined, category: "apikey", }); } diff --git a/tests/unit/free-provider-rankings-catalog-unification.test.ts b/tests/unit/free-provider-rankings-catalog-unification.test.ts new file mode 100644 index 000000000000..4954be994317 --- /dev/null +++ b/tests/unit/free-provider-rankings-catalog-unification.test.ts @@ -0,0 +1,93 @@ +/** + * Ranking follows the documented free-model catalog. + * + * `computeFreeProviderRankings()` must agree with `providerHasFreeModels()`: + * providers whose only claim to a free tier is the static `hasFree` flag but + * which have no documented free model stay out of the ranking, while catalog + * providers with `hasFree: false` stay in. Provider ids and model ids come + * from the production readers (constants + registry), never hand-written. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rankings-catalog-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const intelligenceDb = await import("../../src/lib/db/modelIntelligence.ts"); +const rankings = await import("../../src/lib/freeProviderRankings.ts"); +const { REGISTRY } = await import("../../open-sse/config/providerRegistry.ts"); +const { providerHasFreeModels } = await import("../../src/shared/utils/freeModels.ts"); + +function registryModelIds(providerId: string): string[] { + const entry = REGISTRY[providerId] as { models?: Array<{ id?: string }> } | undefined; + return (entry?.models ?? []) + .map((m) => m.id) + .filter((id): id is string => typeof id === "string"); +} + +function seedScores(modelIds: string[]): void { + for (const model of modelIds) { + intelligenceDb.upsertModelIntelligence({ + model, + source: "arena_elo", + category: "default", + score: 0.85, + eloRaw: 1300, + confidence: "high", + expiresAt: null, + }); + } +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("ranking excludes providers with a raw free flag but no documented free model", async () => { + const outside = ["sarvam", "modal", "openference"]; + for (const id of outside) { + assert.equal(providerHasFreeModels(id), false, `${id} must be outside the free catalog`); + } + for (const id of outside) { + seedScores(registryModelIds(id)); + } + + const ranking = await rankings.computeFreeProviderRankings(undefined, 200, {}); + const ids = new Set(ranking.map((r) => r.id)); + for (const id of outside) { + assert.ok(!ids.has(id), `${id} must stay out of the free ranking`); + } +}); + +test("ranking keeps catalog providers whose static free flag is off", async () => { + const cataloged = ["predibase", "publicai"]; + for (const id of cataloged) { + assert.equal(providerHasFreeModels(id), true, `${id} must be in the free catalog`); + } + for (const id of cataloged) { + seedScores(registryModelIds(id)); + } + + const ranking = await rankings.computeFreeProviderRankings(undefined, 200, {}); + const ids = new Set(ranking.map((r) => r.id)); + for (const id of cataloged) { + assert.ok(ids.has(id), `${id} must appear in the free ranking`); + } +}); + +test("ranking still lists always-free providers from the free catalog", async () => { + const id = "aihorde"; + assert.equal(providerHasFreeModels(id), true, `${id} must be in the free catalog`); + seedScores(registryModelIds(id)); + + const ranking = await rankings.computeFreeProviderRankings(undefined, 200, {}); + assert.ok( + ranking.some((r) => r.id === id), + `${id} must appear in the free ranking` + ); +});