diff --git a/apps/web/src/lib/ai-gateway/forbidden-free-models.test.ts b/apps/web/src/lib/ai-gateway/forbidden-free-models.test.ts new file mode 100644 index 0000000000..c7d1ac91ac --- /dev/null +++ b/apps/web/src/lib/ai-gateway/forbidden-free-models.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from '@jest/globals'; +import { + familyHasForbiddenFreeModel, + isForbiddenFreeModel, +} from '@/lib/ai-gateway/forbidden-free-models'; + +describe('forbidden free models', () => { + test('keeps exact matching for request rejection', () => { + expect(isForbiddenFreeModel('openai/gpt-oss-20b:free')).toBe(true); + expect(isForbiddenFreeModel('openai/gpt-oss-20b')).toBe(false); + }); + + test('matches normalized families for provider metadata', () => { + expect(familyHasForbiddenFreeModel('openai/gpt-oss-20b:free')).toBe(true); + expect(familyHasForbiddenFreeModel('openai/gpt-oss-20b')).toBe(true); + expect(familyHasForbiddenFreeModel('cohere/north-mini-code')).toBe(false); + }); +}); diff --git a/apps/web/src/lib/ai-gateway/forbidden-free-models.ts b/apps/web/src/lib/ai-gateway/forbidden-free-models.ts index a879e7af45..9ce8522da0 100644 --- a/apps/web/src/lib/ai-gateway/forbidden-free-models.ts +++ b/apps/web/src/lib/ai-gateway/forbidden-free-models.ts @@ -1,4 +1,5 @@ import { claude_sonnet_clawsetup_model } from '@/lib/ai-gateway/providers/anthropic.constants'; +import { normalizeModelId } from '@/lib/ai-gateway/model-utils'; const forbiddenFreeModelIds: ReadonlySet = new Set([ 'auto:free', // this is not a free model, OpenRouter can map it to a paid model @@ -56,3 +57,11 @@ const forbiddenFreeModelIds: ReadonlySet = new Set([ export function isForbiddenFreeModel(modelId: string): boolean { return forbiddenFreeModelIds.has(modelId); } + +const forbiddenFreeModelFamilies: ReadonlySet = new Set( + [...forbiddenFreeModelIds].map(normalizeModelId) +); + +export function familyHasForbiddenFreeModel(modelId: string): boolean { + return forbiddenFreeModelFamilies.has(normalizeModelId(modelId)); +} diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.test.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.test.ts new file mode 100644 index 0000000000..8e783dd699 --- /dev/null +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, test } from '@jest/globals'; +import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model'; +import { + applyFreeEndpointDataPolicy, + getOpenRouterFreeEndpoints, +} from '@/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy'; +import type { OpenRouterModel } from '@/lib/ai-gateway/providers/openrouter/openrouter-types'; + +function offering(slug: string, isFree = false): OpenRouterModel { + return { + slug, + name: slug, + author: 'test', + description: '', + context_length: 1, + input_modalities: ['text'], + output_modalities: ['text'], + group: 'test', + updated_at: '2026-07-27T00:00:00.000Z', + endpoint: { + provider_display_name: 'Test', + is_free: isFree, + pricing: { prompt: '0.1', completion: '0.2' }, + data_policy: { training: false, retainsPrompts: false }, + }, + }; +} + +function freeExclusiveModel( + public_id: string, + inference_provider_restriction: KiloExclusiveModel['inference_provider_restriction'] +): KiloExclusiveModel { + return { + public_id, + internal_id: public_id, + display_name: public_id, + description: '', + context_length: 1, + max_completion_tokens: 1, + status: 'public', + flags: [], + gateway: 'openrouter', + pricing: null, + inference_provider_restriction, + }; +} + +function dataCollectionExclusiveModel( + public_id: string, + inference_provider_restriction: KiloExclusiveModel['inference_provider_restriction'] +): KiloExclusiveModel { + const model = freeExclusiveModel(public_id, inference_provider_restriction); + model.pricing = [ + { + start_context_length: 0, + pricing: { + prompt_per_million: 1, + completion_per_million: 1, + input_cache_read_per_million: null, + input_cache_write_per_million: null, + }, + }, + ]; + model.flags = ['requires-data-collection']; + return model; +} + +describe('applyFreeEndpointDataPolicy', () => { + test('applies free OpenRouter endpoint policy by normalized model and provider', () => { + const freeVariant = offering('example/model:free', true); + const matching = offering('example/model'); + const otherProvider = offering('example/model'); + const providerModelData = [ + { provider: { slug: 'novita' }, models: [freeVariant, matching] }, + { provider: { slug: 'deepinfra' }, models: [otherProvider] }, + ]; + const openRouterFreeEndpoints = getOpenRouterFreeEndpoints(providerModelData); + + applyFreeEndpointDataPolicy({ + providerModelData, + openRouterFreeEndpoints, + kiloExclusiveModels: [], + }); + + expect(freeVariant.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + expect(matching.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + expect(otherProvider.endpoint?.data_policy).toEqual({ + training: false, + retainsPrompts: false, + }); + }); + + test('does not infer free status from zero token prices', () => { + const model = offering('example/model'); + if (!model.endpoint) throw new Error('expected endpoint'); + model.endpoint.pricing = { prompt: '0', completion: '0' }; + + applyFreeEndpointDataPolicy({ + providerModelData: [{ provider: { slug: 'novita' }, models: [model] }], + openRouterFreeEndpoints: getOpenRouterFreeEndpoints([ + { provider: { slug: 'novita' }, models: [model] }, + ]), + kiloExclusiveModels: [], + }); + + expect(model.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); + + test('applies an unrestricted free exclusive model to every provider', () => { + const first = offering('example/model'); + const second = offering('example/model'); + + applyFreeEndpointDataPolicy({ + providerModelData: [ + { provider: { slug: 'novita' }, models: [first] }, + { provider: { slug: 'deepinfra' }, models: [second] }, + ], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [freeExclusiveModel('example/model:free', [])], + }); + + expect(first.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + expect(second.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + }); + + test('ignores hidden free exclusive models', () => { + const model = offering('example/model'); + const hidden = freeExclusiveModel('example/model:free', []); + hidden.status = 'hidden'; + + applyFreeEndpointDataPolicy({ + providerModelData: [{ provider: { slug: 'novita' }, models: [model] }], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [hidden], + }); + + expect(model.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); + + test('applies public data-collection exclusive model policy', () => { + const model = offering('example/model'); + + applyFreeEndpointDataPolicy({ + providerModelData: [{ provider: { slug: 'deepseek' }, models: [model] }], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [dataCollectionExclusiveModel('example/model:discounted', ['deepseek'])], + }); + + expect(model.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + }); + + test('respects free exclusive model provider restrictions', () => { + const allowed = offering('example/model'); + const blocked = offering('example/model'); + + applyFreeEndpointDataPolicy({ + providerModelData: [ + { provider: { slug: 'stepfun' }, models: [allowed] }, + { provider: { slug: 'novita' }, models: [blocked] }, + ], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [freeExclusiveModel('example/model:free', ['stepfun'])], + }); + + expect(allowed.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + expect(blocked.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); + + test('combines restrictions from normalized free exclusive variants', () => { + const first = offering('example/model'); + const second = offering('example/model'); + + applyFreeEndpointDataPolicy({ + providerModelData: [ + { provider: { slug: 'stepfun' }, models: [first] }, + { provider: { slug: 'novita' }, models: [second] }, + ], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [ + freeExclusiveModel('example/model:free', ['stepfun']), + freeExclusiveModel('example/model:promo', ['novita']), + ], + }); + + expect(first.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + expect(second.endpoint?.data_policy).toEqual({ training: true, retainsPrompts: true }); + }); + + test('ignores disabled and paid exclusive models without data collection', () => { + const model = offering('example/model'); + const disabled = freeExclusiveModel('example/model:free', []); + disabled.status = 'disabled'; + const paid = freeExclusiveModel('example/model:discounted', []); + paid.pricing = [ + { + start_context_length: 0, + pricing: { + prompt_per_million: 1, + completion_per_million: 1, + input_cache_read_per_million: null, + input_cache_write_per_million: null, + }, + }, + ]; + + applyFreeEndpointDataPolicy({ + providerModelData: [{ provider: { slug: 'novita' }, models: [model] }], + openRouterFreeEndpoints: [], + kiloExclusiveModels: [disabled, paid], + }); + + expect(model.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); + + test('ignores forbidden OpenRouter and exclusive free models', () => { + const freeVariant = offering('openai/gpt-oss-20b', true); + const model = offering('openai/gpt-oss-20b'); + const providerModelData = [{ provider: { slug: 'darkbloom' }, models: [freeVariant, model] }]; + + applyFreeEndpointDataPolicy({ + providerModelData, + openRouterFreeEndpoints: getOpenRouterFreeEndpoints(providerModelData), + kiloExclusiveModels: [freeExclusiveModel('openai/gpt-oss-20b:free', [])], + }); + + expect(freeVariant.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + expect(model.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); + + test('does not collect free keys from offerings injected later', () => { + const model = offering('example/model'); + const providerModelData = [{ provider: { slug: 'novita' }, models: [model] }]; + const openRouterFreeEndpoints = getOpenRouterFreeEndpoints(providerModelData); + providerModelData[0].models.push(offering('example/model:free', true)); + + applyFreeEndpointDataPolicy({ + providerModelData, + openRouterFreeEndpoints, + kiloExclusiveModels: [], + }); + + expect(model.endpoint?.data_policy).toEqual({ training: false, retainsPrompts: false }); + }); +}); diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.ts new file mode 100644 index 0000000000..6d84de91df --- /dev/null +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy.ts @@ -0,0 +1,95 @@ +import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model'; +import { + familyHasForbiddenFreeModel, + isForbiddenFreeModel, +} from '@/lib/ai-gateway/forbidden-free-models'; +import { normalizeModelId } from '@/lib/ai-gateway/model-utils'; +import { normalizeInferenceProviderId } from '@/lib/ai-gateway/providers/openrouter/inference-provider-id'; +import type { OpenRouterModel } from '@/lib/ai-gateway/providers/openrouter/openrouter-types'; + +type ProviderModels = Array<{ + provider: { slug: string }; + models: OpenRouterModel[]; +}>; + +export type OpenRouterFreeEndpoint = { + modelId: string; + providerId: string; +}; + +export function getOpenRouterFreeEndpoints( + providerModelData: ProviderModels +): OpenRouterFreeEndpoint[] { + const endpoints: OpenRouterFreeEndpoint[] = []; + for (const { provider, models } of providerModelData) { + const providerId = normalizeInferenceProviderId(provider.slug); + for (const model of models) { + if (model.endpoint?.is_free && !familyHasForbiddenFreeModel(model.slug)) { + endpoints.push({ modelId: normalizeModelId(model.slug), providerId }); + } + } + } + return endpoints; +} + +function dataCollectingKiloExclusiveModels( + kiloExclusiveModels: ReadonlyArray +): Map | null> { + const models = new Map | null>(); + for (const model of kiloExclusiveModels) { + const collectsData = model.pricing === null || model.flags.includes('requires-data-collection'); + if (model.status !== 'public' || !collectsData || isForbiddenFreeModel(model.public_id)) + continue; + const modelId = normalizeModelId(model.public_id); + if (model.inference_provider_restriction.length === 0) { + models.set(modelId, null); + continue; + } + const existing = models.get(modelId); + if (existing === null) continue; + models.set( + modelId, + new Set([ + ...(existing ?? []), + ...model.inference_provider_restriction.map(providerId => + normalizeInferenceProviderId(providerId) + ), + ]) + ); + } + return models; +} + +export function applyFreeEndpointDataPolicy({ + providerModelData, + openRouterFreeEndpoints, + kiloExclusiveModels, +}: { + providerModelData: ProviderModels; + openRouterFreeEndpoints: ReadonlyArray; + kiloExclusiveModels: ReadonlyArray; +}): void { + const exclusiveModels = dataCollectingKiloExclusiveModels(kiloExclusiveModels); + + for (const { provider, models } of providerModelData) { + const providerId = normalizeInferenceProviderId(provider.slug); + for (const model of models) { + if (!model.endpoint) continue; + + const modelId = normalizeModelId(model.slug); + const restrictions = exclusiveModels.get(modelId); + const hasFreeExclusiveEndpoint = + exclusiveModels.has(modelId) && (restrictions === null || restrictions?.has(providerId)); + const hasFreeOpenRouterEndpoint = openRouterFreeEndpoints.some( + endpoint => endpoint.modelId === modelId && endpoint.providerId === providerId + ); + if (!hasFreeExclusiveEndpoint && !hasFreeOpenRouterEndpoint) continue; + + model.endpoint.data_policy = { + ...model.endpoint.data_policy, + training: true, + retainsPrompts: true, + }; + } + } +} diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/sync-providers.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/sync-providers.ts index 3362d904ec..fab738fb00 100644 --- a/apps/web/src/lib/ai-gateway/providers/openrouter/sync-providers.ts +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/sync-providers.ts @@ -41,6 +41,11 @@ import { openRouterToVercelInferenceProviderId, VercelInferenceProviderIdSchema, } from '@/lib/ai-gateway/providers/openrouter/inference-provider-id'; +import { + applyFreeEndpointDataPolicy, + getOpenRouterFreeEndpoints, +} from '@/lib/ai-gateway/providers/openrouter/free-endpoint-data-policy'; +import { isForbiddenFreeModel } from '@/lib/ai-gateway/forbidden-free-models'; /** * Advisory lock key hashed from a stable identifier. Serializes concurrent @@ -265,6 +270,7 @@ async function syncProviders( }) ) ); + const openRouterFreeEndpoints = getOpenRouterFreeEndpoints(providerModelData); injectExtraProviderModels(vercelModels, providerModelData); @@ -295,6 +301,10 @@ async function syncProviders( prompt: model.pricing.prompt, completion: model.pricing.completion, }, + ...((!kfm.pricing || kfm.flags.includes('requires-data-collection')) && + !isForbiddenFreeModel(kfm.public_id) && { + data_policy: { training: true, retainsPrompts: true }, + }), }, }, provider: inferenceProvider, @@ -313,6 +323,12 @@ async function syncProviders( } } + applyFreeEndpointDataPolicy({ + providerModelData, + openRouterFreeEndpoints, + kiloExclusiveModels, + }); + // Filter out providers with no models const filteredProviderModelData = providerModelData.filter(data => data.models.length > 0);