Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions apps/web/src/lib/ai-gateway/model-api-kinds.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ describe('modelServesAllGatewayChatApis', () => {

it('rejects a Kilo-exclusive model served by a provider without Messages support', () => {
expect(modelServesAllGatewayChatApis('test-exclusive/alibaba-only')).toBe(false);
expect(gatewayChatApisForModel('test-exclusive/alibaba-only')).toEqual([
'chat_completions',
'responses',
]);
expect(gatewayChatApisForModel('test-exclusive/alibaba-only')).toEqual(['chat_completions']);
});

it('treats disabled Kilo-exclusive models like plain OpenRouter models, matching get-provider', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ describe('getAiSdkProvider', () => {
expect(getAiSdkProvider('morph/morph-gpt-compatible', 'morph-byok')).toBe('openai-compatible');
});

test('uses the Anthropic Messages API for MiniMax models through the gateway', () => {
expect(getAiSdkProvider('minimax/minimax-m2.5', null)).toBe('anthropic');
test('uses Chat Completions for MiniMax models through the gateway', () => {
expect(getAiSdkProvider('minimax/minimax-m2.5', null)).toBeUndefined();
});
});
9 changes: 9 additions & 0 deletions apps/web/src/lib/ai-gateway/providers/model-settings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ describe('getOpenRouterDerivedModelVariants', () => {
endpoints: [],
reasoning: { mandatory: false, supported_efforts: supportedEfforts },
},
'moonshotai/kimi-k3': {
id: 'moonshotai/kimi-k3',
name: 'Kimi K3',
endpoints: [],
reasoning: { mandatory: false, supported_efforts: supportedEfforts },
},
})),
}));
const { getOpenRouterDerivedModelVariants } =
Expand All @@ -29,5 +35,8 @@ describe('getOpenRouterDerivedModelVariants', () => {
'max',
]);
expect(supportedEfforts).toEqual(['max', 'high', 'medium', 'low', 'minimal']);

const kimiVariants = await getOpenRouterDerivedModelVariants('moonshotai/kimi-k3');
expect(Object.keys(kimiVariants ?? {})).toEqual(['minimal', 'low', 'medium', 'high', 'max']);
});
});
15 changes: 9 additions & 6 deletions apps/web/src/lib/ai-gateway/providers/model-settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,13 @@ export async function getOpenRouterDerivedModelVariants(
verbosity: useAnthropicProvider ? VerbositySchema.safeParse(effort).data : undefined,
},
]);
if (!reasoning.mandatory && !variants.some(([effort]) => effort === 'none')) {
if (
!reasoning.mandatory &&
!variants.some(([effort]) => effort === 'none') &&
// Kimi does not support non-reasoning despite what OpenRouter says
// https://huggingface.co/moonshotai/Kimi-K3#6-model-usage
model !== 'moonshotai/kimi-k3'
Comment thread
chrarnoldus marked this conversation as resolved.
) {
variants.unshift(['none', { reasoning: { enabled: false, effort: 'none' } }]);
}
return Object.fromEntries(variants);
Expand Down Expand Up @@ -71,14 +77,11 @@ export function getAiSdkProvider(
if (directProviderId === 'opencode-go' && (isMinimaxModel(model) || isQwenModel(model))) {
return 'anthropic';
}
if (
isClaudeModel(model) || // on Vercel AI Gateway, this is necessary to support document attachments
(!directProviderId && isMinimaxModel(model)) // on Vercel AI Gateway, this is necessary for reasoning to show
) {
if (isClaudeModel(model)) {
return 'anthropic';
}
if (isOpenAiModel(model) || isGrokModel(model) || isMuseModel(model)) {
// OpenAI: "While Chat Completions remains supported, Responses is recommended for all new projects.""
// OpenAI: "While Chat Completions remains supported, Responses is recommended for all new projects."
// xAI: "The Responses API is the recommended way to interact with xAI models."
return 'openai';
}
Expand Down
8 changes: 6 additions & 2 deletions apps/web/src/lib/ai-gateway/providers/openrouter/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,12 @@ export type OpenRouterChatCompletionRequest = OpenAI.Chat.ChatCompletionCreatePa
// https://openrouter.ai/docs/use-cases/reasoning-tokens#controlling-reasoning-tokens
reasoning?: OpenRouterReasoningConfig;

// https://platform.minimax.io/docs/api-reference/text-openai-api#4-important-note
reasoning_split?: boolean;
// https://friendli.ai/docs/guides/reasoning
chat_template_kwargs?: {
enable_thinking: boolean;
};
parse_reasoning?: boolean;
include_reasoning?: boolean;
};

export type MessageWithReasoning = {
Expand Down
33 changes: 24 additions & 9 deletions apps/web/src/lib/ai-gateway/providers/partner-routing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ afterAll(() => {
});

function request(
kind: GatewayRequest['kind'] = 'messages',
kind: GatewayRequest['kind'] = 'chat_completions',
provider?: GatewayRequest['body']['provider']
): GatewayRequest {
if (kind === 'responses') {
Expand Down Expand Up @@ -84,37 +84,52 @@ describe('getPercentageRoutedPartnerProvider', () => {
expect(selectPartner({ requestedModel: model })).toBe(expectedProvider);
});

it.each(['chat_completions', 'messages', 'responses'] as const)('routes %s requests', kind => {
expect(selectPartner({ request: request(kind) })).toBe(PROVIDERS.FRIENDLI_GLM);
it('routes chat completions requests', () => {
expect(selectPartner()).toBe(PROVIDERS.FRIENDLI_GLM);
});

it.each(['messages', 'responses'] as const)('does not route untested %s requests', kind => {
expect(selectPartner({ request: request(kind) })).toBeNull();
});

it('allows an empty provider object', () => {
expect(selectPartner({ request: request('messages', {}) })).toBe(PROVIDERS.FRIENDLI_GLM);
expect(selectPartner({ request: request('chat_completions', {}) })).toBe(
PROVIDERS.FRIENDLI_GLM
);
});

it.each([
[FRIENDLI_GLM_PUBLIC_ID, 'friendli', PROVIDERS.FRIENDLI_GLM],
[PERPLEXITY_KIMI_PUBLIC_ID, 'perplexity', PROVIDERS.PERPLEXITY_KIMI],
])('honors provider filters for %s', (requestedModel, providerId, expectedProvider) => {
expect(
selectPartner({ requestedModel, request: request('messages', { only: [providerId] }) })
selectPartner({
requestedModel,
request: request('chat_completions', { only: [providerId] }),
})
).toBe(expectedProvider);
expect(
selectPartner({ requestedModel, request: request('messages', { ignore: ['openai'] }) })
selectPartner({
requestedModel,
request: request('chat_completions', { ignore: ['openai'] }),
})
).toBe(expectedProvider);
expect(
selectPartner({ requestedModel, request: request('messages', { only: ['openai'] }) })
selectPartner({ requestedModel, request: request('chat_completions', { only: ['openai'] }) })
).toBeNull();
expect(
selectPartner({ requestedModel, request: request('messages', { ignore: [providerId] }) })
selectPartner({
requestedModel,
request: request('chat_completions', { ignore: [providerId] }),
})
).toBeNull();
});

it.each([
[FRIENDLI_GLM_PUBLIC_ID, { data_collection: 'deny' } as const, PROVIDERS.FRIENDLI_GLM],
[PERPLEXITY_KIMI_PUBLIC_ID, { zdr: true } as const, PROVIDERS.PERPLEXITY_KIMI],
])('routes ZDR model %s with data policy options', (requestedModel, provider, expected) => {
expect(selectPartner({ requestedModel, request: request('messages', provider) })).toBe(
expect(selectPartner({ requestedModel, request: request('chat_completions', provider) })).toBe(
expected
);
});
Expand Down
45 changes: 38 additions & 7 deletions apps/web/src/lib/ai-gateway/providers/provider-definitions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,21 @@ describe.each([
{
name: 'Friendli GLM',
provider: PROVIDERS.FRIENDLI_GLM,
expectedUrl: 'https://api.friendli.ai/serverless/v1/messages',
expectedUrl: 'https://api.friendli.ai/serverless/v1/chat/completions',
requestedModel: FRIENDLI_GLM_PUBLIC_ID,
upstreamModel: 'zai-org/GLM-5.2',
},
{
name: 'Perplexity Kimi',
provider: PROVIDERS.PERPLEXITY_KIMI,
expectedUrl: 'https://api.perplexity.ai/router/v1/messages',
expectedUrl: 'https://api.perplexity.ai/router/v1/chat/completions',
requestedModel: PERPLEXITY_KIMI_PUBLIC_ID,
upstreamModel: 'perplexity/kimi-k3',
},
])('$name provider', ({ provider, expectedUrl, requestedModel, upstreamModel }) => {
test('supports all gateway chat APIs', () => {
expect(`${provider.apiUrl}/messages`).toBe(expectedUrl);
expect(provider.supportedChatApis).toEqual(['chat_completions', 'messages', 'responses']);
test('supports the chat completions API', () => {
expect(`${provider.apiUrl}/chat/completions`).toBe(expectedUrl);
expect(provider.supportedChatApis).toEqual(['chat_completions']);
});

test('enables the reasoning details response transform', () => {
Expand All @@ -66,10 +66,9 @@ describe.each([

test('hardwires the upstream model and removes provider settings', async () => {
const request: GatewayRequest = {
kind: 'messages',
kind: 'chat_completions',
body: {
model: requestedModel,
max_tokens: 1_024,
messages: [{ role: 'user', content: 'hello' }],
provider: { order: ['friendli'] },
},
Expand All @@ -81,3 +80,35 @@ describe.each([
expect(request.body.provider).toBeUndefined();
});
});

describe('Friendli GLM reasoning', () => {
test.each([
[
{ enabled: false as const, effort: 'none' as const },
{ chat_template_kwargs: { enable_thinking: false } },
],
[
{ enabled: true as const, effort: 'high' as const },
{
chat_template_kwargs: { enable_thinking: true },
reasoning_effort: 'high',
parse_reasoning: true,
include_reasoning: true,
},
],
])('maps reasoning %p to Friendli settings', async (reasoning, expected) => {
const request: GatewayRequest = {
kind: 'chat_completions',
body: {
model: FRIENDLI_GLM_PUBLIC_ID,
messages: [{ role: 'user', content: 'hello' }],
reasoning,
},
};

await PROVIDERS.FRIENDLI_GLM.transformRequest({ request } as TransformRequestContext);

expect(request.body).toMatchObject(expected);
expect(request.body.reasoning).toBeUndefined();
});
});
44 changes: 33 additions & 11 deletions apps/web/src/lib/ai-gateway/providers/provider-definitions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ export default {
apiUrl: 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1',
apiUrlOverrides: {},
apiKey: getEnvVariable('ALIBABA_API_KEY'),
supportedChatApis: ['chat_completions', 'responses'],
supportedChatApis: [
'chat_completions',
// 'responses', // supported, not tested
],
responseTransforms: null,
async transformRequest(context) {
context.request.body.enable_thinking = !isReasoningExplicitlyDisabled(context.request);
Expand All @@ -32,7 +35,10 @@ export default {
apiUrl: 'https://ark.ap-southeast.bytepluses.com/api/v3',
apiUrlOverrides: {},
apiKey: getEnvVariable('BYTEDANCE_API_KEY'),
supportedChatApis: ['chat_completions', 'responses'],
supportedChatApis: [
'chat_completions',
// 'responses', // supported, not tested
],
responseTransforms: null,
async transformRequest(context) {
if (!isReasoningExplicitlyDisabled(context.request)) {
Expand Down Expand Up @@ -90,27 +96,43 @@ export default {
apiUrl: 'https://api.friendli.ai/serverless/v1',
apiUrlOverrides: {},
apiKey: getEnvVariable('FRIENDLI_API_KEY'),
supportedChatApis: ['chat_completions', 'messages', 'responses'],
supportedChatApis: [
'chat_completions',
// 'messages', // supported, not tested
// 'responses', // supported, not tested
],
responseTransforms: { mapGeminiThoughtContent: false, mapReasoningContentToDetails: true },
async transformRequest(context) {
context.request.body.model = 'zai-org/GLM-5.2';
delete context.request.body.provider;

if (context.request.kind === 'chat_completions') {
context.request.body.reasoning_effort = isReasoningExplicitlyDisabled(context.request)
? 'none'
: (context.request.body.reasoning?.effort ??
context.request.body.reasoning_effort ??
undefined);
delete context.request.body.reasoning;
const requestBody = context.request.body;
if (isReasoningExplicitlyDisabled(context.request)) {
requestBody.chat_template_kwargs = { enable_thinking: false };
delete requestBody.reasoning;
delete requestBody.reasoning_effort;
} else {
requestBody.chat_template_kwargs = { enable_thinking: true };
requestBody.reasoning_effort =
requestBody.reasoning?.effort ?? requestBody.reasoning_effort ?? undefined;
requestBody.parse_reasoning = true;
requestBody.include_reasoning = true;
delete requestBody.reasoning;
}
}
delete context.request.body.provider;
},
},
PERPLEXITY_KIMI: {
id: 'perplexity',
apiUrl: 'https://api.perplexity.ai/router/v1',
apiUrlOverrides: {},
apiKey: getEnvVariable('PERPLEXITY_API_KEY'),
supportedChatApis: ['chat_completions', 'messages', 'responses'],
supportedChatApis: [
'chat_completions',
// 'messages', // supported, not tested
// 'responses', // supported, not tested
],
responseTransforms: { mapGeminiThoughtContent: false, mapReasoningContentToDetails: true },
async transformRequest(context) {
context.request.body.model = 'perplexity/kimi-k3';
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/lib/ai-gateway/providers/variants.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe('getFallbackModelVariants', () => {
['google/gemini-3.1-pro-preview', ['minimal', 'low', 'medium', 'high']],
['z-ai/glm-5.2', ['none', 'high', 'xhigh']],
['xai/grok-4.5', ['low', 'medium', 'high']],
['moonshotai/kimi-k3', ['none', 'low', 'high', 'max']],
['moonshotai/kimi-k3', ['low', 'high', 'max']],
['inception/mercury-2', ['instant', 'low', 'medium', 'high']],
['meta/muse-1', ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']],
['nvidia/nemotron-3-super-120b-a12b:free', ['none', 'medium', 'high']],
Expand Down
8 changes: 7 additions & 1 deletion apps/web/src/lib/ai-gateway/providers/variants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,12 @@ export const REASONING_VARIANTS_NONE_LOW_HIGH_MAX = {
max: { reasoning: { enabled: true, effort: 'max' } },
} as const;

export const REASONING_VARIANTS_LOW_HIGH_MAX = {
low: { reasoning: { enabled: true, effort: 'low' } },
high: { reasoning: { enabled: true, effort: 'high' } },
max: { reasoning: { enabled: true, effort: 'max' } },
} as const;

const REASONING_VARIANTS_CLAUDE = {
none: { reasoning: { enabled: false, effort: 'none' } },
low: { reasoning: { enabled: true, effort: 'low' }, verbosity: 'low' },
Expand Down Expand Up @@ -107,7 +113,7 @@ export function getFallbackModelVariants(model: string): OpenCodeSettings['varia
return REASONING_VARIANTS_LOW_MEDIUM_HIGH;
}
if (isKimiModel(model)) {
return REASONING_VARIANTS_NONE_LOW_HIGH_MAX;
return REASONING_VARIANTS_LOW_HIGH_MAX;
}
if (isLongCatModel(model)) {
return REASONING_VARIANTS_BINARY;
Expand Down
2 changes: 0 additions & 2 deletions packages/db/src/schema-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1989,8 +1989,6 @@ export const CustomLlmProviderSchema = z.enum([
'openai', // uses Responses API
'openai-compatible', // uses Chat Completions API with reasoning_content
'openrouter', // uses Chat Completions API with reasoning_details
'alibaba', // identical to openai-compatible, but reports cache write tokens that alibaba bills separately
'mistral', // uses Chat Completions API with possibly complex content objects for e.g. thinking
]);

export type CustomLlmProvider = z.infer<typeof CustomLlmProviderSchema>;
Expand Down