diff --git a/apps/web/env.ts b/apps/web/env.ts index fb50f660de..984872502e 100644 --- a/apps/web/env.ts +++ b/apps/web/env.ts @@ -2,6 +2,16 @@ import { createEnv } from "@t3-oss/env-nextjs"; import { z } from "zod"; +const llmProviderEnum = z.enum([ + "anthropic", + "google", + "openai", + "bedrock", + "openrouter", + "groq", + "ollama", +]); + export const env = createEnv({ server: { NODE_ENV: z.enum(["development", "production", "test"]), @@ -12,36 +22,21 @@ export const env = createEnv({ GOOGLE_CLIENT_SECRET: z.string().min(1), GOOGLE_ENCRYPT_SECRET: z.string(), GOOGLE_ENCRYPT_SALT: z.string(), + DEFAULT_LLM_PROVIDER: z - .enum([ - "anthropic", - // "bedrock", - "google", - "openai", - "openrouter", - "groq", - "ollama", - "custom", - ]) + .enum([...llmProviderEnum.options, "custom"]) .default("anthropic"), DEFAULT_LLM_MODEL: z.string().optional(), - // Economy LLM configuration (for large context windows where cost efficiency matters) - ECONOMY_LLM_PROVIDER: z - .enum([ - "anthropic", - "google", - "openai", - "bedrock", - "openrouter", - "groq", - "ollama", - ]) - .optional() - .default("openrouter"), - ECONOMY_LLM_MODEL: z - .string() - .optional() - .default("google/gemini-2.5-flash-preview-05-20"), + DEFAULT_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for default model (e.g., "Google Vertex,Anthropic") + // Set this to a cheaper model like Gemini Flash + ECONOMY_LLM_PROVIDER: llmProviderEnum.optional(), + ECONOMY_LLM_MODEL: z.string().optional(), + ECONOMY_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for economy model (e.g., "Google Vertex,Anthropic") + // Set this to a fast but strong model like Groq Kimi K2. Leaving blank will fallback to default which is also fine. + CHAT_LLM_PROVIDER: llmProviderEnum.optional(), + CHAT_LLM_MODEL: z.string().optional(), + CHAT_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for chat (e.g., "Google Vertex,Anthropic") + OPENAI_API_KEY: z.string().optional(), ANTHROPIC_API_KEY: z.string().optional(), BEDROCK_ACCESS_KEY: z.string().optional(), diff --git a/apps/web/utils/ai/assistant/chat.ts b/apps/web/utils/ai/assistant/chat.ts index 4507e8ff4a..fbb6291fa2 100644 --- a/apps/web/utils/ai/assistant/chat.ts +++ b/apps/web/utils/ai/assistant/chat.ts @@ -432,6 +432,7 @@ Examples: const result = chatCompletionStream({ userAi: user.user, userEmail: user.email, + modelType: "chat", usageLabel: "assistant-chat", system, messages, diff --git a/apps/web/utils/ai/assistant/process-user-request.ts b/apps/web/utils/ai/assistant/process-user-request.ts index c655901309..1f5757df08 100644 --- a/apps/web/utils/ai/assistant/process-user-request.ts +++ b/apps/web/utils/ai/assistant/process-user-request.ts @@ -200,6 +200,7 @@ ${senderCategory || "No category"} const result = await chatCompletionTools({ userAi: emailAccount.user, + modelType: "chat", messages: allMessages, tools: { update_conditional_operator: tool({ diff --git a/apps/web/utils/ai/knowledge/extract-from-email-history.ts b/apps/web/utils/ai/knowledge/extract-from-email-history.ts index e41246c7a4..28b6f3dd69 100644 --- a/apps/web/utils/ai/knowledge/extract-from-email-history.ts +++ b/apps/web/utils/ai/knowledge/extract-from-email-history.ts @@ -103,7 +103,7 @@ export async function aiExtractFromEmailHistory({ usageLabel: "Email history extraction", userAi: emailAccount.user, userEmail: emailAccount.email, - useEconomyModel: true, + modelType: "economy", }); logger.trace("Output", result.object); diff --git a/apps/web/utils/ai/knowledge/extract.ts b/apps/web/utils/ai/knowledge/extract.ts index 7260203740..2bbbde62e9 100644 --- a/apps/web/utils/ai/knowledge/extract.ts +++ b/apps/web/utils/ai/knowledge/extract.ts @@ -105,7 +105,7 @@ export async function aiExtractRelevantKnowledge({ usageLabel: "Knowledge extraction", userAi: emailAccount.user, userEmail: emailAccount.email, - useEconomyModel: true, + modelType: "economy", }); logger.trace("Output", result.object); diff --git a/apps/web/utils/llms/config.ts b/apps/web/utils/llms/config.ts index 17d5af6226..6692af38b0 100644 --- a/apps/web/utils/llms/config.ts +++ b/apps/web/utils/llms/config.ts @@ -32,6 +32,7 @@ export const Model = { GEMINI_2_0_FLASH_OPENROUTER: "google/gemini-2.0-flash-001", GEMINI_2_5_PRO_OPENROUTER: "google/gemini-2.5-pro-preview-03-25", GROQ_LLAMA_3_3_70B: "llama-3.3-70b-versatile", + KIMI_K2_OPENROUTER: "moonshotai/kimi-k2", ...(supportsOllama ? { OLLAMA: env.NEXT_PUBLIC_OLLAMA_MODEL } : {}), }; diff --git a/apps/web/utils/llms/index.ts b/apps/web/utils/llms/index.ts index b5f8426936..dd97070893 100644 --- a/apps/web/utils/llms/index.ts +++ b/apps/web/utils/llms/index.ts @@ -28,7 +28,7 @@ import { isServiceUnavailableError, } from "@/utils/error"; import { sleep } from "@/utils/sleep"; -import { getModel } from "@/utils/llms/model"; +import { getModel, type ModelType } from "@/utils/llms/model"; import { createScopedLogger } from "@/utils/logger"; const logger = createScopedLogger("llms"); @@ -41,14 +41,14 @@ const commonOptions: { export async function chatCompletion({ userAi, - useEconomyModel, + modelType = "default", prompt, system, userEmail, usageLabel, }: { userAi: UserAIFields; - useEconomyModel?: boolean; + modelType?: ModelType; prompt: string; system?: string; userEmail: string; @@ -57,7 +57,7 @@ export async function chatCompletion({ try { const { provider, model, llmModel, providerOptions } = getModel( userAi, - useEconomyModel, + modelType, ); const result = await generateText({ @@ -87,7 +87,7 @@ export async function chatCompletion({ type ChatCompletionObjectArgs = { userAi: UserAIFields; - useEconomyModel?: boolean; + modelType?: ModelType; schema: z.Schema; userEmail: string; usageLabel: string; @@ -112,7 +112,7 @@ export async function chatCompletionObject( async function chatCompletionObjectInternal({ userAi, - useEconomyModel, + modelType, system, prompt, messages, @@ -123,7 +123,7 @@ async function chatCompletionObjectInternal({ try { const { provider, model, llmModel, providerOptions } = getModel( userAi, - useEconomyModel, + modelType, ); const result = await generateObject({ @@ -155,7 +155,7 @@ async function chatCompletionObjectInternal({ export async function chatCompletionStream({ userAi, - useEconomyModel, + modelType, system, prompt, messages, @@ -167,7 +167,7 @@ export async function chatCompletionStream({ onStepFinish, }: { userAi: UserAIFields; - useEconomyModel?: boolean; + modelType?: ModelType; system?: string; prompt?: string; messages?: Message[]; @@ -184,7 +184,7 @@ export async function chatCompletionStream({ }) { const { provider, model, llmModel, providerOptions } = getModel( userAi, - useEconomyModel, + modelType, ); const result = streamText({ @@ -229,7 +229,7 @@ export async function chatCompletionStream({ type ChatCompletionToolsArgs = { userAi: UserAIFields; - useEconomyModel?: boolean; + modelType?: ModelType; tools: Record; maxSteps?: number; label: string; @@ -253,7 +253,7 @@ export async function chatCompletionTools(options: ChatCompletionToolsArgs) { async function chatCompletionToolsInternal({ userAi, - useEconomyModel, + modelType, system, prompt, messages, @@ -265,7 +265,7 @@ async function chatCompletionToolsInternal({ try { const { provider, model, llmModel, providerOptions } = getModel( userAi, - useEconomyModel, + modelType, ); const result = await generateText({ @@ -300,7 +300,7 @@ async function chatCompletionToolsInternal({ // not in use atm async function streamCompletionTools({ userAi, - useEconomyModel, + modelType, prompt, system, tools, @@ -310,7 +310,7 @@ async function streamCompletionTools({ onFinish, }: { userAi: UserAIFields; - useEconomyModel?: boolean; + modelType?: ModelType; prompt: string; system?: string; tools: Record; @@ -321,7 +321,7 @@ async function streamCompletionTools({ }) { const { provider, model, llmModel, providerOptions } = getModel( userAi, - useEconomyModel, + modelType, ); const result = await streamText({ diff --git a/apps/web/utils/llms/model.test.ts b/apps/web/utils/llms/model.test.ts index 085b0e6745..60be7d2a28 100644 --- a/apps/web/utils/llms/model.test.ts +++ b/apps/web/utils/llms/model.test.ts @@ -38,6 +38,13 @@ vi.mock("ollama-ai-provider", () => ({ vi.mock("@/env", () => ({ env: { DEFAULT_LLM_PROVIDER: "openai", + DEFAULT_OPENROUTER_PROVIDERS: "Google Vertex,Anthropic", + ECONOMY_LLM_PROVIDER: "openrouter", + ECONOMY_LLM_MODEL: "google/gemini-2.5-flash-preview-05-20", + ECONOMY_OPENROUTER_PROVIDERS: "Google Vertex,Anthropic", + CHAT_LLM_PROVIDER: "openrouter", + CHAT_LLM_MODEL: "moonshotai/kimi-k2", + CHAT_OPENROUTER_PROVIDERS: "Google Vertex,Anthropic", OPENAI_API_KEY: "test-openai-key", GOOGLE_API_KEY: "test-google-key", ANTHROPIC_API_KEY: "test-anthropic-key", @@ -211,5 +218,139 @@ describe("Models", () => { expect(() => getModel(userAi)).toThrow("LLM provider not supported"); }); + + it("should use chat model when modelType is 'chat'", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).CHAT_LLM_PROVIDER = "openrouter"; + vi.mocked(env).CHAT_LLM_MODEL = "moonshotai/kimi-k2"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "chat"); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(result.model).toBe("moonshotai/kimi-k2"); + }); + + it("should use OpenRouter with provider options for chat", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).CHAT_LLM_PROVIDER = "openrouter"; + vi.mocked(env).CHAT_LLM_MODEL = "moonshotai/kimi-k2"; + vi.mocked(env).CHAT_OPENROUTER_PROVIDERS = "Google Vertex,Anthropic"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "chat"); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(result.model).toBe("moonshotai/kimi-k2"); + expect(result.providerOptions?.openrouter?.provider?.order).toEqual([ + "Google Vertex", + "Anthropic", + ]); + }); + + it("should use economy model when modelType is 'economy'", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).ECONOMY_LLM_PROVIDER = "openrouter"; + vi.mocked(env).ECONOMY_LLM_MODEL = + "google/gemini-2.5-flash-preview-05-20"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "economy"); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(result.model).toBe("google/gemini-2.5-flash-preview-05-20"); + }); + + it("should use OpenRouter with provider options for economy", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).ECONOMY_LLM_PROVIDER = "openrouter"; + vi.mocked(env).ECONOMY_LLM_MODEL = + "google/gemini-2.5-flash-preview-05-20"; + vi.mocked(env).ECONOMY_OPENROUTER_PROVIDERS = "Google Vertex,Anthropic"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "economy"); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(result.model).toBe("google/gemini-2.5-flash-preview-05-20"); + expect(result.providerOptions?.openrouter?.provider?.order).toEqual([ + "Google Vertex", + "Anthropic", + ]); + }); + + it("should use default model when modelType is 'default'", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + const result = getModel(userAi, "default"); + expect(result.provider).toBe(Provider.OPEN_AI); + expect(result.model).toBe("gpt-4o"); + }); + + it("should use OpenRouter with provider options for default model", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).DEFAULT_LLM_PROVIDER = "openrouter"; + vi.mocked(env).DEFAULT_LLM_MODEL = "anthropic/claude-3.5-sonnet"; + vi.mocked(env).DEFAULT_OPENROUTER_PROVIDERS = "Google Vertex,Anthropic"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "default"); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(result.model).toBe("anthropic/claude-3.5-sonnet"); + expect(result.providerOptions?.openrouter?.provider?.order).toEqual([ + "Google Vertex", + "Anthropic", + ]); + }); + + it("should preserve custom logic and not override with default provider options", () => { + const userAi: UserAIFields = { + aiApiKey: null, + aiProvider: null, + aiModel: null, + }; + + vi.mocked(env).DEFAULT_LLM_PROVIDER = "custom"; + vi.mocked(env).DEFAULT_OPENROUTER_PROVIDERS = "Should Not Override"; + vi.mocked(env).OPENROUTER_API_KEY = "test-openrouter-key"; + + const result = getModel(userAi, "default"); + expect(result.provider).toBe(Provider.OPENROUTER); + // Should have custom logic provider options, not the default ones + expect(result.providerOptions?.openrouter?.provider?.order).toEqual([ + "Google Vertex", + "Google AI Studio", + "Anthropic", + ]); + // Should NOT contain the DEFAULT_OPENROUTER_PROVIDERS value + expect(result.providerOptions?.openrouter?.provider?.order).not.toContain( + "Should Not Override", + ); + }); }); }); diff --git a/apps/web/utils/llms/model.ts b/apps/web/utils/llms/model.ts index 5e2970ecc5..db7558c4aa 100644 --- a/apps/web/utils/llms/model.ts +++ b/apps/web/utils/llms/model.ts @@ -13,13 +13,16 @@ import { createScopedLogger } from "@/utils/logger"; const logger = createScopedLogger("llms/model"); -export function getModel(userAi: UserAIFields, useEconomyModel?: boolean) { - const data = useEconomyModel - ? selectEconomyModel(userAi) - : selectDefaultModel(userAi); +export type ModelType = "default" | "economy" | "chat"; + +export function getModel( + userAi: UserAIFields, + modelType: ModelType = "default", +) { + const data = selectModelByType(userAi, modelType); logger.info("Using model", { - useEconomyModel, + modelType, provider: data.provider, model: data.model, providerOptions: data.providerOptions, @@ -28,6 +31,17 @@ export function getModel(userAi: UserAIFields, useEconomyModel?: boolean) { return data; } +function selectModelByType(userAi: UserAIFields, modelType: ModelType) { + switch (modelType) { + case "economy": + return selectEconomyModel(userAi); + case "chat": + return selectChatModel(userAi); + default: + return selectDefaultModel(userAi); + } +} + function selectModel( { aiProvider, @@ -140,6 +154,21 @@ function selectModel( } } +/** + * Creates OpenRouter provider options from a comma-separated string + */ +function createOpenRouterProviderOptions( + providers: string, +): Record { + return { + openrouter: { + provider: { + order: providers.split(",").map((p: string) => p.trim()), + }, + }, + }; +} + /** * Selects the appropriate economy model for high-volume or context-heavy tasks * By default, uses a cheaper model like Gemini Flash for tasks that don't require the most powerful LLM @@ -160,11 +189,62 @@ function selectEconomyModel(userAi: UserAIFields) { return selectDefaultModel(userAi); } - return selectModel({ - aiProvider: env.ECONOMY_LLM_PROVIDER, - aiModel: env.ECONOMY_LLM_MODEL, - aiApiKey: apiKey, - }); + // Configure OpenRouter provider options if using OpenRouter for economy + let providerOptions: Record | undefined; + if ( + env.ECONOMY_LLM_PROVIDER === Provider.OPENROUTER && + env.ECONOMY_OPENROUTER_PROVIDERS + ) { + providerOptions = createOpenRouterProviderOptions( + env.ECONOMY_OPENROUTER_PROVIDERS, + ); + } + + return selectModel( + { + aiProvider: env.ECONOMY_LLM_PROVIDER, + aiModel: env.ECONOMY_LLM_MODEL, + aiApiKey: apiKey, + }, + providerOptions, + ); + } + + return selectDefaultModel(userAi); +} + +/** + * Selects the appropriate chat model for fast conversational tasks + */ +function selectChatModel(userAi: UserAIFields) { + if (env.CHAT_LLM_PROVIDER && env.CHAT_LLM_MODEL) { + const apiKey = getProviderApiKey(env.CHAT_LLM_PROVIDER); + if (!apiKey) { + logger.warn("Chat LLM provider configured but API key not found", { + provider: env.CHAT_LLM_PROVIDER, + }); + return selectDefaultModel(userAi); + } + + // Configure OpenRouter provider options if using OpenRouter for chat + let providerOptions: Record | undefined; + if ( + env.CHAT_LLM_PROVIDER === Provider.OPENROUTER && + env.CHAT_OPENROUTER_PROVIDERS + ) { + providerOptions = createOpenRouterProviderOptions( + env.CHAT_OPENROUTER_PROVIDERS, + ); + } + + return selectModel( + { + aiProvider: env.CHAT_LLM_PROVIDER, + aiModel: env.CHAT_LLM_MODEL, + aiApiKey: apiKey, + }, + providerOptions, + ); } return selectDefaultModel(userAi); @@ -247,6 +327,19 @@ function selectDefaultModel(userAi: UserAIFields) { } } + // Configure OpenRouter provider options if using OpenRouter for default model + // (but not overriding custom logic which already sets its own provider options) + if ( + aiProvider === Provider.OPENROUTER && + env.DEFAULT_OPENROUTER_PROVIDERS && + !providerOptions.openrouter + ) { + const openRouterOptions = createOpenRouterProviderOptions( + env.DEFAULT_OPENROUTER_PROVIDERS, + ); + Object.assign(providerOptions, openRouterOptions); + } + return selectModel( { aiProvider, @@ -257,17 +350,8 @@ function selectDefaultModel(userAi: UserAIFields) { ); } -function getProviderApiKey( - provider: - | "openai" - | "anthropic" - | "google" - | "groq" - | "openrouter" - | "bedrock" - | "ollama", -) { - const providerApiKeys = { +function getProviderApiKey(provider: string) { + const providerApiKeys: Record = { [Provider.ANTHROPIC]: env.ANTHROPIC_API_KEY, [Provider.OPEN_AI]: env.OPENAI_API_KEY, [Provider.GOOGLE]: env.GOOGLE_API_KEY, diff --git a/apps/web/utils/usage.ts b/apps/web/utils/usage.ts index effd9f6e53..fa3fdad561 100644 --- a/apps/web/utils/usage.ts +++ b/apps/web/utils/usage.ts @@ -126,6 +126,11 @@ const costs: Record< input: 0.2 / 1_000_000, output: 0.85 / 1_000_000, }, + // Kimi K2 Groq via OpenRouter - https://openrouter.ai/moonshotai/kimi-k2 + "moonshotai/kimi-k2": { + input: 1 / 1_000_000, + output: 3 / 1_000_000, + }, // https://groq.com/pricing "llama-3.3-70b-versatile": { input: 0.59 / 1_000_000, diff --git a/version.txt b/version.txt index c133388438..2be866b779 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v1.9.17 \ No newline at end of file +v1.9.18 \ No newline at end of file