Skip to content

Add seperate chat model #577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
elie222 merged 5 commits into from
Jul 18, 2025
Merged

Add seperate chat model #577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dfe92cc
Add seperate chat model
elie222 Jul 17, 2025
fd13809
Optional providers for economy and chat models
elie222 Jul 17, 2025
2546dc4
allow
elie222 Jul 17, 2025
2536564
Merge branch 'main' into feat/chat-model
elie222 Jul 18, 2025
d0f1b08
fix bug from failing test
elie222 Jul 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 22 additions & 27 deletions apps/web/env.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
import { createEnv } from "@t3-oss/env-nextjs";
import { z } from "zod";

const llmProviderEnum = z.enum([
"anthropic",
"google",
"openai",
"bedrock",
"openrouter",
"groq",
"ollama",
]);

export const env = createEnv({
server: {
NODE_ENV: z.enum(["development", "production", "test"]),
Expand All @@ -12,36 +22,21 @@ export const env = createEnv({
GOOGLE_CLIENT_SECRET: z.string().min(1),
GOOGLE_ENCRYPT_SECRET: z.string(),
GOOGLE_ENCRYPT_SALT: z.string(),

DEFAULT_LLM_PROVIDER: z
.enum([
"anthropic",
// "bedrock",
"google",
"openai",
"openrouter",
"groq",
"ollama",
"custom",
])
.enum([...llmProviderEnum.options, "custom"])
.default("anthropic"),
DEFAULT_LLM_MODEL: z.string().optional(),
// Economy LLM configuration (for large context windows where cost efficiency matters)
ECONOMY_LLM_PROVIDER: z
.enum([
"anthropic",
"google",
"openai",
"bedrock",
"openrouter",
"groq",
"ollama",
])
.optional()
.default("openrouter"),
ECONOMY_LLM_MODEL: z
.string()
.optional()
.default("google/gemini-2.5-flash-preview-05-20"),
DEFAULT_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for default model (e.g., "Google Vertex,Anthropic")
// Set this to a cheaper model like Gemini Flash
ECONOMY_LLM_PROVIDER: llmProviderEnum.optional(),
ECONOMY_LLM_MODEL: z.string().optional(),
ECONOMY_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for economy model (e.g., "Google Vertex,Anthropic")
// Set this to a fast but strong model like Groq Kimi K2. Leaving blank will fallback to default which is also fine.
CHAT_LLM_PROVIDER: llmProviderEnum.optional(),
CHAT_LLM_MODEL: z.string().optional(),
CHAT_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for chat (e.g., "Google Vertex,Anthropic")

OPENAI_API_KEY: z.string().optional(),
ANTHROPIC_API_KEY: z.string().optional(),
BEDROCK_ACCESS_KEY: z.string().optional(),
Expand Down
1 change: 1 addition & 0 deletions apps/web/utils/ai/assistant/chat.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,7 @@ Examples:
const result = chatCompletionStream({
userAi: user.user,
userEmail: user.email,
modelType: "chat",
usageLabel: "assistant-chat",
system,
messages,
Expand Down
1 change: 1 addition & 0 deletions apps/web/utils/ai/assistant/process-user-request.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ ${senderCategory || "No category"}

const result = await chatCompletionTools({
userAi: emailAccount.user,
modelType: "chat",
messages: allMessages,
tools: {
update_conditional_operator: tool({
Expand Down
2 changes: 1 addition & 1 deletion apps/web/utils/ai/knowledge/extract-from-email-history.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ export async function aiExtractFromEmailHistory({
usageLabel: "Email history extraction",
userAi: emailAccount.user,
userEmail: emailAccount.email,
useEconomyModel: true,
modelType: "economy",
});

logger.trace("Output", result.object);
Expand Down
2 changes: 1 addition & 1 deletion apps/web/utils/ai/knowledge/extract.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export async function aiExtractRelevantKnowledge({
usageLabel: "Knowledge extraction",
userAi: emailAccount.user,
userEmail: emailAccount.email,
useEconomyModel: true,
modelType: "economy",
});

logger.trace("Output", result.object);
Expand Down
1 change: 1 addition & 0 deletions apps/web/utils/llms/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export const Model = {
GEMINI_2_0_FLASH_OPENROUTER: "google/gemini-2.0-flash-001",
GEMINI_2_5_PRO_OPENROUTER: "google/gemini-2.5-pro-preview-03-25",
GROQ_LLAMA_3_3_70B: "llama-3.3-70b-versatile",
KIMI_K2_OPENROUTER: "moonshotai/kimi-k2",
...(supportsOllama ? { OLLAMA: env.NEXT_PUBLIC_OLLAMA_MODEL } : {}),
};

Expand Down
32 changes: 16 additions & 16 deletions apps/web/utils/llms/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import {
isServiceUnavailableError,
} from "@/utils/error";
import { sleep } from "@/utils/sleep";
import { getModel } from "@/utils/llms/model";
import { getModel, type ModelType } from "@/utils/llms/model";
import { createScopedLogger } from "@/utils/logger";

const logger = createScopedLogger("llms");
Expand All @@ -41,14 +41,14 @@ const commonOptions: {

export async function chatCompletion({
userAi,
useEconomyModel,
modelType = "default",
prompt,
system,
userEmail,
usageLabel,
}: {
userAi: UserAIFields;
useEconomyModel?: boolean;
modelType?: ModelType;
prompt: string;
system?: string;
userEmail: string;
Expand All @@ -57,7 +57,7 @@ export async function chatCompletion({
try {
const { provider, model, llmModel, providerOptions } = getModel(
userAi,
useEconomyModel,
modelType,
);

const result = await generateText({
Expand Down Expand Up @@ -87,7 +87,7 @@ export async function chatCompletion({

type ChatCompletionObjectArgs<T> = {
userAi: UserAIFields;
useEconomyModel?: boolean;
modelType?: ModelType;
schema: z.Schema<T>;
userEmail: string;
usageLabel: string;
Expand All @@ -112,7 +112,7 @@ export async function chatCompletionObject<T>(

async function chatCompletionObjectInternal<T>({
userAi,
useEconomyModel,
modelType,
system,
prompt,
messages,
Expand All @@ -123,7 +123,7 @@ async function chatCompletionObjectInternal<T>({
try {
const { provider, model, llmModel, providerOptions } = getModel(
userAi,
useEconomyModel,
modelType,
);

const result = await generateObject({
Expand Down Expand Up @@ -155,7 +155,7 @@ async function chatCompletionObjectInternal<T>({

export async function chatCompletionStream({
userAi,
useEconomyModel,
modelType,
system,
prompt,
messages,
Expand All @@ -167,7 +167,7 @@ export async function chatCompletionStream({
onStepFinish,
}: {
userAi: UserAIFields;
useEconomyModel?: boolean;
modelType?: ModelType;
system?: string;
prompt?: string;
messages?: Message[];
Expand All @@ -184,7 +184,7 @@ export async function chatCompletionStream({
}) {
const { provider, model, llmModel, providerOptions } = getModel(
userAi,
useEconomyModel,
modelType,
);

const result = streamText({
Expand Down Expand Up @@ -229,7 +229,7 @@ export async function chatCompletionStream({

type ChatCompletionToolsArgs = {
userAi: UserAIFields;
useEconomyModel?: boolean;
modelType?: ModelType;
tools: Record<string, Tool>;
maxSteps?: number;
label: string;
Expand All @@ -253,7 +253,7 @@ export async function chatCompletionTools(options: ChatCompletionToolsArgs) {

async function chatCompletionToolsInternal({
userAi,
useEconomyModel,
modelType,
system,
prompt,
messages,
Expand All @@ -265,7 +265,7 @@ async function chatCompletionToolsInternal({
try {
const { provider, model, llmModel, providerOptions } = getModel(
userAi,
useEconomyModel,
modelType,
);

const result = await generateText({
Expand Down Expand Up @@ -300,7 +300,7 @@ async function chatCompletionToolsInternal({
// not in use atm
async function streamCompletionTools({
userAi,
useEconomyModel,
modelType,
prompt,
system,
tools,
Expand All @@ -310,7 +310,7 @@ async function streamCompletionTools({
onFinish,
}: {
userAi: UserAIFields;
useEconomyModel?: boolean;
modelType?: ModelType;
prompt: string;
system?: string;
tools: Record<string, Tool>;
Expand All @@ -321,7 +321,7 @@ async function streamCompletionTools({
}) {
const { provider, model, llmModel, providerOptions } = getModel(
userAi,
useEconomyModel,
modelType,
);

const result = await streamText({
Expand Down
Loading
Loading