Improved LLM calls

.cursor/rules/llm.mdc

@@ -28,39 +28,20 @@ import { z } from "zod";
 import { createScopedLogger } from "@/utils/logger";
 import { chatCompletionObject } from "@/utils/llms";
 import type { UserEmailWithAI } from "@/utils/llms/types";
-// Import other necessary types and utilities
-
-// 1. Create a scoped logger
-const logger = createScopedLogger("feature-name");
-
-// 2. Define output schema with zod
-export const schema = z.object({
-  // Define expected response fields
-  field1: z.string(),
-  field2: z.number(),
-  nested: z.object({
-    subfield: z.string(),
-  }),
-  array_field: z.array(z.string()),
-});
-
-// 3. Create main function with typed options
+
 export async function featureFunction(options: {
   inputData: InputType;
   user: UserEmailWithAI;
 }) {
   const { inputData, user } = options;
 
-  // 4. Add early validation/returns
   if (!inputData || [other validation conditions]) {
     logger.warn("Invalid input for feature function");
     return null;
   }
 
-  // 5. Define system prompt
   const system = `[Detailed system prompt that defines the LLM's role and task]`;
 
-  // 6. Construct user prompt
   const prompt = `[User prompt with context and specific instructions]
 
 <data>
@@ -69,24 +50,23 @@ export async function featureFunction(options: {
 
 ${user.about ? `<user_info>${user.about}</user_info>` : ""}`;
 
-  // 7. Log inputs
-  logger.trace("Input", { system, prompt });
-
-  // 8. Call LLM with proper configuration
   const result = await chatCompletionObject({
     userAi: user,
     system,
     prompt,
-    schema,
+    schema: z.object({
+      field1: z.string(),
+      field2: z.number(),
+      nested: z.object({
+        subfield: z.string(),
+      }),
+      array_field: z.array(z.string()),
+    }),
     userEmail: user.email,
     usageLabel: "Feature Name",
   });
 
-  // 9. Log outputs
-  logger.trace("Output", { result });
-
-  // 10. Return validated result
-  return result.object;
+return result.object;
 }
 ```
 

apps/web/app/(app)/[emailAccountId]/assistant/RulesPrompt.tsx

@@ -309,7 +309,7 @@ function RulesPromptForm({
 
                         if (result?.data?.rulesPrompt) {
                           editorRef.current?.appendText(
-                            result?.data?.rulesPrompt,
+                            `\n${result?.data?.rulesPrompt || ""}`,
                           );
                         } else {
                           toast.error("Error generating prompt");

apps/web/env.ts

@@ -42,6 +42,11 @@ export const env = createEnv({
     CHAT_LLM_MODEL: z.string().optional(),
     CHAT_OPENROUTER_PROVIDERS: z.string().optional(), // Comma-separated list of OpenRouter providers for chat (e.g., "Google Vertex,Anthropic")
 
+    OPENROUTER_BACKUP_MODEL: z
+      .string()
+      .optional()
+      .default("google/gemini-2.5-flash"),
+
     OPENAI_API_KEY: z.string().optional(),
     ANTHROPIC_API_KEY: z.string().optional(),
     BEDROCK_ACCESS_KEY: z.string().optional(),
@@ -160,9 +165,6 @@ export const env = createEnv({
     NEXT_PUBLIC_BEDROCK_SONNET_MODEL: z
       .string()
       .default("us.anthropic.claude-3-7-sonnet-20250219-v1:0"),
-    NEXT_PUBLIC_BEDROCK_ANTHROPIC_BACKUP_MODEL: z
-      .string()
-      .default("us.anthropic.claude-3-5-sonnet-20241022-v2:0"),
     NEXT_PUBLIC_OLLAMA_MODEL: z.string().optional(),
     NEXT_PUBLIC_APP_HOME_PATH: z.string().default("/setup"),
     NEXT_PUBLIC_DUB_REFER_DOMAIN: z.string().optional(),
@@ -216,8 +218,6 @@ export const env = createEnv({
     NEXT_PUBLIC_AXIOM_TOKEN: process.env.NEXT_PUBLIC_AXIOM_TOKEN,
     NEXT_PUBLIC_BEDROCK_SONNET_MODEL:
       process.env.NEXT_PUBLIC_BEDROCK_SONNET_MODEL,
-    NEXT_PUBLIC_BEDROCK_ANTHROPIC_BACKUP_MODEL:
-      process.env.NEXT_PUBLIC_BEDROCK_ANTHROPIC_BACKUP_MODEL,
     NEXT_PUBLIC_OLLAMA_MODEL: process.env.NEXT_PUBLIC_OLLAMA_MODEL,
     NEXT_PUBLIC_APP_HOME_PATH: process.env.NEXT_PUBLIC_APP_HOME_PATH,
     NEXT_PUBLIC_DUB_REFER_DOMAIN: process.env.NEXT_PUBLIC_DUB_REFER_DOMAIN,

apps/web/utils/actions/ai-rule.ts

@@ -552,7 +552,7 @@ export const generateRulesPromptAction = actionClient
     const result = await aiGenerateRulesPrompt({
       emailAccount,
       lastSentEmails,
-      snippets: snippetsResult.map((snippet) => snippet.text),
+      snippets: snippetsResult.snippets.map((snippet) => snippet.text),
       userLabels: labelsWithCounts.map((label) => label.label),
     });
 

apps/web/utils/ai/assistant/process-user-request.ts

@@ -1,7 +1,7 @@
-import { tool } from "ai";
+import { stepCountIs, tool } from "ai";
 import { z } from "zod";
 import { after } from "next/server";
-import { chatCompletionTools } from "@/utils/llms";
+import { createGenerateText } from "@/utils/llms";
 import { createScopedLogger } from "@/utils/logger";
 import {
   type Category,
@@ -35,6 +35,7 @@ import {
 import { env } from "@/env";
 import { posthogCaptureEvent } from "@/utils/posthog";
 import { getUserCategoriesForNames } from "@/utils/category.server";
+import { getModel } from "@/utils/llms/model";
 
 const logger = createScopedLogger("ai-fix-rules");
 
@@ -198,10 +199,18 @@ ${senderCategory || "No category"}
     }
   }
 
-  const result = await chatCompletionTools({
-    userAi: emailAccount.user,
-    modelType: "chat",
+  const modelOptions = getModel(emailAccount.user, "chat");
+
+  const generateText = createGenerateText({
+    userEmail: emailAccount.email,
+    label: "Process user request",
+    modelOptions,
+  });
+
+  const result = await generateText({
+    ...modelOptions,
     messages: allMessages,
+    stopWhen: stepCountIs(5),
     tools: {
       update_conditional_operator: tool({
         description: "Update the conditional operator of a rule",
@@ -622,9 +631,6 @@ ${senderCategory || "No category"}
         // no execute function - invoking it will terminate the agent
       }),
     },
-    maxSteps: 5,
-    label: "Fix Rule",
-    userEmail: emailAccount.email,
   });
 
   const toolCalls = result.steps.flatMap((step) => step.toolCalls);

apps/web/utils/ai/categorize-sender/ai-categorize-senders.ts

@@ -3,13 +3,9 @@ import { isDefined } from "@/utils/types";
 import type { EmailAccountWithAI } from "@/utils/llms/types";
 import type { Category } from "@prisma/client";
 import { formatCategoriesForPrompt } from "@/utils/ai/categorize-sender/format-categories";
-import { createScopedLogger } from "@/utils/logger";
 import { extractEmailAddress } from "@/utils/email";
 import { getModel } from "@/utils/llms/model";
-import { generateObject } from "ai";
-import { saveAiUsage } from "@/utils/usage";
-
-const logger = createScopedLogger("ai-categorize-senders");
+import { createGenerateObject } from "@/utils/llms";
 
 export const REQUEST_MORE_INFORMATION_CATEGORY = "RequestMoreInformation";
 export const UNKNOWN_CATEGORY = "Unknown";
@@ -87,43 +83,22 @@ ${formatCategoriesForPrompt(categories)}
 - Accuracy is more important than completeness
 - Only use the categories provided above
 - Respond with "Unknown" if unsure
+- Return your response in JSON format
 </important>`;
 
-  logger.trace("Categorize senders", { system, prompt });
-
-  // const aiResponse = await chatCompletionObject({
-  //   userAi: emailAccount.user,
-  //   system,
-  //   prompt,
-  //   schema: categorizeSendersSchema,
-  //   userEmail: emailAccount.email,
-  //   usageLabel: "Categorize senders bulk",
-  // });
+  const modelOptions = getModel(emailAccount.user, "chat");
 
-  const { provider, model, llmModel, providerOptions } = getModel(
-    emailAccount.user,
-  );
+  const generateObject = createGenerateObject({
+    userEmail: emailAccount.email,
+    label: "Categorize senders bulk",
+    modelOptions,
+  });
 
   const aiResponse = await generateObject({
-    model: llmModel,
+    ...modelOptions,
     system,
     prompt,
     schema: categorizeSendersSchema,
-    providerOptions,
-  });
-
-  if (aiResponse.usage) {
-    await saveAiUsage({
-      email: emailAccount.email,
-      usage: aiResponse.usage,
-      provider,
-      model,
-      label: "Categorize senders bulk",
-    });
-  }
-
-  logger.trace("Categorize senders response", {
-    senders: aiResponse.object.senders,
   });
 
   const matchedSenders = matchSendersWithFullEmail(
@@ -143,8 +118,6 @@ ${formatCategoriesForPrompt(categories)}
     return r;
   });
 
-  logger.trace("Categorize senders results", { results });
-
   return results;
 }
 

apps/web/utils/ai/categorize-sender/ai-categorize-single-sender.ts

@@ -1,21 +1,9 @@
-import { generateObject } from "ai";
 import { z } from "zod";
 import type { EmailAccountWithAI } from "@/utils/llms/types";
 import type { Category } from "@prisma/client";
 import { formatCategoriesForPrompt } from "@/utils/ai/categorize-sender/format-categories";
-import { createScopedLogger } from "@/utils/logger";
 import { getModel } from "@/utils/llms/model";
-import { saveAiUsage } from "@/utils/usage";
-
-const logger = createScopedLogger("aiCategorizeSender");
-
-const categorizeSenderSchema = z.object({
-  rationale: z.string().describe("Keep it short. 1-2 sentences max."),
-  category: z.string(),
-  // possibleCategories: z
-  //   .array(z.string())
-  //   .describe("Possible categories when the main category is unknown"),
-});
+import { createGenerateObject } from "@/utils/llms";
 
 export async function aiCategorizeSender({
   emailAccount,
@@ -55,45 +43,29 @@ ${formatCategoriesForPrompt(categories)}
 3. If the category is clear, assign it.
 4. If you're not certain, respond with "Unknown".
 5. If multiple categories are possible, respond with "Unknown".
+6. Return your response in JSON format.
 </instructions>`;
 
-  logger.trace("aiCategorizeSender", { system, prompt });
-
-  // const aiResponse = await chatCompletionObject({
-  //   userAi: emailAccount.user,
-  //   system,
-  //   prompt,
-  //   schema: categorizeSenderSchema,
-  //   userEmail: emailAccount.email,
-  //   usageLabel: "Categorize sender",
-  // });
+  const modelOptions = getModel(emailAccount.user);
 
-  const { provider, model, llmModel, providerOptions } = getModel(
-    emailAccount.user,
-  );
+  const generateObject = createGenerateObject({
+    userEmail: emailAccount.email,
+    label: "Categorize sender",
+    modelOptions,
+  });
 
   const aiResponse = await generateObject({
-    model: llmModel,
+    ...modelOptions,
     system,
     prompt,
-    schema: categorizeSenderSchema,
-    providerOptions,
+    schema: z.object({
+      rationale: z.string().describe("Keep it short. 1-2 sentences max."),
+      category: z.string(),
+    }),
   });
 
-  if (aiResponse.usage) {
-    await saveAiUsage({
-      email: emailAccount.email,
-      usage: aiResponse.usage,
-      provider,
-      model,
-      label: "Categorize sender",
-    });
-  }
-
   if (!categories.find((c) => c.name === aiResponse.object.category))
     return null;
 
-  logger.trace("aiCategorizeSender result", { aiResponse: aiResponse.object });
-
   return aiResponse.object;
 }