diff --git a/.cursorrules b/.cursorrules index a59a791ca5..8b1fec999b 100644 --- a/.cursorrules +++ b/.cursorrules @@ -145,3 +145,16 @@ const logger = createScopedLogger("Rules"); logger.log("Created rule", { userId }); ``` + +- This is how we import Prisma: + +```ts +import prisma from "@/utils/prisma"; +``` + +- If an import is a type then use `type` to import it: + +```ts +import type { Action } from "@prisma/client"; +import { type Action, GroupItemType } from "@prisma/client"; +``` \ No newline at end of file diff --git a/apps/web/__tests__/ai-find-snippets.test.ts b/apps/web/__tests__/ai-find-snippets.test.ts index 5e99b3427e..fffd8d93ce 100644 --- a/apps/web/__tests__/ai-find-snippets.test.ts +++ b/apps/web/__tests__/ai-find-snippets.test.ts @@ -1,7 +1,6 @@ import { describe, expect, test, vi } from "vitest"; import { aiFindSnippets } from "@/utils/ai/snippets/find-snippets"; -import type { EmailForLLM } from "@/utils/ai/choose-rule/stringify-email"; - +import type { EmailForLLM } from "@/utils/types"; // pnpm test-ai ai-find-snippets const isAiTest = process.env.RUN_AI_TESTS === "true"; diff --git a/apps/web/__tests__/ai-process-user-request.test.ts b/apps/web/__tests__/ai-process-user-request.test.ts new file mode 100644 index 0000000000..cf1eed9d58 --- /dev/null +++ b/apps/web/__tests__/ai-process-user-request.test.ts @@ -0,0 +1,536 @@ +import { describe, expect, test, vi } from "vitest"; +import stripIndent from "strip-indent"; +import { processUserRequest } from "@/utils/ai/assistant/process-user-request"; +import type { ParsedMessage, ParsedMessageHeaders } from "@/utils/types"; +import type { RuleWithRelations } from "@/utils/ai/rule/create-prompt-from-rule"; +import { + type Category, + type GroupItem, + type Prisma, + RuleType, +} from "@prisma/client"; +import { GroupItemType, LogicalOperator } from "@prisma/client"; + +// pnpm test-ai ai-process-user-request + +const isAiTest = process.env.RUN_AI_TESTS === "true"; + +vi.mock("server-only", () => ({})); +vi.mock("@/utils/gmail/mail", () => ({ replyToEmail: vi.fn() })); + +describe( + "processUserRequest", + { + timeout: 30_000, + skip: !isAiTest, + }, + () => { + test("should fix a rule with incorrect AI instructions", async () => { + const rule = getRule({ + name: "Partnership Rule", + instructions: "Match emails discussing business opportunities", + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "sales@company.com", + subject: "Special Offer for Your Business", + }, + textPlain: stripIndent(` + Hi there, + + We have an amazing product that could boost your revenue by 300%. + Special discount available this week only! + + Let me know if you'd like a demo. + + Best, + Sales Team + `), + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: "This is a promotional email", + }, + ], + originalEmail, + matchedRule: rule, + categories: null, + senderCategory: null, + }); + + expect(result).toBeDefined(); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + const updateInstructionsToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "update_ai_instructions", + ); + + expect(updateInstructionsToolCall).toBeDefined(); + expect(updateInstructionsToolCall?.args.ruleName).toBe( + "Partnership Rule", + ); + }); + + test("should handle request to refine ai rule instructions", async () => { + const ruleSupport = getRule({ + name: "Support Rule", + instructions: "Match technical support requests", + }); + const ruleUrgent = getRule({ + name: "Urgent Rule", + instructions: "Match urgent requests", + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "user@test.com", + subject: "Help with Login", + }, + textPlain: stripIndent(` + Hello, + + I can't log into my account. Can someone help? + + Thanks, + User + `), + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [ruleSupport, ruleUrgent], + messages: [ + { + role: "user", + content: "This isn't urgent.", + }, + ], + originalEmail, + matchedRule: ruleUrgent, + categories: null, + senderCategory: null, + }); + + expect(result).toBeDefined(); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + + const toolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "update_ai_instructions", + ); + + expect(toolCall).toBeDefined(); + }); + + test("should fix static conditions when user indicates incorrect matching", async () => { + const rule = getRule({ + name: "Receipt Rule", + from: "@amazon.com", + subject: "Order", + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "shipping@amazon.com", + subject: "Order #123 Has Shipped", + }, + textPlain: stripIndent(` + Your order has shipped! + Tracking number: 1234567890 + Expected delivery: Tomorrow + `), + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: "This isn't a receipt, it's a shipping notification.", + }, + ], + originalEmail, + matchedRule: rule, + categories: null, + senderCategory: null, + }); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + const updateStaticConditionsToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "update_static_conditions", + ); + + expect(updateStaticConditionsToolCall).toBeDefined(); + expect(updateStaticConditionsToolCall?.args.ruleName).toBe( + "Receipt Rule", + ); + expect( + updateStaticConditionsToolCall?.args.staticConditions?.subject?.includes( + "shipping", + ) || + updateStaticConditionsToolCall?.args.staticConditions?.subject?.includes( + "Shipped", + ), + ).toBe(true); + }); + + test("should fix group conditions when user reports incorrect matching", async () => { + const group = getGroup({ + name: "Newsletters", + items: [ + getGroupItem({ + id: "1", + type: GroupItemType.FROM, + value: "david@hello.com", + }), + getGroupItem({ + id: "2", + type: GroupItemType.FROM, + value: "@beehiiv.com", + }), + ], + }); + + const rule = getRule({ + name: "Newsletter Rule", + group, + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "david@hello.com", + subject: "Question about your latest post", + }, + textPlain: stripIndent(` + Hey there, + + Thanks for reaching out about my article on microservices. You raised some + really interesting points about the scalability challenges you're facing. + + I actually dealt with a similar issue at my previous company. Would love to + hop on a quick call to discuss it in more detail. + + Best, + David + `), + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: "This isn't a newsletter", + }, + ], + originalEmail, + matchedRule: rule, + categories: null, + senderCategory: null, + }); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + const removeFromGroupToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "remove_from_group", + ); + + expect(removeFromGroupToolCall).toBeDefined(); + expect(removeFromGroupToolCall?.args.value).toBe("david@hello.com"); + }); + + test("should suggest adding sender to group when identified as missing", async () => { + const group = getGroup({ + name: "Newsletters", + items: [ + getGroupItem({ + type: GroupItemType.FROM, + value: "ainewsletter@substack.com", + }), + getGroupItem({ + type: GroupItemType.FROM, + value: "milkroad@beehiiv.com", + }), + ], + }); + + const rule = getRule({ + name: "Newsletter Rule", + group, + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "mattsnews@convertkit.com", + to: "me@ourcompany.com", + subject: "Weekly Developer Digest", + }, + textPlain: stripIndent(` + This Week's Top Stories: + + 1. The Future of TypeScript + 2. React Server Components Deep Dive + 3. Building Scalable Systems + + To unsubscribe, click here + Powered by ConvertKit + `), + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: "This is a newsletter", + }, + ], + originalEmail, + matchedRule: null, // Important: rule didn't match initially + categories: null, + senderCategory: null, + }); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + const addToGroupToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "add_to_group", + ); + + expect(addToGroupToolCall).toBeDefined(); + expect(addToGroupToolCall?.args.type).toBe("from"); + expect(addToGroupToolCall?.args.value).toContain("convertkit.com"); + }); + + test("should fix category filters when user indicates wrong categorization", async () => { + const marketingCategory = getCategory({ + name: "Marketing", + description: "Marketing related emails", + }); + + const rule = getRule({ + name: "Marketing Rule", + categoryFilterType: "INCLUDE", + categoryFilters: [marketingCategory], + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "marketing@company.com", + subject: "Special Offer", + }, + textPlain: "Would you like to purchase our enterprise plan?", + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: "This is actually a sales email, not marketing.", + }, + ], + originalEmail, + matchedRule: rule, + categories: [ + { id: "1", name: "Marketing" }, + { id: "2", name: "Sales" }, + { id: "3", name: "Newsletter" }, + ], + senderCategory: "Marketing", + }); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + const changeSenderCategoryToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "change_sender_category", + ); + + expect(changeSenderCategoryToolCall).toBeDefined(); + expect(changeSenderCategoryToolCall?.args.category).toBe("Sales"); + }); + + test("should handle complex rule fixes with multiple condition types", async () => { + const salesCategory = getCategory({ + name: "Sales", + description: "Sales related emails", + }); + + const rule = getRule({ + name: "Sales Rule", + instructions: "Match sales opportunities", + from: "@enterprise.com", + subject: "Business opportunity", + categoryFilters: [salesCategory], + categoryFilterType: "INCLUDE", + }); + + const originalEmail = getParsedMessage({ + headers: { + from: "contact@enterprise.com", + subject: "Business opportunity - Act now!", + }, + textPlain: "Make millions with this amazing opportunity!", + }); + + const result = await processUserRequest({ + user: getUser(), + rules: [rule], + messages: [ + { + role: "user", + content: + "This is a spam email pretending to be a business opportunity.", + }, + ], + originalEmail, + matchedRule: rule, + categories: [ + { id: "1", name: "Marketing" }, + { id: "2", name: "Sales" }, + { id: "3", name: "Newsletter" }, + ], + senderCategory: "Marketing", + }); + + const toolCalls = result.steps.flatMap((step) => step.toolCalls); + + const updateStaticConditionsToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "update_static_conditions", + ); + const updateAiInstructionsToolCall = toolCalls.find( + (toolCall) => toolCall.toolName === "update_ai_instructions", + ); + + expect( + updateStaticConditionsToolCall || updateAiInstructionsToolCall, + ).toBeDefined(); + if (updateStaticConditionsToolCall) { + expect(updateStaticConditionsToolCall.args.ruleName).toBe("Sales Rule"); + } + if (updateAiInstructionsToolCall) { + expect(updateAiInstructionsToolCall.args.ruleName).toBe("Sales Rule"); + } + }); + }, +); + +function getRule(rule: Partial): RuleWithRelations { + return { + id: "1", + userId: "user1", + name: "Rule name", + + conditionalOperator: LogicalOperator.AND, + // ai instructions + instructions: null, + // static conditions + from: null, + to: null, + subject: null, + body: null, + // group conditions + group: null, + groupId: null, + // category conditions + categoryFilters: [], + categoryFilterType: null, + + // other + actions: [], + automate: true, + runOnThreads: true, + enabled: true, + type: RuleType.AI, + createdAt: new Date(), + updatedAt: new Date(), + ...rule, + }; +} + +function getParsedMessage( + message: Omit, "headers"> & { + headers?: Partial; + }, +): ParsedMessage { + return { + id: "id", + threadId: "thread-id", + snippet: "", + attachments: [], + historyId: "history-id", + sizeEstimate: 100, + internalDate: new Date().toISOString(), + inline: [], + textPlain: "", + ...message, + headers: { + from: "test@example.com", + to: "recipient@example.com", + subject: "", + date: new Date().toISOString(), + references: "", + "message-id": "message-id", + ...message.headers, + }, + }; +} + +function getUser() { + return { + id: "user1", + aiModel: null, + aiProvider: null, + email: "user@test.com", + aiApiKey: null, + about: null, + }; +} + +type Group = Prisma.GroupGetPayload<{ + select: { + id: true; + name: true; + items: { select: { id: true; type: true; value: true } }; + }; +}>; + +function getGroup(group: Partial): Group { + return { + id: "id", + name: "Group name", + items: [], + ...group, + }; +} + +function getGroupItem(item: Partial): GroupItem { + return { + id: "id", + value: "", + type: GroupItemType.FROM, + createdAt: new Date(), + updatedAt: new Date(), + groupId: "group1", + ...item, + }; +} + +function getCategory(category: Partial): Category { + return { + id: "id", + name: "", + description: null, + createdAt: new Date(), + updatedAt: new Date(), + userId: "user1", + ...category, + }; +} diff --git a/apps/web/__tests__/ai-rule-fix.test.ts b/apps/web/__tests__/ai-rule-fix.test.ts index 3ffe4ec43b..f9aa4d8c0c 100644 --- a/apps/web/__tests__/ai-rule-fix.test.ts +++ b/apps/web/__tests__/ai-rule-fix.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test, vi } from "vitest"; import stripIndent from "strip-indent"; import { aiRuleFix } from "@/utils/ai/rule/rule-fix"; -import type { EmailForLLM } from "@/utils/ai/choose-rule/stringify-email"; +import type { EmailForLLM } from "@/utils/types"; // pnpm test-ai ai-rule-fix @@ -41,7 +41,7 @@ describe.skipIf(!isAiTest)("aiRuleFix", () => { console.log(result); expect(result).toBeDefined(); - expect(result.rule).toBe("actual_rule"); + expect(result.ruleToFix).toBe("actual_rule"); expect(result.fixedInstructions).toContain("sales"); expect(result.fixedInstructions).not.toBe(rule.instructions); // The new instructions should be more specific to exclude sales pitches @@ -81,7 +81,7 @@ describe.skipIf(!isAiTest)("aiRuleFix", () => { console.log(result); expect(result).toBeDefined(); - expect(result.rule).toBe("correct_rule"); + expect(result.ruleToFix).toBe("expected_rule"); expect(result.fixedInstructions).toContain("technical"); // The incorrect rule should be more specific to exclude feature requests expect(result.fixedInstructions.toLowerCase()).toMatch( @@ -117,7 +117,7 @@ describe.skipIf(!isAiTest)("aiRuleFix", () => { console.log(result); expect(result).toBeDefined(); - expect(result.rule).toBe("actual_rule"); + expect(result.ruleToFix).toBe("actual_rule"); expect(result.fixedInstructions).toContain("collaboration"); // The fixed rule should exclude newsletters and automated updates expect(result.fixedInstructions.toLowerCase()).toMatch( @@ -153,7 +153,7 @@ describe.skipIf(!isAiTest)("aiRuleFix", () => { console.log(result); expect(result).toBeDefined(); - expect(result.rule).toBe("correct_rule"); + expect(result.ruleToFix).toBe("expected_rule"); expect(result.fixedInstructions).toContain("pric"); // The fixed rule should be more inclusive of various pricing inquiries expect(result.fixedInstructions.toLowerCase()).toMatch( diff --git a/apps/web/app/(app)/automation/ReportMistake.tsx b/apps/web/app/(app)/automation/ReportMistake.tsx index fcd2e82f34..76badf219e 100644 --- a/apps/web/app/(app)/automation/ReportMistake.tsx +++ b/apps/web/app/(app)/automation/ReportMistake.tsx @@ -388,7 +388,6 @@ function RuleMismatch({ rules: RulesResponse; onSelectExpectedRuleId: (ruleId: string | null) => void; }) { - console.log("🚀 ~ result:", result); return (