Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .cursor/rules/llm-test.mdc
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ Tests for LLM-related functionality should follow these guidelines to ensure con

vi.mock("server-only", () => ({}));

const TIMEOUT = 15_000;

// Skip tests unless explicitly running AI tests
const isAiTest = process.env.RUN_AI_TESTS === "true";

Expand All @@ -39,7 +41,7 @@ Tests for LLM-related functionality should follow these guidelines to ensure con
test("test case description", async () => {
// Test implementation
});
}, 15_000);
}, TIMEOUT);
```

## Helper Functions
Expand Down Expand Up @@ -100,9 +102,10 @@ Tests for LLM-related functionality should follow these guidelines to ensure con
1. Set appropriate timeouts for LLM calls:

```typescript
const TIMEOUT = 15_000;
test("handles long-running LLM operations", async () => {
// ...
}, 15_000); // 15 second timeout
}, TIMEOUT);
```

2. Use descriptive console.debug for generated content:
Expand Down
6 changes: 3 additions & 3 deletions apps/mcp-server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
"build"
],
"dependencies": {
"@modelcontextprotocol/sdk": "1.12.1",
"@modelcontextprotocol/sdk": "1.17.4",
"zod": "3.25.46"
},
"devDependencies": {
"@types/node": "22.15.29",
"typescript": "5.8.3"
"@types/node": "24.3.0",
"typescript": "5.9.2"
}
}
12 changes: 6 additions & 6 deletions apps/unsubscriber/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
},
"devDependencies": {
"@types/dotenv": "8.2.3",
"@types/node": "22.15.29",
"playwright": "1.52.0",
"tsx": "4.19.4",
"typescript": "5.8.3"
"@types/node": "24.3.0",
"playwright": "1.55.0",
"tsx": "4.20.5",
"typescript": "5.9.2"
},
Comment on lines +15 to 19
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Remove @types/dotenv (dotenv ships its own types and v17 is ESM).

The legacy typings can conflict with v17 and ESM usage.

   "devDependencies": {
-    "@types/dotenv": "8.2.3",
     "@types/node": "24.3.0",
     "playwright": "1.55.0",
     "tsx": "4.20.5",
     "typescript": "5.9.2"
   },
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
"@types/node": "24.3.0",
"playwright": "1.55.0",
"tsx": "4.20.5",
"typescript": "5.9.2"
},
"devDependencies": {
"@types/node": "24.3.0",
"playwright": "1.55.0",
"tsx": "4.20.5",
"typescript": "5.9.2"
},
🤖 Prompt for AI Agents
In apps/unsubscriber/package.json around lines 15 to 19, remove the legacy
"@types/dotenv" entry from dependencies/devDependencies (dotenv v17 includes its
own types and the @types package can conflict with ESM). After removing the
entry, run your package manager (npm/yarn/pnpm) to update node_modules and the
lockfile, and verify no code imports reference "@types/dotenv"; if CI uses a
cached lockfile update that as well.

"dependencies": {
"@ai-sdk/amazon-bedrock": "2.2.9",
Expand All @@ -25,8 +25,8 @@
"@fastify/cors": "11.0.1",
"@t3-oss/env-core": "0.13.6",
"ai": "4.3.16",
"dotenv": "16.5.0",
"fastify": "5.3.3",
"dotenv": "17.2.1",
"fastify": "5.5.0",
Comment thread
elie222 marked this conversation as resolved.
"zod": "3.25.46"
}
}
2 changes: 2 additions & 0 deletions apps/unsubscriber/src/llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,7 @@ export function getModel(provider: LLMProvider) {
return anthropic("claude-3-7-sonnet-20250219");
case "bedrock":
return bedrock("anthropic.claude-3-7-sonnet-20250219-v1:0");
default:
throw new Error(`Unknown provider: ${provider}`);
}
}
246 changes: 135 additions & 111 deletions apps/web/__tests__/ai-categorize-senders.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import { getEmailAccount } from "@/__tests__/helpers";

const isAiTest = process.env.RUN_AI_TESTS === "true";

const TIMEOUT = 15_000;

vi.mock("server-only", () => ({}));

const emailAccount = getEmailAccount();
Expand Down Expand Up @@ -49,31 +51,37 @@ const testSenders = [

describe.runIf(isAiTest)("AI Sender Categorization", () => {
describe("Bulk Categorization", () => {
it("should categorize senders with snippets using AI", async () => {
const result = await aiCategorizeSenders({
emailAccount,
senders: testSenders,
categories: getEnabledCategories(),
});
it(
"should categorize senders with snippets using AI",
async () => {
const result = await aiCategorizeSenders({
emailAccount,
senders: testSenders,
categories: getEnabledCategories(),
});

expect(result).toHaveLength(testSenders.length);
expect(result).toHaveLength(testSenders.length);

// Test newsletter categorization with snippet
const newsletterResult = result.find(
(r) => r.sender === "newsletter@company.com",
);
expect(newsletterResult?.category).toBe("Newsletter");
// Test newsletter categorization with snippet
const newsletterResult = result.find(
(r) => r.sender === "newsletter@company.com",
);
expect(newsletterResult?.category).toBe("Newsletter");

// Test support categorization with ticket snippet
const supportResult = result.find(
(r) => r.sender === "support@service.com",
);
expect(supportResult?.category).toBe("Support");
// Test support categorization with ticket snippet
const supportResult = result.find(
(r) => r.sender === "support@service.com",
);
expect(supportResult?.category).toBe("Support");

// Test sales categorization with offer snippet
const salesResult = result.find((r) => r.sender === "sales@business.com");
expect(salesResult?.category).toBe("Marketing");
}, 15_000);
// Test sales categorization with offer snippet
const salesResult = result.find(
(r) => r.sender === "sales@business.com",
);
expect(salesResult?.category).toBe("Marketing");
},
TIMEOUT,
);

it("should handle empty senders list", async () => {
const result = await aiCategorizeSenders({
Expand All @@ -85,116 +93,132 @@ describe.runIf(isAiTest)("AI Sender Categorization", () => {
expect(result).toEqual([]);
});

it("should categorize senders for all valid SenderCategory values", async () => {
const senders = getEnabledCategories()
.filter((category) => category.name !== "Unknown")
.map((category) => `${category.name}@example.com`);
it(
"should categorize senders for all valid SenderCategory values",
async () => {
const senders = getEnabledCategories()
.filter((category) => category.name !== "Unknown")
.map((category) => `${category.name}@example.com`);

const result = await aiCategorizeSenders({
emailAccount,
senders: senders.map((sender) => ({
emailAddress: sender,
emails: [],
})),
categories: getEnabledCategories(),
});
const result = await aiCategorizeSenders({
emailAccount,
senders: senders.map((sender) => ({
emailAddress: sender,
emails: [],
})),
categories: getEnabledCategories(),
});

expect(result).toHaveLength(senders.length);
expect(result).toHaveLength(senders.length);

for (const sender of senders) {
const category = sender.split("@")[0];
const senderResult = result.find((r) => r.sender === sender);
expect(senderResult).toBeDefined();
expect(senderResult?.category).toBe(category);
}
}, 15_000);
for (const sender of senders) {
const category = sender.split("@")[0];
const senderResult = result.find((r) => r.sender === sender);
expect(senderResult).toBeDefined();
expect(senderResult?.category).toBe(category);
}
},
TIMEOUT,
);
});

describe("Single Sender Categorization", () => {
it("should categorize individual senders with snippets", async () => {
for (const { emailAddress, emails, expectedCategory } of testSenders) {
it(
"should categorize individual senders with snippets",
async () => {
for (const { emailAddress, emails, expectedCategory } of testSenders) {
const result = await aiCategorizeSender({
emailAccount,
sender: emailAddress,
previousEmails: emails,
categories: getEnabledCategories(),
});

if (expectedCategory === "Unknown") {
expect([REQUEST_MORE_INFORMATION_CATEGORY, "Unknown"]).toContain(
result?.category,
);
} else {
expect(result?.category).toBe(expectedCategory);
}
}
},
TIMEOUT * 2,
);

it(
"should handle unknown sender appropriately",
async () => {
const unknownSender = testSenders.find(
(s) => s.expectedCategory === "Unknown",
);
if (!unknownSender) throw new Error("No unknown sender in test data");

const result = await aiCategorizeSender({
emailAccount,
sender: emailAddress,
previousEmails: emails,
sender: unknownSender.emailAddress,
previousEmails: [],
categories: getEnabledCategories(),
});

if (expectedCategory === "Unknown") {
expect([REQUEST_MORE_INFORMATION_CATEGORY, "Unknown"]).toContain(
result?.category,
);
} else {
expect(result?.category).toBe(expectedCategory);
}
}
}, 30_000);

it("should handle unknown sender appropriately", async () => {
const unknownSender = testSenders.find(
(s) => s.expectedCategory === "Unknown",
);
if (!unknownSender) throw new Error("No unknown sender in test data");

const result = await aiCategorizeSender({
emailAccount,
sender: unknownSender.emailAddress,
previousEmails: [],
categories: getEnabledCategories(),
});

expect([REQUEST_MORE_INFORMATION_CATEGORY, "Unknown"]).toContain(
result?.category,
);
}, 15_000);
expect([REQUEST_MORE_INFORMATION_CATEGORY, "Unknown"]).toContain(
result?.category,
);
},
TIMEOUT,
);
});

describe("Comparison Tests", () => {
it("should produce consistent results between bulk and single categorization", async () => {
// Run bulk categorization
const bulkResults = await aiCategorizeSenders({
emailAccount,
senders: testSenders,
categories: getEnabledCategories(),
});
it(
"should produce consistent results between bulk and single categorization",
async () => {
// Run bulk categorization
const bulkResults = await aiCategorizeSenders({
emailAccount,
senders: testSenders,
categories: getEnabledCategories(),
});

// Run individual categorizations and pair with senders
const singleResults = await Promise.all(
testSenders.map(async ({ emailAddress, emails }) => {
const result = await aiCategorizeSender({
emailAccount,
sender: emailAddress,
previousEmails: emails,
categories: getEnabledCategories(),
});
return {
sender: emailAddress,
category: result?.category,
};
}),
);

// Compare results for each sender
for (const { emailAddress, expectedCategory } of testSenders) {
const bulkResult = bulkResults.find((r) => r.sender === emailAddress);
const singleResult = singleResults.find(
(r) => r.sender === emailAddress,
// Run individual categorizations and pair with senders
const singleResults = await Promise.all(
testSenders.map(async ({ emailAddress, emails }) => {
const result = await aiCategorizeSender({
emailAccount,
sender: emailAddress,
previousEmails: emails,
categories: getEnabledCategories(),
});
return {
sender: emailAddress,
category: result?.category,
};
}),
);

// Both should either have a category or both be undefined
if (bulkResult?.category || singleResult?.category) {
expect(bulkResult?.category).toBeDefined();
expect(singleResult?.category).toBeDefined();
expect(bulkResult?.category).toBe(singleResult?.category);
// Compare results for each sender
for (const { emailAddress, expectedCategory } of testSenders) {
const bulkResult = bulkResults.find((r) => r.sender === emailAddress);
const singleResult = singleResults.find(
(r) => r.sender === emailAddress,
);

// If not Unknown, check against expected category
if (expectedCategory !== "Unknown") {
expect(bulkResult?.category).toBe(expectedCategory);
expect(singleResult?.category).toBe(expectedCategory);
// Both should either have a category or both be undefined
if (bulkResult?.category || singleResult?.category) {
expect(bulkResult?.category).toBeDefined();
expect(singleResult?.category).toBeDefined();
expect(bulkResult?.category).toBe(singleResult?.category);

// If not Unknown, check against expected category
if (expectedCategory !== "Unknown") {
expect(bulkResult?.category).toBe(expectedCategory);
expect(singleResult?.category).toBe(expectedCategory);
}
}
}
}
}, 30_000);
},
TIMEOUT * 2,
);
});
});

Expand Down
Loading
Loading