feat(billing): add deep-ish research task

apps/billing/package.json

@@ -5,8 +5,8 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "dev": "pnpm dlx trigger.dev@latest dev",
-    "trigger:deploy": "pnpm dlx trigger.dev@latest deploy",
+    "dev": "pnpm dlx trigger.dev@latest dev --skip-update-check",
+    "trigger:deploy": "pnpm dlx trigger.dev@latest deploy --skip-update-check",
     "db:push": "dotenv -e .env drizzle-kit push",
     "db:studio": "dotenv -e .env drizzle-kit studio --verbose",
     "db:generate": "dotenv -e .env drizzle-kit generate",
@@ -18,22 +18,19 @@
   "license": "ISC",
   "dependencies": {
     "@agentic/serper": "^7.0.0",
+    "@ai-sdk/google": "^1.1.19",
     "@ai-sdk/openai": "^0.0.63",
-    "@clerk/nextjs": "^4.29.10",
     "@mendable/firecrawl-js": "^1.5.2",
     "@octokit/rest": "^21.0.2",
     "@planetscale/database": "^1.16.0",
-    "@trigger.dev/nextjs": "3.3.1",
-    "@trigger.dev/sdk": "3.3.1",
-    "@trigger.dev/slack": "3.3.1",
+    "@trigger.dev/nextjs": "3.3.12",
+    "@trigger.dev/sdk": "3.3.12",
+    "@trigger.dev/slack": "3.3.12",
     "@types/js-yaml": "^4.0.9",
-    "@unkey/error": "workspace:^",
-    "@unkey/id": "workspace:^",
-    "@unkey/resend": "workspace:^",
-    "@unkey/schema": "workspace:^",
     "ai": "^3.4.7",
     "drizzle-orm": "^0.33.0",
     "drizzle-zod": "^0.5.1",
+    "exa-js": "^1.4.10",
     "github-slugger": "^2.0.0",
     "js-yaml": "^4.1.0",
     "react-dom": "^18",

apps/billing/src/lib/db-marketing/schemas/entries.ts

@@ -30,12 +30,12 @@ export const entries = mysqlTable(
   "entries",
   {
     id: int("id").primaryKey().autoincrement(),
-    inputTerm: varchar("input_term", { length: 255 }).notNull(),
-    githubPrUrl: varchar("github_pr_url", { length: 255 }),
+    inputTerm: varchar("input_term", { length: 767 }).notNull(),
+    githubPrUrl: text("github_pr_url"),
     dynamicSectionsContent: text("dynamic_sections_content"),
-    metaTitle: varchar("meta_title", { length: 255 }),
-    metaDescription: varchar("meta_description", { length: 255 }),
-    metaH1: varchar("meta_h1", { length: 255 }),
+    metaTitle: text("meta_title"),
+    metaDescription: text("meta_description"),
+    metaH1: text("meta_h1"),
     categories: json("linking_categories").$type<string[]>().default([]),
     status: mysqlEnum("status", entryStatus),
     takeaways: json("content_takeaways").$type<Takeaways>(),
@@ -47,7 +47,7 @@ export const entries = mysqlTable(
       .$onUpdate(() => new Date()),
   },
   (table) => ({
-    inputTermIdx: index("input_term_idx").on(table.inputTerm),
+    inputTermHashIdx: index("input_term_idx").on(table.inputTerm),
   }),
 );
 

apps/billing/src/lib/db-marketing/schemas/evals.ts

@@ -1,10 +1,10 @@
 import { relations } from "drizzle-orm";
-import { index, int, mysqlEnum, mysqlTable, text, timestamp } from "drizzle-orm/mysql-core";
+import { index, int, mysqlTable, text, timestamp, varchar } from "drizzle-orm/mysql-core";
 import { createInsertSchema, createSelectSchema } from "drizzle-zod";
 import { z } from "zod";
 import { entries } from "./entries";
 
-export const evalTypes = ["technical", "seo", "editorial"] as const;
+export const evalTypes = ["technical", "seo", "editorial", "brand_bias"] as const;
 export type EvalType = (typeof evalTypes)[number];
 
 export const evals = mysqlTable(
@@ -14,7 +14,7 @@ export const evals = mysqlTable(
     entryId: int("entry_id")
       .notNull()
       .references(() => entries.id),
-    type: mysqlEnum("type", evalTypes).notNull(),
+    type: varchar("type", { enum: evalTypes, length: 1024 }),
     ratings: text("ratings").notNull(), // JSON stringified ratings
     recommendations: text("recommendations").notNull().default("[]"), // Add default empty array
     outline: text("outline").default("[]"), // Add outline field
@@ -70,6 +70,19 @@ export const recommendationsSchema = z.object({
   ),
 });
 
+// schemas for brand bias evaluation to be used with LLM
+export const brandBiasRatingSchema = z.object({
+  commercialBias: z.number().min(0).max(10),
+  neutralityScore: z.number().min(0).max(10),
+  educationalValue: z.number().min(0).max(10),
+});
+
+export const brandBiasRecommendationSchema = z.object({
+  recommendation: z.enum(["use_current", "fetch_neutral"]),
+  dominantBrands: z.array(z.string()),
+  reasoning: z.string(),
+});
+
 // DB schemas
 export const insertEvalSchema = createInsertSchema(evals).extend({}).omit({ id: true });
 export const selectEvalSchema = createSelectSchema(evals);

apps/billing/src/lib/db-marketing/schemas/firecrawl.ts

@@ -23,18 +23,18 @@ export const firecrawlResponses = mysqlTable(
     markdown: text("markdown"),
     sourceUrl: varchar("source_url", { length: 767 }).notNull(),
     statusCode: int("status_code"),
-    title: varchar("title", { length: 767 }),
+    title: text("title"),
     description: text("description"),
-    language: varchar("language", { length: 255 }),
-    ogTitle: varchar("og_title", { length: 767 }),
-    ogDescription: varchar("og_description", { length: 767 }),
+    language: text("language"),
+    ogTitle: text("og_title"),
+    ogDescription: text("og_description"),
     ogUrl: text("og_url"),
-    ogImage: varchar("og_image", { length: 767 }),
-    ogSiteName: varchar("og_site_name", { length: 767 }),
+    ogImage: text("og_image"),
+    ogSiteName: text("og_site_name"),
     createdAt: timestamp("created_at").defaultNow().notNull(),
     updatedAt: timestamp("updated_at").onUpdateNow(),
     error: text("error"),
-    inputTerm: varchar("input_term", { length: 255 }),
+    inputTerm: varchar("input_term", { length: 767 }),
     summary: text("summary"),
   },
   (table) => ({

apps/billing/src/lib/db-marketing/schemas/index.ts

@@ -5,3 +5,4 @@ export * from "./keywords";
 export * from "./firecrawl";
 export * from "./sections";
 export * from "./evals";
+export * from "./takeaways-schema";

apps/billing/src/lib/db-marketing/schemas/keywords.ts

@@ -10,17 +10,21 @@ export const keywords = mysqlTable(
   "keywords",
   {
     id: int("id").primaryKey().autoincrement(),
-    inputTerm: varchar("input_term", { length: 255 }).notNull(),
-    keyword: varchar("keyword", { length: 255 }).notNull(),
-    source: varchar("source", { length: 255 }).notNull(),
+    inputTerm: varchar("input_term", { length: 767 }).notNull(),
+    keyword: varchar("keyword", { length: 767 }).notNull(),
+    inputTermAndKeywordHash: varchar("input_term_and_keyword_hash", { length: 64 }).notNull(), // so that we avoid having to halve the inputTerm/keywords length
+    source: varchar("source", { length: 767 }).notNull(),
     sourceUrl: varchar("source_url", { length: 767 }),
     createdAt: timestamp("created_at").defaultNow().notNull(),
     updatedAt: timestamp("updated_at").defaultNow().notNull(),
   },
   (table) => ({
     inputTermIdx: index("input_term_idx").on(table.inputTerm),
+    keywordIdx: index("keyword_idx").on(table.keyword),
     sourceUrlIdx: index("source_url_idx").on(table.sourceUrl),
-    uniqueKeyword: unique("keywords_input_term_keyword_unique").on(table.inputTerm, table.keyword),
+    uniqueKeyword: unique("keywords_input_term_keyword_hash_unique").on(
+      table.inputTermAndKeywordHash,
+    ),
   }),
 );
 

apps/billing/src/lib/db-marketing/schemas/searchQuery.ts

@@ -4,6 +4,7 @@ import {
   index,
   int,
   mysqlTable,
+  text,
   timestamp,
   unique,
   varchar,
@@ -18,11 +19,11 @@ export const searchQueries = mysqlTable(
   "search_queries",
   {
     id: int("id").primaryKey().autoincrement(),
-    inputTerm: varchar("input_term", { length: 255 }).notNull(),
-    query: varchar("query", { length: 255 }).notNull(),
+    inputTerm: varchar("input_term", { length: 767 }).notNull(),
+    query: text("query").notNull(),
     isTermAsQueryAmbiguous: boolean("is_term_as_query_ambiguous").notNull().default(false),
-    ambiguityReason: varchar("ambiguity_reason", { length: 255 }).notNull().default(""),
-    clarifyingContext: varchar("clarifying_context", { length: 255 }).notNull().default(""),
+    ambiguityReason: text("ambiguity_reason").notNull().default(""),
+    clarifyingContext: text("clarifying_context").notNull().default(""),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
   },

apps/billing/src/lib/db-marketing/schemas/sections.ts

@@ -1,13 +1,5 @@
 import { relations } from "drizzle-orm";
-import {
-  int,
-  mysqlEnum,
-  mysqlTable,
-  primaryKey,
-  text,
-  timestamp,
-  varchar,
-} from "drizzle-orm/mysql-core";
+import { int, mysqlEnum, mysqlTable, primaryKey, text, timestamp } from "drizzle-orm/mysql-core";
 import { createInsertSchema, createSelectSchema } from "drizzle-zod";
 import type { z } from "zod";
 import { entries } from "./entries";
@@ -18,7 +10,7 @@ export const sections = mysqlTable("sections", {
   entryId: int("entry_id")
     .notNull()
     .references(() => entries.id),
-  heading: varchar("heading", { length: 255 }).notNull(),
+  heading: text("heading").notNull(),
   description: text("description").notNull(),
   order: int("order").notNull(),
   markdown: text("markdown"),

apps/billing/src/lib/db-marketing/schemas/serper.ts

@@ -10,7 +10,7 @@ export const serperSearchResponses = mysqlTable(
   "serper_search_responses",
   {
     id: int("id").primaryKey().autoincrement(),
-    inputTerm: varchar("input_term", { length: 255 }).notNull(),
+    inputTerm: varchar("input_term", { length: 767 }).notNull(),
     searchParameters: json("search_parameters").notNull(),
     answerBox: json("answer_box"),
     knowledgeGraph: json("knowledge_graph"),
@@ -47,11 +47,11 @@ export const serperOrganicResults = mysqlTable(
     id: int("id").primaryKey().autoincrement(),
     searchResponseId: int("search_response_id").notNull(),
     firecrawlResponseId: int("firecrawl_response_id"),
-    title: varchar("title", { length: 255 }).notNull(),
+    title: text("title").notNull(),
     link: varchar("link", { length: 767 }).notNull(),
     snippet: text("snippet").notNull(),
     position: int("position").notNull(),
-    imageUrl: varchar("image_url", { length: 767 }),
+    imageUrl: text("image_url"),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
   },
@@ -60,12 +60,15 @@ export const serperOrganicResults = mysqlTable(
     linkIdx: index("link_idx").on(table.link),
   }),
 );
-export const serperOrganicResultsRelations = relations(serperOrganicResults, ({ one, many }) => ({
+
+export const serperOrganicResultSchema = createSelectSchema(serperOrganicResults);
+export type SerperOrganicResult = z.infer<typeof serperOrganicResultSchema>;
+
+export const serperOrganicResultsRelations = relations(serperOrganicResults, ({ one }) => ({
   searchResponse: one(serperSearchResponses, {
     fields: [serperOrganicResults.searchResponseId],
     references: [serperSearchResponses.id],
   }),
-  sitelinks: many(serperSitelinks),
   firecrawlResponse: one(firecrawlResponses, {
     fields: [serperOrganicResults.link],
     references: [firecrawlResponses.sourceUrl],
@@ -83,7 +86,7 @@ export const serperSitelinks = mysqlTable(
   {
     id: int("id").primaryKey().autoincrement(),
     organicResultId: int("organic_result_id").notNull(),
-    title: varchar("title", { length: 255 }).notNull(),
+    title: text("title").notNull(),
     link: varchar("link", { length: 767 }).notNull(),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
@@ -115,11 +118,11 @@ export const serperTopStories = mysqlTable(
   {
     id: int("id").primaryKey().autoincrement(),
     searchResponseId: int("search_response_id").notNull(),
-    title: varchar("title", { length: 255 }).notNull(),
+    title: text("title").notNull(),
     link: varchar("link", { length: 767 }).notNull(),
-    source: varchar("source", { length: 255 }).notNull(),
-    date: varchar("date", { length: 255 }).notNull(),
-    imageUrl: varchar("image_url", { length: 767 }).notNull(),
+    source: text("source").notNull(),
+    date: text("date").notNull(),
+    imageUrl: text("image_url").notNull(),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
   },
@@ -148,9 +151,9 @@ export const serperPeopleAlsoAsk = mysqlTable(
   {
     id: int("id").primaryKey().autoincrement(),
     searchResponseId: int("search_response_id").notNull(),
-    question: varchar("question", { length: 255 }).notNull(),
+    question: text("question").notNull(),
     snippet: text("snippet").notNull(),
-    title: varchar("title", { length: 255 }).notNull(),
+    title: text("title").notNull(),
     link: varchar("link", { length: 767 }).notNull(),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
@@ -182,7 +185,7 @@ export const serperRelatedSearches = mysqlTable(
   {
     id: int("id").primaryKey().autoincrement(),
     searchResponseId: int("search_response_id").notNull(),
-    query: varchar("query", { length: 255 }).notNull(),
+    query: varchar("query", { length: 767 }).notNull(),
     createdAt: timestamp("created_at").notNull().defaultNow(),
     updatedAt: timestamp("updated_at").notNull().defaultNow().onUpdateNow(),
   },