Link A/B testing

apps/web/app/api/cron/links/[linkId]/complete-tests/route.ts

@@ -0,0 +1,48 @@
+import { handleAndReturnErrorResponse } from "@/lib/api/errors";
+import { completeABTests } from "@/lib/api/links/complete-ab-tests";
+import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
+import { prisma } from "@dub/prisma";
+
+// POST - /api/cron/links/[linkId]/complete-tests
+// Completes a link's AB tests if they're ready, scheduled by QStash
+export async function POST(
+  req: Request,
+  {
+    params: { linkId },
+  }: {
+    params: { linkId: string };
+  },
+) {
+  try {
+    const rawBody = await req.text();
+    await verifyQstashSignature({ req, rawBody });
+
+    const link = await prisma.link.findUnique({
+      where: {
+        id: linkId,
+      },
+      include: {
+        project: true,
+      },
+    });
+
+    if (!link) {
+      return new Response(`Link ${linkId} not found. Skipping...`);
+    }
+
+    if (
+      link.testVariants &&
+      link.testCompletedAt &&
+      link.testCompletedAt < new Date() &&
+      Date.now() - link.testCompletedAt.getTime() < 2 * 60 * 60 * 1000 // Limit to two hour window
+    ) {
+      await completeABTests(link as any);
+
+      return new Response(`Tests completed for link ${linkId}.`);
+    }
+
+    return new Response(`No test completion necessary for link ${linkId}.`);
+  } catch (error) {
+    return handleAndReturnErrorResponse(error);
+  }
+}

apps/web/app/api/links/[linkId]/route.ts

@@ -108,6 +108,7 @@ export const PATCH = withWorkspace(
           ? link.expiresAt.toISOString()
           : link.expiresAt,
       geo: link.geo as NewLinkProps["geo"],
+
       ...body,
       // for UTM tags, we only pass them to processLink if they have changed from their previous value
       // or else they will override any changes to the UTM params in the destination URL

apps/web/app/api/links/bulk/route.ts

@@ -389,6 +389,15 @@ export const PATCH = withWorkspace(
                 ? link.expiresAt.toISOString()
                 : link.expiresAt,
             geo: link.geo as NewLinkProps["geo"],
+            testVariants: link.testVariants as NewLinkProps["testVariants"],
+            testCompletedAt:
+              link.testCompletedAt instanceof Date
+                ? link.testCompletedAt.toISOString()
+                : link.testCompletedAt,
+            testStartedAt:
+              link.testStartedAt instanceof Date
+                ? link.testStartedAt.toISOString()
+                : link.testStartedAt,
             ...data,
           },
           workspace,

apps/web/app/api/links/upsert/route.ts

@@ -62,6 +62,15 @@ export const PUT = withWorkspace(
             ? link.expiresAt.toISOString()
             : link.expiresAt,
         geo: link.geo as NewLinkProps["geo"],
+        testVariants: link.testVariants as NewLinkProps["testVariants"],
+        testCompletedAt:
+          link.testCompletedAt instanceof Date
+            ? link.testCompletedAt.toISOString()
+            : link.testCompletedAt,
+        testStartedAt:
+          link.testStartedAt instanceof Date
+            ? link.testStartedAt.toISOString()
+            : link.testStartedAt,
         ...body,
       };
 

apps/web/app/api/partner-profile/programs/[programId]/links/[linkId]/route.ts

@@ -69,6 +69,15 @@ export const PATCH = withPartnerProfile(
             ? link.expiresAt.toISOString()
             : link.expiresAt,
         geo: link.geo as NewLinkProps["geo"],
+        testVariants: link.testVariants as NewLinkProps["testVariants"],
+        testCompletedAt:
+          link.testCompletedAt instanceof Date
+            ? link.testCompletedAt.toISOString()
+            : link.testCompletedAt,
+        testStartedAt:
+          link.testStartedAt instanceof Date
+            ? link.testStartedAt.toISOString()
+            : link.testStartedAt,
 
         // merge in new props
         key: key || undefined,

apps/web/app/api/partners/links/upsert/route.ts

@@ -85,6 +85,15 @@ export const PUT = withWorkspace(
             ? link.expiresAt.toISOString()
             : link.expiresAt,
         geo: link.geo as NewLinkProps["geo"],
+        testVariants: link.testVariants as NewLinkProps["testVariants"],
+        testCompletedAt:
+          link.testCompletedAt instanceof Date
+            ? link.testCompletedAt.toISOString()
+            : link.testCompletedAt,
+        testStartedAt:
+          link.testStartedAt instanceof Date
+            ? link.testStartedAt.toISOString()
+            : link.testStartedAt,
         // merge in new props
         ...linkProps,
         // set default fields

apps/web/lib/api/links/ab-test-scheduler.ts

@@ -0,0 +1,52 @@
+import { qstash } from "@/lib/cron";
+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
+import { ExpandedLink } from "./utils";
+
+// Schedules a job to complete a link's AB test
+export async function scheduleABTestCompletion(
+  link: Pick<ExpandedLink, "id" | "testVariants" | "testCompletedAt">,
+) {
+  await cancelScheduledABTestCompletion(link.id);
+
+  if (!link.testVariants || !link.testCompletedAt) {
+    return;
+  }
+
+  const url = `${APP_DOMAIN_WITH_NGROK}/api/cron/links/${link.id}/complete-tests`;
+  const testCompletedAt = new Date(link.testCompletedAt);
+
+  // Tests are not complete yet, schedule a job for completion
+  if (testCompletedAt > new Date()) {
+    await qstash.publishJSON({
+      url,
+      delay: (testCompletedAt.getTime() - new Date().getTime()) / 1000,
+      deduplicationId: link.id,
+    });
+  }
+}
+
+// Cancels a scheduled AB test completion for a link (Eg: if the link is updated or deleted)
+export async function cancelScheduledABTestCompletion(linkId: string) {
+  const url = `${APP_DOMAIN_WITH_NGROK}/api/cron/links/${linkId}/complete-tests`;
+
+  const response = await fetch("https://qstash.upstash.io/v2/messages", {
+    method: "DELETE",
+    headers: {
+      Authorization: `Bearer ${process.env.QSTASH_TOKEN}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      url,
+    }),
+  });
+
+  if (!response.ok) {
+    console.error(
+      `Failed to cancel scheduled AB test completion for link ${linkId}`,
+      {
+        status: response.status,
+        statusText: response.statusText,
+      },
+    );
+  }
+}

apps/web/lib/api/links/bulk-create-links.ts

@@ -1,6 +1,7 @@
 import { ProcessedLinkProps } from "@/lib/types";
 import { prisma } from "@dub/prisma";
 import { getParamsFromURL, linkConstructorSimple, truncate } from "@dub/utils";
+import { Prisma } from "@prisma/client";
 import { waitUntil } from "@vercel/functions";
 import { createId } from "../create-id";
 import { combineTagIds } from "../tags/combine-tag-ids";
@@ -71,6 +72,7 @@ export async function bulkCreateLinks({
         utm_content,
         expiresAt: link.expiresAt ? new Date(link.expiresAt) : null,
         geo: link.geo || undefined,
+        testVariants: link.testVariants || Prisma.JsonNull,
       };
     }),
     skipDuplicates: true,

apps/web/lib/api/links/bulk-update-links.ts

@@ -25,6 +25,7 @@ export async function bulkUpdateLinks(
     proxy,
     expiresAt,
     geo,
+    testVariants,
     tagId,
     tagIds,
     tagNames,
@@ -54,6 +55,8 @@ export async function bulkUpdateLinks(
               : image,
           expiresAt: expiresAt ? new Date(expiresAt) : null,
           geo: geo || Prisma.JsonNull,
+          testVariants: testVariants || Prisma.JsonNull,
+
           ...(url && getParamsFromURL(url)),
           // Associate tags by tagNames
           ...(tagNames &&

apps/web/lib/api/links/complete-ab-tests.ts

@@ -0,0 +1,121 @@
+import { getAnalytics } from "@/lib/analytics/get-analytics";
+import { NewLinkProps, WorkspaceProps } from "@/lib/types";
+import { sendWorkspaceWebhook } from "@/lib/webhook/publish";
+import { ABTestVariantsSchema, linkEventSchema } from "@/lib/zod/schemas/links";
+import { Link, Project } from "@prisma/client";
+import { waitUntil } from "@vercel/functions";
+import { DubApiError, ErrorCodes } from "../errors";
+import { processLink } from "./process-link";
+import { updateLink } from "./update-link";
+
+export async function completeABTests(link: Link & { project: Project }) {
+  if (!link.testVariants || !link.testCompletedAt || !link.projectId) {
+    return;
+  }
+
+  const testVariants = ABTestVariantsSchema.parse(link.testVariants);
+
+  const analytics: { url: string; leads: number }[] = await getAnalytics({
+    event: "leads",
+    groupBy: "top_urls",
+    linkId: link.id,
+    workspaceId: link.projectId,
+    dataAvailableFrom: link.project.createdAt,
+    start: link.testStartedAt ? new Date(link.testStartedAt) : undefined,
+    end: link.testCompletedAt,
+  });
+
+  const max = Math.max(
+    ...testVariants.map(
+      (test) => analytics.find(({ url }) => url === test.url)?.leads || 0,
+    ),
+  );
+
+  // There are no leads generated for any test variant, do nothing
+  if (max === 0) {
+    console.log(
+      `AB Test completed but all results are zero for ${link.id}, doing nothing.`,
+    );
+
+    return;
+  }
+
+  const winners = testVariants.filter(
+    (test) =>
+      (analytics.find(({ url }) => url === test.url)?.leads || 0) === max,
+  );
+
+  if (winners.length === 0) {
+    throw new Error(
+      `AB Test completed but failed to find winners based on max leads for link ${link.id}.`,
+    );
+  }
+
+  const winner = winners[Math.floor(Math.random() * winners.length)];
+
+  if (winner.url === link.url) {
+    return;
+  }
+
+  // Update the link's URL to the winner
+  const { project, ...originalLink } = link;
+
+  const updatedLink = {
+    ...originalLink,
+    expiresAt:
+      link.expiresAt instanceof Date
+        ? link.expiresAt.toISOString()
+        : link.expiresAt,
+    geo: link.geo as NewLinkProps["geo"],
+    testVariants: link.testVariants as NewLinkProps["testVariants"],
+    testCompletedAt:
+      link.testCompletedAt instanceof Date
+        ? link.testCompletedAt.toISOString()
+        : link.testCompletedAt,
+    testStartedAt:
+      link.testStartedAt instanceof Date
+        ? link.testStartedAt.toISOString()
+        : link.testStartedAt,
+    url: winner.url,
+    ...(link.key === "_root" && {
+      domain: link.domain,
+      key: link.key,
+    }),
+  };
+
+  const {
+    link: processedLink,
+    error,
+    code,
+  } = await processLink({
+    payload: updatedLink,
+    workspace: link.project as WorkspaceProps,
+    skipKeyChecks: true,
+    skipExternalIdChecks: true,
+    skipFolderChecks: true,
+  });
+
+  if (error) {
+    throw new DubApiError({
+      code: code as ErrorCodes,
+      message: error,
+    });
+  }
+
+  const response = await updateLink({
+    oldLink: {
+      domain: link.domain,
+      key: link.key,
+      image: link.image,
+    },
+    updatedLink: processedLink,
+  });
+
+  waitUntil(
+    sendWorkspaceWebhook({
+      trigger: "link.updated",
+      workspace: link.project,
+      data: linkEventSchema.parse(response),
+    }),
+  );
+}

apps/web/lib/api/links/create-link.ts

@@ -15,6 +15,7 @@ import { linkConstructorSimple } from "@dub/utils/src/functions/link-constructor
 import { waitUntil } from "@vercel/functions";
 import { createId } from "../create-id";
 import { combineTagIds } from "../tags/combine-tag-ids";
+import { scheduleABTestCompletion } from "./ab-test-scheduler";
 import { linkCache } from "./cache";
 import { encodeKeyIfCaseSensitive } from "./case-sensitivity";
 import { includeTags } from "./include-tags";
@@ -32,6 +33,9 @@ export async function createLink(link: ProcessedLinkProps) {
     proxy,
     geo,
     publicStats,
+    testVariants,
+    testStartedAt,
+    testCompletedAt,
   } = link;
 
   const combinedTagIds = combineTagIds(link);
@@ -64,6 +68,10 @@ export async function createLink(link: ProcessedLinkProps) {
       expiresAt: expiresAt ? new Date(expiresAt) : null,
       geo: geo || Prisma.JsonNull,
 
+      testVariants: testVariants || Prisma.JsonNull,
+      testCompletedAt: testCompletedAt ? new Date(testCompletedAt) : null,
+      testStartedAt: testStartedAt ? new Date(testStartedAt) : null,
+
       // Associate tags by tagNames
       ...(tagNames?.length &&
         link.projectId && {
@@ -173,6 +181,8 @@ export async function createLink(link: ProcessedLinkProps) {
         propagateWebhookTriggerChanges({
           webhookIds,
         }),
+
+      testVariants && testCompletedAt && scheduleABTestCompletion(response),
     ]),
   );
 

apps/web/lib/api/links/delete-link.ts

@@ -3,6 +3,7 @@ import { recordLinkTB, transformLinkTB } from "@/lib/tinybird";
 import { prisma } from "@dub/prisma";
 import { R2_URL } from "@dub/utils";
 import { waitUntil } from "@vercel/functions";
+import { cancelScheduledABTestCompletion } from "./ab-test-scheduler";
 import { linkCache } from "./cache";
 import { includeTags } from "./include-tags";
 import { transformLink } from "./utils";
@@ -32,6 +33,7 @@ export async function deleteLink(linkId: string) {
         ...transformLinkTB(link),
         deleted: true,
       }),
+
       link.projectId &&
         prisma.project.update({
           where: {
@@ -41,6 +43,10 @@ export async function deleteLink(linkId: string) {
             totalLinks: { decrement: 1 },
           },
         }),
+
+      link.testVariants &&
+        link.testCompletedAt &&
+        cancelScheduledABTestCompletion(link.id),
     ]),
   );