Optimize analytics endpoints

apps/web/app/api/workspaces/[idOrSlug]/billing/usage/route.ts

@@ -9,7 +9,7 @@ export const GET = withWorkspace(async ({ searchParams, workspace }) => {
     usageQuerySchema.parse(searchParams);
 
   const pipe = tb.buildPipe({
-    pipe: "v2_usage",
+    pipe: "v3_usage",
     // we extend this here since we don't need to include all the additional parameters
     // in the actual request query schema
     parameters: usageQuerySchema.extend({

apps/web/lib/analytics/get-analytics.ts

@@ -80,14 +80,15 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
 
   // Create a Tinybird pipe
   const pipe = tb.buildPipe({
-    pipe: `v2_${UTM_TAGS_PLURAL_LIST.includes(groupBy) ? "utms" : groupBy}`,
+    pipe: `v3_${UTM_TAGS_PLURAL_LIST.includes(groupBy) ? "utms" : groupBy}`,
     parameters: analyticsFilterTB,
     data:
       groupBy === "top_links" || UTM_TAGS_PLURAL_LIST.includes(groupBy)
         ? z.any()
         : analyticsResponse[groupBy],
   });
 
+  console.time(`pipe (${groupBy}, ${event})`);
   const response = await pipe({
     ...params,
     ...(UTM_TAGS_PLURAL_LIST.includes(groupBy)
@@ -104,6 +105,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
     country,
     region,
   });
+  console.timeEnd(`pipe (${groupBy}, ${event})`);
 
   if (groupBy === "count") {
     // Return the count value for deprecated endpoints

apps/web/lib/analytics/get-events.ts

@@ -67,7 +67,7 @@ export const getEvents = async (params: EventsFilters) => {
   }
 
   const pipe = tb.buildPipe({
-    pipe: "v2_events",
+    pipe: "v3_events",
     parameters: eventsFilterTB,
     data:
       {

apps/web/scripts/backfill-total-counts.ts

@@ -0,0 +1,58 @@
+import { prisma } from "@dub/prisma";
+import { Prisma } from "@dub/prisma/client";
+import "dotenv-flow/config";
+
+async function main() {
+  const where: Prisma.ProjectWhereInput = {
+    linksUsage: 0,
+    totalLinks: 0,
+    // not within the last 2 hours
+    updatedAt: {
+      lt: new Date(Date.now() - 2 * 60 * 60 * 1000),
+    },
+  };
+  const projectsToUpdate = await prisma.project.findMany({
+    where,
+    select: {
+      id: true,
+    },
+    orderBy: {
+      createdAt: "asc",
+    },
+    take: 50,
+  });
+
+  const results = await Promise.all(
+    projectsToUpdate.map(async (project) => {
+      const totalLinks = await prisma.link.count({
+        where: {
+          projectId: project.id,
+        },
+      });
+
+      return await prisma.project.update({
+        where: {
+          id: project.id,
+        },
+        data: {
+          totalLinks,
+          ...(totalLinks === 0 && {
+            linksUsage: 0,
+          }),
+        },
+        select: {
+          slug: true,
+          totalLinks: true,
+        },
+      });
+    }),
+  );
+
+  console.table(results);
+  const projectLeft = await prisma.project.count({
+    where,
+  });
+  console.log(`${projectLeft} projects left to update`);
+}
+
+main();

packages/tinybird/datasources/dub_enriched_click_events_mv.datasource

@@ -0,0 +1,35 @@
+# Data Source created from Pipe 'dub_enriched_click_events_pipe'
+
+SCHEMA >
+    `timestamp` DateTime64(3),
+    `click_id` String,
+    `link_id` String,
+    `url` String,
+    `continent` LowCardinality(String),
+    `country` LowCardinality(String),
+    `city` String,
+    `region` String,
+    `latitude` String,
+    `longitude` String,
+    `device` LowCardinality(String),
+    `browser` LowCardinality(String),
+    `os` LowCardinality(String),
+    `engine` LowCardinality(String),
+    `ua` String,
+    `identity_hash` String,
+    `referer` String,
+    `referer_url` String,
+    `qr` UInt8,
+    `ip` String,
+    `workspace_id` String,
+    `folder_id` String,
+    `tag_ids` Array(String),
+    `domain` String,
+    `key` String,
+    `program_id` String,
+    `tenant_id` String,
+    `partner_id` String
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMMDD(timestamp)"
+ENGINE_SORTING_KEY "workspace_id, timestamp, link_id"

packages/tinybird/datasources/dub_enriched_lead_events_mv.datasource

@@ -0,0 +1,37 @@
+# Data Source created from Pipe 'dub_enriched_lead_events_pipe'
+
+SCHEMA >
+    `timestamp` DateTime64(3),
+    `event_id` String,
+    `event_name` String,
+    `click_id` String,
+    `link_id` String,
+    `customer_id` String,
+    `url` String,
+    `continent` LowCardinality(String),
+    `country` LowCardinality(String),
+    `city` String,
+    `region` String,
+    `latitude` String,
+    `longitude` String,
+    `device` LowCardinality(String),
+    `browser` LowCardinality(String),
+    `os` LowCardinality(String),
+    `engine` LowCardinality(String),
+    `ua` String,
+    `referer` String,
+    `referer_url` String,
+    `qr` UInt8,
+    `ip` String,
+    `workspace_id` String,
+    `folder_id` String,
+    `tag_ids` Array(String),
+    `domain` String,
+    `key` String,
+    `program_id` String,
+    `tenant_id` String,
+    `partner_id` String
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMMDD(timestamp)"
+ENGINE_SORTING_KEY "workspace_id, timestamp, link_id"

packages/tinybird/datasources/dub_enriched_sale_events_mv.datasource

@@ -0,0 +1,40 @@
+# Data Source created from Pipe 'dub_enriched_sale_events_pipe'
+
+SCHEMA >
+    `timestamp` DateTime64(3),
+    `event_id` String,
+    `event_name` String,
+    `click_id` String,
+    `link_id` String,
+    `customer_id` String,
+    `payment_processor` LowCardinality(String),
+    `invoice_id` String,
+    `amount` UInt32,
+    `url` String,
+    `continent` LowCardinality(String),
+    `country` LowCardinality(String),
+    `city` String,
+    `region` String,
+    `latitude` String,
+    `longitude` String,
+    `device` LowCardinality(String),
+    `browser` LowCardinality(String),
+    `os` LowCardinality(String),
+    `engine` LowCardinality(String),
+    `ua` String,
+    `referer` String,
+    `referer_url` String,
+    `qr` UInt8,
+    `ip` String,
+    `workspace_id` String,
+    `folder_id` String,
+    `tag_ids` Array(String),
+    `domain` String,
+    `key` String,
+    `program_id` String,
+    `tenant_id` String,
+    `partner_id` String
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMMDD(timestamp)"
+ENGINE_SORTING_KEY "workspace_id, timestamp, link_id"

packages/tinybird/pipes/dub_enriched_click_events_pipe.pipe

@@ -0,0 +1,40 @@
+NODE mv
+SQL >
+
+    SELECT DISTINCT ON (e.click_id)
+        e.timestamp,
+        e.click_id,
+        e.link_id,
+        e.url,
+        e.continent,
+        e.country,
+        e.city,
+        e.region,
+        e.latitude,
+        e.longitude,
+        e.device,
+        e.browser,
+        e.os,
+        e.engine,
+        e.ua,
+        coalesce(e.identity_hash, '') as identity_hash,
+        e.referer,
+        e.referer_url,
+        e.qr,
+        e.ip,
+
+        l.workspace_id,
+        l.folder_id,
+        if(l.tag_ids IS NULL, [], l.tag_ids) as tag_ids,
+        l.domain,
+        l.key,
+        l.program_id,
+        l.tenant_id,
+        l.partner_id,
+    FROM dub_click_events_mv e
+    INNER JOIN dub_links_metadata_latest l FINAL ON e.link_id = l.link_id
+
+TYPE materialized
+DATASOURCE dub_enriched_click_events_mv
+
+

packages/tinybird/pipes/dub_enriched_lead_events_pipe.pipe

@@ -0,0 +1,42 @@
+NODE mv
+SQL >
+
+    SELECT DISTINCT ON (e.event_id)
+        e.timestamp,
+        e.event_id,
+        e.event_name,
+        e.click_id,
+        e.link_id,
+        e.customer_id,
+        e.url,
+        e.continent,
+        e.country,
+        e.city,
+        e.region,
+        e.latitude,
+        e.longitude,
+        e.device,
+        e.browser,
+        e.os,
+        e.engine,
+        e.ua,
+        e.referer,
+        e.referer_url,
+        e.qr,
+        e.ip,
+
+        l.workspace_id,
+        l.folder_id,
+        if(l.tag_ids IS NULL, [], l.tag_ids) as tag_ids,
+        l.domain,
+        l.key,
+        l.program_id,
+        l.tenant_id,
+        l.partner_id,
+    FROM dub_lead_events_mv e
+    INNER JOIN dub_links_metadata_latest l FINAL ON e.link_id = l.link_id
+
+TYPE materialized
+DATASOURCE dub_enriched_lead_events_mv
+
+

packages/tinybird/pipes/dub_enriched_sale_events_pipe.pipe

@@ -0,0 +1,45 @@
+NODE mv
+SQL >
+
+    SELECT DISTINCT ON (e.event_id)
+        e.timestamp,
+        e.event_id,
+        e.event_name,
+        e.click_id,
+        e.link_id,
+        e.customer_id,
+        e.payment_processor,
+        e.invoice_id,
+        e.amount,
+        e.url,
+        e.continent,
+        e.country,
+        e.city,
+        e.region,
+        e.latitude,
+        e.longitude,
+        e.device,
+        e.browser,
+        e.os,
+        e.engine,
+        e.ua,
+        e.referer,
+        e.referer_url,
+        e.qr,
+        e.ip,
+
+        l.workspace_id,
+        l.folder_id,
+        if(l.tag_ids IS NULL, [], l.tag_ids) as tag_ids,
+        l.domain,
+        l.key,
+        l.program_id,
+        l.tenant_id,
+        l.partner_id,
+    FROM dub_sale_events_mv e
+    INNER JOIN dub_links_metadata_latest l FINAL ON e.link_id = l.link_id
+
+TYPE materialized
+DATASOURCE dub_enriched_sale_events_mv
+
+