feat: add minute granularity - This should be merged before key details

apps/dashboard/lib/trpc/routers/utils/granularity.ts

@@ -52,6 +52,8 @@ export const getTimeseriesGranularity = <TContext extends TimeseriesContext>(
   context: TContext,
   startTime?: number | null,
   endTime?: number | null,
+  // INFO: We'll delete this once key details is merged. Required for accumulating data for minutely charts before merging the key details.
+  allowMinutelyForVerifications = false,
 ): TimeseriesConfig<TContext> => {
   const now = Date.now();
   const WEEK_IN_MS = DAY_IN_MS * 7;
@@ -62,7 +64,6 @@ export const getTimeseriesGranularity = <TContext extends TimeseriesContext>(
   if (!startTime && !endTime) {
     const defaultGranularity = DEFAULT_GRANULARITY[context];
     const defaultDuration = context === "forVerifications" ? DAY_IN_MS : HOUR_IN_MS;
-
     return {
       granularity: defaultGranularity as TimeseriesGranularityMap[TContext],
       startTime: now - defaultDuration,
@@ -76,9 +77,7 @@ export const getTimeseriesGranularity = <TContext extends TimeseriesContext>(
   // Set default start time if missing (defaults vary by context)
   const defaultDuration = context === "forVerifications" ? DAY_IN_MS : HOUR_IN_MS;
   const effectiveStartTime = startTime ?? effectiveEndTime - defaultDuration;
-
   const timeRange = effectiveEndTime - effectiveStartTime;
-
   let granularity: CompoundTimeseriesGranularity;
 
   if (context === "forVerifications") {
@@ -93,7 +92,25 @@ export const getTimeseriesGranularity = <TContext extends TimeseriesContext>(
     } else if (timeRange >= WEEK_IN_MS) {
       granularity = "perHour";
     } else {
-      granularity = "perHour";
+      // Use the minutely granularity only if allowMinutelyForVerifications is true
+      if (allowMinutelyForVerifications) {
+        if (timeRange >= DAY_IN_MS * 3) {
+          granularity = "perHour";
+        } else if (timeRange >= DAY_IN_MS) {
+          granularity = "perHour";
+        } else if (timeRange >= HOUR_IN_MS * 16) {
+          granularity = "perHour";
+        } else if (timeRange >= HOUR_IN_MS * 8) {
+          granularity = "per30Minutes";
+        } else if (timeRange >= HOUR_IN_MS * 4) {
+          granularity = "per5Minutes";
+        } else {
+          granularity = "perMinute";
+        }
+      } else {
+        // Fall back to hourly granularity if minutely is disabled
+        granularity = "perHour";
+      }
     }
   } else {
     if (timeRange >= DAY_IN_MS * 7) {

go/pkg/clickhouse/schema/databases/001_verifications/021_key_verifications_per_minute_v1.sql

@@ -0,0 +1,16 @@
+CREATE TABLE verifications.key_verifications_per_minute_v1
+(
+  time          DateTime,
+  workspace_id  String,
+  key_space_id  String,
+  identity_id   String,
+  key_id        String,
+  outcome       LowCardinality(String),
+  tags          Array(String),
+  count         Int64
+)
+ENGINE = SummingMergeTree()
+ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome)
+;
+
+

go/pkg/clickhouse/schema/databases/001_verifications/022_key_verifications_per_minute_mv_v1.sql

@@ -0,0 +1,22 @@
+CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1
+TO verifications.key_verifications_per_minute_v1
+AS
+SELECT
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  count(*) as count,
+  toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time,
+  tags
+FROM verifications.raw_key_verifications_v1
+GROUP BY
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  time,
+  tags
+;

internal/clickhouse/schema/050_create_verifications.key_verifications_per_minute_v1.sql

@@ -0,0 +1,19 @@
+-- +goose up
+CREATE TABLE verifications.key_verifications_per_minute_v1
+(
+  time          DateTime,
+  workspace_id  String,
+  key_space_id  String,
+  identity_id   String,
+  key_id        String,
+  outcome       LowCardinality(String),
+  tags          Array(String),
+  count         Int64
+)
+ENGINE = SummingMergeTree()
+ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome)
+;
+
+
+-- +goose down
+DROP TABLE verifications.key_verifications_per_minute_v1;

internal/clickhouse/schema/051_create_verifications.key_verifications_per_minute_mv_v1.sql

@@ -0,0 +1,26 @@
+-- +goose up
+CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1
+TO verifications.key_verifications_per_minute_v1
+AS
+SELECT
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  count(*) as count,
+  toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time,
+  tags
+FROM verifications.raw_key_verifications_v1
+GROUP BY
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  time,
+  tags
+;
+
+-- +goose down
+DROP VIEW verifications.key_verifications_per_minute_mv_v1;

internal/clickhouse/src/index.ts

@@ -2,10 +2,13 @@ import { getBillableRatelimits, getBillableVerifications } from "./billing";
 import { Client, type Inserter, Noop, type Querier } from "./client";
 import {
   getDailyActiveKeysTimeseries,
+  getFiveMinutelyActiveKeysTimeseries,
   getFourHourlyActiveKeysTimeseries,
   getHourlyActiveKeysTimeseries,
+  getMinutelyActiveKeysTimeseries,
   getMonthlyActiveKeysTimeseries,
   getSixHourlyActiveKeysTimeseries,
+  getThirtyMinutelyActiveKeysTimeseries,
   getThreeDayActiveKeysTimeseries,
   getTwelveHourlyActiveKeysTimeseries,
   getTwoHourlyActiveKeysTimeseries,
@@ -55,10 +58,13 @@ import { getActiveWorkspacesPerMonth } from "./success";
 import { insertSDKTelemetry } from "./telemetry";
 import {
   getDailyVerificationTimeseries,
+  getFiveMinutelyVerificationTimeseries,
   getFourHourlyVerificationTimeseries,
   getHourlyVerificationTimeseries,
+  getMinutelyVerificationTimeseries,
   getMonthlyVerificationTimeseries,
   getSixHourlyVerificationTimeseries,
+  getThirtyMinutelyVerificationTimeseries,
   getThreeDayVerificationTimeseries,
   getTwelveHourlyVerificationTimeseries,
   getTwoHourlyVerificationTimeseries,
@@ -112,25 +118,39 @@ export class ClickHouse {
       perMonth: getVerificationsPerMonth(this.querier),
       latest: getLatestVerifications(this.querier),
       timeseries: {
+        // Minute-based granularity
+        perMinute: getMinutelyVerificationTimeseries(this.querier),
+        per5Minutes: getFiveMinutelyVerificationTimeseries(this.querier),
+        per30Minutes: getThirtyMinutelyVerificationTimeseries(this.querier),
+        // Hour-based granularity
         perHour: getHourlyVerificationTimeseries(this.querier),
         per2Hours: getTwoHourlyVerificationTimeseries(this.querier),
         per4Hours: getFourHourlyVerificationTimeseries(this.querier),
         per6Hours: getSixHourlyVerificationTimeseries(this.querier),
         per12Hours: getTwelveHourlyVerificationTimeseries(this.querier),
+        // Day-based granularity
         perDay: getDailyVerificationTimeseries(this.querier),
         per3Days: getThreeDayVerificationTimeseries(this.querier),
         perWeek: getWeeklyVerificationTimeseries(this.querier),
+        // Month-based granularity
         perMonth: getMonthlyVerificationTimeseries(this.querier),
       },
       activeKeysTimeseries: {
+        // Minute-based granularity
+        perMinute: getMinutelyActiveKeysTimeseries(this.querier),
+        per5Minutes: getFiveMinutelyActiveKeysTimeseries(this.querier),
+        per30Minutes: getThirtyMinutelyActiveKeysTimeseries(this.querier),
+        // Hour-based granularity
         perHour: getHourlyActiveKeysTimeseries(this.querier),
         per2Hours: getTwoHourlyActiveKeysTimeseries(this.querier),
         per4Hours: getFourHourlyActiveKeysTimeseries(this.querier),
         per6Hours: getSixHourlyActiveKeysTimeseries(this.querier),
         per12Hours: getTwelveHourlyActiveKeysTimeseries(this.querier),
+        // Day-based granularity
         perDay: getDailyActiveKeysTimeseries(this.querier),
         per3Days: getThreeDayActiveKeysTimeseries(this.querier),
         perWeek: getWeeklyActiveKeysTimeseries(this.querier),
+        // Month-based granularity
         perMonth: getMonthlyActiveKeysTimeseries(this.querier),
       },
     };

internal/clickhouse/src/keys/active_keys.ts

@@ -60,81 +60,97 @@ type TimeInterval = {
 };
 
 const ACTIVE_KEYS_INTERVALS: Record<string, TimeInterval> = {
+  // Minute-based intervals
+  minute: {
+    table: "verifications.key_verifications_per_minute_v1",
+    step: "MINUTE",
+    stepSize: 1,
+  },
+  fiveMinutes: {
+    table: "verifications.key_verifications_per_minute_v1",
+    step: "MINUTE",
+    stepSize: 5,
+  },
+  thirtyMinutes: {
+    table: "verifications.key_verifications_per_minute_v1",
+    step: "MINUTE",
+    stepSize: 30,
+  },
+  // Hour-based intervals
   hour: {
     table: "verifications.key_verifications_per_hour_v3",
     step: "HOUR",
     stepSize: 1,
   },
   twoHours: {
     table: "verifications.key_verifications_per_hour_v3",
-    step: "HOURS",
+    step: "HOUR",
     stepSize: 2,
   },
   fourHours: {
     table: "verifications.key_verifications_per_hour_v3",
-    step: "HOURS",
+    step: "HOUR",
     stepSize: 4,
   },
   sixHours: {
     table: "verifications.key_verifications_per_hour_v3",
-    step: "HOURS",
+    step: "HOUR",
     stepSize: 6,
   },
   twelveHours: {
     table: "verifications.key_verifications_per_hour_v3",
-    step: "HOURS",
+    step: "HOUR",
     stepSize: 12,
   },
+  // Day-based intervals
   day: {
     table: "verifications.key_verifications_per_day_v3",
     step: "DAY",
     stepSize: 1,
   },
   threeDays: {
     table: "verifications.key_verifications_per_day_v3",
-    step: "DAYS",
+    step: "DAY",
     stepSize: 3,
   },
   week: {
     table: "verifications.key_verifications_per_day_v3",
-    step: "DAYS",
+    step: "DAY",
     stepSize: 7,
   },
   twoWeeks: {
     table: "verifications.key_verifications_per_day_v3",
-    step: "DAYS",
+    step: "DAY",
     stepSize: 14,
   },
+  // Monthly-based intervals
   month: {
     table: "verifications.key_verifications_per_month_v3",
     step: "MONTH",
     stepSize: 1,
   },
   quarter: {
     table: "verifications.key_verifications_per_month_v3",
-    step: "MONTHS",
+    step: "MONTH",
     stepSize: 3,
   },
 } as const;
 
 function createActiveKeysTimeseriesQuery(interval: TimeInterval, whereClause: string) {
   const intervalUnit = {
+    MINUTE: "minute",
     HOUR: "hour",
-    HOURS: "hour",
     DAY: "day",
     DAYS: "day",
     MONTH: "month",
-    MONTHS: "month",
   }[interval.step];
 
   // For millisecond step calculation
   const msPerUnit = {
+    MINUTE: 60_000,
     HOUR: 3600_000,
-    HOURS: 3600_000,
     DAY: 86400_000,
-    DAYS: 86400_000,
     MONTH: 2592000_000,
-    MONTHS: 2592000_000,
   }[interval.step];
 
   const stepMs = msPerUnit! * interval.stepSize;
@@ -264,7 +280,18 @@ function createActiveKeysTimeseriesQuerier(interval: TimeInterval) {
     })(parameters);
   };
 }
+// Minute-based timeseries
+export const getMinutelyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
+  ACTIVE_KEYS_INTERVALS.minute,
+);
+export const getFiveMinutelyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
+  ACTIVE_KEYS_INTERVALS.fiveMinutes,
+);
+export const getThirtyMinutelyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
+  ACTIVE_KEYS_INTERVALS.thirtyMinutes,
+);
 
+// Hour-based timeseries
 export const getHourlyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
   ACTIVE_KEYS_INTERVALS.hour,
 );
@@ -280,6 +307,8 @@ export const getSixHourlyActiveKeysTimeseries = createActiveKeysTimeseriesQuerie
 export const getTwelveHourlyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
   ACTIVE_KEYS_INTERVALS.twelveHours,
 );
+
+// Day-based timeseries
 export const getDailyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
   ACTIVE_KEYS_INTERVALS.day,
 );
@@ -292,6 +321,8 @@ export const getWeeklyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
 export const getTwoWeeklyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
   ACTIVE_KEYS_INTERVALS.twoWeeks,
 );
+
+// Month-based timeseries
 export const getMonthlyActiveKeysTimeseries = createActiveKeysTimeseriesQuerier(
   ACTIVE_KEYS_INTERVALS.month,
 );