fix: overflowed keyId issue

apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema.ts

@@ -2,19 +2,22 @@ import { KEY_VERIFICATION_OUTCOMES } from "@unkey/clickhouse/src/keys/keys";
 import { z } from "zod";
 import { keysOverviewFilterOperatorEnum } from "../../../filters.schema";
 
+export const MAX_KEYID_COUNT = 15;
 export const keysOverviewQueryTimeseriesPayload = z.object({
   startTime: z.number().int(),
   endTime: z.number().int(),
   since: z.string(),
   apiId: z.string(),
   keyIds: z
     .object({
-      filters: z.array(
-        z.object({
-          operator: keysOverviewFilterOperatorEnum,
-          value: z.string(),
-        }),
-      ),
+      filters: z
+        .array(
+          z.object({
+            operator: keysOverviewFilterOperatorEnum,
+            value: z.string(),
+          }),
+        )
+        .max(MAX_KEYID_COUNT),
     })
     .nullable(),
   names: z

apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/query-logs.schema.ts

@@ -1,5 +1,6 @@
 import { KEY_VERIFICATION_OUTCOMES } from "@unkey/clickhouse/src/keys/keys";
 import { z } from "zod";
+import { MAX_KEYID_COUNT } from "../charts/bar-chart/query-timeseries.schema";
 
 export const keysQueryOverviewLogsPayload = z.object({
   limit: z.number().int(),
@@ -30,6 +31,7 @@ export const keysQueryOverviewLogsPayload = z.object({
         value: z.string(),
       }),
     )
+    .max(MAX_KEYID_COUNT)
     .optional()
     .nullable(),
   names: z

apps/dashboard/app/(app)/apis/_components/hooks/use-query-timeseries.ts

@@ -55,8 +55,8 @@ export const useFetchVerificationTimeseries = (keyspaceId: string | null) => {
     enabled,
   });
 
-  const timeseries = data?.timeseries.map((ts) => ({
-    displayX: formatTimestampForChart(ts.x, data.granularity),
+  const timeseries = (data?.timeseries ?? []).map((ts) => ({
+    displayX: formatTimestampForChart(ts.x, data?.granularity ?? "per12Hours"),
     originalTimestamp: ts.x,
     valid: ts.y.valid,
     total: ts.y.total,

apps/dashboard/lib/trpc/routers/api/keys/api-query.ts

@@ -326,3 +326,27 @@ export function extractRolesAndPermissions(key: any) {
     permissions,
   };
 }
+
+export const getApi = async (apiId: string, workspaceId: string) => {
+  const api = await db.query.apis
+    .findFirst({
+      where: (api, { and, eq, isNull }) =>
+        and(eq(api.id, apiId), eq(api.workspaceId, workspaceId), isNull(api.deletedAtM)),
+      with: {
+        keyAuth: {
+          columns: {
+            id: true,
+          },
+        },
+      },
+    })
+    .catch((err) => {
+      console.error("Database query error:", err);
+      throw new TRPCError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "Failed to retrieve API information.",
+      });
+    });
+
+  return api;
+};

apps/dashboard/lib/trpc/routers/api/keys/query-active-keys-timeseries/index.ts

@@ -2,39 +2,82 @@ import { keysOverviewQueryTimeseriesPayload } from "@/app/(app)/apis/[apiId]/_ov
 import { clickhouse } from "@/lib/clickhouse";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
-import { queryApiKeys } from "../api-query";
+import { getApi, queryApiKeys } from "../api-query";
 import { transformVerificationFilters } from "../timeseries.utils";
 
 export const activeKeysTimeseries = rateLimitedProcedure(ratelimit.read)
   .input(keysOverviewQueryTimeseriesPayload)
   .query(async ({ ctx, input }) => {
-    const { params: transformedInputs, granularity } = transformVerificationFilters(input);
+    const api = await getApi(input.apiId, ctx.workspace.id);
+    if (!api || !api.keyAuth?.id) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "API not found or does not have key authentication enabled",
+      });
+    }
+    const keyspaceId = api.keyAuth.id;
 
-    const { keyspaceId, keyIds } = await queryApiKeys({
-      apiId: input.apiId,
-      workspaceId: ctx.workspace.id,
-      keyIds: transformedInputs.keyIds,
-      names: transformedInputs.names,
-      identities: transformedInputs.identities,
-    });
+    const { params: transformedInputs, granularity } = transformVerificationFilters(input);
 
-    const result = await clickhouse.verifications.activeKeysTimeseries[granularity]({
+    const clickhouseResult = await clickhouse.verifications.activeKeysTimeseries[granularity]({
       ...transformedInputs,
       workspaceId: ctx.workspace.id,
       keyspaceId: keyspaceId,
-      keyIds: (keyIds ?? []).map((x) => ({
-        value: String(x.value),
-        operator: x.operator as "is" | "contains",
-      })),
+      keyIds: input.keyIds ? transformedInputs.keyIds : null,
     });
 
-    if (result.err) {
+    if (!clickhouseResult || clickhouseResult.err) {
       throw new TRPCError({
         code: "INTERNAL_SERVER_ERROR",
-        message:
-          "Failed to retrieve active keys timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.",
+        message: "Something went wrong when fetching data from ClickHouse.",
+      });
+    }
+
+    const timeseriesWithKeys = clickhouseResult.val || [];
+    if (timeseriesWithKeys.length === 0) {
+      return {
+        timeseries: null,
+        granularity,
+      };
+    }
+
+    if (input.names?.filters?.length || input.identities?.filters?.length) {
+      const allKeyIds = new Set();
+      timeseriesWithKeys.forEach((point) => {
+        (point.key_ids ?? []).forEach((id) => allKeyIds.add(id));
+      });
+
+      const { keys } = await queryApiKeys({
+        apiId: input.apiId,
+        workspaceId: ctx.workspace.id,
+        keyIds: Array.from(allKeyIds).map((id) => ({
+          operator: "is",
+          value: id as string,
+        })),
+        names: input.names?.filters || null,
+        identities: input.identities?.filters || null,
+      });
+
+      const filteredKeyIdSet = new Set(keys.map((key) => key.id));
+
+      const filteredTimeseries = timeseriesWithKeys.map((point) => {
+        const filteredKeys = (point.key_ids ?? []).filter((id) => filteredKeyIdSet.has(id));
+
+        return {
+          x: point.x,
+          y: {
+            keys: filteredKeys.length,
+          },
+        };
       });
+
+      return { timeseries: filteredTimeseries, granularity };
     }
 
-    return { timeseries: result.val, granularity };
+    const timeseriesData = timeseriesWithKeys.map((point) => ({
+      x: point.x,
+      y: point.y,
+    }));
+
+    return { timeseries: timeseriesData, granularity };
   });

apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/index.ts

@@ -4,7 +4,7 @@ import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
 import { keysOverviewLogs as keysLogs } from "@unkey/clickhouse/src/keys/keys";
 import { z } from "zod";
-import { createKeyDetailsMap, queryApiKeys } from "../api-query";
+import { createKeyDetailsMap, getApi, queryApiKeys } from "../api-query";
 import { transformKeysFilters } from "./utils";
 
 const KeysOverviewLogsResponse = z.object({
@@ -20,57 +20,82 @@ const KeysOverviewLogsResponse = z.object({
 
 type KeysOverviewLogsResponse = z.infer<typeof KeysOverviewLogsResponse>;
 
+/**
+ * This procedure queries keys overview logs by:
+ * 1. First querying ClickHouse with relevant filters
+ * 2. Then filtering the results with SQL
+ * 3. Finally merging with key details
+ */
 export const queryKeysOverviewLogs = rateLimitedProcedure(ratelimit.read)
   .input(keysQueryOverviewLogsPayload)
   .output(KeysOverviewLogsResponse)
   .query(async ({ ctx, input }) => {
-    const transformedInputs = transformKeysFilters(input);
-
-    const { keyspaceId, keys, keyIds } = await queryApiKeys({
-      apiId: input.apiId,
-      workspaceId: ctx.workspace.id,
-      keyIds: transformedInputs.keyIds,
-      names: transformedInputs.names,
-      identities: transformedInputs.identities,
-    });
+    const api = await getApi(input.apiId, ctx.workspace.id);
+    if (!api || !api.keyAuth?.id) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "API not found or does not have key authentication enabled",
+      });
+    }
+    const keyspaceId = api.keyAuth.id;
 
-    const keyDetailsMap = createKeyDetailsMap(keys);
+    const transformedInputs = transformKeysFilters(input);
 
-    const result = await clickhouse.api.keys.logs({
+    const clickhouseResult = await clickhouse.api.keys.logs({
       ...transformedInputs,
       cursorRequestId: input.cursor?.requestId ?? null,
       cursorTime: input.cursor?.time ?? null,
       workspaceId: ctx.workspace.id,
       keyspaceId: keyspaceId,
-      keyIds: (keyIds ?? []).map((x) => ({
-        value: String(x.value),
-        operator: x.operator as "is" | "contains",
-      })),
+      // Only include keyIds filters if explicitly provided in the input
+      keyIds: input.keyIds ? transformedInputs.keyIds : null,
+      // Nullify these as we'll filter in the database
+      names: null,
+      identities: null,
     });
 
-    if (!result || result.err) {
+    if (!clickhouseResult || clickhouseResult.err) {
       throw new TRPCError({
         code: "INTERNAL_SERVER_ERROR",
-        message: "Something went wrong when fetching data from clickhouse.",
+        message: "Something went wrong when fetching data from ClickHouse.",
       });
     }
 
-    const logs = result.val || [];
+    const logs = clickhouseResult.val || [];
     if (logs.length === 0) {
       return {
         keysOverviewLogs: [],
         hasMore: false,
       };
     }
 
-    const keysOverviewLogs = logs.map((log) => ({
-      ...log,
-      key_details: keyDetailsMap.get(log.key_id) || null,
-    }));
+    const keyIdsFromLogs = logs.map((log) => log.key_id);
+
+    // This ensures we only get keys that exist in both ClickHouse and the database
+    const { keys } = await queryApiKeys({
+      apiId: input.apiId,
+      workspaceId: ctx.workspace.id,
+      // Pass the key IDs from ClickHouse logs as "is" filters
+      keyIds: keyIdsFromLogs.map((id) => ({ operator: "is", value: id })),
+      // Still apply any name or identity filters from the original input
+      names: input.names || null,
+      identities: input.identities || null,
+    });
+
+    const keyDetailsMap = createKeyDetailsMap(keys);
+    const filteredKeyIds = Array.from(keyDetailsMap.keys());
+
+    // Only include logs for keys that exist in the database and passed all filters
+    const keysOverviewLogs = logs
+      .filter((log) => filteredKeyIds.includes(log.key_id))
+      .map((log) => ({
+        ...log,
+        key_details: keyDetailsMap.get(log.key_id) || null,
+      }));
 
     const response: KeysOverviewLogsResponse = {
       keysOverviewLogs,
-      hasMore: logs.length === input.limit,
+      hasMore: logs.length === input.limit && keysOverviewLogs.length > 0,
       nextCursor:
         logs.length === input.limit
           ? {

apps/dashboard/lib/trpc/routers/api/keys/query-overview-timeseries/index.ts

@@ -1,22 +1,45 @@
 import { keysOverviewQueryTimeseriesPayload } from "@/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema";
+import type { KeysOverviewFilterUrlValue } from "@/app/(app)/apis/[apiId]/_overview/filters.schema";
 import { clickhouse } from "@/lib/clickhouse";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
-import { queryApiKeys } from "../api-query";
+import { getApi, queryApiKeys } from "../api-query";
 import { transformVerificationFilters } from "../timeseries.utils";
 
 export const keyVerificationsTimeseries = rateLimitedProcedure(ratelimit.read)
   .input(keysOverviewQueryTimeseriesPayload)
   .query(async ({ ctx, input }) => {
+    const api = await getApi(input.apiId, ctx.workspace.id);
+    if (!api || !api.keyAuth?.id) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "API not found or does not have key authentication enabled",
+      });
+    }
+
+    const keyspaceId = api.keyAuth.id;
     const { params: transformedInputs, granularity } = transformVerificationFilters(input);
 
-    const { keyspaceId, keyIds } = await queryApiKeys({
-      apiId: input.apiId,
-      workspaceId: ctx.workspace.id,
-      keyIds: transformedInputs.keyIds,
-      names: transformedInputs.names,
-      identities: transformedInputs.identities,
-    });
+    // Check if we have any key-related filters
+    const hasKeyFilters =
+      (transformedInputs.keyIds !== null && transformedInputs.keyIds.length > 0) ||
+      (transformedInputs.names !== null && transformedInputs.names.length > 0) ||
+      (transformedInputs.identities !== null && transformedInputs.identities.length > 0);
+
+    let keyIds: KeysOverviewFilterUrlValue[] | null = [];
+
+    // Only query API keys if we have key-related filters
+    if (hasKeyFilters) {
+      const apiKeysResult = await queryApiKeys({
+        apiId: input.apiId,
+        workspaceId: ctx.workspace.id,
+        keyIds: transformedInputs.keyIds,
+        names: transformedInputs.names,
+        identities: transformedInputs.identities,
+      });
+
+      keyIds = apiKeysResult.keyIds || [];
+    }
 
     const result = await clickhouse.verifications.timeseries[granularity]({
       ...transformedInputs,
@@ -28,13 +51,8 @@ export const keyVerificationsTimeseries = rateLimitedProcedure(ratelimit.read)
       })),
     });
 
-    if (result.err) {
-      throw new TRPCError({
-        code: "INTERNAL_SERVER_ERROR",
-        message:
-          "Failed to retrieve key verification timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.",
-      });
-    }
-
-    return { timeseries: result.val, granularity };
+    return {
+      timeseries: result,
+      granularity,
+    };
   });

apps/dashboard/lib/trpc/routers/api/overview/query-timeseries/index.ts

@@ -1,7 +1,6 @@
 import { verificationQueryTimeseriesPayload } from "@/app/(app)/apis/_components/hooks/query-timeseries.schema";
 import { clickhouse } from "@/lib/clickhouse";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
-import { TRPCError } from "@trpc/server";
 import { transformVerificationFilters } from "./utils";
 
 export const queryVerificationTimeseries = rateLimitedProcedure(ratelimit.read)
@@ -15,13 +14,8 @@ export const queryVerificationTimeseries = rateLimitedProcedure(ratelimit.read)
       keyspaceId: input.keyspaceId,
     });
 
-    if (result.err) {
-      throw new TRPCError({
-        code: "INTERNAL_SERVER_ERROR",
-        message:
-          "Failed to retrieve ratelimit timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.",
-      });
-    }
-
-    return { timeseries: result.val, granularity };
+    return {
+      timeseries: result,
+      granularity,
+    };
   });