diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema.ts index 40804b6bd3..c11546c629 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema.ts +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema.ts @@ -2,6 +2,7 @@ import { KEY_VERIFICATION_OUTCOMES } from "@unkey/clickhouse/src/keys/keys"; import { z } from "zod"; import { keysOverviewFilterOperatorEnum } from "../../../filters.schema"; +export const MAX_KEYID_COUNT = 15; export const keysOverviewQueryTimeseriesPayload = z.object({ startTime: z.number().int(), endTime: z.number().int(), @@ -9,12 +10,14 @@ export const keysOverviewQueryTimeseriesPayload = z.object({ apiId: z.string(), keyIds: z .object({ - filters: z.array( - z.object({ - operator: keysOverviewFilterOperatorEnum, - value: z.string(), - }), - ), + filters: z + .array( + z.object({ + operator: keysOverviewFilterOperatorEnum, + value: z.string(), + }), + ) + .max(MAX_KEYID_COUNT), }) .nullable(), names: z diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/query-logs.schema.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/query-logs.schema.ts index 2546ee50b3..c041436b27 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/query-logs.schema.ts +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/query-logs.schema.ts @@ -1,5 +1,6 @@ import { KEY_VERIFICATION_OUTCOMES } from "@unkey/clickhouse/src/keys/keys"; import { z } from "zod"; +import { MAX_KEYID_COUNT } from "../charts/bar-chart/query-timeseries.schema"; export const keysQueryOverviewLogsPayload = z.object({ limit: z.number().int(), @@ -30,6 +31,7 @@ export const keysQueryOverviewLogsPayload = z.object({ value: z.string(), }), ) + .max(MAX_KEYID_COUNT) .optional() .nullable(), names: z diff --git a/apps/dashboard/app/(app)/apis/_components/hooks/use-query-timeseries.ts b/apps/dashboard/app/(app)/apis/_components/hooks/use-query-timeseries.ts index 5e434ae1ee..58b4b0aca8 100644 --- a/apps/dashboard/app/(app)/apis/_components/hooks/use-query-timeseries.ts +++ b/apps/dashboard/app/(app)/apis/_components/hooks/use-query-timeseries.ts @@ -55,8 +55,8 @@ export const useFetchVerificationTimeseries = (keyspaceId: string | null) => { enabled, }); - const timeseries = data?.timeseries.map((ts) => ({ - displayX: formatTimestampForChart(ts.x, data.granularity), + const timeseries = (data?.timeseries ?? []).map((ts) => ({ + displayX: formatTimestampForChart(ts.x, data?.granularity ?? "per12Hours"), originalTimestamp: ts.x, valid: ts.y.valid, total: ts.y.total, diff --git a/apps/dashboard/lib/trpc/routers/api/keys/api-query.ts b/apps/dashboard/lib/trpc/routers/api/keys/api-query.ts index e3f001c73b..edd76887e5 100644 --- a/apps/dashboard/lib/trpc/routers/api/keys/api-query.ts +++ b/apps/dashboard/lib/trpc/routers/api/keys/api-query.ts @@ -326,3 +326,27 @@ export function extractRolesAndPermissions(key: any) { permissions, }; } + +export const getApi = async (apiId: string, workspaceId: string) => { + const api = await db.query.apis + .findFirst({ + where: (api, { and, eq, isNull }) => + and(eq(api.id, apiId), eq(api.workspaceId, workspaceId), isNull(api.deletedAtM)), + with: { + keyAuth: { + columns: { + id: true, + }, + }, + }, + }) + .catch((err) => { + console.error("Database query error:", err); + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: "Failed to retrieve API information.", + }); + }); + + return api; +}; diff --git a/apps/dashboard/lib/trpc/routers/api/keys/query-active-keys-timeseries/index.ts b/apps/dashboard/lib/trpc/routers/api/keys/query-active-keys-timeseries/index.ts index fbfe50f742..aadcc77b87 100644 --- a/apps/dashboard/lib/trpc/routers/api/keys/query-active-keys-timeseries/index.ts +++ b/apps/dashboard/lib/trpc/routers/api/keys/query-active-keys-timeseries/index.ts @@ -2,39 +2,82 @@ import { keysOverviewQueryTimeseriesPayload } from "@/app/(app)/apis/[apiId]/_ov import { clickhouse } from "@/lib/clickhouse"; import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure"; import { TRPCError } from "@trpc/server"; -import { queryApiKeys } from "../api-query"; +import { getApi, queryApiKeys } from "../api-query"; import { transformVerificationFilters } from "../timeseries.utils"; export const activeKeysTimeseries = rateLimitedProcedure(ratelimit.read) .input(keysOverviewQueryTimeseriesPayload) .query(async ({ ctx, input }) => { - const { params: transformedInputs, granularity } = transformVerificationFilters(input); + const api = await getApi(input.apiId, ctx.workspace.id); + if (!api || !api.keyAuth?.id) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "API not found or does not have key authentication enabled", + }); + } + const keyspaceId = api.keyAuth.id; - const { keyspaceId, keyIds } = await queryApiKeys({ - apiId: input.apiId, - workspaceId: ctx.workspace.id, - keyIds: transformedInputs.keyIds, - names: transformedInputs.names, - identities: transformedInputs.identities, - }); + const { params: transformedInputs, granularity } = transformVerificationFilters(input); - const result = await clickhouse.verifications.activeKeysTimeseries[granularity]({ + const clickhouseResult = await clickhouse.verifications.activeKeysTimeseries[granularity]({ ...transformedInputs, workspaceId: ctx.workspace.id, keyspaceId: keyspaceId, - keyIds: (keyIds ?? []).map((x) => ({ - value: String(x.value), - operator: x.operator as "is" | "contains", - })), + keyIds: input.keyIds ? transformedInputs.keyIds : null, }); - if (result.err) { + if (!clickhouseResult || clickhouseResult.err) { throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", - message: - "Failed to retrieve active keys timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.", + message: "Something went wrong when fetching data from ClickHouse.", + }); + } + + const timeseriesWithKeys = clickhouseResult.val || []; + if (timeseriesWithKeys.length === 0) { + return { + timeseries: null, + granularity, + }; + } + + if (input.names?.filters?.length || input.identities?.filters?.length) { + const allKeyIds = new Set(); + timeseriesWithKeys.forEach((point) => { + (point.key_ids ?? []).forEach((id) => allKeyIds.add(id)); + }); + + const { keys } = await queryApiKeys({ + apiId: input.apiId, + workspaceId: ctx.workspace.id, + keyIds: Array.from(allKeyIds).map((id) => ({ + operator: "is", + value: id as string, + })), + names: input.names?.filters || null, + identities: input.identities?.filters || null, + }); + + const filteredKeyIdSet = new Set(keys.map((key) => key.id)); + + const filteredTimeseries = timeseriesWithKeys.map((point) => { + const filteredKeys = (point.key_ids ?? []).filter((id) => filteredKeyIdSet.has(id)); + + return { + x: point.x, + y: { + keys: filteredKeys.length, + }, + }; }); + + return { timeseries: filteredTimeseries, granularity }; } - return { timeseries: result.val, granularity }; + const timeseriesData = timeseriesWithKeys.map((point) => ({ + x: point.x, + y: point.y, + })); + + return { timeseries: timeseriesData, granularity }; }); diff --git a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/index.ts b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/index.ts index a07ac8f468..f555321b26 100644 --- a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/index.ts +++ b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/index.ts @@ -4,7 +4,7 @@ import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure"; import { TRPCError } from "@trpc/server"; import { keysOverviewLogs as keysLogs } from "@unkey/clickhouse/src/keys/keys"; import { z } from "zod"; -import { createKeyDetailsMap, queryApiKeys } from "../api-query"; +import { createKeyDetailsMap, getApi, queryApiKeys } from "../api-query"; import { transformKeysFilters } from "./utils"; const KeysOverviewLogsResponse = z.object({ @@ -20,42 +20,48 @@ const KeysOverviewLogsResponse = z.object({ type KeysOverviewLogsResponse = z.infer; +/** + * This procedure queries keys overview logs by: + * 1. First querying ClickHouse with relevant filters + * 2. Then filtering the results with SQL + * 3. Finally merging with key details + */ export const queryKeysOverviewLogs = rateLimitedProcedure(ratelimit.read) .input(keysQueryOverviewLogsPayload) .output(KeysOverviewLogsResponse) .query(async ({ ctx, input }) => { - const transformedInputs = transformKeysFilters(input); - - const { keyspaceId, keys, keyIds } = await queryApiKeys({ - apiId: input.apiId, - workspaceId: ctx.workspace.id, - keyIds: transformedInputs.keyIds, - names: transformedInputs.names, - identities: transformedInputs.identities, - }); + const api = await getApi(input.apiId, ctx.workspace.id); + if (!api || !api.keyAuth?.id) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "API not found or does not have key authentication enabled", + }); + } + const keyspaceId = api.keyAuth.id; - const keyDetailsMap = createKeyDetailsMap(keys); + const transformedInputs = transformKeysFilters(input); - const result = await clickhouse.api.keys.logs({ + const clickhouseResult = await clickhouse.api.keys.logs({ ...transformedInputs, cursorRequestId: input.cursor?.requestId ?? null, cursorTime: input.cursor?.time ?? null, workspaceId: ctx.workspace.id, keyspaceId: keyspaceId, - keyIds: (keyIds ?? []).map((x) => ({ - value: String(x.value), - operator: x.operator as "is" | "contains", - })), + // Only include keyIds filters if explicitly provided in the input + keyIds: input.keyIds ? transformedInputs.keyIds : null, + // Nullify these as we'll filter in the database + names: null, + identities: null, }); - if (!result || result.err) { + if (!clickhouseResult || clickhouseResult.err) { throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", - message: "Something went wrong when fetching data from clickhouse.", + message: "Something went wrong when fetching data from ClickHouse.", }); } - const logs = result.val || []; + const logs = clickhouseResult.val || []; if (logs.length === 0) { return { keysOverviewLogs: [], @@ -63,14 +69,33 @@ export const queryKeysOverviewLogs = rateLimitedProcedure(ratelimit.read) }; } - const keysOverviewLogs = logs.map((log) => ({ - ...log, - key_details: keyDetailsMap.get(log.key_id) || null, - })); + const keyIdsFromLogs = logs.map((log) => log.key_id); + + // This ensures we only get keys that exist in both ClickHouse and the database + const { keys } = await queryApiKeys({ + apiId: input.apiId, + workspaceId: ctx.workspace.id, + // Pass the key IDs from ClickHouse logs as "is" filters + keyIds: keyIdsFromLogs.map((id) => ({ operator: "is", value: id })), + // Still apply any name or identity filters from the original input + names: input.names || null, + identities: input.identities || null, + }); + + const keyDetailsMap = createKeyDetailsMap(keys); + const filteredKeyIds = Array.from(keyDetailsMap.keys()); + + // Only include logs for keys that exist in the database and passed all filters + const keysOverviewLogs = logs + .filter((log) => filteredKeyIds.includes(log.key_id)) + .map((log) => ({ + ...log, + key_details: keyDetailsMap.get(log.key_id) || null, + })); const response: KeysOverviewLogsResponse = { keysOverviewLogs, - hasMore: logs.length === input.limit, + hasMore: logs.length === input.limit && keysOverviewLogs.length > 0, nextCursor: logs.length === input.limit ? { diff --git a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-timeseries/index.ts b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-timeseries/index.ts index 5b1e7ca276..7c403d9bbe 100644 --- a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-timeseries/index.ts +++ b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-timeseries/index.ts @@ -1,22 +1,45 @@ import { keysOverviewQueryTimeseriesPayload } from "@/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/query-timeseries.schema"; +import type { KeysOverviewFilterUrlValue } from "@/app/(app)/apis/[apiId]/_overview/filters.schema"; import { clickhouse } from "@/lib/clickhouse"; import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure"; import { TRPCError } from "@trpc/server"; -import { queryApiKeys } from "../api-query"; +import { getApi, queryApiKeys } from "../api-query"; import { transformVerificationFilters } from "../timeseries.utils"; export const keyVerificationsTimeseries = rateLimitedProcedure(ratelimit.read) .input(keysOverviewQueryTimeseriesPayload) .query(async ({ ctx, input }) => { + const api = await getApi(input.apiId, ctx.workspace.id); + if (!api || !api.keyAuth?.id) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "API not found or does not have key authentication enabled", + }); + } + + const keyspaceId = api.keyAuth.id; const { params: transformedInputs, granularity } = transformVerificationFilters(input); - const { keyspaceId, keyIds } = await queryApiKeys({ - apiId: input.apiId, - workspaceId: ctx.workspace.id, - keyIds: transformedInputs.keyIds, - names: transformedInputs.names, - identities: transformedInputs.identities, - }); + // Check if we have any key-related filters + const hasKeyFilters = + (transformedInputs.keyIds !== null && transformedInputs.keyIds.length > 0) || + (transformedInputs.names !== null && transformedInputs.names.length > 0) || + (transformedInputs.identities !== null && transformedInputs.identities.length > 0); + + let keyIds: KeysOverviewFilterUrlValue[] | null = []; + + // Only query API keys if we have key-related filters + if (hasKeyFilters) { + const apiKeysResult = await queryApiKeys({ + apiId: input.apiId, + workspaceId: ctx.workspace.id, + keyIds: transformedInputs.keyIds, + names: transformedInputs.names, + identities: transformedInputs.identities, + }); + + keyIds = apiKeysResult.keyIds || []; + } const result = await clickhouse.verifications.timeseries[granularity]({ ...transformedInputs, @@ -28,13 +51,8 @@ export const keyVerificationsTimeseries = rateLimitedProcedure(ratelimit.read) })), }); - if (result.err) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - "Failed to retrieve key verification timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.", - }); - } - - return { timeseries: result.val, granularity }; + return { + timeseries: result, + granularity, + }; }); diff --git a/apps/dashboard/lib/trpc/routers/api/overview/query-timeseries/index.ts b/apps/dashboard/lib/trpc/routers/api/overview/query-timeseries/index.ts index 5f53a9af44..90195cf3af 100644 --- a/apps/dashboard/lib/trpc/routers/api/overview/query-timeseries/index.ts +++ b/apps/dashboard/lib/trpc/routers/api/overview/query-timeseries/index.ts @@ -1,7 +1,6 @@ import { verificationQueryTimeseriesPayload } from "@/app/(app)/apis/_components/hooks/query-timeseries.schema"; import { clickhouse } from "@/lib/clickhouse"; import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure"; -import { TRPCError } from "@trpc/server"; import { transformVerificationFilters } from "./utils"; export const queryVerificationTimeseries = rateLimitedProcedure(ratelimit.read) @@ -15,13 +14,8 @@ export const queryVerificationTimeseries = rateLimitedProcedure(ratelimit.read) keyspaceId: input.keyspaceId, }); - if (result.err) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - "Failed to retrieve ratelimit timeseries analytics due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.", - }); - } - - return { timeseries: result.val, granularity }; + return { + timeseries: result, + granularity, + }; }); diff --git a/internal/clickhouse/src/keys/active_keys.ts b/internal/clickhouse/src/keys/active_keys.ts index 8808a67618..1548c44613 100644 --- a/internal/clickhouse/src/keys/active_keys.ts +++ b/internal/clickhouse/src/keys/active_keys.ts @@ -47,6 +47,7 @@ export const activeKeysTimeseriesDataPoint = z.object({ y: z.object({ keys: z.number().int().default(0), }), + key_ids: z.array(z.string()).optional(), }); export type ActiveKeysTimeseriesDataPoint = z.infer; @@ -143,7 +144,8 @@ function createActiveKeysTimeseriesQuery(interval: TimeInterval, whereClause: st toUnixTimestamp64Milli(CAST(toStartOfInterval(time, INTERVAL ${interval.stepSize} ${intervalUnit}) AS DateTime64(3))) as x, map( 'keys', count(DISTINCT key_id) - ) as y + ) as y, + groupArray(DISTINCT key_id) as key_ids FROM ${interval.table} ${whereClause} GROUP BY x diff --git a/internal/clickhouse/src/verifications.ts b/internal/clickhouse/src/verifications.ts index c95c92a166..1654adbb9c 100644 --- a/internal/clickhouse/src/verifications.ts +++ b/internal/clickhouse/src/verifications.ts @@ -400,24 +400,123 @@ function createVerificationTimeseriesQuerier(interval: TimeInterval) { }; } -export const getHourlyVerificationTimeseries = createVerificationTimeseriesQuerier(INTERVALS.hour); -export const getTwoHourlyVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.twoHours, -); -export const getFourHourlyVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.fourHours, -); -export const getSixHourlyVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.sixHours, -); -export const getTwelveHourlyVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.twelveHours, -); -export const getDailyVerificationTimeseries = createVerificationTimeseriesQuerier(INTERVALS.day); -export const getThreeDayVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.threeDays, -); -export const getWeeklyVerificationTimeseries = createVerificationTimeseriesQuerier(INTERVALS.week); -export const getMonthlyVerificationTimeseries = createVerificationTimeseriesQuerier( - INTERVALS.month, -); +async function batchVerificationTimeseries( + ch: Querier, + interval: TimeInterval, + args: VerificationTimeseriesParams, + maxBatchSize = 15, +) { + if (!args.keyIds || args.keyIds.length === 0 || args.keyIds.length <= maxBatchSize) { + return (await createVerificationTimeseriesQuerier(interval)(ch)(args)).val; + } + + const keyIdBatches: any[] = []; + for (let i = 0; i < args.keyIds.length; i += maxBatchSize) { + keyIdBatches.push(args.keyIds.slice(i, i + maxBatchSize)); + } + + const batchResults = await Promise.allSettled( + keyIdBatches.map(async (batchKeyIds, batchIndex) => { + const batchArgs = { + ...args, + keyIds: batchKeyIds, + }; + try { + const res = await createVerificationTimeseriesQuerier(interval)(ch)(batchArgs); + if (res?.val) { + return res.val; + } + return res; // Return res directly if no .val + } catch (error) { + console.error(`Batch ${batchIndex} query failed:`, error); + return []; + } + }), + ); + + const successfulResults = batchResults + .filter((result) => result.status === "fulfilled") + .map((result) => (result as PromiseFulfilledResult).value) + .filter((value) => Array.isArray(value)); + + return mergeVerificationTimeseriesResults(successfulResults); +} + +function mergeVerificationTimeseriesResults( + results: VerificationTimeseriesDataPoint[][], +): VerificationTimeseriesDataPoint[] { + const mergedMap = new Map(); + + results.forEach((resultBatch) => { + resultBatch.forEach((dataPoint) => { + if (!dataPoint) { + return; // Skip undefined or null points + } + const existingPoint = mergedMap.get(dataPoint.x); + + if (!existingPoint) { + mergedMap.set(dataPoint.x, dataPoint); + } else { + mergedMap.set(dataPoint.x, { + x: dataPoint.x, + y: { + total: (existingPoint.y.total ?? 0) + (dataPoint.y.total ?? 0), + valid: (existingPoint.y.valid ?? 0) + (dataPoint.y.valid ?? 0), + valid_count: (existingPoint.y.valid_count ?? 0) + (dataPoint.y.valid_count ?? 0), + rate_limited_count: + (existingPoint.y.rate_limited_count ?? 0) + (dataPoint.y.rate_limited_count ?? 0), + insufficient_permissions_count: + (existingPoint.y.insufficient_permissions_count ?? 0) + + (dataPoint.y.insufficient_permissions_count ?? 0), + forbidden_count: + (existingPoint.y.forbidden_count ?? 0) + (dataPoint.y.forbidden_count ?? 0), + disabled_count: + (existingPoint.y.disabled_count ?? 0) + (dataPoint.y.disabled_count ?? 0), + expired_count: (existingPoint.y.expired_count ?? 0) + (dataPoint.y.expired_count ?? 0), + usage_exceeded_count: + (existingPoint.y.usage_exceeded_count ?? 0) + (dataPoint.y.usage_exceeded_count ?? 0), + }, + }); + } + }); + }); + + // Convert map back to sorted array + return Array.from(mergedMap.values()).sort((a, b) => a.x - b.x); +} + +export const getHourlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.hour, args); + +export const getTwoHourlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.twoHours, args); + +export const getFourHourlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.fourHours, args); + +export const getSixHourlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.sixHours, args); + +export const getTwelveHourlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.twelveHours, args); + +export const getDailyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.day, args); + +export const getThreeDayVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.threeDays, args); + +export const getWeeklyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.week, args); + +export const getMonthlyVerificationTimeseries = + (ch: Querier) => (args: VerificationTimeseriesParams) => + batchVerificationTimeseries(ch, INTERVALS.month, args);