From 62121ff717b5ab0e7a2da36128eb9b545cdf3987 Mon Sep 17 00:00:00 2001 From: ogzhanolguncu Date: Tue, 18 Mar 2025 13:54:41 +0300 Subject: [PATCH 1/3] feat: add outcome sort to ratelimit overview --- .../_overview/components/table/logs-table.tsx | 44 +++-- .../components/table/query-logs.schema.ts | 18 +- internal/clickhouse/src/ratelimits.ts | 160 ++++++++++++------ 3 files changed, 157 insertions(+), 65 deletions(-) diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx index 7c6e1de51d..466980fdca 100644 --- a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx +++ b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx @@ -15,7 +15,11 @@ import { LogsTableAction } from "./components/logs-actions"; import { IdentifierColumn } from "./components/override-indicator"; import { useRatelimitOverviewLogsQuery } from "./hooks/use-logs-query"; import { useSort } from "./hooks/use-sort"; -import { STATUS_STYLES, getRowClassName, getStatusStyle } from "./utils/get-row-class"; +import { + STATUS_STYLES, + getRowClassName, + getStatusStyle, +} from "./utils/get-row-class"; // const MAX_LATENCY = 10; export const RatelimitOverviewLogsTable = ({ @@ -25,9 +29,10 @@ export const RatelimitOverviewLogsTable = ({ }) => { const [selectedLog, setSelectedLog] = useState(); const { getSortDirection, toggleSort } = useSort(); - const { historicalLogs, isLoading, isLoadingMore, loadMore } = useRatelimitOverviewLogsQuery({ - namespaceId, - }); + const { historicalLogs, isLoading, isLoadingMore, loadMore } = + useRatelimitOverviewLogsQuery({ + namespaceId, + }); const columns = (namespaceId: string): Column[] => { return [ @@ -52,6 +57,13 @@ export const RatelimitOverviewLogsTable = ({ key: "passed", header: "Passed", width: "7.5%", + sort: { + direction: getSortDirection("passed"), + sortable: true, + onSort() { + toggleSort("passed", false); + }, + }, render: (log) => { return (
@@ -60,7 +72,7 @@ export const RatelimitOverviewLogsTable = ({ "uppercase px-[6px] rounded-md font-mono whitespace-nowrap", selectedLog?.request_id === log.request_id ? STATUS_STYLES.success.badge.selected - : STATUS_STYLES.success.badge.default, + : STATUS_STYLES.success.badge.default )} title={`${log.passed_count.toLocaleString()} Passed requests`} > @@ -78,6 +90,13 @@ export const RatelimitOverviewLogsTable = ({ key: "blocked", header: "Blocked", width: "7.5%", + sort: { + direction: getSortDirection("blocked"), + sortable: true, + onSort() { + toggleSort("blocked", false); + }, + }, render: (log) => { const style = getStatusStyle(log); return ( @@ -87,7 +106,7 @@ export const RatelimitOverviewLogsTable = ({ "uppercase px-[6px] rounded-md font-mono whitespace-nowrap gap-[6px]", selectedLog?.request_id === log.request_id ? style.badge.selected - : style.badge.default, + : style.badge.default )} title={`${log.blocked_count.toLocaleString()} Blocked requests`} > @@ -166,7 +185,9 @@ export const RatelimitOverviewLogsTable = ({ value={log.time} className={cn( "font-mono group-hover:underline decoration-dotted", - selectedLog && selectedLog.request_id !== log.request_id && "pointer-events-none", + selectedLog && + selectedLog.request_id !== log.request_id && + "pointer-events-none" )} /> ), @@ -198,15 +219,18 @@ export const RatelimitOverviewLogsTable = ({ onLoadMore={loadMore} columns={columns(namespaceId)} keyExtractor={(log) => log.identifier} - rowClassName={(rowLog) => getRowClassName(rowLog, selectedLog as RatelimitOverviewLog)} + rowClassName={(rowLog) => + getRowClassName(rowLog, selectedLog as RatelimitOverviewLog) + } emptyState={
Logs - No rate limit data to show. Once requests are made, you'll see a summary of passed and - blocked requests for each rate limit identifier. + No rate limit data to show. Once requests are made, you'll see a + summary of passed and blocked requests for each rate limit + identifier. ; +export type RatelimitQueryOverviewLogsPayload = z.infer< + typeof ratelimitQueryOverviewLogsPayload +>; diff --git a/internal/clickhouse/src/ratelimits.ts b/internal/clickhouse/src/ratelimits.ts index d63e23bd1f..ad581dd9a9 100644 --- a/internal/clickhouse/src/ratelimits.ts +++ b/internal/clickhouse/src/ratelimits.ts @@ -26,7 +26,7 @@ export const ratelimitLogsTimeseriesParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }), + }) ) .nullable(), }); @@ -39,8 +39,12 @@ export const ratelimitLogsTimeseriesDataPoint = z.object({ }), }); -export type RatelimitLogsTimeseriesDataPoint = z.infer; -export type RatelimitLogsTimeseriesParams = z.infer; +export type RatelimitLogsTimeseriesDataPoint = z.infer< + typeof ratelimitLogsTimeseriesDataPoint +>; +export type RatelimitLogsTimeseriesParams = z.infer< + typeof ratelimitLogsTimeseriesParams +>; type TimeInterval = { table: string; @@ -132,7 +136,7 @@ function createTimeseriesQuery(interval: TimeInterval, whereClause: string) { function getRatelimitLogsTimeseriesWhereClause( params: RatelimitLogsTimeseriesParams, - additionalConditions: string[] = [], + additionalConditions: string[] = [] ): { whereClause: string; paramSchema: z.ZodType } { const conditions = [ "workspace_id = {workspaceId: String}", @@ -160,17 +164,21 @@ function getRatelimitLogsTimeseriesWhereClause( } return { - whereClause: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + whereClause: + conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", paramSchema: ratelimitLogsTimeseriesParams.extend(paramSchemaExtension), }; } function createTimeseriesQuerier(interval: TimeInterval) { return (ch: Querier) => async (args: RatelimitLogsTimeseriesParams) => { - const { whereClause, paramSchema } = getRatelimitLogsTimeseriesWhereClause(args, [ - "time >= fromUnixTimestamp64Milli({startTime: Int64})", - "time <= fromUnixTimestamp64Milli({endTime: Int64})", - ]); + const { whereClause, paramSchema } = getRatelimitLogsTimeseriesWhereClause( + args, + [ + "time >= fromUnixTimestamp64Milli({startTime: Int64})", + "time <= fromUnixTimestamp64Milli({endTime: Int64})", + ] + ); const parameters = { ...args, @@ -180,7 +188,7 @@ function createTimeseriesQuerier(interval: TimeInterval) { ...acc, [`identifierValue_${index}`]: i.value, }), - {}, + {} ) ?? {}), }; @@ -192,18 +200,36 @@ function createTimeseriesQuerier(interval: TimeInterval) { }; } -export const getMinutelyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.minute); -export const getFiveMinuteRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.fiveMinutes); +export const getMinutelyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.minute +); +export const getFiveMinuteRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.fiveMinutes +); export const getFifteenMinuteRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.fifteenMinutes, + INTERVALS.fifteenMinutes +); +export const getThirtyMinuteRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.thirtyMinutes +); +export const getHourlyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.hour +); +export const getTwoHourlyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.twoHours +); +export const getFourHourlyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.fourHours +); +export const getSixHourlyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.sixHours +); +export const getDailyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.day +); +export const getMonthlyRatelimitTimeseries = createTimeseriesQuerier( + INTERVALS.month ); -export const getThirtyMinuteRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.thirtyMinutes); -export const getHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.hour); -export const getTwoHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.twoHours); -export const getFourHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.fourHours); -export const getSixHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.sixHours); -export const getDailyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.day); -export const getMonthlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.month); const getRatelimitLastUsedParameters = z.object({ workspaceId: z.string(), @@ -223,7 +249,11 @@ export function getRatelimitLastUsed(ch: Querier) { WHERE workspace_id = {workspaceId: String} AND namespace_id = {namespaceId: String} - ${args.identifier ? "AND multiSearchAny(identifier, {identifier: Array(String)}) > 0" : ""} + ${ + args.identifier + ? "AND multiSearchAny(identifier, {identifier: Array(String)}) > 0" + : "" + } GROUP BY identifier ORDER BY time DESC LIMIT {limit: Int} @@ -251,7 +281,7 @@ export const ratelimitLogsParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }), + }) ) .nullable(), status: z @@ -259,7 +289,7 @@ export const ratelimitLogsParams = z.object({ z.object({ value: z.enum(["blocked", "passed"]), operator: z.literal("is"), - }), + }) ) .nullable(), cursorTime: z.number().int().nullable(), @@ -300,7 +330,8 @@ export function getRatelimitLogs(ch: Querier) { const hasRequestIds = args.requestIds && args.requestIds.length > 0; const hasStatusFilters = args.status && args.status.length > 0; - const hasIdentifierFilters = args.identifiers && args.identifiers.length > 0; + const hasIdentifierFilters = + args.identifiers && args.identifiers.length > 0; const statusCondition = !hasStatusFilters ? "TRUE" @@ -336,7 +367,8 @@ export function getRatelimitLogs(ch: Querier) { .filter(Boolean) .join(" OR ") || "TRUE"; - const extendedParamsSchema = ratelimitLogsParams.extend(paramSchemaExtension); + const extendedParamsSchema = + ratelimitLogsParams.extend(paramSchemaExtension); const query = ch.query({ query: ` @@ -417,7 +449,7 @@ export const ratelimitOverviewLogsParams = z.object({ z.object({ value: z.enum(["blocked", "passed"]), operator: z.literal("is"), - }), + }) ) .nullable(), identifiers: z @@ -425,7 +457,7 @@ export const ratelimitOverviewLogsParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }), + }) ) .nullable(), cursorTime: z.number().int().nullable(), @@ -434,9 +466,15 @@ export const ratelimitOverviewLogsParams = z.object({ sorts: z .array( z.object({ - column: z.enum(["time", "avg_latency", "p99_latency"]), + column: z.enum([ + "time", + "avg_latency", + "p99_latency", + "blocked", + "passed", + ]), direction: z.enum(["asc", "desc"]), - }), + }) ) .nullable(), }); @@ -461,7 +499,9 @@ export const ratelimitOverviewLogs = z.object({ }); export type RatelimitOverviewLog = z.infer; -export type RatelimitOverviewLogsParams = z.infer; +export type RatelimitOverviewLogsParams = z.infer< + typeof ratelimitOverviewLogsParams +>; interface ExtendedParamsOverviewLogs extends RatelimitOverviewLogsParams { [key: string]: unknown; @@ -472,7 +512,8 @@ export function getRatelimitOverviewLogs(ch: Querier) { const paramSchemaExtension: Record = {}; const parameters: ExtendedParamsOverviewLogs = { ...args }; - const hasIdentifierFilters = args.identifiers && args.identifiers.length > 0; + const hasIdentifierFilters = + args.identifiers && args.identifiers.length > 0; const hasStatusFilters = args.status && args.status.length > 0; const hasSortingRules = args.sorts && args.sorts.length > 0; @@ -514,6 +555,8 @@ export function getRatelimitOverviewLogs(ch: Querier) { ["time", "last_request_time"], ["avg_latency", "avg_latency"], ["p99_latency", "p99_latency"], + ["passed", "passed_count"], + ["blocked", "blocked_count"], ]); const orderBy = @@ -523,7 +566,8 @@ export function getRatelimitOverviewLogs(ch: Querier) { // Only add to ORDER BY if it's an allowed column to prevent injection if (column) { const direction = - sort.direction.toUpperCase() === "ASC" || sort.direction.toUpperCase() === "DESC" + sort.direction.toUpperCase() === "ASC" || + sort.direction.toUpperCase() === "DESC" ? sort.direction.toUpperCase() : "DESC"; acc.push(`${column} ${direction}`); @@ -534,10 +578,13 @@ export function getRatelimitOverviewLogs(ch: Querier) { // If time is explicitly sorted ASC, maintain that direction const timeSort = args.sorts?.find((s) => s.column === "time"); - const timeDirection = timeSort?.direction.toUpperCase() === "ASC" ? "ASC" : "DESC"; + const timeDirection = + timeSort?.direction.toUpperCase() === "ASC" ? "ASC" : "DESC"; // Remove any existing time sort from the orderBy array - const orderByWithoutTime = orderBy.filter((clause) => !clause.startsWith("last_request_time")); + const orderByWithoutTime = orderBy.filter( + (clause) => !clause.startsWith("last_request_time") + ); // Construct final ORDER BY clause with time and request_id always at the end const orderByClause = @@ -547,7 +594,8 @@ export function getRatelimitOverviewLogs(ch: Querier) { `request_id ${timeDirection}`, ].join(", ") || "last_request_time DESC, request_id DESC"; // Fallback if empty - const extendedParamsSchema = ratelimitOverviewLogsParams.extend(paramSchemaExtension); + const extendedParamsSchema = + ratelimitOverviewLogsParams.extend(paramSchemaExtension); const query = ch.query({ query: `WITH filtered_ratelimits AS ( SELECT @@ -603,7 +651,7 @@ export const ratelimitLatencyTimeseriesParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }), + }) ) .nullable(), }); @@ -619,7 +667,9 @@ export const ratelimitLatencyTimeseriesDataPoint = z.object({ export type RatelimitLatencyTimeseriesDataPoint = z.infer< typeof ratelimitLatencyTimeseriesDataPoint >; -export type RatelimitLatencyTimeseriesParams = z.infer; +export type RatelimitLatencyTimeseriesParams = z.infer< + typeof ratelimitLatencyTimeseriesParams +>; const LATENCY_INTERVALS: Record = { minute: { @@ -669,7 +719,10 @@ const LATENCY_INTERVALS: Record = { }, } as const; -function createLatencyTimeseriesQuery(interval: TimeInterval, whereClause: string) { +function createLatencyTimeseriesQuery( + interval: TimeInterval, + whereClause: string +) { // Map step to ClickHouse interval unit const intervalUnit = { MINUTE: "minute", @@ -711,7 +764,9 @@ function createLatencyTimeseriesQuery(interval: TimeInterval, whereClause: strin `; } -function getRatelimitLatencyTimeseriesWhereClause(params: RatelimitLatencyTimeseriesParams): { +function getRatelimitLatencyTimeseriesWhereClause( + params: RatelimitLatencyTimeseriesParams +): { whereClause: string; paramSchema: z.ZodType; } { @@ -742,7 +797,8 @@ function getRatelimitLatencyTimeseriesWhereClause(params: RatelimitLatencyTimese function createLatencyTimeseriesQuerier(interval: TimeInterval) { return (ch: Querier) => async (args: RatelimitLatencyTimeseriesParams) => { - const { whereClause, paramSchema } = getRatelimitLatencyTimeseriesWhereClause(args); + const { whereClause, paramSchema } = + getRatelimitLatencyTimeseriesWhereClause(args); const parameters = { ...args, @@ -752,7 +808,7 @@ function createLatencyTimeseriesQuerier(interval: TimeInterval) { ...acc, [`identifierValue_${index}`]: i.value, }), - {}, + {} ) ?? {}), }; @@ -765,25 +821,29 @@ function createLatencyTimeseriesQuerier(interval: TimeInterval) { } export const getMinutelyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.minute, + LATENCY_INTERVALS.minute ); export const getFiveMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fiveMinutes, + LATENCY_INTERVALS.fiveMinutes ); export const getFifteenMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fifteenMinutes, + LATENCY_INTERVALS.fifteenMinutes ); export const getThirtyMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.thirtyMinutes, + LATENCY_INTERVALS.thirtyMinutes +); +export const getHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( + LATENCY_INTERVALS.hour ); -export const getHourlyLatencyTimeseries = createLatencyTimeseriesQuerier(LATENCY_INTERVALS.hour); export const getTwoHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.twoHours, + LATENCY_INTERVALS.twoHours ); export const getFourHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fourHours, + LATENCY_INTERVALS.fourHours ); export const getSixHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.sixHours, + LATENCY_INTERVALS.sixHours +); +export const getDailyLatencyTimeseries = createLatencyTimeseriesQuerier( + LATENCY_INTERVALS.day ); -export const getDailyLatencyTimeseries = createLatencyTimeseriesQuerier(LATENCY_INTERVALS.day); From b577b40dcbfc89d853183984390a6bba187b48e0 Mon Sep 17 00:00:00 2001 From: ogzhanolguncu Date: Tue, 18 Mar 2025 13:55:01 +0300 Subject: [PATCH 2/3] chore: run fmt --- .../_overview/components/table/logs-table.tsx | 30 ++-- .../components/table/query-logs.schema.ts | 18 +- internal/clickhouse/src/ratelimits.ts | 158 ++++++------------ 3 files changed, 65 insertions(+), 141 deletions(-) diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx index 466980fdca..d2a4ec2102 100644 --- a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx +++ b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx @@ -15,11 +15,7 @@ import { LogsTableAction } from "./components/logs-actions"; import { IdentifierColumn } from "./components/override-indicator"; import { useRatelimitOverviewLogsQuery } from "./hooks/use-logs-query"; import { useSort } from "./hooks/use-sort"; -import { - STATUS_STYLES, - getRowClassName, - getStatusStyle, -} from "./utils/get-row-class"; +import { STATUS_STYLES, getRowClassName, getStatusStyle } from "./utils/get-row-class"; // const MAX_LATENCY = 10; export const RatelimitOverviewLogsTable = ({ @@ -29,10 +25,9 @@ export const RatelimitOverviewLogsTable = ({ }) => { const [selectedLog, setSelectedLog] = useState(); const { getSortDirection, toggleSort } = useSort(); - const { historicalLogs, isLoading, isLoadingMore, loadMore } = - useRatelimitOverviewLogsQuery({ - namespaceId, - }); + const { historicalLogs, isLoading, isLoadingMore, loadMore } = useRatelimitOverviewLogsQuery({ + namespaceId, + }); const columns = (namespaceId: string): Column[] => { return [ @@ -72,7 +67,7 @@ export const RatelimitOverviewLogsTable = ({ "uppercase px-[6px] rounded-md font-mono whitespace-nowrap", selectedLog?.request_id === log.request_id ? STATUS_STYLES.success.badge.selected - : STATUS_STYLES.success.badge.default + : STATUS_STYLES.success.badge.default, )} title={`${log.passed_count.toLocaleString()} Passed requests`} > @@ -106,7 +101,7 @@ export const RatelimitOverviewLogsTable = ({ "uppercase px-[6px] rounded-md font-mono whitespace-nowrap gap-[6px]", selectedLog?.request_id === log.request_id ? style.badge.selected - : style.badge.default + : style.badge.default, )} title={`${log.blocked_count.toLocaleString()} Blocked requests`} > @@ -185,9 +180,7 @@ export const RatelimitOverviewLogsTable = ({ value={log.time} className={cn( "font-mono group-hover:underline decoration-dotted", - selectedLog && - selectedLog.request_id !== log.request_id && - "pointer-events-none" + selectedLog && selectedLog.request_id !== log.request_id && "pointer-events-none", )} /> ), @@ -219,18 +212,15 @@ export const RatelimitOverviewLogsTable = ({ onLoadMore={loadMore} columns={columns(namespaceId)} keyExtractor={(log) => log.identifier} - rowClassName={(rowLog) => - getRowClassName(rowLog, selectedLog as RatelimitOverviewLog) - } + rowClassName={(rowLog) => getRowClassName(rowLog, selectedLog as RatelimitOverviewLog)} emptyState={
Logs - No rate limit data to show. Once requests are made, you'll see a - summary of passed and blocked requests for each rate limit - identifier. + No rate limit data to show. Once requests are made, you'll see a summary of passed and + blocked requests for each rate limit identifier. ; +export type RatelimitQueryOverviewLogsPayload = z.infer; diff --git a/internal/clickhouse/src/ratelimits.ts b/internal/clickhouse/src/ratelimits.ts index ad581dd9a9..3c427dce71 100644 --- a/internal/clickhouse/src/ratelimits.ts +++ b/internal/clickhouse/src/ratelimits.ts @@ -26,7 +26,7 @@ export const ratelimitLogsTimeseriesParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }) + }), ) .nullable(), }); @@ -39,12 +39,8 @@ export const ratelimitLogsTimeseriesDataPoint = z.object({ }), }); -export type RatelimitLogsTimeseriesDataPoint = z.infer< - typeof ratelimitLogsTimeseriesDataPoint ->; -export type RatelimitLogsTimeseriesParams = z.infer< - typeof ratelimitLogsTimeseriesParams ->; +export type RatelimitLogsTimeseriesDataPoint = z.infer; +export type RatelimitLogsTimeseriesParams = z.infer; type TimeInterval = { table: string; @@ -136,7 +132,7 @@ function createTimeseriesQuery(interval: TimeInterval, whereClause: string) { function getRatelimitLogsTimeseriesWhereClause( params: RatelimitLogsTimeseriesParams, - additionalConditions: string[] = [] + additionalConditions: string[] = [], ): { whereClause: string; paramSchema: z.ZodType } { const conditions = [ "workspace_id = {workspaceId: String}", @@ -164,21 +160,17 @@ function getRatelimitLogsTimeseriesWhereClause( } return { - whereClause: - conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + whereClause: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", paramSchema: ratelimitLogsTimeseriesParams.extend(paramSchemaExtension), }; } function createTimeseriesQuerier(interval: TimeInterval) { return (ch: Querier) => async (args: RatelimitLogsTimeseriesParams) => { - const { whereClause, paramSchema } = getRatelimitLogsTimeseriesWhereClause( - args, - [ - "time >= fromUnixTimestamp64Milli({startTime: Int64})", - "time <= fromUnixTimestamp64Milli({endTime: Int64})", - ] - ); + const { whereClause, paramSchema } = getRatelimitLogsTimeseriesWhereClause(args, [ + "time >= fromUnixTimestamp64Milli({startTime: Int64})", + "time <= fromUnixTimestamp64Milli({endTime: Int64})", + ]); const parameters = { ...args, @@ -188,7 +180,7 @@ function createTimeseriesQuerier(interval: TimeInterval) { ...acc, [`identifierValue_${index}`]: i.value, }), - {} + {}, ) ?? {}), }; @@ -200,36 +192,18 @@ function createTimeseriesQuerier(interval: TimeInterval) { }; } -export const getMinutelyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.minute -); -export const getFiveMinuteRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.fiveMinutes -); +export const getMinutelyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.minute); +export const getFiveMinuteRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.fiveMinutes); export const getFifteenMinuteRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.fifteenMinutes -); -export const getThirtyMinuteRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.thirtyMinutes -); -export const getHourlyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.hour -); -export const getTwoHourlyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.twoHours -); -export const getFourHourlyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.fourHours -); -export const getSixHourlyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.sixHours -); -export const getDailyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.day -); -export const getMonthlyRatelimitTimeseries = createTimeseriesQuerier( - INTERVALS.month + INTERVALS.fifteenMinutes, ); +export const getThirtyMinuteRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.thirtyMinutes); +export const getHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.hour); +export const getTwoHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.twoHours); +export const getFourHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.fourHours); +export const getSixHourlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.sixHours); +export const getDailyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.day); +export const getMonthlyRatelimitTimeseries = createTimeseriesQuerier(INTERVALS.month); const getRatelimitLastUsedParameters = z.object({ workspaceId: z.string(), @@ -249,11 +223,7 @@ export function getRatelimitLastUsed(ch: Querier) { WHERE workspace_id = {workspaceId: String} AND namespace_id = {namespaceId: String} - ${ - args.identifier - ? "AND multiSearchAny(identifier, {identifier: Array(String)}) > 0" - : "" - } + ${args.identifier ? "AND multiSearchAny(identifier, {identifier: Array(String)}) > 0" : ""} GROUP BY identifier ORDER BY time DESC LIMIT {limit: Int} @@ -281,7 +251,7 @@ export const ratelimitLogsParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }) + }), ) .nullable(), status: z @@ -289,7 +259,7 @@ export const ratelimitLogsParams = z.object({ z.object({ value: z.enum(["blocked", "passed"]), operator: z.literal("is"), - }) + }), ) .nullable(), cursorTime: z.number().int().nullable(), @@ -330,8 +300,7 @@ export function getRatelimitLogs(ch: Querier) { const hasRequestIds = args.requestIds && args.requestIds.length > 0; const hasStatusFilters = args.status && args.status.length > 0; - const hasIdentifierFilters = - args.identifiers && args.identifiers.length > 0; + const hasIdentifierFilters = args.identifiers && args.identifiers.length > 0; const statusCondition = !hasStatusFilters ? "TRUE" @@ -367,8 +336,7 @@ export function getRatelimitLogs(ch: Querier) { .filter(Boolean) .join(" OR ") || "TRUE"; - const extendedParamsSchema = - ratelimitLogsParams.extend(paramSchemaExtension); + const extendedParamsSchema = ratelimitLogsParams.extend(paramSchemaExtension); const query = ch.query({ query: ` @@ -449,7 +417,7 @@ export const ratelimitOverviewLogsParams = z.object({ z.object({ value: z.enum(["blocked", "passed"]), operator: z.literal("is"), - }) + }), ) .nullable(), identifiers: z @@ -457,7 +425,7 @@ export const ratelimitOverviewLogsParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }) + }), ) .nullable(), cursorTime: z.number().int().nullable(), @@ -466,15 +434,9 @@ export const ratelimitOverviewLogsParams = z.object({ sorts: z .array( z.object({ - column: z.enum([ - "time", - "avg_latency", - "p99_latency", - "blocked", - "passed", - ]), + column: z.enum(["time", "avg_latency", "p99_latency", "blocked", "passed"]), direction: z.enum(["asc", "desc"]), - }) + }), ) .nullable(), }); @@ -499,9 +461,7 @@ export const ratelimitOverviewLogs = z.object({ }); export type RatelimitOverviewLog = z.infer; -export type RatelimitOverviewLogsParams = z.infer< - typeof ratelimitOverviewLogsParams ->; +export type RatelimitOverviewLogsParams = z.infer; interface ExtendedParamsOverviewLogs extends RatelimitOverviewLogsParams { [key: string]: unknown; @@ -512,8 +472,7 @@ export function getRatelimitOverviewLogs(ch: Querier) { const paramSchemaExtension: Record = {}; const parameters: ExtendedParamsOverviewLogs = { ...args }; - const hasIdentifierFilters = - args.identifiers && args.identifiers.length > 0; + const hasIdentifierFilters = args.identifiers && args.identifiers.length > 0; const hasStatusFilters = args.status && args.status.length > 0; const hasSortingRules = args.sorts && args.sorts.length > 0; @@ -566,8 +525,7 @@ export function getRatelimitOverviewLogs(ch: Querier) { // Only add to ORDER BY if it's an allowed column to prevent injection if (column) { const direction = - sort.direction.toUpperCase() === "ASC" || - sort.direction.toUpperCase() === "DESC" + sort.direction.toUpperCase() === "ASC" || sort.direction.toUpperCase() === "DESC" ? sort.direction.toUpperCase() : "DESC"; acc.push(`${column} ${direction}`); @@ -578,13 +536,10 @@ export function getRatelimitOverviewLogs(ch: Querier) { // If time is explicitly sorted ASC, maintain that direction const timeSort = args.sorts?.find((s) => s.column === "time"); - const timeDirection = - timeSort?.direction.toUpperCase() === "ASC" ? "ASC" : "DESC"; + const timeDirection = timeSort?.direction.toUpperCase() === "ASC" ? "ASC" : "DESC"; // Remove any existing time sort from the orderBy array - const orderByWithoutTime = orderBy.filter( - (clause) => !clause.startsWith("last_request_time") - ); + const orderByWithoutTime = orderBy.filter((clause) => !clause.startsWith("last_request_time")); // Construct final ORDER BY clause with time and request_id always at the end const orderByClause = @@ -594,8 +549,7 @@ export function getRatelimitOverviewLogs(ch: Querier) { `request_id ${timeDirection}`, ].join(", ") || "last_request_time DESC, request_id DESC"; // Fallback if empty - const extendedParamsSchema = - ratelimitOverviewLogsParams.extend(paramSchemaExtension); + const extendedParamsSchema = ratelimitOverviewLogsParams.extend(paramSchemaExtension); const query = ch.query({ query: `WITH filtered_ratelimits AS ( SELECT @@ -651,7 +605,7 @@ export const ratelimitLatencyTimeseriesParams = z.object({ z.object({ operator: z.enum(["is", "contains"]), value: z.string(), - }) + }), ) .nullable(), }); @@ -667,9 +621,7 @@ export const ratelimitLatencyTimeseriesDataPoint = z.object({ export type RatelimitLatencyTimeseriesDataPoint = z.infer< typeof ratelimitLatencyTimeseriesDataPoint >; -export type RatelimitLatencyTimeseriesParams = z.infer< - typeof ratelimitLatencyTimeseriesParams ->; +export type RatelimitLatencyTimeseriesParams = z.infer; const LATENCY_INTERVALS: Record = { minute: { @@ -719,10 +671,7 @@ const LATENCY_INTERVALS: Record = { }, } as const; -function createLatencyTimeseriesQuery( - interval: TimeInterval, - whereClause: string -) { +function createLatencyTimeseriesQuery(interval: TimeInterval, whereClause: string) { // Map step to ClickHouse interval unit const intervalUnit = { MINUTE: "minute", @@ -764,9 +713,7 @@ function createLatencyTimeseriesQuery( `; } -function getRatelimitLatencyTimeseriesWhereClause( - params: RatelimitLatencyTimeseriesParams -): { +function getRatelimitLatencyTimeseriesWhereClause(params: RatelimitLatencyTimeseriesParams): { whereClause: string; paramSchema: z.ZodType; } { @@ -797,8 +744,7 @@ function getRatelimitLatencyTimeseriesWhereClause( function createLatencyTimeseriesQuerier(interval: TimeInterval) { return (ch: Querier) => async (args: RatelimitLatencyTimeseriesParams) => { - const { whereClause, paramSchema } = - getRatelimitLatencyTimeseriesWhereClause(args); + const { whereClause, paramSchema } = getRatelimitLatencyTimeseriesWhereClause(args); const parameters = { ...args, @@ -808,7 +754,7 @@ function createLatencyTimeseriesQuerier(interval: TimeInterval) { ...acc, [`identifierValue_${index}`]: i.value, }), - {} + {}, ) ?? {}), }; @@ -821,29 +767,25 @@ function createLatencyTimeseriesQuerier(interval: TimeInterval) { } export const getMinutelyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.minute + LATENCY_INTERVALS.minute, ); export const getFiveMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fiveMinutes + LATENCY_INTERVALS.fiveMinutes, ); export const getFifteenMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fifteenMinutes + LATENCY_INTERVALS.fifteenMinutes, ); export const getThirtyMinuteLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.thirtyMinutes -); -export const getHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.hour + LATENCY_INTERVALS.thirtyMinutes, ); +export const getHourlyLatencyTimeseries = createLatencyTimeseriesQuerier(LATENCY_INTERVALS.hour); export const getTwoHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.twoHours + LATENCY_INTERVALS.twoHours, ); export const getFourHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.fourHours + LATENCY_INTERVALS.fourHours, ); export const getSixHourlyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.sixHours -); -export const getDailyLatencyTimeseries = createLatencyTimeseriesQuerier( - LATENCY_INTERVALS.day + LATENCY_INTERVALS.sixHours, ); +export const getDailyLatencyTimeseries = createLatencyTimeseriesQuerier(LATENCY_INTERVALS.day); From 6331350e207b465577ff7c0b18d15c4af213b950 Mon Sep 17 00:00:00 2001 From: ogzhanolguncu Date: Tue, 18 Mar 2025 15:32:29 +0300 Subject: [PATCH 3/3] fix: ordering --- .../_overview/components/table/logs-table.tsx | 2 +- internal/clickhouse/src/ratelimits.ts | 49 +++++++++++++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx index d2a4ec2102..10abf72a49 100644 --- a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx +++ b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/_overview/components/table/logs-table.tsx @@ -172,7 +172,7 @@ export const RatelimitOverviewLogsTable = ({ direction: getSortDirection("time"), sortable: true, onSort() { - toggleSort("time", true); + toggleSort("time", false); }, }, render: (log) => ( diff --git a/internal/clickhouse/src/ratelimits.ts b/internal/clickhouse/src/ratelimits.ts index 3c427dce71..82d7f82f85 100644 --- a/internal/clickhouse/src/ratelimits.ts +++ b/internal/clickhouse/src/ratelimits.ts @@ -534,9 +534,23 @@ export function getRatelimitOverviewLogs(ch: Querier) { }, []) : []; - // If time is explicitly sorted ASC, maintain that direction + // Check if we have custom sorts + const hasAvgLatencySort = args.sorts?.some((s) => s.column === "avg_latency"); + const hasP99LatencySort = args.sorts?.some((s) => s.column === "p99_latency"); + const hasPassedSort = args.sorts?.some((s) => s.column === "passed"); + const hasBlockedSort = args.sorts?.some((s) => s.column === "blocked"); + const hasCustomSort = hasAvgLatencySort || hasP99LatencySort || hasPassedSort || hasBlockedSort; + + // Get explicit time sort if it exists const timeSort = args.sorts?.find((s) => s.column === "time"); - const timeDirection = timeSort?.direction.toUpperCase() === "ASC" ? "ASC" : "DESC"; + + // If we have custom sort (avg_latency, p99_latency, passed, blocked), always use ASC for better pagination + // Otherwise use explicit time direction or default to DESC + const timeDirection = hasCustomSort + ? "ASC" + : timeSort?.direction.toUpperCase() === "ASC" + ? "ASC" + : "DESC"; // Remove any existing time sort from the orderBy array const orderByWithoutTime = orderBy.filter((clause) => !clause.startsWith("last_request_time")); @@ -549,6 +563,33 @@ export function getRatelimitOverviewLogs(ch: Querier) { `request_id ${timeDirection}`, ].join(", ") || "last_request_time DESC, request_id DESC"; // Fallback if empty + // Create cursor condition based on time direction + let cursorCondition: string; + + // For first page or no cursor provided + if (!args.cursorTime || !args.cursorRequestId) { + cursorCondition = ` + AND ({cursorTime: Nullable(UInt64)} IS NULL AND {cursorRequestId: Nullable(String)} IS NULL) + `; + } else { + // For subsequent pages, use cursor based on time direction + if (timeDirection === "ASC") { + cursorCondition = ` + AND ( + (time = {cursorTime: Nullable(UInt64)} AND request_id > {cursorRequestId: Nullable(String)}) + OR time > {cursorTime: Nullable(UInt64)} + ) + `; + } else { + cursorCondition = ` + AND ( + (time = {cursorTime: Nullable(UInt64)} AND request_id < {cursorRequestId: Nullable(String)}) + OR time < {cursorTime: Nullable(UInt64)} + ) + `; + } + } + const extendedParamsSchema = ratelimitOverviewLogsParams.extend(paramSchemaExtension); const query = ch.query({ query: `WITH filtered_ratelimits AS ( @@ -563,9 +604,7 @@ export function getRatelimitOverviewLogs(ch: Querier) { AND time BETWEEN {startTime: UInt64} AND {endTime: UInt64} AND (${identifierConditions}) AND (${statusCondition}) - AND (({cursorTime: Nullable(UInt64)} IS NULL AND {cursorRequestId: Nullable(String)} IS NULL) - OR (time = {cursorTime: Nullable(UInt64)} AND request_id < {cursorRequestId: Nullable(String)}) - OR time < {cursorTime: Nullable(UInt64)}) + ${cursorCondition} ), aggregated_data AS ( SELECT