diff --git a/apps/agent/pkg/clickhouse/schema/requests.go b/apps/agent/pkg/clickhouse/schema/requests.go index 9df975648e..d3c0c4991d 100644 --- a/apps/agent/pkg/clickhouse/schema/requests.go +++ b/apps/agent/pkg/clickhouse/schema/requests.go @@ -15,12 +15,14 @@ type ApiRequestV1 struct { } type KeyVerificationRequestV1 struct { - RequestID string `ch:"request_id"` - Time int64 `ch:"time"` - WorkspaceID string `ch:"workspace_id"` - KeySpaceID string `ch:"key_space_id"` - KeyID string `ch:"key_id"` - Region string `ch:"region"` - Outcome string `ch:"outcome"` - IdentityID string `ch:"identity_id"` + RequestID string `ch:"request_id"` + Time int64 `ch:"time"` + WorkspaceID string `ch:"workspace_id"` + KeySpaceID string `ch:"key_space_id"` + KeyID string `ch:"key_id"` + Region string `ch:"region"` + Outcome string `ch:"outcome"` + IdentityID string `ch:"identity_id"` + SpentCredits int64 `ch:"spent_credits"` + Tags []string `ch:"tags"` } diff --git a/apps/api/src/pkg/analytics.ts b/apps/api/src/pkg/analytics.ts index 7a8ac8b258..9e2ea2cadb 100644 --- a/apps/api/src/pkg/analytics.ts +++ b/apps/api/src/pkg/analytics.ts @@ -72,6 +72,7 @@ export class Analytics { outcome: string; identity_id?: string; tags?: string[]; + spent_credits?: number; }) => { return await wrap( // biome-ignore lint/style/noNonNullAssertion: proxyClient existence verified above diff --git a/apps/api/src/pkg/auth/root_key.ts b/apps/api/src/pkg/auth/root_key.ts index ffee62ae72..7567f6b48f 100644 --- a/apps/api/src/pkg/auth/root_key.ts +++ b/apps/api/src/pkg/auth/root_key.ts @@ -63,6 +63,7 @@ export async function rootKeyAuth(c: Context, permissionQuery?: Permiss region: c.req.cf?.region, request_id: c.get("requestId"), tags: [], + spent_credits: 0, // Root key verifications don't consume credits }), ); diff --git a/apps/api/src/pkg/clickhouse-proxy.ts b/apps/api/src/pkg/clickhouse-proxy.ts index 50c81b805e..9ff8524735 100644 --- a/apps/api/src/pkg/clickhouse-proxy.ts +++ b/apps/api/src/pkg/clickhouse-proxy.ts @@ -25,6 +25,7 @@ export class ClickHouseProxyClient { outcome: string; identity_id?: string; tags?: string[]; + spent_credits?: number; }>, ): Promise { await this.sendEvents("/_internal/chproxy/verifications", events); diff --git a/apps/api/src/pkg/keys/service.ts b/apps/api/src/pkg/keys/service.ts index f0499a91e1..b69042b877 100644 --- a/apps/api/src/pkg/keys/service.ts +++ b/apps/api/src/pkg/keys/service.ts @@ -56,6 +56,7 @@ type NotFoundResponse = { api?: never; ratelimit?: never; remaining?: never; + spentCredits?: number; }; type InvalidResponse = { @@ -84,6 +85,7 @@ type InvalidResponse = { permissions: string[]; roles: string[]; message?: string; + spentCredits?: number; }; type ValidResponse = { @@ -109,6 +111,7 @@ type ValidResponse = { authorizedWorkspaceId: string; permissions: string[]; roles: string[]; + spentCredits?: number; }; type VerifyKeyResult = NotFoundResponse | InvalidResponse | ValidResponse; @@ -396,7 +399,7 @@ export class KeyService { } if (!data) { - return Ok({ valid: false, code: "NOT_FOUND" }); + return Ok({ valid: false, code: "NOT_FOUND", spentCredits: 0 }); } // Quick fix @@ -436,6 +439,7 @@ export class KeyService { permissions: data.permissions, roles: data.roles, message: "the key is disabled", + spentCredits: 0, }); } @@ -449,6 +453,7 @@ export class KeyService { permissions: data.permissions, roles: data.roles, message: `the key does not belong to ${req.apiId}`, + spentCredits: 0, }); } @@ -469,6 +474,7 @@ export class KeyService { permissions: data.permissions, roles: data.roles, message: `the key has expired on ${new Date(expires).toISOString()}`, + spentCredits: 0, }); } } @@ -485,6 +491,7 @@ export class KeyService { code: "FORBIDDEN", permissions: data.permissions, roles: data.roles, + spentCredits: 0, }); } @@ -498,6 +505,7 @@ export class KeyService { code: "FORBIDDEN", permissions: data.permissions, roles: data.roles, + spentCredits: 0, }); } } @@ -542,6 +550,7 @@ export class KeyService { permissions: data.permissions, roles: data.roles, message: rbacResp.val.message, + spentCredits: 0, }); } } @@ -612,10 +621,13 @@ export class KeyService { ratelimit, permissions: data.permissions, roles: data.roles, + spentCredits: 0, }); } let remaining: number | undefined = undefined; + let spentCredits = 0; + if (data.key.remaining !== null) { const t0 = performance.now(); const cost = req.remaining?.cost ?? DEFAULT_REMAINING_COST; @@ -636,7 +648,10 @@ export class KeyService { }); remaining = limited.remaining; - if (!limited.valid) { + if (limited.valid) { + // Credits were successfully spent + spentCredits = cost; + } else { return Ok({ key: data.key, api: data.api, @@ -653,6 +668,7 @@ export class KeyService { authorizedWorkspaceId: data.key.forWorkspaceId ?? data.key.workspaceId, permissions: data.permissions, roles: data.roles, + spentCredits: 0, // No credits spent if usage exceeded }); } } @@ -672,6 +688,7 @@ export class KeyService { authorizedWorkspaceId: data.key.forWorkspaceId ?? data.key.workspaceId, permissions: data.permissions, roles: data.roles, + spentCredits, }); } diff --git a/apps/api/src/routes/v1_keys_verifyKey.ts b/apps/api/src/routes/v1_keys_verifyKey.ts index 63f6625aea..b4df37f43a 100644 --- a/apps/api/src/routes/v1_keys_verifyKey.ts +++ b/apps/api/src/routes/v1_keys_verifyKey.ts @@ -2,6 +2,7 @@ import { UnkeyApiError, openApiErrorResponses } from "@/pkg/errors"; import type { App } from "@/pkg/hono/app"; import { DisabledWorkspaceError, MissingRatelimitError } from "@/pkg/keys/service"; import { createRoute, z } from "@hono/zod-openapi"; + import { SchemaError } from "@unkey/error"; import { permissionQuerySchema } from "@unkey/rbac"; @@ -331,7 +332,6 @@ export const registerV1KeysVerifyKey = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); const { keyService, analytics, logger } = c.get("services"); - const { val, err } = await keyService.verifyKey(c, { key: req.key, apiId: req.apiId, @@ -340,7 +340,6 @@ export const registerV1KeysVerifyKey = (app: App) => ratelimits: req.ratelimits, remaining: req.remaining, }); - if (err) { switch (true) { case err instanceof SchemaError || err instanceof MissingRatelimitError: @@ -409,6 +408,7 @@ export const registerV1KeysVerifyKey = (app: App) => outcome: val.code, identity_id: val.identity?.id, tags: req.tags ?? [], + spent_credits: val.spentCredits ?? 0, }) .then(({ err }) => { if (!err) { @@ -419,6 +419,5 @@ export const registerV1KeysVerifyKey = (app: App) => }); }), ); - return c.json(responseBody); }); diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/hooks/use-fetch-timeseries.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/hooks/use-fetch-timeseries.ts index 2a6874ae6f..dc36a350f0 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/hooks/use-fetch-timeseries.ts +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/bar-chart/hooks/use-fetch-timeseries.ts @@ -159,6 +159,7 @@ export const useFetchVerificationTimeseries = (apiId: string | null) => { return { ...result, ...outcomeFields, + spent_credits: ts.y.spent_credits ?? 0, }; }); }, [data]); diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/credit-spend-chart/hooks/use-fetch-timeseries.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/credit-spend-chart/hooks/use-fetch-timeseries.ts new file mode 100644 index 0000000000..a5feb8192e --- /dev/null +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/credit-spend-chart/hooks/use-fetch-timeseries.ts @@ -0,0 +1,149 @@ +import { formatTimestampForChart } from "@/components/logs/chart/utils/format-timestamp"; +import { HISTORICAL_DATA_WINDOW } from "@/components/logs/constants"; +import { trpc } from "@/lib/trpc/client"; +import { useQueryTime } from "@/providers/query-time-provider"; +import { useMemo } from "react"; +import { keysOverviewFilterFieldConfig } from "../../../../filters.schema"; +import { useFilters } from "../../../../hooks/use-filters"; +import type { KeysOverviewQueryTimeseriesPayload } from "../../bar-chart/query-timeseries.schema"; + +export const useFetchCreditSpendTimeseries = (apiId: string | null) => { + const { filters } = useFilters(); + const { queryTime: timestamp } = useQueryTime(); + + const queryParams = useMemo(() => { + const params: KeysOverviewQueryTimeseriesPayload = { + startTime: timestamp - HISTORICAL_DATA_WINDOW, + endTime: timestamp, + keyIds: { filters: [] }, + outcomes: { filters: [] }, + names: { filters: [] }, + identities: { filters: [] }, + tags: null, + apiId: apiId ?? "", + since: "", + }; + + if (!apiId) { + return params; + } + + filters.forEach((filter) => { + if (!(filter.field in keysOverviewFilterFieldConfig)) { + return; + } + + const fieldConfig = keysOverviewFilterFieldConfig[filter.field]; + const validOperators = fieldConfig.operators; + + const operator = validOperators.includes(filter.operator) + ? filter.operator + : validOperators[0]; + + switch (filter.field) { + case "startTime": + case "endTime": { + const numValue = + typeof filter.value === "number" + ? filter.value + : typeof filter.value === "string" + ? Number(filter.value) + : Number.NaN; + + if (!Number.isNaN(numValue)) { + params[filter.field] = numValue; + } + break; + } + + case "since": { + if (typeof filter.value === "string") { + params.since = filter.value; + } + break; + } + + case "keyIds": { + if (typeof filter.value === "string" && filter.value.trim()) { + const keyIdOperator = operator === "is" || operator === "contains" ? operator : "is"; + + params.keyIds?.filters?.push({ + operator: keyIdOperator, + value: filter.value, + }); + } + break; + } + + case "names": + case "identities": { + if (typeof filter.value === "string" && filter.value.trim()) { + params[filter.field]?.filters?.push({ + operator, + value: filter.value, + }); + } + break; + } + + case "outcomes": { + // For credit spend, we might want to include all outcomes to show credit consumption patterns + if (typeof filter.value === "string") { + params.outcomes?.filters?.push({ + operator: "is", + value: filter.value as + | "VALID" + | "INSUFFICIENT_PERMISSIONS" + | "RATE_LIMITED" + | "FORBIDDEN" + | "DISABLED" + | "EXPIRED" + | "USAGE_EXCEEDED" + | "", + }); + } + break; + } + + case "tags": { + if (typeof filter.value === "string" && filter.value.trim()) { + params.tags = { + operator, + value: filter.value, + }; + } + break; + } + } + }); + + return params; + }, [filters, timestamp, apiId]); + + const { data, isLoading, isError } = trpc.api.keys.timeseries.useQuery(queryParams, { + refetchInterval: queryParams.endTime === timestamp ? 10_000 : false, + enabled: Boolean(apiId), + }); + + const timeseries = useMemo(() => { + if (!data?.timeseries) { + return []; + } + + return data.timeseries.map((ts) => { + return { + displayX: formatTimestampForChart(ts.x, data.granularity), + originalTimestamp: ts.x, + spent_credits: ts.y.spent_credits ?? 0, + total: ts.y.spent_credits ?? 0, + }; + }); + }, [data]); + + return { + timeseries: timeseries || [], + isLoading, + isError, + granularity: data?.granularity, + }; +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/index.tsx b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/index.tsx index 1fae9b215d..bdc4b01ccd 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/index.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/charts/index.tsx @@ -1,9 +1,12 @@ import { OverviewAreaChart } from "@/components/logs/overview-charts/overview-area-chart"; import { OverviewBarChart } from "@/components/logs/overview-charts/overview-bar-chart"; import { getTimeBufferForGranularity } from "@/lib/trpc/routers/utils/granularity"; +import { useEffect, useRef } from "react"; import { useFilters } from "../../hooks/use-filters"; +import { useMetricType } from "../../hooks/use-metric-type"; import { useFetchVerificationTimeseries } from "./bar-chart/hooks/use-fetch-timeseries"; import { createOutcomeChartConfig } from "./bar-chart/utils"; +import { useFetchCreditSpendTimeseries } from "./credit-spend-chart/hooks/use-fetch-timeseries"; import { useFetchActiveKeysTimeseries } from "./line-chart/hooks/use-fetch-timeseries"; export const KeysOverviewLogsCharts = ({ @@ -14,6 +17,8 @@ export const KeysOverviewLogsCharts = ({ onMount: (distanceToTop: number) => void; }) => { const { filters, updateFilters } = useFilters(); + const { isCreditSpendMode } = useMetricType(); + const chartContainerRef = useRef(null); const { timeseries: verificationTimeseries, @@ -21,6 +26,12 @@ export const KeysOverviewLogsCharts = ({ isError: verificationIsError, } = useFetchVerificationTimeseries(apiId); + const { + timeseries: creditSpendTimeseries, + isLoading: creditSpendIsLoading, + isError: creditSpendIsError, + } = useFetchCreditSpendTimeseries(apiId); + const { timeseries: activeKeysTimeseries, isLoading: activeKeysIsLoading, @@ -81,6 +92,63 @@ export const KeysOverviewLogsCharts = ({ reverse: true, }; + const creditSpendChartConfig = { + spent_credits: { + label: "Credits Spent", + color: "hsl(var(--success-9))", + }, + }; + + // Call onMount for credit spend mode + useEffect(() => { + if (isCreditSpendMode && chartContainerRef.current) { + const rect = chartContainerRef.current.getBoundingClientRect(); + onMount(rect.top + window.scrollY); + } + }, [isCreditSpendMode, onMount]); + + if (isCreditSpendMode) { + return ( +
+
+ +
+ {/* Only show active keys chart if it has data in credit spend mode */} + {activeKeysTimeseries.length > 0 && ( +
+ +
+ )} +
+ ); + } + return (
diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/components/logs-metric-type.tsx b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/components/logs-metric-type.tsx new file mode 100644 index 0000000000..8eab3f925c --- /dev/null +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/components/logs-metric-type.tsx @@ -0,0 +1,45 @@ +"use client"; + +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/utils"; +import { ChevronDown } from "@unkey/icons"; +import { Button } from "@unkey/ui"; +import { METRIC_TYPE_LABELS, type MetricType, useMetricType } from "../../../hooks/use-metric-type"; + +export const LogsMetricType = () => { + const { metricType, setMetricType } = useMetricType(); + + return ( + + + + + + {(Object.keys(METRIC_TYPE_LABELS) as MetricType[]).map((type) => ( + setMetricType(type)} + className={cn("cursor-pointer", metricType === type && "bg-gray-3")} + > + {METRIC_TYPE_LABELS[type]} + + ))} + + + ); +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/index.tsx b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/index.tsx index 6dcbf43e93..64397de9c7 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/index.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/controls/index.tsx @@ -5,6 +5,7 @@ import { } from "@/components/logs/controls-container"; import { LogsDateTime } from "./components/logs-datetime"; import { LogsFilters } from "./components/logs-filters"; +import { LogsMetricType } from "./components/logs-metric-type"; import { LogsRefresh } from "./components/logs-refresh"; import { LogsSearch } from "./components/logs-search"; @@ -14,6 +15,7 @@ export function KeysOverviewLogsControls({ apiId }: { apiId: string }) { + diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/components/override-indicator.tsx b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/components/override-indicator.tsx index 4afcf97a09..3629b6f78f 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/components/override-indicator.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/components/override-indicator.tsx @@ -5,7 +5,7 @@ import type { KeysOverviewLog } from "@unkey/clickhouse/src/keys/keys"; import { TriangleWarning2 } from "@unkey/icons"; import { InfoTooltip, Loading } from "@unkey/ui"; import Link from "next/link"; -import { useRouter } from "next/navigation"; +import { useRouter, useSearchParams } from "next/navigation"; import { useCallback, useState } from "react"; import { getErrorPercentage, getErrorSeverity } from "../utils/calculate-blocked-percentage"; @@ -45,12 +45,47 @@ const getWarningMessage = (severity: string, errorRate: number) => { export const KeyIdentifierColumn = ({ log, apiId, onNavigate }: KeyIdentifierColumnProps) => { const router = useRouter(); + const searchParams = useSearchParams(); const errorPercentage = getErrorPercentage(log); const severity = getErrorSeverity(log); const hasErrors = severity !== "none"; const [isNavigating, setIsNavigating] = useState(false); + const buildKeyDetailUrl = useCallback(() => { + const baseUrl = `/apis/${apiId}/keys/${log.key_details?.key_auth_id}/${log.key_id}`; + const params = new URLSearchParams(); + + if (searchParams) { + // Preserve metricType parameter + const metricType = searchParams.get("metricType"); + if (metricType) { + params.set("metricType", metricType); + } + + // Preserve since parameter + const since = searchParams.get("since"); + if (since) { + params.set("since", since); + } + + // Preserve tags parameter (compatible with key details page) + const tags = searchParams.get("tags"); + if (tags) { + params.set("tags", tags); + } + + // Preserve outcomes parameter (compatible with key details page) + const outcomes = searchParams.get("outcomes"); + if (outcomes) { + params.set("outcomes", outcomes); + } + } + + const queryString = params.toString(); + return queryString ? `${baseUrl}?${queryString}` : baseUrl; + }, [apiId, log.key_details?.key_auth_id, log.key_id, searchParams]); + const handleLinkClick = useCallback( (e: React.MouseEvent) => { e.preventDefault(); @@ -58,9 +93,9 @@ export const KeyIdentifierColumn = ({ log, apiId, onNavigate }: KeyIdentifierCol onNavigate?.(); - router.push(`/apis/${apiId}/keys/${log.key_details?.key_auth_id}/${log.key_id}`); + router.push(buildKeyDetailUrl()); }, - [apiId, log.key_id, log.key_details?.key_auth_id, onNavigate, router.push], + [onNavigate, router.push, buildKeyDetailUrl], ); return ( @@ -83,7 +118,7 @@ export const KeyIdentifierColumn = ({ log, apiId, onNavigate }: KeyIdentifierCol
diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/hooks/use-logs-query.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/hooks/use-logs-query.ts index c5d6ac96be..a1295a9164 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/hooks/use-logs-query.ts +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/hooks/use-logs-query.ts @@ -6,6 +6,7 @@ import { KEY_VERIFICATION_OUTCOMES, type KeysOverviewLog } from "@unkey/clickhou import { useEffect, useMemo, useState } from "react"; import { keysOverviewFilterFieldConfig } from "../../../filters.schema"; import { useFilters } from "../../../hooks/use-filters"; +import { useMetricType } from "../../../hooks/use-metric-type"; import type { KeysQueryOverviewLogsPayload, SortFields } from "../query-logs.schema"; type UseLogsQueryParams = { @@ -20,6 +21,7 @@ export function useKeysOverviewLogsQuery({ apiId, limit = 50 }: UseLogsQueryPara const { filters } = useFilters(); const { sorts } = useSort(); + const { isCreditSpendMode } = useMetricType(); const historicalLogs = useMemo(() => Array.from(historicalLogsMap.values()), [historicalLogsMap]); @@ -38,6 +40,7 @@ export function useKeysOverviewLogsQuery({ apiId, limit = 50 }: UseLogsQueryPara apiId, since: "", sorts: sorts.length > 0 ? sorts : null, + creditSpendMode: isCreditSpendMode, }; filters.forEach((filter) => { @@ -119,7 +122,7 @@ export function useKeysOverviewLogsQuery({ apiId, limit = 50 }: UseLogsQueryPara }); return params; - }, [filters, limit, timestamp, apiId, sorts]); + }, [filters, limit, timestamp, apiId, sorts, isCreditSpendMode]); // Main query for historical data const { @@ -139,6 +142,7 @@ export function useKeysOverviewLogsQuery({ apiId, limit = 50 }: UseLogsQueryPara useEffect(() => { if (initialData) { const newMap = new Map(); + initialData.pages.forEach((page) => { page.keysOverviewLogs.forEach((log) => { // Use request_id as the unique key since key_id might not be unique across different requests diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/logs-table.tsx b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/logs-table.tsx index a72f135512..3d7691718c 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/logs-table.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/components/table/logs-table.tsx @@ -8,6 +8,7 @@ import { Badge, Button, Empty, TimestampInfo } from "@unkey/ui"; import { useSort } from "@/components/logs/hooks/use-sort"; import { formatNumber } from "@/lib/fmt"; +import { useMetricType } from "../../hooks/use-metric-type"; import { OutcomesPopover } from "./components/outcome-popover"; import { KeyIdentifierColumn } from "./components/override-indicator"; import { useKeysOverviewLogsQuery } from "./hooks/use-logs-query"; @@ -23,6 +24,7 @@ type Props = { export const KeysOverviewLogsTable = ({ apiId, setSelectedLog, log: selectedLog }: Props) => { const { getSortDirection, toggleSort } = useSort(); + const { isCreditSpendMode } = useMetricType(); const { historicalLogs, isLoading, isLoadingMore, loadMore } = useKeysOverviewLogsQuery({ apiId, }); @@ -32,7 +34,7 @@ export const KeysOverviewLogsTable = ({ apiId, setSelectedLog, log: selectedLog { key: "key_id", header: "ID", - width: "15%", + width: isCreditSpendMode ? "25%" : "15%", headerClassName: "pl-12", render: (log) => ( setSelectedLog(null)} /> @@ -41,12 +43,18 @@ export const KeysOverviewLogsTable = ({ apiId, setSelectedLog, log: selectedLog { key: "name", header: "Name", - width: "15%", + width: isCreditSpendMode ? "22%" : "15%", render: (log) => { const name = log.key_details?.name || "—"; return (
-
+
{name}
@@ -56,96 +64,139 @@ export const KeysOverviewLogsTable = ({ apiId, setSelectedLog, log: selectedLog { key: "external_id", header: "External ID", - width: "15%", + width: isCreditSpendMode ? "20%" : "15%", render: (log) => { const externalId = (log.key_details?.identity?.external_id ?? log.key_details?.owner_id) || "—"; return (
-
- {externalId} -
-
- ); - }, - }, - { - key: "valid", - header: "Valid", - width: "15%", - sort: { - direction: getSortDirection("valid"), - sortable: true, - onSort() { - toggleSort("valid", false); - }, - }, - render: (log) => { - const successPercentage = getSuccessPercentage(log); - return ( -
- - {formatNumber(log.valid_count)} - -
- ); - }, - }, - { - key: "invalid", - header: "Invalid", - width: "15%", - sort: { - direction: getSortDirection("invalid"), - sortable: true, - onSort() { - toggleSort("invalid", false); - }, - }, - render: (log) => { - const style = getStatusStyle(log); - const errorPercentage = getErrorPercentage(log); - - return ( -
-
- - - - - - {formatNumber(log.error_count)} - - -
-
- + {externalId}
); }, }, + ...(isCreditSpendMode + ? [] + : [ + { + key: "valid", + header: "Valid", + width: "15%", + sort: { + direction: getSortDirection("valid"), + sortable: true, + onSort() { + toggleSort("valid", false); + }, + }, + render: (log: KeysOverviewLog) => { + const successPercentage = getSuccessPercentage(log); + return ( +
+ + {formatNumber(log.valid_count)} + +
+ ); + }, + }, + { + key: "invalid", + header: "Invalid", + width: "15%", + sort: { + direction: getSortDirection("invalid"), + sortable: true, + onSort() { + toggleSort("invalid", false); + }, + }, + render: (log: KeysOverviewLog) => { + const style = getStatusStyle(log); + const errorPercentage = getErrorPercentage(log); + + return ( +
+
+ + + + + + {formatNumber(log.error_count)} + + +
+
+ +
+
+ ); + }, + }, + ]), + ...(isCreditSpendMode + ? [ + { + key: "spent_credits" as keyof KeysOverviewLog, + header: "Credits Spent", + width: "18%", + sort: { + direction: getSortDirection("spent_credits"), + sortable: true, + onSort() { + toggleSort("spent_credits", false); + }, + }, + render: (log: KeysOverviewLog) => ( +
+ + {(log.spent_credits || 0).toLocaleString()} + +
+ ), + }, + ] + : []), { key: "lastUsed", header: "Last Used", @@ -188,10 +239,13 @@ export const KeysOverviewLogsTable = ({ apiId, setSelectedLog, log: selectedLog
- Key Verification Logs + + {isCreditSpendMode ? "Credit Spend Logs" : "Key Verification Logs"} + - No key verification data to show. Once requests are made with API keys, you'll see a - summary of successful and failed verification attempts. + {isCreditSpendMode + ? "No credit spend data to show. Once API keys with credit tracking are used, you'll see a summary of credit consumption." + : "No key verification data to show. Once requests are made with API keys, you'll see a summary of successful and failed verification attempts."} {" "} ; export const keysQueryOverviewLogsPayload = z.object({ @@ -12,6 +12,7 @@ export const keysQueryOverviewLogsPayload = z.object({ apiId: z.string(), since: z.string(), cursor: z.number().nullable().optional().nullable(), + creditSpendMode: z.boolean().optional().default(false), outcomes: z .array( z.object({ diff --git a/apps/dashboard/app/(app)/apis/[apiId]/_overview/hooks/use-metric-type.ts b/apps/dashboard/app/(app)/apis/[apiId]/_overview/hooks/use-metric-type.ts new file mode 100644 index 0000000000..d1551264c1 --- /dev/null +++ b/apps/dashboard/app/(app)/apis/[apiId]/_overview/hooks/use-metric-type.ts @@ -0,0 +1,25 @@ +"use client"; + +import { parseAsStringLiteral, useQueryState } from "nuqs"; + +const METRIC_TYPES = ["requests", "creditSpend"] as const; +export type MetricType = (typeof METRIC_TYPES)[number]; + +export const METRIC_TYPE_LABELS: Record = { + requests: "Requests", + creditSpend: "Credit Spend", +}; + +export const useMetricType = () => { + const [metricType, setMetricType] = useQueryState( + "metricType", + parseAsStringLiteral(METRIC_TYPES).withDefault("requests"), + ); + + return { + metricType, + setMetricType, + isRequestsMode: metricType === "requests", + isCreditSpendMode: metricType === "creditSpend", + }; +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/bar-chart/hooks/use-fetch-timeseries.ts b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/bar-chart/hooks/use-fetch-timeseries.ts index 5ce4c4fe82..060f06e22f 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/bar-chart/hooks/use-fetch-timeseries.ts +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/bar-chart/hooks/use-fetch-timeseries.ts @@ -129,6 +129,7 @@ export const useFetchVerificationTimeseries = (keyId: string, keyspaceId: string return { ...result, ...outcomeFields, + spent_credits: ts.y.spent_credits ?? 0, }; }); }, [data]); diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/index.tsx b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/index.tsx index e7bbf13cb3..169e1d201b 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/index.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/charts/index.tsx @@ -1,6 +1,8 @@ import { OverviewBarChart } from "@/components/logs/overview-charts/overview-bar-chart"; import { getTimeBufferForGranularity } from "@/lib/trpc/routers/utils/granularity"; +import { useEffect, useRef } from "react"; import { useFilters } from "../../hooks/use-filters"; +import { useMetricType } from "../../hooks/use-metric-type"; import { useFetchVerificationTimeseries } from "./bar-chart/hooks/use-fetch-timeseries"; import { createOutcomeChartConfig } from "./bar-chart/utils"; @@ -14,6 +16,8 @@ export const KeyDetailsLogsChart = ({ onMount: (distanceToTop: number) => void; }) => { const { filters, updateFilters } = useFilters(); + const { isCreditSpendMode } = useMetricType(); + const chartContainerRef = useRef(null); const { timeseries: verificationTimeseries, @@ -54,6 +58,47 @@ export const KeyDetailsLogsChart = ({ ]); }; + const creditSpendChartConfig = { + spent_credits: { + label: "Credits Spent", + color: "hsl(var(--success-9))", + }, + }; + + // Call onMount for credit spend mode + useEffect(() => { + if (isCreditSpendMode && chartContainerRef.current) { + const rect = chartContainerRef.current.getBoundingClientRect(); + onMount(rect.top + window.scrollY); + } + }, [isCreditSpendMode, onMount]); + + if (isCreditSpendMode) { + return ( +
+ +
+ ); + } + return (
{ + const { metricType, setMetricType } = useMetricType(); + + return ( + + + + + + {(Object.keys(METRIC_TYPE_LABELS) as MetricType[]).map((type) => ( + setMetricType(type)} + className={cn("cursor-pointer", metricType === type && "bg-gray-3")} + > + {METRIC_TYPE_LABELS[type]} + + ))} + + + ); +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/controls/index.tsx b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/controls/index.tsx index 92f6d9e16f..51c7bb9f91 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/controls/index.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/controls/index.tsx @@ -4,14 +4,16 @@ import { ControlsLeft, ControlsRight, } from "@/components/logs/controls-container"; -import { formatNumber } from "@/lib/fmt"; +import { formatRawNumber } from "@/lib/fmt"; import { trpc } from "@/lib/trpc/client"; -import { Coins } from "@unkey/icons"; +import { ChartUsage, Coins } from "@unkey/icons"; import { Separator } from "@unkey/ui"; import { AnimatePresence, motion } from "framer-motion"; +import { useSpentCredits } from "../../hooks/use-spent-credits"; import { LogsDateTime } from "./components/logs-datetime"; import { LogsFilters } from "./components/logs-filters"; import { LogsLiveSwitch } from "./components/logs-live-switch"; +import { LogsMetricType } from "./components/logs-metric-type"; import { LogsRefresh } from "./components/logs-refresh"; import { LogsSearch } from "./components/logs-search"; @@ -29,15 +31,26 @@ export function KeysDetailsLogsControls({ keyspaceId, }); - // Safe access to remaining credit with fallback + const { + spentCredits, + isLoading: spentCreditsLoading, + isError: spentCreditsError, + } = useSpentCredits(keyId, keyspaceId); + const hasRemainingCredit = data?.remainingCredit !== null && data?.remainingCredit !== undefined && !isLoading && !error; + const hasSpentCreditsData = !spentCreditsLoading && !spentCreditsError && spentCredits !== 0; + + // Show credit spent when spent credits data is available (regardless of amount or remaining credits) + const shouldShowSpentCredits = hasSpentCreditsData && (hasRemainingCredit || spentCredits > 0); + return ( + {hasRemainingCredit ? ( @@ -81,7 +94,7 @@ export function KeysDetailsLogsControls({ } /> @@ -111,6 +124,78 @@ export function KeysDetailsLogsControls({ ) : null} + + {shouldShowSpentCredits ? ( + + +
+ + Credits Spent: + + {spentCredits > 0 ? ( + + } + /> + + ) : ( + + } + /> + + )} +
+ + ) : null} + diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/table/logs-table.tsx b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/table/logs-table.tsx index a8f8a7672b..2b17df47c6 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/table/logs-table.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/components/table/logs-table.tsx @@ -19,6 +19,7 @@ import { import { Badge, Button, CopyButton, Empty, InfoTooltip, TimestampInfo } from "@unkey/ui"; import { useCallback, useState } from "react"; import { useKeyDetailsLogsContext } from "../../context/logs"; +import { useMetricType } from "../../hooks/use-metric-type"; import { StatusBadge } from "./components/status-badge"; import { useKeyDetailsLogsQuery } from "./hooks/use-logs-query"; @@ -181,6 +182,7 @@ type Props = { export const KeyDetailsLogsTable = ({ keyspaceId, keyId, selectedLog, onLogSelect }: Props) => { const { isLive } = useKeyDetailsLogsContext(); + const { isCreditSpendMode } = useMetricType(); const { realtimeLogs, historicalLogs, isLoading, isLoadingMore, loadMore, hasMore, totalCount } = useKeyDetailsLogsQuery({ keyId, @@ -304,7 +306,7 @@ export const KeyDetailsLogsTable = ({ keyspaceId, keyId, selectedLog, onLogSelec { key: "tags", header: "Tags", - width: "20%", + width: isCreditSpendMode ? "15%" : "20%", render: (log) => { return (
@@ -425,6 +427,30 @@ export const KeyDetailsLogsTable = ({ keyspaceId, keyId, selectedLog, onLogSelec ); }, }, + ...(isCreditSpendMode + ? [ + { + key: "spent_credits", + header: "Credits Spent", + width: "10%", + render: (log: KeyDetailsLog) => ( +
+ + {(log.spent_credits || 0).toLocaleString()} + +
+ ), + }, + ] + : []), ]; }; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-metric-type.ts b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-metric-type.ts new file mode 100644 index 0000000000..d1551264c1 --- /dev/null +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-metric-type.ts @@ -0,0 +1,25 @@ +"use client"; + +import { parseAsStringLiteral, useQueryState } from "nuqs"; + +const METRIC_TYPES = ["requests", "creditSpend"] as const; +export type MetricType = (typeof METRIC_TYPES)[number]; + +export const METRIC_TYPE_LABELS: Record = { + requests: "Requests", + creditSpend: "Credit Spend", +}; + +export const useMetricType = () => { + const [metricType, setMetricType] = useQueryState( + "metricType", + parseAsStringLiteral(METRIC_TYPES).withDefault("requests"), + ); + + return { + metricType, + setMetricType, + isRequestsMode: metricType === "requests", + isCreditSpendMode: metricType === "creditSpend", + }; +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-spent-credits.ts b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-spent-credits.ts new file mode 100644 index 0000000000..682dbef160 --- /dev/null +++ b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/hooks/use-spent-credits.ts @@ -0,0 +1,125 @@ +import { HISTORICAL_DATA_WINDOW } from "@/components/logs/constants"; +import { trpc } from "@/lib/trpc/client"; +import { getTimestampFromRelative } from "@/lib/utils"; +import { useQueryTime } from "@/providers/query-time-provider"; +import { KEY_VERIFICATION_OUTCOMES } from "@unkey/clickhouse/src/keys/keys"; +import { useMemo } from "react"; +import { keyDetailsFilterFieldConfig } from "../filters.schema"; +import { useFilters } from "./use-filters"; + +export const useSpentCredits = (keyId: string, keyspaceId: string) => { + const { filters } = useFilters(); + const { queryTime: timestamp } = useQueryTime(); + + const queryParams = useMemo(() => { + let startTime = timestamp - HISTORICAL_DATA_WINDOW; + let endTime = timestamp; + let hasSinceFilter = false; + + const params = { + keyId, + keyspaceId, + startTime, + endTime, + outcomes: [] as Array<{ + value: + | "VALID" + | "RATE_LIMITED" + | "INSUFFICIENT_PERMISSIONS" + | "FORBIDDEN" + | "DISABLED" + | "EXPIRED" + | "USAGE_EXCEEDED"; + operator: "is"; + }>, + tags: null as { + operator: "is" | "contains" | "startsWith" | "endsWith"; + value: string; + } | null, + }; + + filters.forEach((filter) => { + if (!(filter.field in keyDetailsFilterFieldConfig)) { + return; + } + + switch (filter.field) { + case "tags": { + if (typeof filter.value === "string" && filter.value.trim()) { + const fieldConfig = keyDetailsFilterFieldConfig[filter.field]; + const validOperators = fieldConfig.operators; + + const operator = validOperators.includes(filter.operator) + ? filter.operator + : validOperators[0]; + + params.tags = { + operator, + value: filter.value, + }; + } + break; + } + + case "startTime": + case "endTime": { + const numValue = + typeof filter.value === "number" + ? filter.value + : typeof filter.value === "string" + ? Number(filter.value) + : Number.NaN; + + if (!Number.isNaN(numValue)) { + params[filter.field] = numValue; + } + break; + } + + case "outcomes": { + type ValidOutcome = (typeof KEY_VERIFICATION_OUTCOMES)[number]; + if ( + typeof filter.value === "string" && + filter.value !== "" && + KEY_VERIFICATION_OUTCOMES.includes(filter.value as ValidOutcome) + ) { + params.outcomes.push({ + operator: "is" as const, + value: filter.value as Exclude, + }); + } + break; + } + + case "since": + if (typeof filter.value === "string") { + try { + startTime = getTimestampFromRelative(filter.value); + endTime = Date.now(); + hasSinceFilter = true; + } catch { + // Invalid since format, ignore + } + } + break; + } + }); + + return { + ...params, + startTime: hasSinceFilter ? startTime : params.startTime, + endTime: hasSinceFilter ? endTime : params.endTime, + outcomes: params.outcomes.length > 0 ? params.outcomes : null, + }; + }, [filters, timestamp, keyId, keyspaceId]); + + const { data, isLoading, isError } = trpc.key.spentCredits.useQuery(queryParams, { + refetchInterval: queryParams.endTime === timestamp ? 10_000 : false, + }); + + return { + spentCredits: data?.spentCredits ?? 0, + isLoading, + isError, + }; +}; diff --git a/apps/dashboard/app/(app)/apis/[apiId]/page.tsx b/apps/dashboard/app/(app)/apis/[apiId]/page.tsx index 3fbb896df6..e1b3b90a54 100644 --- a/apps/dashboard/app/(app)/apis/[apiId]/page.tsx +++ b/apps/dashboard/app/(app)/apis/[apiId]/page.tsx @@ -1,9 +1,14 @@ "use client"; +import { + METRIC_TYPE_LABELS, + useMetricType, +} from "@/app/(app)/apis/[apiId]/_overview/hooks/use-metric-type"; import { LogsClient } from "@/app/(app)/apis/[apiId]/_overview/logs-client"; import { ApisNavbar } from "./api-id-navbar"; export default function ApiPage(props: { params: { apiId: string } }) { const apiId = props.params.apiId; + const { metricType } = useMetricType(); return (
@@ -11,7 +16,7 @@ export default function ApiPage(props: { params: { apiId: string } }) { apiId={apiId} activePage={{ href: `/apis/${apiId}`, - text: "Requests", + text: METRIC_TYPE_LABELS[metricType], }} /> diff --git a/apps/dashboard/components/logs/overview-charts/overview-bar-chart.tsx b/apps/dashboard/components/logs/overview-charts/overview-bar-chart.tsx index 3ff7862a6d..eee7eaf3fb 100644 --- a/apps/dashboard/components/logs/overview-charts/overview-bar-chart.tsx +++ b/apps/dashboard/components/logs/overview-charts/overview-bar-chart.tsx @@ -8,7 +8,7 @@ import { ChartTooltip, ChartTooltipContent, } from "@/components/ui/chart"; -import { formatNumber } from "@/lib/fmt"; +import { formatNumber, formatRawNumber } from "@/lib/fmt"; import { Grid } from "@unkey/icons"; import { useEffect, useRef, useState } from "react"; import { Bar, BarChart, CartesianGrid, ReferenceArea, ResponsiveContainer, YAxis } from "recharts"; @@ -40,6 +40,10 @@ type OverviewBarChartProps = { labels: ChartLabels; tooltipItems?: ChartTooltipItem[]; onMount?: (distanceToTop: number) => void; + showLabels?: boolean; + hideTotal?: boolean; + tooltipPrefix?: string | null; + hideTooltipTotal?: boolean; }; export function OverviewBarChart({ @@ -52,6 +56,10 @@ export function OverviewBarChart({ labels, tooltipItems = [], onMount, + showLabels = true, + hideTotal = false, + tooltipPrefix = "All", + hideTooltipTotal = false, }: OverviewBarChartProps) { const chartRef = useRef(null); const [selection, setSelection] = useState({ start: "", end: "" }); @@ -121,8 +129,13 @@ export function OverviewBarChart({ } // Calculate totals based on the provided keys + const hasSecondaryData = + labels.secondaryLabel && labels.secondaryKey && labels.secondaryKey !== labels.primaryKey; const totalCount = (data ?? []).reduce( - (acc, crr) => acc + (crr[labels.primaryKey] as number) + (crr[labels.secondaryKey] as number), + (acc, crr) => + acc + + (crr[labels.primaryKey] as number) + + (hasSecondaryData ? (crr[labels.secondaryKey] as number) : 0), 0, ); const primaryCount = (data ?? []).reduce( @@ -134,36 +147,49 @@ export function OverviewBarChart({ 0, ); + // Check if this is a credit-related chart + const isCreditChart = labels.primaryKey === "spent_credits" || labels.title.includes("CREDIT"); + return (
{labels.title}
-
- {formatNumber(totalCount)} -
-
- -
-
-
-
-
{labels.primaryLabel}
-
+ {!hideTotal && (
- {formatNumber(primaryCount)} + {isCreditChart ? formatRawNumber(totalCount) : formatNumber(totalCount)}
-
-
-
-
-
{labels.secondaryLabel}
-
-
- {formatNumber(secondaryCount)} + )} +
+ + {showLabels && ( +
+
+
+
+
{labels.primaryLabel}
+
+
+ {isCreditChart ? formatRawNumber(primaryCount) : formatNumber(primaryCount)} +
+ {labels.secondaryLabel && + labels.secondaryKey && + labels.secondaryKey !== labels.primaryKey && ( +
+
+
+
+ {labels.secondaryLabel} +
+
+
+ {isCreditChart ? formatRawNumber(secondaryCount) : formatNumber(secondaryCount)} +
+
+ )}
-
+ )}
@@ -199,60 +225,80 @@ export function OverviewBarChart({ if (!active || !payload?.length || payload?.[0]?.payload.total === 0) { return null; } + const hasBottomContent = + (!hideTotal && !hideTooltipTotal) || tooltipItems.length > 0; + return ( -
- -
-
- - All - - Total -
-
- - {formatNumber(payload[0]?.payload?.total)} - + hasBottomContent ? ( +
+ {!hideTotal && !hideTooltipTotal && ( +
+ +
+
+ {tooltipPrefix && ( + + {tooltipPrefix} + + )} + Total +
+
+ + {isCreditChart + ? formatRawNumber(payload[0]?.payload?.total) + : formatNumber(payload[0]?.payload?.total)} + +
+
-
-
+ )} - {/* Dynamic tooltip items */} - {tooltipItems.map((item, index) => ( -
- -
-
- - All - - - {item.label} - -
-
- - {formatNumber(payload[0]?.payload?.[item.dataKey])} - + {/* Dynamic tooltip items */} + {tooltipItems.map((item, index) => ( +
+ +
+
+ {tooltipPrefix && ( + + {tooltipPrefix} + + )} + + {item.label} + +
+
+ + {isCreditChart + ? formatRawNumber(payload[0]?.payload?.[item.dataKey]) + : formatNumber(payload[0]?.payload?.[item.dataKey])} + +
-
- ))} -
+ ))} +
+ ) : undefined } className="rounded-lg shadow-lg border border-gray-4" labelFormatter={(_, tooltipPayload) => - //@ts-expect-error safe to ignore for now - createTimeIntervalFormatter(data, "HH:mm")(tooltipPayload) + createTimeIntervalFormatter( + data, + "HH:mm", + )( + // biome-ignore lint/suspicious/noExplicitAny: Recharts type mismatch requires any cast + tooltipPayload as any, + ) } /> ); diff --git a/apps/dashboard/lib/fmt.ts b/apps/dashboard/lib/fmt.ts index e159f17b6d..d6950fee86 100644 --- a/apps/dashboard/lib/fmt.ts +++ b/apps/dashboard/lib/fmt.ts @@ -1,3 +1,7 @@ export function formatNumber(n: number): string { return Intl.NumberFormat("en", { notation: "compact" }).format(n); } + +export function formatRawNumber(n: number): string { + return Intl.NumberFormat("en").format(n); +} diff --git a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/utils.ts b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/utils.ts index 0e30e0b5a1..02856955c3 100644 --- a/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/utils.ts +++ b/apps/dashboard/lib/trpc/routers/api/keys/query-overview-logs/utils.ts @@ -60,5 +60,6 @@ export function transformKeysFilters( outcomes, cursorTime: params.cursor ?? null, sorts, + creditSpendMode: params.creditSpendMode ?? false, }; } diff --git a/apps/dashboard/lib/trpc/routers/index.ts b/apps/dashboard/lib/trpc/routers/index.ts index 0724773e7b..15812ec9a8 100644 --- a/apps/dashboard/lib/trpc/routers/index.ts +++ b/apps/dashboard/lib/trpc/routers/index.ts @@ -9,6 +9,7 @@ import { keyUsageTimeseries } from "./api/keys/query-key-usage-timeseries"; import { keyLastVerificationTime } from "./api/keys/query-latest-verification"; import { queryKeysOverviewLogs } from "./api/keys/query-overview-logs"; import { keyVerificationsTimeseries } from "./api/keys/query-overview-timeseries"; + import { enableKey } from "./api/keys/toggle-key-enabled"; import { overviewApiSearch } from "./api/overview-api-search"; import { queryApisOverview } from "./api/overview/query-overview"; @@ -51,6 +52,7 @@ import { createRootKey } from "./key/createRootKey"; import { deleteKeys } from "./key/delete"; import { fetchKeyPermissions } from "./key/fetch-key-permissions"; import { queryKeyDetailsLogs } from "./key/query-logs"; +import { queryKeySpentCredits } from "./key/query-spent-credits"; import { keyDetailsVerificationsTimeseries } from "./key/query-timeseries"; import { getConnectedRolesAndPerms } from "./key/rbac/connected-roles-and-perms"; import { getPermissionSlugs } from "./key/rbac/get-permission-slugs"; @@ -131,6 +133,7 @@ export const router = t.router({ query: queryKeyDetailsLogs, timeseries: keyDetailsVerificationsTimeseries, }), + spentCredits: queryKeySpentCredits, update: t.router({ enabled: updateKeysEnabled, expiration: updateKeyExpiration, @@ -189,6 +192,7 @@ export const router = t.router({ enableKey: enableKey, usageTimeseries: keyUsageTimeseries, latestVerification: keyLastVerificationTime, + spentCredits: queryKeySpentCredits, }), overview: t.router({ timeseries: queryVerificationTimeseries, diff --git a/apps/dashboard/lib/trpc/routers/key/query-spent-credits/index.ts b/apps/dashboard/lib/trpc/routers/key/query-spent-credits/index.ts new file mode 100644 index 0000000000..5041effd7d --- /dev/null +++ b/apps/dashboard/lib/trpc/routers/key/query-spent-credits/index.ts @@ -0,0 +1,86 @@ +import { clickhouse } from "@/lib/clickhouse"; +import { db, isNull } from "@/lib/db"; +import { ratelimit, requireUser, requireWorkspace, t, withRatelimit } from "@/lib/trpc/trpc"; +import { TRPCError } from "@trpc/server"; +import { z } from "zod"; + +const querySpentCreditsSchema = z.object({ + keyId: z.string(), + keyspaceId: z.string(), + startTime: z.number().int(), + endTime: z.number().int(), + outcomes: z + .array( + z.object({ + value: z.enum([ + "VALID", + "RATE_LIMITED", + "INSUFFICIENT_PERMISSIONS", + "FORBIDDEN", + "DISABLED", + "EXPIRED", + "USAGE_EXCEEDED", + ]), + operator: z.literal("is"), + }), + ) + .nullable() + .optional(), + tags: z + .object({ + operator: z.enum(["is", "contains", "startsWith", "endsWith"]), + value: z.string(), + }) + .nullable() + .optional(), +}); + +export const queryKeySpentCredits = t.procedure + .use(requireUser) + .use(requireWorkspace) + .use(withRatelimit(ratelimit.read)) + .input(querySpentCreditsSchema) + .query(async ({ ctx, input }) => { + // Verify the key belongs to the workspace + const key = await db.query.keys + .findFirst({ + where: (table, { and, eq }) => + and( + eq(table.id, input.keyId), + eq(table.keyAuthId, input.keyspaceId), + eq(table.workspaceId, ctx.workspace.id), + isNull(table.deletedAtM), + ), + columns: { + id: true, + }, + }) + .catch((_err) => { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + "Failed to retrieve key details due to an error. If this issue persists, please contact support@unkey.dev with the time this occurred.", + }); + }); + + if (!key) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Key not found or does not belong to your workspace", + }); + } + + const result = await clickhouse.verifications.spentCreditsTotal({ + workspaceId: ctx.workspace.id, + keyspaceId: input.keyspaceId, + keyId: input.keyId, + startTime: input.startTime, + endTime: input.endTime, + outcomes: input.outcomes || null, + tags: input.tags || null, + }); + + return { + spentCredits: result.val?.[0]?.spent_credits ?? 0, + }; + }); diff --git a/go/internal/services/keys/validation.go b/go/internal/services/keys/validation.go index 548aa93f80..25705a1231 100644 --- a/go/internal/services/keys/validation.go +++ b/go/internal/services/keys/validation.go @@ -46,6 +46,9 @@ func (k *KeyVerifier) withCredits(ctx context.Context, cost int32) error { k.Key.RemainingRequests = sql.NullInt32{Int32: usage.Remaining, Valid: true} if !usage.Valid { k.setInvalid(StatusUsageExceeded, "Key usage limit exceeded.") + } else { + // Only track spent credits if the usage was valid (credits were actually deducted) + k.spentCredits = int64(cost) } return nil diff --git a/go/internal/services/keys/verifier.go b/go/internal/services/keys/verifier.go index 0f001c6d75..f4ada9d80b 100644 --- a/go/internal/services/keys/verifier.go +++ b/go/internal/services/keys/verifier.go @@ -41,8 +41,9 @@ type KeyVerifier struct { isRootKey bool // Whether this is a root key (special handling) - message string // Internal message for validation failures - tags []string // Tags associated with this verification + message string // Internal message for validation failures + tags []string // Tags associated with this verification + spentCredits int64 // The number of credits that were actually spent during verification session *zen.Session // The current request session region string // Geographic region identifier @@ -122,15 +123,16 @@ func (k *KeyVerifier) Verify(ctx context.Context, opts ...VerifyOption) error { func (k *KeyVerifier) log() { k.clickhouse.BufferKeyVerification(schema.KeyVerificationRequestV1{ - RequestID: k.session.RequestID(), - WorkspaceID: k.Key.WorkspaceID, - Time: time.Now().UnixMilli(), - Outcome: string(k.Status), - KeySpaceID: k.Key.KeyAuthID, - KeyID: k.Key.ID, - IdentityID: k.Key.IdentityID.String, - Tags: k.tags, - Region: k.region, + RequestID: k.session.RequestID(), + WorkspaceID: k.Key.WorkspaceID, + Time: time.Now().UnixMilli(), + Outcome: string(k.Status), + KeySpaceID: k.Key.KeyAuthID, + KeyID: k.Key.ID, + IdentityID: k.Key.IdentityID.String, + SpentCredits: k.spentCredits, + Tags: k.tags, + Region: k.region, }) keyType := "key" diff --git a/go/pkg/clickhouse/schema/requests.go b/go/pkg/clickhouse/schema/requests.go index 084ae39b79..9886506c20 100644 --- a/go/pkg/clickhouse/schema/requests.go +++ b/go/pkg/clickhouse/schema/requests.go @@ -88,6 +88,9 @@ type KeyVerificationRequestV1 struct { // IdentityID links the key to a specific identity, if applicable IdentityID string `ch:"identity_id" json:"identity_id"` + // SpentCredits is the number of credits that were actually deducted during this verification + SpentCredits int64 `ch:"spent_credits" json:"spent_credits"` + Tags []string `ch:"tags" json:"tags"` } diff --git a/internal/clickhouse/schema/052_add_spent_credits_to_verifications.raw_key_verifications_v1.sql b/internal/clickhouse/schema/052_add_spent_credits_to_verifications.raw_key_verifications_v1.sql new file mode 100644 index 0000000000..effc4e9732 --- /dev/null +++ b/internal/clickhouse/schema/052_add_spent_credits_to_verifications.raw_key_verifications_v1.sql @@ -0,0 +1,7 @@ +-- +goose up +ALTER TABLE verifications.raw_key_verifications_v1 +ADD COLUMN IF NOT EXISTS spent_credits Int64 DEFAULT 0; + +-- +goose down +ALTER TABLE verifications.raw_key_verifications_v1 +DROP COLUMN IF EXISTS spent_credits; diff --git a/internal/clickhouse/schema/053_create_verifications.key_verifications_per_hour_v4.sql b/internal/clickhouse/schema/053_create_verifications.key_verifications_per_hour_v4.sql new file mode 100644 index 0000000000..d86165d3f6 --- /dev/null +++ b/internal/clickhouse/schema/053_create_verifications.key_verifications_per_hour_v4.sql @@ -0,0 +1,19 @@ +-- +goose up +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v4 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; + +-- +goose down +DROP TABLE IF EXISTS verifications.key_verifications_per_hour_v4; diff --git a/internal/clickhouse/schema/054_create_verifications.key_verifications_per_hour_mv_v4.sql b/internal/clickhouse/schema/054_create_verifications.key_verifications_per_hour_mv_v4.sql new file mode 100644 index 0000000000..e3b9a971b6 --- /dev/null +++ b/internal/clickhouse/schema/054_create_verifications.key_verifications_per_hour_mv_v4.sql @@ -0,0 +1,27 @@ +-- +goose up +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v4 +TO verifications.key_verifications_per_hour_v4 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; + +-- +goose down +DROP VIEW IF EXISTS verifications.key_verifications_per_hour_mv_v4; diff --git a/internal/clickhouse/schema/055_create_verifications.key_verifications_per_day_v4.sql b/internal/clickhouse/schema/055_create_verifications.key_verifications_per_day_v4.sql new file mode 100644 index 0000000000..e32d1bc04e --- /dev/null +++ b/internal/clickhouse/schema/055_create_verifications.key_verifications_per_day_v4.sql @@ -0,0 +1,19 @@ +-- +goose up +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v4 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; + +-- +goose down +DROP TABLE IF EXISTS verifications.key_verifications_per_day_v4; diff --git a/internal/clickhouse/schema/056_create_verifications.key_verifications_per_day_mv_v4.sql b/internal/clickhouse/schema/056_create_verifications.key_verifications_per_day_mv_v4.sql new file mode 100644 index 0000000000..515d193b51 --- /dev/null +++ b/internal/clickhouse/schema/056_create_verifications.key_verifications_per_day_mv_v4.sql @@ -0,0 +1,27 @@ +-- +goose up +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v4 +TO verifications.key_verifications_per_day_v4 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; + +-- +goose down +DROP VIEW IF EXISTS verifications.key_verifications_per_day_mv_v4; diff --git a/internal/clickhouse/schema/057_create_verifications.key_verifications_per_month_v4.sql b/internal/clickhouse/schema/057_create_verifications.key_verifications_per_month_v4.sql new file mode 100644 index 0000000000..2949ec6583 --- /dev/null +++ b/internal/clickhouse/schema/057_create_verifications.key_verifications_per_month_v4.sql @@ -0,0 +1,19 @@ +-- +goose up +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v4 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; + +-- +goose down +DROP TABLE IF EXISTS verifications.key_verifications_per_month_v4; diff --git a/internal/clickhouse/schema/058_create_verifications.key_verifications_per_month_mv_v4.sql b/internal/clickhouse/schema/058_create_verifications.key_verifications_per_month_mv_v4.sql new file mode 100644 index 0000000000..89e78fd079 --- /dev/null +++ b/internal/clickhouse/schema/058_create_verifications.key_verifications_per_month_mv_v4.sql @@ -0,0 +1,27 @@ +-- +goose up +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v4 +TO verifications.key_verifications_per_month_v4 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; + +-- +goose down +DROP VIEW IF EXISTS verifications.key_verifications_per_month_mv_v4; diff --git a/internal/clickhouse/schema/059_create_verifications.key_verifications_per_minute_v2.sql b/internal/clickhouse/schema/059_create_verifications.key_verifications_per_minute_v2.sql new file mode 100644 index 0000000000..128fa24863 --- /dev/null +++ b/internal/clickhouse/schema/059_create_verifications.key_verifications_per_minute_v2.sql @@ -0,0 +1,19 @@ +-- +goose up +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_minute_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; + +-- +goose down +DROP TABLE IF EXISTS verifications.key_verifications_per_minute_v2; diff --git a/internal/clickhouse/schema/060_create_verifications.key_verifications_per_minute_mv_v2.sql b/internal/clickhouse/schema/060_create_verifications.key_verifications_per_minute_mv_v2.sql new file mode 100644 index 0000000000..ea7fc1aa3c --- /dev/null +++ b/internal/clickhouse/schema/060_create_verifications.key_verifications_per_minute_mv_v2.sql @@ -0,0 +1,27 @@ +-- +goose up +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v2 +TO verifications.key_verifications_per_minute_v2 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; + +-- +goose down +DROP VIEW IF EXISTS verifications.key_verifications_per_minute_mv_v2; diff --git a/internal/clickhouse/src/index.ts b/internal/clickhouse/src/index.ts index 41e6d84faa..c24852e155 100644 --- a/internal/clickhouse/src/index.ts +++ b/internal/clickhouse/src/index.ts @@ -65,6 +65,7 @@ import { getMinutelyVerificationTimeseries, getMonthlyVerificationTimeseries, getSixHourlyVerificationTimeseries, + getSpentCreditsTotal, getThirtyMinutelyVerificationTimeseries, getThreeDayVerificationTimeseries, getTwelveHourlyVerificationTimeseries, @@ -111,6 +112,7 @@ export class ClickHouse { return { insert: insertVerification(this.inserter), latest: getLatestVerifications(this.querier), + spentCreditsTotal: getSpentCreditsTotal(this.querier), timeseries: { // Minute-based granularity perMinute: getMinutelyVerificationTimeseries(this.querier), diff --git a/internal/clickhouse/src/keys/keys.test.ts b/internal/clickhouse/src/keys/keys.test.ts new file mode 100644 index 0000000000..9c4b41c6bc --- /dev/null +++ b/internal/clickhouse/src/keys/keys.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { getKeysOverviewLogs, keysOverviewLogsParams } from "./keys"; + +// Mock Querier type for testing +type MockQuerier = { + query: (params: any) => (queryParams: any) => Promise<{ val: any[]; err?: any }>; +}; + +describe("getKeysOverviewLogs", () => { + it("should include creditSpendMode filter when enabled", async () => { + let capturedQuery = ""; + + // Mock ClickHouse querier that captures the generated query + const mockQuerier: MockQuerier = { + query: (params) => { + capturedQuery = params.query; + return async () => ({ + val: [ + { + key_id: "test_key_1", + time: 1234567890, + request_id: "req_1", + tags: [], + valid_count: 5, + error_count: 0, + spent_credits: 10, + outcome_counts_array: [["VALID", 5]], + }, + ], + }); + }, + }; + + const logsFunction = getKeysOverviewLogs(mockQuerier); + + // Test with creditSpendMode enabled + const paramsWithCreditMode = { + workspaceId: "ws_test", + keyspaceId: "ks_test", + limit: 50, + startTime: 1234567000, + endTime: 1234568000, + creditSpendMode: true, + outcomes: null, + names: null, + identities: null, + keyIds: null, + tags: null, + cursorTime: null, + sorts: null, + }; + + await logsFunction(paramsWithCreditMode); + + // Verify that the query includes the credit spend filter + expect(capturedQuery).toContain("spent_credits > 0"); + expect(capturedQuery).toContain("AND (spent_credits > 0)"); + }); + + it("should not include creditSpendMode filter when disabled", async () => { + let capturedQuery = ""; + + // Mock ClickHouse querier that captures the generated query + const mockQuerier: MockQuerier = { + query: (params) => { + capturedQuery = params.query; + return async () => ({ + val: [ + { + key_id: "test_key_1", + time: 1234567890, + request_id: "req_1", + tags: [], + valid_count: 5, + error_count: 0, + spent_credits: 0, + outcome_counts_array: [["VALID", 5]], + }, + ], + }); + }, + }; + + const logsFunction = getKeysOverviewLogs(mockQuerier); + + // Test with creditSpendMode disabled + const paramsWithoutCreditMode = { + workspaceId: "ws_test", + keyspaceId: "ks_test", + limit: 50, + startTime: 1234567000, + endTime: 1234568000, + creditSpendMode: false, + outcomes: null, + names: null, + identities: null, + keyIds: null, + tags: null, + cursorTime: null, + sorts: null, + }; + + await logsFunction(paramsWithoutCreditMode); + + // Verify that the query uses TRUE instead of credit filter + expect(capturedQuery).toContain("AND (TRUE)"); + expect(capturedQuery).not.toContain("spent_credits > 0"); + }); + + it("should validate creditSpendMode parameter in schema", () => { + // Test that the schema accepts creditSpendMode parameter + const validParams = { + workspaceId: "ws_test", + keyspaceId: "ks_test", + limit: 50, + startTime: 1234567000, + endTime: 1234568000, + creditSpendMode: true, + outcomes: null, + names: null, + identities: null, + keyIds: null, + tags: null, + cursorTime: null, + sorts: null, + }; + + const result = keysOverviewLogsParams.safeParse(validParams); + expect(result.success).toBe(true); + + if (result.success) { + expect(result.data.creditSpendMode).toBe(true); + } + }); + + it("should default creditSpendMode to false when not provided", () => { + const paramsWithoutCreditMode = { + workspaceId: "ws_test", + keyspaceId: "ks_test", + limit: 50, + startTime: 1234567000, + endTime: 1234568000, + outcomes: null, + names: null, + identities: null, + keyIds: null, + tags: null, + cursorTime: null, + sorts: null, + }; + + const result = keysOverviewLogsParams.safeParse(paramsWithoutCreditMode); + expect(result.success).toBe(true); + + if (result.success) { + expect(result.data.creditSpendMode).toBe(false); + } + }); + + it("should combine credit spend filter with other filters correctly", async () => { + let capturedQuery = ""; + + const mockQuerier: MockQuerier = { + query: (params) => { + capturedQuery = params.query; + return async () => ({ + val: [], + }); + }, + }; + + const logsFunction = getKeysOverviewLogs(mockQuerier); + + // Test with creditSpendMode and other filters + const paramsWithMultipleFilters = { + workspaceId: "ws_test", + keyspaceId: "ks_test", + limit: 50, + startTime: 1234567000, + endTime: 1234568000, + creditSpendMode: true, + outcomes: [{ value: "VALID" as const, operator: "is" as const }], + names: null, + identities: null, + keyIds: [{ value: "test_key", operator: "contains" as const }], + tags: null, + cursorTime: null, + sorts: null, + }; + + await logsFunction(paramsWithMultipleFilters); + + // Verify that both credit spend filter and other filters are applied + expect(capturedQuery).toContain("spent_credits > 0"); + expect(capturedQuery).toContain("outcome = {outcomeValue_0: String}"); + expect(capturedQuery).toContain("like(key_id, CONCAT('%', {keyIdValue_0: String}, '%'))"); + }); +}); diff --git a/internal/clickhouse/src/keys/keys.ts b/internal/clickhouse/src/keys/keys.ts index 9a2a5d4604..42783e8838 100644 --- a/internal/clickhouse/src/keys/keys.ts +++ b/internal/clickhouse/src/keys/keys.ts @@ -17,6 +17,7 @@ export const keysOverviewLogsParams = z.object({ limit: z.number().int(), startTime: z.number().int(), endTime: z.number().int(), + creditSpendMode: z.boolean().optional().default(false), outcomes: z .array( z.object({ @@ -61,7 +62,7 @@ export const keysOverviewLogsParams = z.object({ sorts: z .array( z.object({ - column: z.enum(["time", "valid", "invalid"]), + column: z.enum(["time", "valid", "invalid", "spent_credits"]), direction: z.enum(["asc", "desc"]), }), ) @@ -113,6 +114,7 @@ export const rawKeysOverviewLogs = z.object({ error_count: z.number().int(), outcome_counts: z.record(z.string(), z.number().int()), tags: z.array(z.string()).optional(), + spent_credits: z.number().int().default(0), }); export const keysOverviewLogs = rawKeysOverviewLogs.extend({ @@ -194,10 +196,14 @@ export function getKeysOverviewLogs(ch: Querier) { .join(" OR ") || "TRUE" : "TRUE"; + // Credit spend filtering condition + const creditSpendCondition = args.creditSpendMode ? "spent_credits > 0" : "TRUE"; + const allowedColumns = new Map([ ["time", "time"], ["valid", "valid_count"], ["invalid", "error_count"], + ["spent_credits", "spent_credits"], ]); const orderBy = @@ -272,7 +278,8 @@ WITH time, key_id, tags, - outcome + outcome, + spent_credits FROM verifications.raw_key_verifications_v1 WHERE workspace_id = {workspaceId: String} AND key_space_id = {keyspaceId: String} @@ -283,6 +290,8 @@ WITH AND (${outcomeCondition}) -- Apply dynamic tag filtering AND (${tagConditions}) + -- Apply credit spend mode filtering + AND (${creditSpendCondition}) -- Handle pagination using only time as cursor ${cursorCondition} ), @@ -300,7 +309,9 @@ WITH -- Count valid verifications countIf(outcome = 'VALID') as valid_count, -- Count all non-valid verifications - countIf(outcome != 'VALID') as error_count + countIf(outcome != 'VALID') as error_count, + -- Sum total spent credits for this key + sum(spent_credits) as spent_credits FROM filtered_keys GROUP BY key_id ), @@ -323,6 +334,7 @@ WITH a.tags, a.valid_count, a.error_count, + a.spent_credits, -- Create an array of tuples containing all outcomes and their counts -- This will be transformed into an object in the application code groupArray((o.outcome, o.count)) as outcome_counts_array @@ -335,7 +347,8 @@ WITH a.last_request_id, a.tags, a.valid_count, - a.error_count + a.error_count, + a.spent_credits -- Sort results with most recent verification first ORDER BY ${orderByClause} -- Limit results for pagination @@ -372,6 +385,7 @@ WITH tags: result.tags, valid_count: result.valid_count, error_count: result.error_count, + spent_credits: result.spent_credits || 0, outcome_counts: outcomeCountsObj, }; }), diff --git a/internal/clickhouse/src/verifications.ts b/internal/clickhouse/src/verifications.ts index 2759dc4d34..7080cb574d 100644 --- a/internal/clickhouse/src/verifications.ts +++ b/internal/clickhouse/src/verifications.ts @@ -24,6 +24,7 @@ export function insertVerification(ch: Inserter) { "INSUFFICIENT_PERMISSIONS", ]), identity_id: z.string().optional().default(""), + spent_credits: z.number().int().optional().default(0), }), }); } @@ -61,6 +62,7 @@ export const keyDetailsLog = z.object({ region: z.string(), outcome: z.enum(KEY_VERIFICATION_OUTCOMES), tags: z.array(z.string()), + spent_credits: z.number().int().default(0), }); export type KeyDetailsLog = z.infer; @@ -161,7 +163,8 @@ export function getKeyDetailsLogs(ch: Querier) { time, region, outcome, - tags + tags, + spent_credits FROM verifications.raw_key_verifications_v1 WHERE ${baseConditions} -- Handle pagination using time as cursor @@ -246,6 +249,7 @@ export const verificationTimeseriesDataPoint = z.object({ disabled_count: z.number().int().default(0), expired_count: z.number().int().default(0), usage_exceeded_count: z.number().int().default(0), + spent_credits: z.number().int().default(0), }), }); @@ -261,75 +265,75 @@ type TimeInterval = { const INTERVALS: Record = { // Minute-based intervals minute: { - table: "verifications.key_verifications_per_minute_v1", + table: "verifications.key_verifications_per_minute_v2", step: "MINUTE", stepSize: 1, }, fiveMinutes: { - table: "verifications.key_verifications_per_minute_v1", + table: "verifications.key_verifications_per_minute_v2", step: "MINUTE", stepSize: 5, }, thirtyMinutes: { - table: "verifications.key_verifications_per_minute_v1", + table: "verifications.key_verifications_per_minute_v2", step: "MINUTE", stepSize: 30, }, // Hour-based intervals hour: { - table: "verifications.key_verifications_per_hour_v3", + table: "verifications.key_verifications_per_hour_v4", step: "HOUR", stepSize: 1, }, twoHours: { - table: "verifications.key_verifications_per_hour_v3", + table: "verifications.key_verifications_per_hour_v4", step: "HOUR", stepSize: 2, }, fourHours: { - table: "verifications.key_verifications_per_hour_v3", + table: "verifications.key_verifications_per_hour_v4", step: "HOUR", stepSize: 4, }, sixHours: { - table: "verifications.key_verifications_per_hour_v3", + table: "verifications.key_verifications_per_hour_v4", step: "HOUR", stepSize: 6, }, twelveHours: { - table: "verifications.key_verifications_per_hour_v3", + table: "verifications.key_verifications_per_hour_v4", step: "HOUR", stepSize: 12, }, // Day-based intervals day: { - table: "verifications.key_verifications_per_day_v3", + table: "verifications.key_verifications_per_day_v4", step: "DAY", stepSize: 1, }, threeDays: { - table: "verifications.key_verifications_per_day_v3", + table: "verifications.key_verifications_per_day_v4", step: "DAY", stepSize: 3, }, week: { - table: "verifications.key_verifications_per_day_v3", + table: "verifications.key_verifications_per_day_v4", step: "DAY", stepSize: 7, }, twoWeeks: { - table: "verifications.key_verifications_per_day_v3", + table: "verifications.key_verifications_per_day_v4", step: "DAY", stepSize: 14, }, // Monthly-based intervals month: { - table: "verifications.key_verifications_per_month_v3", + table: "verifications.key_verifications_per_month_v4", step: "MONTH", stepSize: 1, }, quarter: { - table: "verifications.key_verifications_per_month_v3", + table: "verifications.key_verifications_per_month_v4", step: "MONTH", stepSize: 3, }, @@ -370,7 +374,8 @@ function createVerificationTimeseriesQuery(interval: TimeInterval, whereClause: 'forbidden_count', SUM(IF(outcome = 'FORBIDDEN', count, 0)), 'disabled_count', SUM(IF(outcome = 'DISABLED', count, 0)) , 'expired_count',SUM(IF(outcome = 'EXPIRED', count, 0)) , - 'usage_exceeded_count', SUM(IF(outcome = 'USAGE_EXCEEDED', count, 0)) + 'usage_exceeded_count', SUM(IF(outcome = 'USAGE_EXCEEDED', count, 0)), + 'spent_credits', SUM(spent_credits) ) as y FROM ${interval.table} ${whereClause} @@ -593,6 +598,7 @@ function mergeVerificationTimeseriesResults( expired_count: (existingPoint.y.expired_count ?? 0) + (dataPoint.y.expired_count ?? 0), usage_exceeded_count: (existingPoint.y.usage_exceeded_count ?? 0) + (dataPoint.y.usage_exceeded_count ?? 0), + spent_credits: (existingPoint.y.spent_credits ?? 0) + (dataPoint.y.spent_credits ?? 0), }, }); } else { @@ -605,6 +611,124 @@ function mergeVerificationTimeseriesResults( return Array.from(mergedMap.values()).sort((a, b) => a.x - b.x); } +// Schema for spent credits total query +const spentCreditsParams = z.object({ + workspaceId: z.string(), + keyspaceId: z.string(), + keyId: z.string().optional(), + startTime: z.number().int(), + endTime: z.number().int(), + outcomes: z + .array( + z.object({ + value: z.enum(KEY_VERIFICATION_OUTCOMES), + operator: z.literal("is"), + }), + ) + .nullable(), + tags: z + .object({ + operator: z.enum(["is", "contains", "startsWith", "endsWith"]), + value: z.string(), + }) + .nullable(), +}); + +const spentCreditsResult = z.object({ + spent_credits: z.number().int().default(0), +}); + +export type SpentCreditsParams = z.infer; +export type SpentCreditsResult = z.infer; + +/** + * Get total spent credits for a key within a time range + */ +function createSpentCreditsQuerier() { + return (ch: Querier) => async (args: SpentCreditsParams) => { + const conditions = [ + "workspace_id = {workspaceId: String}", + "key_space_id = {keyspaceId: String}", + "time >= {startTime: Int64}", + "time <= {endTime: Int64}", + ]; + + let paramSchemaExtension = {}; + + // Add key filter if specified + if (args.keyId) { + conditions.push("key_id = {keyId: String}"); + } + + // Add outcome filters + if (args.outcomes && args.outcomes.length > 0) { + const outcomeConditions = args.outcomes.map( + (_, index) => `outcome = {outcomeValue_${index}: String}`, + ); + conditions.push(`(${outcomeConditions.join(" OR ")})`); + + paramSchemaExtension = { + ...paramSchemaExtension, + ...args.outcomes.reduce( + (acc, _filter, index) => { + acc[`outcomeValue_${index}`] = z.string(); + return acc; + }, + {} as Record, + ), + }; + } + + // Add tag filters + if (args.tags) { + const tagCondition = + args.tags.operator === "is" + ? "has(tags, {tagValue: String})" + : "arrayExists(tag -> tag LIKE {tagValue: String}, tags)"; + conditions.push(tagCondition); + + paramSchemaExtension = { + ...paramSchemaExtension, + tagValue: z.string(), + }; + } + + const whereClause = `WHERE ${conditions.join(" AND ")}`; + + const query = ` + SELECT COALESCE(SUM(spent_credits), 0) as spent_credits + FROM verifications.raw_key_verifications_v1 + ${whereClause} + `; + + const parameters = { + workspaceId: args.workspaceId, + keyspaceId: args.keyspaceId, + startTime: args.startTime, + endTime: args.endTime, + outcomes: args.outcomes, + tags: args.tags, + ...(args.keyId ? { keyId: args.keyId } : {}), + ...(args.outcomes?.reduce( + (acc, filter, index) => { + acc[`outcomeValue_${index}`] = filter.value; + return acc; + }, + {} as Record, + ) ?? {}), + ...(args.tags ? { tagValue: args.tags.value } : {}), + }; + + return ch.query({ + query, + params: spentCreditsParams.extend(paramSchemaExtension), + schema: spentCreditsResult, + })(parameters); + }; +} + +export const getSpentCreditsTotal = createSpentCreditsQuerier(); + // Minute-based timeseries export const getMinutelyVerificationTimeseries = (ch: Querier) => (args: VerificationTimeseriesParams) => diff --git a/tools/local/src/cmd/seed/batch-operations.ts b/tools/local/src/cmd/seed/batch-operations.ts index 3bfba22dbe..624eb230e8 100644 --- a/tools/local/src/cmd/seed/batch-operations.ts +++ b/tools/local/src/cmd/seed/batch-operations.ts @@ -36,6 +36,13 @@ export async function insertVerificationEvents( // Track usage stats for reporting const keyUsageCounter = new Map(); + const creditStats = { + totalCreditsSpent: 0, + verificationsWith0Credits: 0, + verificationsWith1Credit: 0, + verificationsWithMultipleCredits: 0, + keysWithUsageLimits: sortedKeys.filter((key) => key.hasUsageLimit).length, + }; sortedKeys.forEach((key) => keyUsageCounter.set(key.id, 0)); try { @@ -64,6 +71,16 @@ export async function insertVerificationEvents( const verificationEvent = biasVerificationOutcome(key, workspaceId, keyAuthId, requestId); batchOfVerificationRecords.push(verificationEvent); + // Track credit statistics + creditStats.totalCreditsSpent += verificationEvent.spent_credits; + if (verificationEvent.spent_credits === 0) { + creditStats.verificationsWith0Credits++; + } else if (verificationEvent.spent_credits === 1) { + creditStats.verificationsWith1Credit++; + } else { + creditStats.verificationsWithMultipleCredits++; + } + // If needed, create a matching API request if (createApiRequestLog) { const apiRequest = generateMatchingApiRequestForVerification( @@ -98,13 +115,18 @@ export async function insertVerificationEvents( }.`, ); - return { keyUsageStats: Object.fromEntries(keyUsageCounter) }; + return { + keyUsageStats: Object.fromEntries(keyUsageCounter), + creditStats, + }; } catch (error: unknown) { // End progress with a newline process.stdout.write("\n"); console.error( - `❌ Error inserting data during batch ${progress.batchNumber}: ${(error as { message: string }).message}`, + `❌ Error inserting data during batch ${progress.batchNumber}: ${ + (error as { message: string }).message + }`, ); console.error("ClickHouse Insert Error Details:", error); throw error; @@ -189,7 +211,9 @@ export async function insertRatelimitEvents( process.stdout.write("\n"); console.error( - `❌ Error inserting data during batch ${progress.batchNumber}: ${(error as { message: string }).message}`, + `❌ Error inserting data during batch ${progress.batchNumber}: ${ + (error as { message: string }).message + }`, ); console.error("ClickHouse Insert Error Details:", error); throw error; diff --git a/tools/local/src/cmd/seed/event-generator.ts b/tools/local/src/cmd/seed/event-generator.ts index 0cba74a4c0..45f2b4fec8 100644 --- a/tools/local/src/cmd/seed/event-generator.ts +++ b/tools/local/src/cmd/seed/event-generator.ts @@ -1,4 +1,4 @@ -import crypto from "node:crypto"; +import { createHash } from "node:crypto"; import { generateMetadata, generateRandomApiRequest, @@ -31,6 +31,7 @@ export type VerificationEvent = { tags: string[]; outcome: string; identity_id: string; + spent_credits: number; }; export type RatelimitEvent = { @@ -78,7 +79,7 @@ export function selectKeyWithNormalDistribution(keys: KeyInfo[]): KeyInfo { * Generates a hash for a key */ export function generateKeyHash(keyContent: string): string { - return crypto.createHash("sha256").update(keyContent).digest("hex"); + return createHash("sha256").update(keyContent).digest("hex"); } /** @@ -174,6 +175,9 @@ export function generateVerificationEvent( // Optionally include identity_id (30% of the time) const identityId = Math.random() < 0.3 ? `ident_${generateRandomString(24)}` : ""; + // Default spent_credits to 0 - will be set by biasVerificationOutcome based on key properties + const spent_credits = 0; + return { request_id: generatedRequestId, time, @@ -184,9 +188,34 @@ export function generateVerificationEvent( tags, outcome, identity_id: identityId, + spent_credits, }; } +/** + * Calculates credit cost for a verification event + */ +function calculateCreditCost(key: KeyInfo, outcome: string): number { + // Rule 3: If request is rejected, cost should be 0 + if (outcome !== "VALID") { + return 0; + } + + // Only calculate cost if key has usage limits (remaining enabled) + if (!key.hasUsageLimit) { + return 0; + } + + // Rule 1: Default cost is 1 credit + // Rule 2: 15% of VALID verifications should have cost > 1 (2-5 credits) + // Since we only get here for VALID outcomes, we can apply the 15% rule directly + if (Math.random() < 0.15) { + return 2 + Math.floor(Math.random() * 4); // Random between 2-5 + } + + return 1; +} + /** * Biases verification event outcome based on key properties */ @@ -218,6 +247,9 @@ export function biasVerificationOutcome( verificationEvent = generateVerificationEvent(workspaceId, keyAuthId, key.id, requestId); } + // Calculate and set the credit cost based on the outcome and key properties + verificationEvent.spent_credits = calculateCreditCost(key, verificationEvent.outcome); + return verificationEvent; } diff --git a/tools/local/src/cmd/seed/utils.ts b/tools/local/src/cmd/seed/utils.ts index e9dc738e65..7d9bd0d310 100644 --- a/tools/local/src/cmd/seed/utils.ts +++ b/tools/local/src/cmd/seed/utils.ts @@ -4,7 +4,7 @@ import { eq, mysqlDrizzle, schema } from "@unkey/db"; import mysql from "mysql2/promise"; export function generateUuid() { - return crypto.randomUUID(); + return globalThis.crypto.randomUUID(); } function env() {