From b3ac029c726772092de41d0bc49e241e7b3e127b Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 21 Sep 2024 16:04:11 +0200 Subject: [PATCH 1/2] feat: clickhouse container collecting api metrics --- Taskfile.yml | 4 +- .../schema/001_create_requests_table.sql | 8 +- .../002_create_key_verifications_table.sql | 4 +- apps/api/package.json | 1 + apps/api/src/pkg/analytics.ts | 61 +++++++++++++ apps/api/src/pkg/env.ts | 3 + apps/api/src/pkg/hono/env.ts | 2 + apps/api/src/pkg/keys/service.ts | 1 + apps/api/src/pkg/middleware/init.ts | 8 ++ apps/api/src/pkg/middleware/metrics.ts | 32 +++++++ ..._keys_verifyKey.ratelimit_accuracy.test.ts | 2 +- apps/api/src/routes/v1_keys_verifyKey.ts | 16 ++++ apps/dashboard/app/(app)/logs/page.tsx | 28 ++++++ apps/dashboard/lib/clickhouse/index.ts | 52 +++++++++++ apps/dashboard/lib/env.ts | 2 + apps/dashboard/package.json | 1 + deployment/docker-compose.yaml | 10 ++- internal/clickhouse-zod/package.json | 17 ++++ internal/clickhouse-zod/src/client.ts | 87 +++++++++++++++++++ internal/clickhouse-zod/src/index.ts | 1 + pnpm-lock.yaml | 51 ++++++++--- tools/local/src/cmd/dashboard.ts | 3 + tools/local/src/main.ts | 2 +- 23 files changed, 376 insertions(+), 20 deletions(-) create mode 100644 apps/dashboard/app/(app)/logs/page.tsx create mode 100644 apps/dashboard/lib/clickhouse/index.ts create mode 100644 internal/clickhouse-zod/package.json create mode 100644 internal/clickhouse-zod/src/client.ts create mode 100644 internal/clickhouse-zod/src/index.ts diff --git a/Taskfile.yml b/Taskfile.yml index fb3bbc389c..4e44a5160b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -14,7 +14,7 @@ tasks: - docker compose -f ./deployment/docker-compose.yaml up -d - seed: + migrate: cmds: - task: migrate-db - task: migrate-clickhouse @@ -22,7 +22,7 @@ tasks: migrate-clickhouse: env: GOOSE_DRIVER: clickhouse - GOOSE_DBSTRING: "tcp://127.0.0.1:9000" + GOOSE_DBSTRING: "tcp://default:password@127.0.0.1:9000" GOOSE_MIGRATION_DIR: ./apps/agent/pkg/clickhouse/schema cmds: - goose up diff --git a/apps/agent/pkg/clickhouse/schema/001_create_requests_table.sql b/apps/agent/pkg/clickhouse/schema/001_create_requests_table.sql index c0f22f1215..e8db0d3b0c 100644 --- a/apps/agent/pkg/clickhouse/schema/001_create_requests_table.sql +++ b/apps/agent/pkg/clickhouse/schema/001_create_requests_table.sql @@ -21,7 +21,13 @@ CREATE TABLE default.raw_api_requests_v1( response_headers Array(String), response_body String, -- internal err.Error() string, empty if no error - error String + error String, + + -- milliseconds + service_latency Int64, + + user_agent String, + ip_address String ) ENGINE = MergeTree() diff --git a/apps/agent/pkg/clickhouse/schema/002_create_key_verifications_table.sql b/apps/agent/pkg/clickhouse/schema/002_create_key_verifications_table.sql index add2e9351b..58ead77dc0 100644 --- a/apps/agent/pkg/clickhouse/schema/002_create_key_verifications_table.sql +++ b/apps/agent/pkg/clickhouse/schema/002_create_key_verifications_table.sql @@ -12,7 +12,7 @@ CREATE TABLE default.raw_key_verifications_v1( -- Right now this is a 3 character airport code, but when we move to aws, -- this will be the region code such as `us-east-1` - region String, + region LowCardinality(String), -- Examples: -- - "VALID" @@ -24,6 +24,8 @@ CREATE TABLE default.raw_key_verifications_v1( -- Empty string if the key has no identity identity_id String, + + ) ENGINE = MergeTree() ORDER BY (workspace_id, key_space_id, key_id, time) diff --git a/apps/api/package.json b/apps/api/package.json index 040eae0b38..2022dc6a34 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -36,6 +36,7 @@ "@unkey/logs": "workspace:^", "@unkey/metrics": "workspace:^", "@unkey/rbac": "workspace:^", + "@unkey/clickhouse-zod": "workspace:^", "@unkey/schema": "workspace:^", "@unkey/worker-logging": "workspace:^", "hono": "^4.5.8", diff --git a/apps/api/src/pkg/analytics.ts b/apps/api/src/pkg/analytics.ts index bc834fa887..6da709f7b0 100644 --- a/apps/api/src/pkg/analytics.ts +++ b/apps/api/src/pkg/analytics.ts @@ -1,4 +1,5 @@ import { NoopTinybird, Tinybird } from "@chronark/zod-bird"; +import { Clickhouse } from "@unkey/clickhouse-zod"; import { newId } from "@unkey/id"; import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog"; import { ratelimitSchemaV1 } from "@unkey/schema/src/ratelimit-tinybird"; @@ -17,6 +18,7 @@ const dateToUnixMilli = z.string().transform((t) => new Date(t.split(" ").at(0) export class Analytics { public readonly readClient: Tinybird | NoopTinybird; public readonly writeClient: Tinybird | NoopTinybird; + private clickhouse: Clickhouse; constructor(opts: { tinybirdToken?: string; @@ -24,6 +26,9 @@ export class Analytics { url: string; token: string; }; + clickhouse?: { + url: string; + }; }) { this.readClient = opts.tinybirdToken ? new Tinybird({ token: opts.tinybirdToken }) @@ -32,6 +37,14 @@ export class Analytics { this.writeClient = opts.tinybirdProxy ? new Tinybird({ token: opts.tinybirdProxy.token, baseUrl: opts.tinybirdProxy.url }) : this.readClient; + + this.clickhouse = new Clickhouse( + opts.clickhouse?.url + ? { + url: opts.clickhouse.url, + } + : { noop: true }, + ); } public get ingestSdkTelemetry() { @@ -93,6 +106,54 @@ export class Analytics { }); } + public get insertKeyVerification() { + return this.clickhouse.insert({ + table: "default.raw_key_verifications_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + key_space_id: z.string(), + key_id: z.string(), + region: z.string(), + outcome: z.enum([ + "VALID", + "RATE_LIMITED", + "EXPIRED", + "DISABLED", + "FORBIDDEN", + "USAGE_EXCEEDED", + "DISABLED", + "INSUFFICIENT_PERMISSIONS", + ]), + identity_id: z.string().optional().default(""), + }), + }); + } + + public get insertApiRequest() { + return this.clickhouse.insert({ + table: "default.raw_api_requests_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + host: z.string(), + method: z.string(), + path: z.string(), + request_headers: z.array(z.string()), + request_body: z.string(), + response_status: z.number().int(), + response_headers: z.array(z.string()), + response_body: z.string(), + error: z.string().optional().default(""), + service_latency: z.number().int(), + user_agent: z.string(), + ip_address: z.string(), + }), + }); + } + public get ingestKeyVerification() { return this.writeClient.buildIngestEndpoint({ datasource: "key_verifications__v2", diff --git a/apps/api/src/pkg/env.ts b/apps/api/src/pkg/env.ts index 2e0a33bbd8..6ce33d7a81 100644 --- a/apps/api/src/pkg/env.ts +++ b/apps/api/src/pkg/env.ts @@ -28,6 +28,9 @@ export const zEnv = z.object({ }), AGENT_URL: z.string().url(), AGENT_TOKEN: z.string(), + + CLICKHOUSE_URL: z.string().optional(), + SYNC_RATELIMIT_ON_NO_DATA: z .string() .optional() diff --git a/apps/api/src/pkg/hono/env.ts b/apps/api/src/pkg/hono/env.ts index 991193f7b7..34ccb8dd89 100644 --- a/apps/api/src/pkg/hono/env.ts +++ b/apps/api/src/pkg/hono/env.ts @@ -29,6 +29,8 @@ export type HonoEnv = { isolateId: string; isolateCreatedAt: number; requestId: string; + requestStartedAt: number; + workspaceId?: string; metricsContext: { keyId?: string; [key: string]: unknown; diff --git a/apps/api/src/pkg/keys/service.ts b/apps/api/src/pkg/keys/service.ts index 407c34965b..15c03d0223 100644 --- a/apps/api/src/pkg/keys/service.ts +++ b/apps/api/src/pkg/keys/service.ts @@ -151,6 +151,7 @@ export class KeyService { }); return res; } + c.set("workspaceId", res.val.key?.forWorkspaceId ?? res.val.key?.workspaceId); this.metrics.emit({ metric: "metric.key.verification", diff --git a/apps/api/src/pkg/middleware/init.ts b/apps/api/src/pkg/middleware/init.ts index d78a0dea78..23289f7876 100644 --- a/apps/api/src/pkg/middleware/init.ts +++ b/apps/api/src/pkg/middleware/init.ts @@ -1,5 +1,6 @@ import { Analytics } from "@/pkg/analytics"; import { createConnection } from "@/pkg/db"; + import { KeyService } from "@/pkg/keys/service"; import { AgentRatelimiter } from "@/pkg/ratelimit"; import { DurableUsageLimiter, NoopUsageLimiter } from "@/pkg/usagelimit"; @@ -45,6 +46,8 @@ export function init(): MiddlewareHandler { const requestId = newId("request"); c.set("requestId", requestId); + c.set("requestStartedAt", Date.now()); + c.res.headers.set("Unkey-Request-Id", requestId); const logger = new ConsoleLogger({ @@ -104,6 +107,11 @@ export function init(): MiddlewareHandler { const analytics = new Analytics({ tinybirdProxy, tinybirdToken: c.env.TINYBIRD_TOKEN, + clickhouse: c.env.CLICKHOUSE_URL + ? { + url: c.env.CLICKHOUSE_URL, + } + : undefined, }); const rateLimiter = new AgentRatelimiter({ agent: { url: c.env.AGENT_URL, token: c.env.AGENT_TOKEN }, diff --git a/apps/api/src/pkg/middleware/metrics.ts b/apps/api/src/pkg/middleware/metrics.ts index 72d37be3ca..f612c1d72d 100644 --- a/apps/api/src/pkg/middleware/metrics.ts +++ b/apps/api/src/pkg/middleware/metrics.ts @@ -8,6 +8,8 @@ export function metrics(): MiddlewareHandler { return async (c, next) => { const { metrics, analytics, logger } = c.get("services"); + let requestBody = await c.req.raw.clone().text(); + requestBody = requestBody.replaceAll(/"key":\s*"[a-zA-Z0-9_]+"/g, '"key": ""'); const start = performance.now(); const m = { isolateId: c.get("isolateId"), @@ -79,6 +81,36 @@ export function metrics(): MiddlewareHandler { c.res.headers.append("Unkey-Version", c.env.VERSION); metrics.emit(m); c.executionCtx.waitUntil(metrics.flush()); + + const responseHeaders: Array = []; + c.res.headers.forEach((v, k) => { + responseHeaders.push(`${k}: ${v}`); + }); + + c.executionCtx.waitUntil( + analytics.insertApiRequest({ + request_id: c.get("requestId"), + time: c.get("requestStartedAt"), + workspace_id: c.get("workspaceId") ?? "", + host: new URL(c.req.url).host, + method: c.req.method, + path: c.req.path, + request_headers: Object.entries(c.req.header()).map(([k, v]) => { + if (k.toLowerCase() === "authorization") { + return `${k}: `; + } + return `${k}: ${v}`; + }), + request_body: requestBody, + response_status: c.res.status, + response_headers: responseHeaders, + response_body: await c.res.clone().text(), + error: m.error ?? "", + service_latency: Date.now() - c.get("requestStartedAt"), + ip_address: c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP") ?? "", + user_agent: c.req.header("User-Agent") ?? "", + }), + ); } }; } diff --git a/apps/api/src/routes/v1_keys_verifyKey.ratelimit_accuracy.test.ts b/apps/api/src/routes/v1_keys_verifyKey.ratelimit_accuracy.test.ts index bed2efaf97..8cdf91c439 100644 --- a/apps/api/src/routes/v1_keys_verifyKey.ratelimit_accuracy.test.ts +++ b/apps/api/src/routes/v1_keys_verifyKey.ratelimit_accuracy.test.ts @@ -31,7 +31,7 @@ const testCases: { }, { limit: 20, - duration: 5000, + duration: 30000, rps: 50, seconds: 60, }, diff --git a/apps/api/src/routes/v1_keys_verifyKey.ts b/apps/api/src/routes/v1_keys_verifyKey.ts index 5cd232b3d0..da2d904564 100644 --- a/apps/api/src/routes/v1_keys_verifyKey.ts +++ b/apps/api/src/routes/v1_keys_verifyKey.ts @@ -349,6 +349,22 @@ export const registerV1KeysVerifyKey = (app: App) => : undefined, }; c.executionCtx.waitUntil( + // new clickhouse + analytics.insertKeyVerification({ + request_id: c.get("requestId"), + time: Date.now(), + workspace_id: val.key.workspaceId, + key_space_id: val.key.keyAuthId, + key_id: val.key.id, + // @ts-expect-error + region: c.req.raw.cf.colo ?? "", + outcome: val.code ?? "VALID", + identity_id: val.identity?.id, + }), + ); + + c.executionCtx.waitUntil( + // old tinybird analytics.ingestKeyVerification({ workspaceId: val.key.workspaceId, apiId: val.api.id, diff --git a/apps/dashboard/app/(app)/logs/page.tsx b/apps/dashboard/app/(app)/logs/page.tsx new file mode 100644 index 0000000000..a8a40fbe32 --- /dev/null +++ b/apps/dashboard/app/(app)/logs/page.tsx @@ -0,0 +1,28 @@ +import { PageHeader } from "@/components/dashboard/page-header"; +import { getTenantId } from "@/lib/auth"; +import { getLogs } from "@/lib/clickhouse"; +import { db } from "@/lib/db"; + +export const revalidate = 0; + +export default async function Page() { + const tenantId = getTenantId(); + + const workspace = await db.query.workspaces.findFirst({ + where: (table, { and, eq, isNull }) => + and(eq(table.tenantId, tenantId), isNull(table.deletedAt)), + }); + if (!workspace) { + return
Workspace with tenantId: {tenantId} not found
; + } + + const logs = await getLogs({ workspaceId: workspace.id, limit: 10 }); + + return ( +
+ + + 
{JSON.stringify(logs, null, 2)}
+
+ ); +} diff --git a/apps/dashboard/lib/clickhouse/index.ts b/apps/dashboard/lib/clickhouse/index.ts new file mode 100644 index 0000000000..744d42b3bd --- /dev/null +++ b/apps/dashboard/lib/clickhouse/index.ts @@ -0,0 +1,52 @@ +import { Clickhouse } from "@unkey/clickhouse-zod"; +import { z } from "zod"; +import { env } from "../env"; + +// dummy example of how to query stuff from clickhouse +export async function getLogs(args: { workspaceId: string; limit: number }) { + const { CLICKHOUSE_URL } = env(); + + const ch = new Clickhouse(CLICKHOUSE_URL ? { url: CLICKHOUSE_URL } : { noop: true }); + const query = ch.query({ + query: ` + SELECT + request_id, + time, + workspace_id, + host, + method, + path, + request_headers, + request_body, + response_status, + response_headers, + response_body, + error, + service_latency + FROM default.raw_api_requests_v1 + WHERE workspace_id = {workspaceId: String} + ORDER BY time DESC + LIMIT {limit: Int}`, + params: z.object({ + workspaceId: z.string(), + limit: z.number().int(), + }), + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + host: z.string(), + method: z.string(), + path: z.string(), + request_headers: z.array(z.string()), + request_body: z.string(), + response_status: z.number().int(), + response_headers: z.array(z.string()), + response_body: z.string(), + error: z.string(), + service_latency: z.number().int(), + }), + }); + + return query(args); +} diff --git a/apps/dashboard/lib/env.ts b/apps/dashboard/lib/env.ts index 259f629e50..f06e937326 100644 --- a/apps/dashboard/lib/env.ts +++ b/apps/dashboard/lib/env.ts @@ -37,6 +37,8 @@ export const env = () => // - `ratelimit.*.create_namespace` // - `ratelimit.*.limit` UNKEY_ROOT_KEY: z.string().optional(), + + CLICKHOUSE_URL: z.string().optional(), }) .parse(process.env); diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index 986825c782..9b146a70c9 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -42,6 +42,7 @@ "@trpc/react-query": "^10.45.2", "@trpc/server": "^10.45.2", "@unkey/billing": "workspace:^", + "@unkey/clickhouse-zod": "workspace:^", "@unkey/db": "workspace:^", "@unkey/encryption": "workspace:^", "@unkey/error": "workspace:^", diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index dbc7cd4210..40656f9a90 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -55,7 +55,7 @@ services: VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password" VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" TINYBIRD_TOKEN: "I can't wait until we use clickhouse for local development" - CLICKHOUSE_URL: "clickhouse://default:@clickhouse:9000" + CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000" agent_lb: @@ -73,7 +73,8 @@ services: image: bitnami/clickhouse:latest container_name: clickhouse environment: - - ALLOW_EMPTY_PASSWORD=yes + CLICKHOUSE_ADMIN_USER: default + CLICKHOUSE_ADMIN_PASSWORD: password ports: - '8123:8123' - '9000:9000' @@ -119,12 +120,15 @@ services: "--var=EMIT_METRICS_LOGS:false", "--var=TINYBIRD_PROXY_URL:http://agent_lb:8080", "--var=TINYBIRD_PROXY_TOKEN:agent-auth-secret", - "--var=SYNC_RATELIMIT_ON_NO_DATA:1.0" + "--var=SYNC_RATELIMIT_ON_NO_DATA:1.0", + "--var=CLICKHOUSE_URL:http://default:password@clickhouse:8123", + ] depends_on: - planetscale - agent_lb + - clickhouse prometheus: image: prom/prometheus diff --git a/internal/clickhouse-zod/package.json b/internal/clickhouse-zod/package.json new file mode 100644 index 0000000000..19e5ea3f59 --- /dev/null +++ b/internal/clickhouse-zod/package.json @@ -0,0 +1,17 @@ +{ + "name": "@unkey/clickhouse-zod", + "version": "1.0.0", + "description": "", + "main": "./src/index.ts", + "types": "./src/index.ts", + "keywords": [], + "author": "Andreas Thomas", + "license": "AGPL-3.0", + "devDependencies": { + "typescript": "^5.5.3" + }, + "dependencies": { + "@clickhouse/client-web": "^1.6.0", + "zod": "^3.23.8" + } +} diff --git a/internal/clickhouse-zod/src/client.ts b/internal/clickhouse-zod/src/client.ts new file mode 100644 index 0000000000..a8cc8a9728 --- /dev/null +++ b/internal/clickhouse-zod/src/client.ts @@ -0,0 +1,87 @@ +import { type ClickHouseClient, createClient } from "@clickhouse/client-web"; +import type { z } from "zod"; + +export type Config = + | { + url: string; + noop?: never; + } + | { + url?: never; + noop: true; + }; + +export class Clickhouse { + private readonly client: ClickHouseClient; + private readonly noop: boolean; + + constructor(config: Config) { + this.noop = config.noop || false; + this.client = createClient({ + url: config.url, + clickhouse_settings: { + async_insert: 1, + wait_for_async_insert: 1, + output_format_json_quote_64bit_integers: 0, + output_format_json_quote_64bit_floats: 0, + }, + }); + } + + public query, TOut extends z.ZodSchema>(req: { + // The SQL query to run. + // Use {paramName: Type} to define parameters + // Example: `SELECT * FROM table WHERE id = {id: String}` + query: string; + // The schema of the parameters + // Example: z.object({ id: z.string() }) + params?: TIn; + // The schema of the output of each row + // Example: z.object({ id: z.string() }) + schema: TOut; + }): (params: z.input) => Promise[]> { + return async (params: z.input): Promise[]> => { + if (this.noop) { + return []; + } + + const res = await this.client.query({ + query: req.query, + query_params: params, + format: "JSONEachRow", + }); + const rows = await res.json(); + return rows.map((row) => req.schema.parse(row)); + }; + } + + public insert>(req: { + table: string; + schema: TSchema; + }): ( + events: z.input | z.input[], + ) => Promise<{ executed: boolean; query_id: string }> { + return async (events: z.input | z.input[]) => { + let validatedEvents: z.output | z.output[] | undefined = undefined; + if (req.schema) { + const v = Array.isArray(events) + ? req.schema.array().safeParse(events) + : req.schema.safeParse(events); + if (!v.success) { + throw new Error(v.error.message); + } + validatedEvents = v.data; + } + + if (this.noop) { + return { executed: true, query_id: "noop" }; + } + + return await this.client.insert({ + table: req.table, + format: "JSONEachRow", + values: Array.isArray(validatedEvents) ? validatedEvents : [validatedEvents], + }); + }; + } +} diff --git a/internal/clickhouse-zod/src/index.ts b/internal/clickhouse-zod/src/index.ts new file mode 100644 index 0000000000..5ec76921e1 --- /dev/null +++ b/internal/clickhouse-zod/src/index.ts @@ -0,0 +1 @@ +export * from "./client"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 35141a4b3f..85ec48b071 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,6 +79,9 @@ importers: '@unkey/cache': specifier: workspace:^ version: link:../../packages/cache + '@unkey/clickhouse-zod': + specifier: workspace:^ + version: link:../../internal/clickhouse-zod '@unkey/db': specifier: workspace:^ version: link:../../internal/db @@ -321,6 +324,9 @@ importers: '@unkey/billing': specifier: workspace:^ version: link:../../internal/billing + '@unkey/clickhouse-zod': + specifier: workspace:^ + version: link:../../internal/clickhouse-zod '@unkey/db': specifier: workspace:^ version: link:../../internal/db @@ -1226,7 +1232,7 @@ importers: devDependencies: checkly: specifier: latest - version: 4.9.0(@types/node@20.14.9)(typescript@5.5.3) + version: 4.8.1(@types/node@20.14.9)(typescript@5.5.3) ts-node: specifier: 10.9.1 version: 10.9.1(@types/node@20.14.9)(typescript@5.5.3) @@ -1234,6 +1240,19 @@ importers: specifier: 5.5.3 version: 5.5.3 + internal/clickhouse-zod: + dependencies: + '@clickhouse/client-web': + specifier: ^1.6.0 + version: 1.6.0 + zod: + specifier: ^3.23.8 + version: 3.23.8 + devDependencies: + typescript: + specifier: ^5.5.3 + version: 5.5.3 + internal/db: dependencies: '@planetscale/database': @@ -2856,6 +2875,16 @@ packages: csstype: 3.1.1 dev: true + /@clickhouse/client-common@1.6.0: + resolution: {integrity: sha512-DPdfPUIrBumWff8JdE2oLqnOIN1DwxXt48juV+tB6zaonufQR1QImtUGEhIFIqRWJU7zErtQ1eKfkwjLY0MYDg==} + dev: false + + /@clickhouse/client-web@1.6.0: + resolution: {integrity: sha512-1N65jzmN7cbAusay6sdy4FJIZpP+z3tW0nhE3KmxDiJDYrcTj5FOailYJKy51vWkqECJezgC8g8FLZSHm1OoOg==} + dependencies: + '@clickhouse/client-common': 1.6.0 + dev: false + /@cloudflare/kv-asset-handler@0.3.2: resolution: {integrity: sha512-EeEjMobfuJrwoctj7FA1y1KEbM0+Q1xSjobIEyie9k4haVEBB7vkDvsasw1pM3rO39mL2akxIAzLMUAtrMHZhA==} engines: {node: '>=16.13'} @@ -6998,7 +7027,7 @@ packages: resolution: {integrity: sha512-coUfuoMeIB7B8/NMekxaDzLhaYmp0HZNPEjYRm9goRou8UZIC3z21s0sL9AWoCw4EG876QyO3kYrc61WNF9B/w==} engines: {node: '>=8.0.0'} dependencies: - tslib: 2.4.1 + tslib: 2.7.0 dev: false /@peculiar/webcrypto@1.4.1: @@ -7008,7 +7037,7 @@ packages: '@peculiar/asn1-schema': 2.3.13 '@peculiar/json-schema': 1.1.12 pvtsutils: 1.3.5 - tslib: 2.4.1 + tslib: 2.7.0 webcrypto-core: 1.8.0 dev: false @@ -13185,8 +13214,8 @@ packages: get-func-name: 2.0.2 dev: true - /checkly@4.9.0(@types/node@20.14.9)(typescript@5.5.3): - resolution: {integrity: sha512-LqohEntErF7dJaJPsEpjvr/O9wUfzBRac6DOXgFDMEw+dNi19oBAcspdOqVGjPjMoCZ9/s5b5tSJI1pusY4mJQ==} + /checkly@4.8.1(@types/node@20.14.9)(typescript@5.5.3): + resolution: {integrity: sha512-LyVxHVOqjZ6k/QXGZQhZgnG7stL5mYfzu+Vl5hkdh2ZKDm/MLk4Q+gRKAyMLOjMN66jrJN1ZM1K3zlLt2/EJcg==} engines: {node: '>=16.0.0'} hasBin: true dependencies: @@ -14485,7 +14514,7 @@ packages: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} dependencies: no-case: 3.0.4 - tslib: 2.4.1 + tslib: 2.7.0 dev: false /dot-prop@6.0.1: @@ -15693,7 +15722,7 @@ packages: engines: {node: '>= 10.17.0'} hasBin: true dependencies: - debug: 4.3.4 + debug: 4.3.7(supports-color@8.1.1) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -18183,7 +18212,7 @@ packages: /lower-case@2.0.2: resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} dependencies: - tslib: 2.4.1 + tslib: 2.7.0 dev: false /lowercase-keys@3.0.0: @@ -20083,7 +20112,7 @@ packages: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} dependencies: lower-case: 2.0.2 - tslib: 2.4.1 + tslib: 2.7.0 dev: false /node-addon-api@7.1.1: @@ -22869,7 +22898,7 @@ packages: resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==} dependencies: dot-case: 3.0.4 - tslib: 2.4.1 + tslib: 2.7.0 dev: false /snakecase-keys@3.2.1: @@ -23565,7 +23594,7 @@ packages: dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 - chokidar: 3.5.3 + chokidar: 3.6.0 didyoumean: 1.2.2 dlv: 1.1.3 fast-glob: 3.3.2 diff --git a/tools/local/src/cmd/dashboard.ts b/tools/local/src/cmd/dashboard.ts index 6ca5a3888e..69402334a8 100644 --- a/tools/local/src/cmd/dashboard.ts +++ b/tools/local/src/cmd/dashboard.ts @@ -56,6 +56,9 @@ which you need in to copy in the next step.`, AGENT_URL: "http://localhost:8080", AGENT_TOKEN: "agent-auth-secret", }, + Clickhouse: { + CLICKHOUSE_URL: "http://default:password@clickhouse:8123", + }, }); if (fs.existsSync(envPath)) { diff --git a/tools/local/src/main.ts b/tools/local/src/main.ts index dce7a202e6..831a913df2 100644 --- a/tools/local/src/main.ts +++ b/tools/local/src/main.ts @@ -56,7 +56,7 @@ async function main() { break; } case "dashboard": { - await startContainers(["planetscale", "agent", "agent_lb"]); + await startContainers(["planetscale", "clickhouse", "agent", "agent_lb"]); const resources = await prepareDatabase(); !skipEnv && (await bootstrapDashboard(resources)); From 901266b8e4d5959b0ad362bba4395100e7d905ba Mon Sep 17 00:00:00 2001 From: chronark Date: Mon, 23 Sep 2024 09:41:03 +0200 Subject: [PATCH 2/2] style: clean up --- apps/api/src/pkg/analytics.ts | 13 ++----- apps/dashboard/lib/clickhouse/index.ts | 4 +-- internal/clickhouse-zod/src/client.ts | 45 ++++++++---------------- internal/clickhouse-zod/src/index.ts | 2 ++ internal/clickhouse-zod/src/interface.ts | 23 ++++++++++++ internal/clickhouse-zod/src/noop.ts | 39 ++++++++++++++++++++ 6 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 internal/clickhouse-zod/src/interface.ts create mode 100644 internal/clickhouse-zod/src/noop.ts diff --git a/apps/api/src/pkg/analytics.ts b/apps/api/src/pkg/analytics.ts index 6da709f7b0..66907d5b04 100644 --- a/apps/api/src/pkg/analytics.ts +++ b/apps/api/src/pkg/analytics.ts @@ -1,5 +1,5 @@ import { NoopTinybird, Tinybird } from "@chronark/zod-bird"; -import { Clickhouse } from "@unkey/clickhouse-zod"; +import * as ch from "@unkey/clickhouse-zod"; import { newId } from "@unkey/id"; import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog"; import { ratelimitSchemaV1 } from "@unkey/schema/src/ratelimit-tinybird"; @@ -18,7 +18,7 @@ const dateToUnixMilli = z.string().transform((t) => new Date(t.split(" ").at(0) export class Analytics { public readonly readClient: Tinybird | NoopTinybird; public readonly writeClient: Tinybird | NoopTinybird; - private clickhouse: Clickhouse; + private clickhouse: ch.Clickhouse; constructor(opts: { tinybirdToken?: string; @@ -38,13 +38,7 @@ export class Analytics { ? new Tinybird({ token: opts.tinybirdProxy.token, baseUrl: opts.tinybirdProxy.url }) : this.readClient; - this.clickhouse = new Clickhouse( - opts.clickhouse?.url - ? { - url: opts.clickhouse.url, - } - : { noop: true }, - ); + this.clickhouse = opts.clickhouse ? new ch.Client({ url: opts.clickhouse.url }) : new ch.Noop(); } public get ingestSdkTelemetry() { @@ -123,7 +117,6 @@ export class Analytics { "DISABLED", "FORBIDDEN", "USAGE_EXCEEDED", - "DISABLED", "INSUFFICIENT_PERMISSIONS", ]), identity_id: z.string().optional().default(""), diff --git a/apps/dashboard/lib/clickhouse/index.ts b/apps/dashboard/lib/clickhouse/index.ts index 744d42b3bd..77b44b606b 100644 --- a/apps/dashboard/lib/clickhouse/index.ts +++ b/apps/dashboard/lib/clickhouse/index.ts @@ -1,4 +1,4 @@ -import { Clickhouse } from "@unkey/clickhouse-zod"; +import { type Clickhouse, Client, Noop } from "@unkey/clickhouse-zod"; import { z } from "zod"; import { env } from "../env"; @@ -6,7 +6,7 @@ import { env } from "../env"; export async function getLogs(args: { workspaceId: string; limit: number }) { const { CLICKHOUSE_URL } = env(); - const ch = new Clickhouse(CLICKHOUSE_URL ? { url: CLICKHOUSE_URL } : { noop: true }); + const ch: Clickhouse = CLICKHOUSE_URL ? new Client({ url: CLICKHOUSE_URL }) : new Noop(); const query = ch.query({ query: ` SELECT diff --git a/internal/clickhouse-zod/src/client.ts b/internal/clickhouse-zod/src/client.ts index a8cc8a9728..06ce9b7a8c 100644 --- a/internal/clickhouse-zod/src/client.ts +++ b/internal/clickhouse-zod/src/client.ts @@ -1,22 +1,15 @@ import { type ClickHouseClient, createClient } from "@clickhouse/client-web"; -import type { z } from "zod"; +import { z } from "zod"; +import type { Clickhouse } from "./interface"; -export type Config = - | { - url: string; - noop?: never; - } - | { - url?: never; - noop: true; - }; +export type Config = { + url: string; +}; -export class Clickhouse { +export class Client implements Clickhouse { private readonly client: ClickHouseClient; - private readonly noop: boolean; constructor(config: Config) { - this.noop = config.noop || false; this.client = createClient({ url: config.url, clickhouse_settings: { @@ -41,17 +34,13 @@ export class Clickhouse { schema: TOut; }): (params: z.input) => Promise[]> { return async (params: z.input): Promise[]> => { - if (this.noop) { - return []; - } - const res = await this.client.query({ query: req.query, - query_params: params, + query_params: req.params?.safeParse(params), format: "JSONEachRow", }); const rows = await res.json(); - return rows.map((row) => req.schema.parse(row)); + return z.array(req.schema).parse(rows); }; } @@ -63,19 +52,13 @@ export class Clickhouse { ) => Promise<{ executed: boolean; query_id: string }> { return async (events: z.input | z.input[]) => { let validatedEvents: z.output | z.output[] | undefined = undefined; - if (req.schema) { - const v = Array.isArray(events) - ? req.schema.array().safeParse(events) - : req.schema.safeParse(events); - if (!v.success) { - throw new Error(v.error.message); - } - validatedEvents = v.data; - } - - if (this.noop) { - return { executed: true, query_id: "noop" }; + const v = Array.isArray(events) + ? req.schema.array().safeParse(events) + : req.schema.safeParse(events); + if (!v.success) { + throw new Error(v.error.message); } + validatedEvents = v.data; return await this.client.insert({ table: req.table, diff --git a/internal/clickhouse-zod/src/index.ts b/internal/clickhouse-zod/src/index.ts index 5ec76921e1..0092e390e1 100644 --- a/internal/clickhouse-zod/src/index.ts +++ b/internal/clickhouse-zod/src/index.ts @@ -1 +1,3 @@ export * from "./client"; +export * from "./noop"; +export * from "./interface"; diff --git a/internal/clickhouse-zod/src/interface.ts b/internal/clickhouse-zod/src/interface.ts new file mode 100644 index 0000000000..e950fabc60 --- /dev/null +++ b/internal/clickhouse-zod/src/interface.ts @@ -0,0 +1,23 @@ +import type { z } from "zod"; + +export interface Clickhouse { + query, TOut extends z.ZodSchema>(req: { + // The SQL query to run. + // Use {paramName: Type} to define parameters + // Example: `SELECT * FROM table WHERE id = {id: String}` + query: string; + // The schema of the parameters + // Example: z.object({ id: z.string() }) + params?: TIn; + // The schema of the output of each row + // Example: z.object({ id: z.string() }) + schema: TOut; + }): (params: z.input) => Promise[]>; + + insert>(req: { + table: string; + schema: TSchema; + }): ( + events: z.input | z.input[], + ) => Promise<{ executed: boolean; query_id: string }>; +} diff --git a/internal/clickhouse-zod/src/noop.ts b/internal/clickhouse-zod/src/noop.ts new file mode 100644 index 0000000000..c5164b328f --- /dev/null +++ b/internal/clickhouse-zod/src/noop.ts @@ -0,0 +1,39 @@ +import type { z } from "zod"; +import type { Clickhouse } from "./interface"; +export class Noop implements Clickhouse { + public query, TOut extends z.ZodSchema>(req: { + // The SQL query to run. + // Use {paramName: Type} to define parameters + // Example: `SELECT * FROM table WHERE id = {id: String}` + query: string; + // The schema of the parameters + // Example: z.object({ id: z.string() }) + params?: TIn; + // The schema of the output of each row + // Example: z.object({ id: z.string() }) + schema: TOut; + }): (params: z.input) => Promise[]> { + return async (params: z.input): Promise[]> => { + req.params?.safeParse(params); + return []; + }; + } + + public insert>(req: { + table: string; + schema: TSchema; + }): ( + events: z.input | z.input[], + ) => Promise<{ executed: boolean; query_id: string }> { + return async (events: z.input | z.input[]) => { + const v = Array.isArray(events) + ? req.schema.array().safeParse(events) + : req.schema.safeParse(events); + if (!v.success) { + throw new Error(v.error.message); + } + + return { executed: true, query_id: "noop" }; + }; + } +}