feat: analytics api

.github/workflows/job_test_api_local.yaml

@@ -23,6 +23,7 @@ jobs:
         uses: ./.github/actions/install
         with:
           ts: true
+          go: true
 
       - name: Build
         run: pnpm turbo run build --filter=./apps/api...
@@ -34,6 +35,13 @@ jobs:
         env:
           DRIZZLE_DATABASE_URL: "mysql://unkey:password@localhost:3306/unkey"
 
+      - name: Migrate ClickHouse
+        run: goose up
+        env:
+          GOOSE_DRIVER: clickhouse
+          GOOSE_DBSTRING: "tcp://default:password@127.0.0.1:9000"
+          GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema
+
       - name: Test
         run: pnpm vitest run -c vitest.integration.ts
         working-directory: apps/api

apps/api/src/pkg/analytics.ts

@@ -46,4 +46,11 @@ export class Analytics {
   public get getVerificationsDaily() {
     return this.clickhouse.verifications.perDay;
   }
+
+  /**
+   * Use this sparingly, mostly for quick iterations
+   */
+  public get internalQuerier() {
+    return this.clickhouse.querier;
+  }
 }

apps/api/src/pkg/cache/index.ts

@@ -71,6 +71,7 @@ export function initCache(c: Context<HonoEnv>, metrics: Metrics): C<CacheNamespa
       c.executionCtx,
       defaultOpts,
     ),
+    identityById: new Namespace<CacheNamespaces["identityById"]>(c.executionCtx, defaultOpts),
     auditLogBucketByWorkspaceIdAndName: new Namespace<
       CacheNamespaces["auditLogBucketByWorkspaceIdAndName"]
     >(c.executionCtx, defaultOpts),

apps/api/src/pkg/cache/namespaces.ts

@@ -63,6 +63,7 @@ export type CacheNamespaces = {
     total: number;
   };
   identityByExternalId: Identity | null;
+  identityById: Identity | null;
   // uses a compound key of [workspaceId, name]
   auditLogBucketByWorkspaceIdAndName: {
     id: string;

apps/api/src/pkg/testutil/env.ts

@@ -4,6 +4,7 @@ export const databaseEnv = z.object({
   DATABASE_HOST: z.string().default("localhost:3900"),
   DATABASE_USERNAME: z.string().default("unkey"),
   DATABASE_PASSWORD: z.string().default("password"),
+  CLICKHOUSE_URL: z.string().default("http://default:password@localhost:8123"),
 });
 
 export const integrationTestEnv = databaseEnv.merge(

apps/api/src/pkg/testutil/harness.ts

@@ -1,4 +1,5 @@
 import { Client } from "@planetscale/database";
+import { ClickHouse } from "@unkey/clickhouse";
 import { sha256 } from "@unkey/hash";
 import { newId } from "@unkey/id";
 import { KeyV1 } from "@unkey/keys";
@@ -27,10 +28,12 @@ export type Resources = {
 
 export abstract class Harness {
   public readonly db: { primary: Database; readonly: Database };
+  public readonly ch: ClickHouse;
   public resources: Resources;
 
   constructor(t: TaskContext) {
-    const { DATABASE_HOST, DATABASE_PASSWORD, DATABASE_USERNAME } = databaseEnv.parse(process.env);
+    const { DATABASE_HOST, DATABASE_PASSWORD, DATABASE_USERNAME, CLICKHOUSE_URL } =
+      databaseEnv.parse(process.env);
 
     const db = drizzle(
       new Client({
@@ -51,6 +54,7 @@ export abstract class Harness {
     );
 
     this.db = { primary: db, readonly: db };
+    this.ch = new ClickHouse({ url: CLICKHOUSE_URL });
     this.resources = this.createResources();
 
     t.onTestFinished(async () => {
@@ -131,7 +135,11 @@ export abstract class Harness {
       name: string;
       permissions?: string[];
     }[];
-  }): Promise<{ keyId: string; key: string }> {
+  }): Promise<{
+    keyId: string;
+    key: string;
+    identityId?: string;
+  }> {
     /**
      * Prepare the key we'll use
      */
@@ -186,6 +194,7 @@ export abstract class Harness {
     return {
       keyId,
       key,
+      identityId: opts?.identityId,
     };
   }
 

apps/api/src/pkg/testutil/request.ts

@@ -3,6 +3,7 @@ export type StepRequest<TRequestBody> = {
   url: string;
   method: "POST" | "GET" | "PUT" | "DELETE";
   headers?: Record<string, string>;
+  searchparams?: Record<string, string | string[]>;
   body?: TRequestBody;
 };
 export type StepResponse<TBody = unknown> = {
@@ -14,7 +15,18 @@ export type StepResponse<TBody = unknown> = {
 export async function step<TRequestBody = unknown, TResponseBody = unknown>(
   req: StepRequest<TRequestBody>,
 ): Promise<StepResponse<TResponseBody>> {
-  const res = await fetch(req.url, {
+  const url = new URL(req.url);
+  for (const [k, vv] of Object.entries(req.searchparams ?? {})) {
+    if (Array.isArray(vv)) {
+      for (const v of vv) {
+        url.searchParams.append(k, v);
+      }
+    } else {
+      url.searchParams.append(k, vv);
+    }
+  }
+
+  const res = await fetch(url, {
     method: req.method,
     headers: req.headers,
     body: JSON.stringify(req.body),
@@ -28,7 +40,7 @@ export async function step<TRequestBody = unknown, TResponseBody = unknown>(
       body: JSON.parse(body),
     };
   } catch {
-    console.error(`${req.url} didn't return json, received: ${body}`);
+    console.error(`${url.toString()} didn't return json, received: ${body}`);
     return {} as StepResponse<TResponseBody>;
   }
 }