-
Notifications
You must be signed in to change notification settings - Fork 544
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: replace more tinybird, add tests for clickhouse
- Loading branch information
Showing
20 changed files
with
990 additions
and
223 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import { expect, test } from "vitest"; | ||
| import { ClickHouse } from "./index"; | ||
|
|
||
| import { ClickHouseContainer } from "./testutil"; | ||
|
|
||
| test( | ||
| "inserts a single row", | ||
| { | ||
| timeout: 300_000, | ||
| }, | ||
| async (t) => { | ||
| const container = await ClickHouseContainer.start(t); | ||
|
|
||
| const ch = new ClickHouse({ url: container.url() }); | ||
|
|
||
| const verification = { | ||
| request_id: "1", | ||
| time: Date.now(), | ||
| workspace_id: "workspace_id", | ||
| key_space_id: "key_space_id", | ||
| key_id: "key_id", | ||
| outcome: "VALID", | ||
| region: "test", | ||
| } as const; | ||
|
|
||
| await ch.verifications.insert(verification); | ||
|
|
||
| const latestVerifications = await ch.verifications.logs({ | ||
| workspaceId: verification.workspace_id, | ||
| keySpaceId: verification.key_space_id, | ||
| keyId: verification.key_id, | ||
| }); | ||
|
|
||
| expect(latestVerifications.length).toBe(1); | ||
| expect(latestVerifications[0].time).toBe(verification.time); | ||
| expect(latestVerifications[0].outcome).toBe("VALID"); | ||
| expect(latestVerifications[0].region).toBe(verification.region); | ||
| }, | ||
| ); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| import { expect, test } from "vitest"; | ||
| import { ClickHouse } from "./index"; | ||
|
|
||
| import { randomUUID } from "node:crypto"; | ||
| import { ClickHouseContainer } from "./testutil"; | ||
|
|
||
| test( | ||
| "returns the correct amount of billable ratelimits", | ||
| { | ||
| timeout: 300_000, | ||
| }, | ||
| async (t) => { | ||
| const container = await ClickHouseContainer.start(t); | ||
|
|
||
| const ch = new ClickHouse({ url: container.url() }); | ||
|
|
||
| const workspaceId = randomUUID(); | ||
| const namespaceId = randomUUID(); | ||
| const now = new Date(); | ||
| const year = now.getUTCFullYear(); | ||
| const month = now.getUTCMonth() + 1; // 1 = January | ||
| const endTime = now.getTime(); | ||
| const startTime = now.setUTCDate(1); | ||
|
|
||
| let billable = 0; | ||
| for (let i = 0; i < 100; i++) { | ||
| const ratelimits = new Array(10_000).fill(null).map(() => { | ||
| const passed = Math.random() > 0.2; | ||
| if (passed) { | ||
| billable++; | ||
| } | ||
| return { | ||
| workspace_id: workspaceId, | ||
| namespace_id: namespaceId, | ||
| identifier: randomUUID(), | ||
| passed, | ||
| time: Math.floor(startTime + Math.random() * (endTime - startTime)), | ||
| request_id: randomUUID(), | ||
| }; | ||
| }); | ||
|
|
||
| await ch.ratelimits.insert(ratelimits); | ||
| } | ||
| // give clickhouse time to process all writes and update the materialized views | ||
| await new Promise((r) => setTimeout(r, 10_000)); | ||
|
|
||
| const billableRatelimits = await ch.billing.billableRatelimits({ | ||
| workspaceId, | ||
| year, | ||
| month, | ||
| }); | ||
|
|
||
| expect(billableRatelimits).toBe(billable); | ||
| }, | ||
| ); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| import { | ||
| GenericContainer, | ||
| GenericContainerBuilder, | ||
| Network, | ||
| type StartedTestContainer, | ||
| Wait, | ||
| } from "testcontainers"; | ||
| import type { TaskContext } from "vitest"; | ||
|
|
||
| export class ClickHouseContainer { | ||
| static readonly username = "default"; | ||
| static readonly password = "password"; | ||
| private readonly container: StartedTestContainer; | ||
|
|
||
| private constructor(container: StartedTestContainer) { | ||
| this.container = container; | ||
| } | ||
|
|
||
| public url(): string { | ||
| return `http://${ClickHouseContainer.username}:${ | ||
| ClickHouseContainer.password | ||
| }@${this.container.getHost()}:${this.container.getMappedPort(8123)}`; | ||
| } | ||
|
|
||
| // Stops the container | ||
| // | ||
| // You do not need to call this every time. | ||
| // The container is automatically stopped when the test ends | ||
| public async stop(): Promise<void> { | ||
| await this.container.stop(); | ||
| } | ||
|
|
||
| static async start(t: TaskContext): Promise<ClickHouseContainer> { | ||
| const network = await new Network().start(); | ||
|
|
||
| const container = await new GenericContainer("bitnami/clickhouse:latest") | ||
| .withEnvironment({ | ||
| CLICKHOUSE_ADMIN_USER: ClickHouseContainer.username, | ||
| CLICKHOUSE_ADMIN_PASSWORD: ClickHouseContainer.password, | ||
| }) | ||
| .withNetworkMode(network.getName()) | ||
| .withExposedPorts(8123, 9000) | ||
| .start(); | ||
| t.onTestFinished(async () => { | ||
| await container.stop(); | ||
| }); | ||
| const dsn = `tcp://${ClickHouseContainer.username}:${ClickHouseContainer.password}@${container | ||
| .getName() | ||
| .replace(/^\//, "")}:9000`; | ||
|
|
||
| const migratorImage = await new GenericContainerBuilder(".", "Dockerfile").build(); | ||
|
|
||
| const migrator = await migratorImage | ||
| .withEnvironment({ | ||
| GOOSE_DBSTRING: dsn, | ||
| }) | ||
| .withNetworkMode(network.getName()) | ||
| .withDefaultLogDriver() | ||
| .withWaitStrategy(Wait.forLogMessage("successfully migrated database")) | ||
| .start(); | ||
|
|
||
| t.onTestFinished(async () => { | ||
| await migrator.stop(); | ||
| }); | ||
|
|
||
| return new ClickHouseContainer(container); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| import { expect, test } from "vitest"; | ||
| import { ClickHouse } from "./index"; | ||
|
|
||
| import { randomUUID } from "node:crypto"; | ||
| import { ClickHouseContainer } from "./testutil"; | ||
|
|
||
| test( | ||
| "returns the correct amount of billable verifications", | ||
| { | ||
| timeout: 300_000, | ||
| }, | ||
| async (t) => { | ||
| const container = await ClickHouseContainer.start(t); | ||
|
|
||
| const ch = new ClickHouse({ url: container.url() }); | ||
|
|
||
| const workspaceId = randomUUID(); | ||
| const keySpaceId = randomUUID(); | ||
| const keyId = randomUUID(); | ||
| const now = new Date(); | ||
| const year = now.getUTCFullYear(); | ||
| const month = now.getUTCMonth() + 1; // 1 = January | ||
| const endTime = now.getTime(); | ||
| const startTime = now.setUTCDate(1); | ||
|
|
||
| const outcomes = ["VALID", "RATE_LIMITED", "DISABLED"] as const; | ||
| let valid = 0; | ||
| for (let i = 0; i < 100; i++) { | ||
| console.log({ i }); | ||
| const verifications = new Array(10_000).fill(null).map(() => { | ||
| const outcome = outcomes[Math.floor(Math.random() * outcomes.length)]; | ||
| if (outcome === "VALID") { | ||
| valid++; | ||
| } | ||
| return { | ||
| workspace_id: workspaceId, | ||
| key_space_id: keySpaceId, | ||
| key_id: keyId, | ||
| outcome, | ||
| time: Math.floor(startTime + Math.random() * (endTime - startTime)), | ||
| region: "test", | ||
| request_id: randomUUID(), | ||
| }; | ||
| }); | ||
|
|
||
| await ch.verifications.insert(verifications); | ||
| } | ||
| // give clickhouse time to process all writes and update the materialized views | ||
| await new Promise((r) => setTimeout(r, 10_000)); | ||
|
|
||
| const billableVerifications = await ch.billing.billableVerifications({ | ||
| workspaceId, | ||
| year, | ||
| month, | ||
| }); | ||
|
|
||
| expect(billableVerifications).toBe(valid); | ||
| }, | ||
| ); |
Oops, something went wrong.