diff --git a/.github/workflows/job_test_api_local.yaml b/.github/workflows/job_test_api_local.yaml index 6677a22900..f7dded374f 100644 --- a/.github/workflows/job_test_api_local.yaml +++ b/.github/workflows/job_test_api_local.yaml @@ -30,8 +30,6 @@ jobs: uses: ./.github/actions/setup-go with: github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Goose - run: go install github.com/pressly/goose/v3/cmd/goose@latest - name: Build run: pnpm turbo run build --filter=./apps/api... env: @@ -42,12 +40,6 @@ jobs: env: DRIZZLE_DATABASE_URL: "mysql://unkey:password@localhost:3306/unkey" CI: 1 - - name: Migrate ClickHouse - run: goose up - env: - GOOSE_DRIVER: clickhouse - GOOSE_DBSTRING: "tcp://default:password@127.0.0.1:9000" - GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema - name: Test run: pnpm vitest run -c vitest.integration.ts --bail=1 working-directory: apps/api diff --git a/.github/workflows/job_test_unit.yaml b/.github/workflows/job_test_unit.yaml index 73303b47f5..f6f80f02a7 100644 --- a/.github/workflows/job_test_unit.yaml +++ b/.github/workflows/job_test_unit.yaml @@ -15,12 +15,6 @@ jobs: run: | sudo chmod 666 /var/run/docker.sock docker version - - name: Install goose - run: | - wget -qO- https://github.com/pressly/goose/releases/download/v3.20.0/goose_linux_x86_64 > /tmp/goose - chmod +x /tmp/goose - sudo mv /tmp/goose /usr/local/bin/goose - goose --version - name: Setup Node uses: ./.github/actions/setup-node with: diff --git a/Makefile b/Makefile index f756647b8a..b8e197a221 100644 --- a/Makefile +++ b/Makefile @@ -12,18 +12,6 @@ down: up: down build docker compose -f ./deployment/docker-compose.yaml up -d -migrate-clickhouse: - @export GOOSE_DRIVER=clickhouse && \ - export GOOSE_DBSTRING="tcp://default:password@127.0.0.1:9000" && \ - export GOOSE_MIGRATION_DIR=./internal/clickhouse/schema && \ - goose up - -migrate-clickhouse-reset: - @export GOOSE_DRIVER=clickhouse && \ - export GOOSE_DBSTRING="tcp://default:password@127.0.0.1:9000" && \ - export GOOSE_MIGRATION_DIR=./internal/clickhouse/schema && \ - goose down-to 0 - integration: up @cd apps/api && \ $(MAKE) seed && \ diff --git a/apps/engineering/content/docs/contributing/testing.mdx b/apps/engineering/content/docs/contributing/testing.mdx index 9431ca6a33..1201ae6f08 100644 --- a/apps/engineering/content/docs/contributing/testing.mdx +++ b/apps/engineering/content/docs/contributing/testing.mdx @@ -25,15 +25,3 @@ Full end to end tests require a running API either locally or in a deployed envi UNKEY_ROOT_KEY= pnpm test:integration ``` - - -### Clickhouse Tests - -`/internal/clickhouse` is where we store our ClickHouse queries. To run tests locally, please install [goose](https://pressly.github.io/goose). - -To install `goose`: - -```bash -go install github.com/pressly/goose/v4/cmd/goose@latest -``` - diff --git a/deployment/Dockerfile.clickhouse b/deployment/Dockerfile.clickhouse new file mode 100644 index 0000000000..a143627b8a --- /dev/null +++ b/deployment/Dockerfile.clickhouse @@ -0,0 +1,8 @@ +FROM bitnami/clickhouse:25.6.4 + +# Copy ClickHouse schemas +COPY go/pkg/clickhouse/schema/databases/ /opt/clickhouse-schemas/ + +# Create initialization script that will execute our SQL files on first run +# (script is already made executable on host) +COPY deployment/init-clickhouse.sh /docker-entrypoint-initdb.d/init-clickhouse.sh \ No newline at end of file diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 3b2bba38de..423088bf6e 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -17,7 +17,8 @@ services: volumes: - mysql:/var/lib/mysql healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-proot"] + test: + ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-proot"] timeout: 20s retries: 10 start_period: 40s @@ -25,7 +26,14 @@ services: planetscale: container_name: planetscale image: ghcr.io/mattrobenolt/ps-http-sim:v0.0.12 - command: ["-listen-port=3900", "-mysql-dbname=unkey", "-mysql-addr=mysql", "-mysql-max-rows=100000", "-mysql-idle-timeout=1s"] + command: + [ + "-listen-port=3900", + "-mysql-dbname=unkey", + "-mysql-addr=mysql", + "-mysql-max-rows=100000", + "-mysql-idle-timeout=1s", + ] depends_on: mysql: condition: service_healthy @@ -102,8 +110,10 @@ services: VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000" clickhouse: - image: bitnami/clickhouse:25.6.4 container_name: clickhouse + build: + context: .. + dockerfile: deployment/Dockerfile.clickhouse environment: CLICKHOUSE_ADMIN_USER: default CLICKHOUSE_ADMIN_PASSWORD: password @@ -113,18 +123,23 @@ services: volumes: - clickhouse:/bitnami/clickhouse healthcheck: - test: ["CMD", "clickhouse-client", "--host", "localhost", "--user", "default", "--password", "password", "--query", "SELECT 1"] + test: + [ + "CMD", + "clickhouse-client", + "--host", + "localhost", + "--user", + "default", + "--password", + "password", + "--query", + "SELECT 1", + ] timeout: 10s retries: 10 start_period: 30s interval: 5s - clickhouse_migrator: - container_name: clickhouse_migrator - build: - context: ../internal/clickhouse - dockerfile: ./Dockerfile - depends_on: - - clickhouse s3: container_name: s3 image: bitnami/minio:2025.4.3 @@ -151,7 +166,24 @@ services: dockerfile: ./apps/api/Dockerfile.dev ports: - 8787:8787 - command: ["pnpm", "wrangler", "dev", "--ip=0.0.0.0", "--var=VERSION:1", "--var=DATABASE_HOST:planetscale:3900", "--var=DATABASE_USERNAME:unkey", "--var=DATABASE_PASSWORD:password", "--var=AGENT_URL:http://agent:8080", "--var=AGENT_TOKEN:agent-auth-secret", "--var=EMIT_METRICS_LOGS:false", "--var=SYNC_RATELIMIT_ON_NO_DATA:1.0", "--var=CLICKHOUSE_URL:http://default:password@clickhouse:8123", "--var=CLICKHOUSE_INSERT_URL:http://apiv2:7070", "--var=CLICKHOUSE_PROXY_TOKEN:chproxy-test-token-123"] + command: + [ + "pnpm", + "wrangler", + "dev", + "--ip=0.0.0.0", + "--var=VERSION:1", + "--var=DATABASE_HOST:planetscale:3900", + "--var=DATABASE_USERNAME:unkey", + "--var=DATABASE_PASSWORD:password", + "--var=AGENT_URL:http://agent:8080", + "--var=AGENT_TOKEN:agent-auth-secret", + "--var=EMIT_METRICS_LOGS:false", + "--var=SYNC_RATELIMIT_ON_NO_DATA:1.0", + "--var=CLICKHOUSE_URL:http://default:password@clickhouse:8123", + "--var=CLICKHOUSE_INSERT_URL:http://apiv2:7070", + "--var=CLICKHOUSE_PROXY_TOKEN:chproxy-test-token-123", + ] depends_on: - planetscale - agent diff --git a/deployment/init-clickhouse.sh b/deployment/init-clickhouse.sh new file mode 100755 index 0000000000..2c3a1411c2 --- /dev/null +++ b/deployment/init-clickhouse.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +echo "Initializing ClickHouse schemas..." + +# Execute SQL files in order from our schemas directory +for db_dir in /opt/clickhouse-schemas/*/; do + if [ -d "$db_dir" ]; then + echo "Processing database directory: $db_dir" + + # Execute SQL files in numerical order + for sql_file in "$db_dir"*.sql; do + if [ -f "$sql_file" ] && [[ "$sql_file" == *.sql ]]; then + echo "Executing: $sql_file" + + if ! clickhouse-client --host localhost --user "$CLICKHOUSE_ADMIN_USER" --password "$CLICKHOUSE_ADMIN_PASSWORD" --queries-file "$sql_file"; then + echo "Error executing $sql_file - stopping initialization" + exit 1 + fi + fi + done + fi +done + +echo "ClickHouse schema initialization complete!" \ No newline at end of file diff --git a/go/Makefile b/go/Makefile index 1628da1954..7d482569d5 100644 --- a/go/Makefile +++ b/go/Makefile @@ -25,17 +25,6 @@ pull: up: pull @docker compose -f ../deployment/docker-compose.yaml up -d planetscale mysql redis clickhouse s3 otel - @echo "Starting ClickHouse migrations (will retry if ClickHouse isn't ready)..." - @for i in {1..10}; do \ - echo "Migration attempt $$i..."; \ - if docker compose -f ../deployment/docker-compose.yaml run --rm clickhouse_migrator; then \ - echo "Migrations completed successfully!"; \ - break; \ - else \ - echo "Migration failed, retrying in 5 seconds..."; \ - sleep 5; \ - fi; \ - done clean: @docker compose -f ../deployment/docker-compose.yaml down --volumes diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/002_raw_key_verifications_v2.sql b/go/pkg/clickhouse/schema/databases/001_verifications/002_raw_key_verifications_v2.sql new file mode 100644 index 0000000000..b8bf62d09e --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/002_raw_key_verifications_v2.sql @@ -0,0 +1,45 @@ +CREATE TABLE IF NOT EXISTS verifications.raw_key_verifications_v2( + -- the api request id, so we can correlate the verification with traces and logs + request_id String, + + -- unix milli + time Int64 CODEC(Delta, LZ4), + + workspace_id String, + key_space_id String, + -- Empty string if the key has no identity + identity_id String, + key_id String, + + -- Right now this is a 3 character airport code, but when we move to aws, + -- this will be the region code such as `us-east-1` + region LowCardinality(String), + + -- Examples: + -- - "VALID" + -- - "RATE_LIMITED" + -- - "EXPIRED" + -- - "DISABLED + outcome LowCardinality(String), + + + tags Array(String) DEFAULT [], + + -- The number of credits spent on this verification + -- 0 means no credits were spent + spent_credits Int64, + + -- Latency in milliseconds for this verification + latency Float64 + + +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(time)) +ORDER BY (workspace_id, time, key_space_id, identity_id, key_id) +TTL fromUnixTimestamp64Milli(time) + INTERVAL 100 DAY +SETTINGS non_replicated_deduplication_window = 10000 +; + +ALTER TABLE verifications.raw_key_verifications_v2 +ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/003_key_verifications_per_minute_v2.sql b/go/pkg/clickhouse/schema/databases/001_verifications/003_key_verifications_per_minute_v2.sql new file mode 100644 index 0000000000..cee4de4359 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/003_key_verifications_per_minute_v2.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_minute_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, key_space_id, identity_id, key_id, tags, outcome) +; \ No newline at end of file diff --git a/internal/clickhouse/schema/051_create_verifications.key_verifications_per_minute_mv_v1.sql b/go/pkg/clickhouse/schema/databases/001_verifications/004_key_verifications_per_minute_mv_v2.sql similarity index 52% rename from internal/clickhouse/schema/051_create_verifications.key_verifications_per_minute_mv_v1.sql rename to go/pkg/clickhouse/schema/databases/001_verifications/004_key_verifications_per_minute_mv_v2.sql index 2a358d855c..14ee0f659e 100644 --- a/internal/clickhouse/schema/051_create_verifications.key_verifications_per_minute_mv_v1.sql +++ b/go/pkg/clickhouse/schema/databases/001_verifications/004_key_verifications_per_minute_mv_v2.sql @@ -1,6 +1,5 @@ --- +goose up -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1 -TO verifications.key_verifications_per_minute_v1 +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v2 +TO verifications.key_verifications_per_minute_v2 AS SELECT workspace_id, @@ -9,9 +8,13 @@ SELECT key_id, outcome, count(*) as count, + sum(spent_credits) as spent_credits, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, tags -FROM verifications.raw_key_verifications_v1 +FROM verifications.raw_key_verifications_v2 GROUP BY workspace_id, key_space_id, @@ -20,7 +23,4 @@ GROUP BY outcome, time, tags -; - --- +goose down -DROP VIEW verifications.key_verifications_per_minute_mv_v1; +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/005_key_verifications_per_hour_v2.sql b/go/pkg/clickhouse/schema/databases/001_verifications/005_key_verifications_per_hour_v2.sql new file mode 100644 index 0000000000..af84baef28 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/005_key_verifications_per_hour_v2.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, key_space_id, identity_id, key_id) +; diff --git a/internal/clickhouse/schema/029_create_verifications.key_verifications_per_hour_mv_v2.sql b/go/pkg/clickhouse/schema/databases/001_verifications/006_key_verifications_per_hour_mv_v2.sql similarity index 50% rename from internal/clickhouse/schema/029_create_verifications.key_verifications_per_hour_mv_v2.sql rename to go/pkg/clickhouse/schema/databases/001_verifications/006_key_verifications_per_hour_mv_v2.sql index 23bd8f56d2..c4f94a573d 100644 --- a/internal/clickhouse/schema/029_create_verifications.key_verifications_per_hour_mv_v2.sql +++ b/go/pkg/clickhouse/schema/databases/001_verifications/006_key_verifications_per_hour_mv_v2.sql @@ -1,5 +1,4 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_hour_mv_v2 +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v2 TO verifications.key_verifications_per_hour_v2 AS SELECT @@ -9,9 +8,13 @@ SELECT key_id, outcome, count(*) as count, + sum(spent_credits) as spent_credits, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags -FROM verifications.raw_key_verifications_v1 +FROM verifications.raw_key_verifications_v2 GROUP BY workspace_id, key_space_id, @@ -21,7 +24,3 @@ GROUP BY time, tags ; - - --- +goose down -DROP VIEW verifications.key_verifications_per_hour_mv_v2; diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/007_key_verifications_per_day_v4.sql b/go/pkg/clickhouse/schema/databases/001_verifications/007_key_verifications_per_day_v4.sql new file mode 100644 index 0000000000..5da6b805b9 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/007_key_verifications_per_day_v4.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v4 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, key_space_id, identity_id, key_id, tags, outcome) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/008_key_verifications_per_day_mv_v4.sql b/go/pkg/clickhouse/schema/databases/001_verifications/008_key_verifications_per_day_mv_v4.sql new file mode 100644 index 0000000000..19b1ad3e90 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/008_key_verifications_per_day_mv_v4.sql @@ -0,0 +1,26 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v4 +TO verifications.key_verifications_per_day_v4 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v2 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/009_key_verifications_per_month_v4.sql b/go/pkg/clickhouse/schema/databases/001_verifications/009_key_verifications_per_month_v4.sql new file mode 100644 index 0000000000..6b595b7a95 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/009_key_verifications_per_month_v4.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v4 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64, + spent_credits Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, key_space_id, identity_id, key_id, tags, outcome) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/010_key_verifications_per_month_mv_v4.sql b/go/pkg/clickhouse/schema/databases/001_verifications/010_key_verifications_per_month_mv_v4.sql new file mode 100644 index 0000000000..57a97f9dac --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/010_key_verifications_per_month_mv_v4.sql @@ -0,0 +1,26 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v4 +TO verifications.key_verifications_per_month_v4 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + sum(spent_credits) as spent_credits, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v2 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/011_temp_sync_v1_to_v2.sql b/go/pkg/clickhouse/schema/databases/001_verifications/011_temp_sync_v1_to_v2.sql new file mode 100644 index 0000000000..109b9605bb --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/011_temp_sync_v1_to_v2.sql @@ -0,0 +1,20 @@ +-- Temporary materialized view to sync new writes from v1 to v2 during migration +-- This ensures zero-downtime migration by duplicating all new inserts +-- DROP this view after migration is complete and application switches to v2 + +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.temp_sync_v1_to_v2 +TO verifications.raw_key_verifications_v2 +AS +SELECT + request_id, + time, + workspace_id, + key_space_id, + identity_id, + key_id, + region, + outcome, + tags, + 0 as spent_credits, -- v1 doesn't have this column, default to 0 + 0.0 as latency -- v1 doesn't have this column, default to 0.0 +FROM verifications.raw_key_verifications_v1; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/001_verifications/MIGRATION_v1_to_v2.md b/go/pkg/clickhouse/schema/databases/001_verifications/MIGRATION_v1_to_v2.md new file mode 100644 index 0000000000..a3e5aeb1e2 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/001_verifications/MIGRATION_v1_to_v2.md @@ -0,0 +1,150 @@ +# Migration Guide: Raw Key Verifications v1 to v2 + +This guide explains how to migrate from `raw_key_verifications_v1` to `raw_key_verifications_v2` with zero downtime. + +## What's Changed in v2 + +- ✅ **Partitioning**: Monthly partitions for better performance and lifecycle management +- ✅ **TTL**: Automatic data cleanup after 3 months +- ✅ **Deduplication**: Protection against duplicate inserts during retries +- ✅ **Sub-millisecond latency**: Float64 for precise latency measurements +- ✅ **Credit tracking**: New `spent_credits` column +- ✅ **Better ordering**: Optimized ORDER BY for time-series queries + +## Migration Steps + +### Phase 1: Deploy New Schema (Zero Downtime) + +1. **Deploy the new schema files** - This creates v2 tables and temporary sync: + ```bash + # The auto-migration will create: + # - raw_key_verifications_v2 (new table) + # - All new aggregation tables (minute/hour/day/month v2/v4) + # - temp_sync_v1_to_v2 (materialized view for live sync) + ``` + +2. **Verify deployment** - Check that all new tables exist: + ```sql + SHOW TABLES FROM verifications LIKE '%v2%'; + SHOW TABLES FROM verifications LIKE '%v4%'; + ``` + +### Phase 2: Backfill Historical Data + +**⚠️ Important**: The temporary materialized view (`temp_sync_v1_to_v2`) is now active and syncing all NEW data from v1 to v2. You can safely backfill historical data while new writes continue. + +Run these queries **manually** to backfill historical data in chunks: + +#### Step 1: Find your data range +```sql +SELECT + toDateTime(min(time)/1000) as earliest_data, + toDateTime(max(time)/1000) as latest_data, + count(*) as total_rows +FROM verifications.raw_key_verifications_v1; +``` + +#### Step 2: Backfill month by month +Replace the date ranges based on your actual data from Step 1: + +```sql +-- January 2024 (example - adjust dates) +INSERT INTO verifications.raw_key_verifications_v2 +SELECT + request_id, + time, + workspace_id, + key_space_id, + identity_id, + key_id, + region, + outcome, + tags, + 0 as spent_credits, -- v1 doesn't have this column, default to 0 + 0.0 as latency -- v1 doesn't have this column, default to 0.0 +FROM verifications.raw_key_verifications_v1 +WHERE time >= toUnixTimestamp64Milli(toDateTime('2024-01-01 00:00:00')) + AND time < toUnixTimestamp64Milli(toDateTime('2024-02-01 00:00:00')); + +-- February 2024 +INSERT INTO verifications.raw_key_verifications_v2 +SELECT + request_id, + time, + workspace_id, + key_space_id, + identity_id, + key_id, + region, + outcome, + tags, + 0 as spent_credits, + 0.0 as latency +FROM verifications.raw_key_verifications_v1 +WHERE time >= toUnixTimestamp64Milli(toDateTime('2024-02-01 00:00:00')) + AND time < toUnixTimestamp64Milli(toDateTime('2024-03-01 00:00:00')); + +-- Continue for each month until you reach current data... +``` + +#### Step 3: Monitor progress +```sql +-- Check backfill progress +SELECT + 'v1' as version, + toDateTime(min(time)/1000) as earliest, + toDateTime(max(time)/1000) as latest, + count(*) as rows +FROM verifications.raw_key_verifications_v1 +UNION ALL +SELECT + 'v2' as version, + toDateTime(min(time)/1000) as earliest, + toDateTime(max(time)/1000) as latest, + count(*) as rows +FROM verifications.raw_key_verifications_v2 +ORDER BY version; +``` + +### Phase 3: Switch Application to v2 + +1. **Update your application** to write to `raw_key_verifications_v2` instead of `v1` +2. **Include the new fields**: + - `spent_credits` (Int64) - credits consumed by this verification + - `latency` (Float64) - latency in milliseconds with sub-ms precision + +### Phase 4: Cleanup (After Confirming v2 Works) + +1. **Drop the temporary sync view**: + ```sql + DROP VIEW verifications.temp_sync_v1_to_v2; + ``` + +2. **Optional: Keep v1 as backup** or drop it: + ```sql + -- To drop v1 (only after confirming v2 works perfectly) + DROP TABLE verifications.raw_key_verifications_v1; + ``` + +## Rollback Plan + +If you need to rollback: + +1. **Switch application** back to writing to `v1` +2. **Keep the temp sync view** active +3. **Investigate issues** with v2 +4. **Re-run migration** when ready + +## Important Notes + +- 🔄 **Zero downtime**: New data automatically flows to v2 via the temp sync view +- 📊 **Historical data**: Will have `spent_credits=0` and `latency=0.0` (can be updated later if needed) +- 🗓️ **TTL**: v2 data will auto-delete after 3 months +- 🔍 **Deduplication**: Protects against duplicate inserts during retries +- 📈 **Performance**: Monthly partitions improve query performance on time ranges + +## Questions? + +- Check the temp sync view is active: `SELECT * FROM system.tables WHERE name = 'temp_sync_v1_to_v2'` +- Monitor both tables during migration to ensure data consistency +- The aggregation tables (minute/hour/day/month) will automatically populate from v2 data \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/002_raw_ratelimits_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/002_raw_ratelimits_v2.sql new file mode 100644 index 0000000000..6231179fc8 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/002_raw_ratelimits_v2.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS ratelimits.raw_ratelimits_v2( + -- the request id for correlation with traces and logs + request_id String, + + -- unix milli + time Int64 CODEC(Delta, LZ4), + + workspace_id String, + namespace_id String, + identifier String, + + -- whether the ratelimit check passed or was blocked + passed Bool, + + -- Latency in milliseconds for this ratelimit check + latency Float64 + + +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(time)) +ORDER BY (workspace_id, time, namespace_id, identifier) +TTL fromUnixTimestamp64Milli(time) + INTERVAL 100 DAY +SETTINGS non_replicated_deduplication_window = 10000 +; + +ALTER TABLE ratelimits.raw_ratelimits_v2 +ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/003_ratelimits_per_minute_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/003_ratelimits_per_minute_v2.sql new file mode 100644 index 0000000000..e006375dc0 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/003_ratelimits_per_minute_v2.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_minute_v2 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + passed Int64, + total Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, namespace_id, identifier) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/004_ratelimits_per_minute_mv_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/004_ratelimits_per_minute_mv_v2.sql new file mode 100644 index 0000000000..2c125d1ae4 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/004_ratelimits_per_minute_mv_v2.sql @@ -0,0 +1,20 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_minute_mv_v2 +TO ratelimits.ratelimits_per_minute_v2 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v2 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/005_ratelimits_per_hour_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/005_ratelimits_per_hour_v2.sql new file mode 100644 index 0000000000..f85f1656b7 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/005_ratelimits_per_hour_v2.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_hour_v2 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + passed Int64, + total Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, namespace_id, identifier) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/006_ratelimits_per_hour_mv_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/006_ratelimits_per_hour_mv_v2.sql new file mode 100644 index 0000000000..c91bbd42ec --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/006_ratelimits_per_hour_mv_v2.sql @@ -0,0 +1,20 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_hour_mv_v2 +TO ratelimits.ratelimits_per_hour_v2 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v2 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/007_ratelimits_per_day_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/007_ratelimits_per_day_v2.sql new file mode 100644 index 0000000000..5a4defb924 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/007_ratelimits_per_day_v2.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_day_v2 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + passed Int64, + total Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, namespace_id, identifier) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/008_ratelimits_per_day_mv_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/008_ratelimits_per_day_mv_v2.sql new file mode 100644 index 0000000000..f6068c3eec --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/008_ratelimits_per_day_mv_v2.sql @@ -0,0 +1,20 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_day_mv_v2 +TO ratelimits.ratelimits_per_day_v2 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v2 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/009_ratelimits_per_month_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/009_ratelimits_per_month_v2.sql new file mode 100644 index 0000000000..8427763c18 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/009_ratelimits_per_month_v2.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_month_v2 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + passed Int64, + total Int64, + latency_avg Float64, + latency_p75 Float64, + latency_p99 Float64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, time, namespace_id, identifier) +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/010_ratelimits_per_month_mv_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/010_ratelimits_per_month_mv_v2.sql new file mode 100644 index 0000000000..4cbf92df66 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/010_ratelimits_per_month_mv_v2.sql @@ -0,0 +1,20 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_month_mv_v2 +TO ratelimits.ratelimits_per_month_v2 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + avg(latency) as latency_avg, + quantileTDigest(0.75)(latency) as latency_p75, + quantileTDigest(0.99)(latency) as latency_p99, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v2 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/011_ratelimits_last_used_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/011_ratelimits_last_used_v2.sql new file mode 100644 index 0000000000..6ed0f1b870 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/011_ratelimits_last_used_v2.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_last_used_v2 +( + time Int64, + workspace_id String, + namespace_id String, + identifier String + +) +ENGINE = AggregatingMergeTree() +ORDER BY (workspace_id, time, namespace_id, identifier) +; diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/012_ratelimits_last_used_mv_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/012_ratelimits_last_used_mv_v2.sql new file mode 100644 index 0000000000..46b5dc7a21 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/012_ratelimits_last_used_mv_v2.sql @@ -0,0 +1,14 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_last_used_mv_v2 +TO ratelimits.ratelimits_last_used_v2 +AS +SELECT + workspace_id, + namespace_id, + identifier, + maxState(time) as time +FROM ratelimits.raw_ratelimits_v2 +GROUP BY + workspace_id, + namespace_id, + identifier +; diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/013_temp_sync_v1_to_v2.sql b/go/pkg/clickhouse/schema/databases/002_ratelimits/013_temp_sync_v1_to_v2.sql new file mode 100644 index 0000000000..62ee377044 --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/013_temp_sync_v1_to_v2.sql @@ -0,0 +1,16 @@ +-- Temporary materialized view to sync new writes from v1 to v2 during migration +-- This ensures zero-downtime migration by duplicating all new inserts +-- DROP this view after migration is complete and application switches to v2 + +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.temp_sync_v1_to_v2 +TO ratelimits.raw_ratelimits_v2 +AS +SELECT + request_id, + time, + workspace_id, + namespace_id, + identifier, + passed, + 0.0 as latency -- v1 doesn't have this column, default to 0.0 +FROM ratelimits.raw_ratelimits_v1; \ No newline at end of file diff --git a/go/pkg/clickhouse/schema/databases/002_ratelimits/MIGRATION_v1_to_v2.md b/go/pkg/clickhouse/schema/databases/002_ratelimits/MIGRATION_v1_to_v2.md new file mode 100644 index 0000000000..286a6f009c --- /dev/null +++ b/go/pkg/clickhouse/schema/databases/002_ratelimits/MIGRATION_v1_to_v2.md @@ -0,0 +1,150 @@ +# Migration Guide: Raw Ratelimits v1 to v2 + +This guide explains how to migrate from `raw_ratelimits_v1` to `raw_ratelimits_v2` with zero downtime. + +## What's Changed in v2 + +- ✅ **Latency tracking**: New `latency` column for ratelimit check performance +- ✅ **Partitioning**: Monthly partitions for better performance and lifecycle management +- ✅ **TTL**: Automatic data cleanup after 3 months +- ✅ **Deduplication**: Protection against duplicate inserts during retries +- ✅ **Better ordering**: Optimized ORDER BY for time-series queries +- ✅ **Latency aggregations**: avg, p75, p99 latency metrics in all aggregation tables + +## Migration Steps + +### Phase 1: Deploy New Schema (Zero Downtime) + +1. **Deploy the new schema files** - This creates v2 tables and temporary sync: + ```bash + # The auto-migration will create: + # - raw_ratelimits_v2 (new table) + # - All new aggregation tables (minute/hour/day/month v2) + # - temp_sync_v1_to_v2 (materialized view for live sync) + ``` + +2. **Verify deployment** - Check that all new tables exist: + ```sql + SHOW TABLES FROM ratelimits LIKE '%v2%'; + ``` + +### Phase 2: Backfill Historical Data + +**⚠️ Important**: The temporary materialized view (`temp_sync_v1_to_v2`) is now active and syncing all NEW data from v1 to v2. You can safely backfill historical data while new writes continue. + +Run these queries **manually** to backfill historical data in chunks: + +#### Step 1: Find your data range +```sql +SELECT + toDateTime(min(time)/1000) as earliest_data, + toDateTime(max(time)/1000) as latest_data, + count(*) as total_rows +FROM ratelimits.raw_ratelimits_v1; +``` + +#### Step 2: Backfill month by month +Replace the date ranges based on your actual data from Step 1: + +```sql +-- January 2024 (example - adjust dates) +INSERT INTO ratelimits.raw_ratelimits_v2 +SELECT + request_id, + time, + workspace_id, + namespace_id, + identifier, + passed, + 0.0 as latency -- v1 doesn't have this column, default to 0.0 +FROM ratelimits.raw_ratelimits_v1 +WHERE time >= toUnixTimestamp64Milli(toDateTime('2024-01-01 00:00:00')) + AND time < toUnixTimestamp64Milli(toDateTime('2024-02-01 00:00:00')); + +-- February 2024 +INSERT INTO ratelimits.raw_ratelimits_v2 +SELECT + request_id, + time, + workspace_id, + namespace_id, + identifier, + passed, + 0.0 as latency +FROM ratelimits.raw_ratelimits_v1 +WHERE time >= toUnixTimestamp64Milli(toDateTime('2024-02-01 00:00:00')) + AND time < toUnixTimestamp64Milli(toDateTime('2024-03-01 00:00:00')); + +-- Continue for each month until you reach current data... +``` + +#### Step 3: Monitor progress +```sql +-- Check backfill progress +SELECT + 'v1' as version, + toDateTime(min(time)/1000) as earliest, + toDateTime(max(time)/1000) as latest, + count(*) as rows +FROM ratelimits.raw_ratelimits_v1 +UNION ALL +SELECT + 'v2' as version, + toDateTime(min(time)/1000) as earliest, + toDateTime(max(time)/1000) as latest, + count(*) as rows +FROM ratelimits.raw_ratelimits_v2 +ORDER BY version; +``` + +### Phase 3: Switch Application to v2 + +1. **Update your application** to write to `raw_ratelimits_v2` instead of `v1` +2. **Include the new field**: + - `latency` (Float64) - ratelimit check latency in milliseconds with sub-ms precision + +### Phase 4: Cleanup (After Confirming v2 Works) + +1. **Drop the temporary sync view**: + ```sql + DROP VIEW ratelimits.temp_sync_v1_to_v2; + ``` + +2. **Optional: Keep v1 as backup** or drop it: + ```sql + -- To drop v1 (only after confirming v2 works perfectly) + DROP TABLE ratelimits.raw_ratelimits_v1; + ``` + +## Rollback Plan + +If you need to rollback: + +1. **Switch application** back to writing to `v1` +2. **Keep the temp sync view** active +3. **Investigate issues** with v2 +4. **Re-run migration** when ready + +## Important Notes + +- 🔄 **Zero downtime**: New data automatically flows to v2 via the temp sync view +- 📊 **Historical data**: Will have `latency=0.0` (can be updated later when you add latency tracking) +- 🗓️ **TTL**: v2 data will auto-delete after 3 months +- 🔍 **Deduplication**: Protects against duplicate inserts during retries +- 📈 **Performance**: Monthly partitions improve query performance on time ranges +- ⚡ **TDigest quantiles**: Efficient approximate percentile calculations for latency + +## New Aggregation Metrics Available + +With v2, all aggregation tables now include: +- `latency_avg`: Average ratelimit check latency +- `latency_p75`: 75th percentile latency +- `latency_p99`: 99th percentile latency + +Perfect for monitoring ratelimit performance and identifying bottlenecks! + +## Questions? + +- Check the temp sync view is active: `SELECT * FROM system.tables WHERE name = 'temp_sync_v1_to_v2'` +- Monitor both tables during migration to ensure data consistency +- The aggregation tables (minute/hour/day/month) will automatically populate from v2 data \ No newline at end of file diff --git a/go/pkg/db/retry.go b/go/pkg/db/retry.go index e9bc9f95cf..fa768a53a1 100644 --- a/go/pkg/db/retry.go +++ b/go/pkg/db/retry.go @@ -32,7 +32,7 @@ func WithRetry[T any](fn func() (T, error)) (T, error) { // Predefined backoff delays: 50ms, 100ms, 200ms delays := []time.Duration{ DefaultBackoff, // 50ms for attempt 1 - DefaultBackoff * 2, // 100ms for attempt 2 + DefaultBackoff * 2, // 100ms for attempt 2 DefaultBackoff * 4, // 200ms for attempt 3 } if n <= 0 || n > len(delays) { diff --git a/go/pkg/retry/retry_test.go b/go/pkg/retry/retry_test.go index 5f6234026c..259ff4b29d 100644 --- a/go/pkg/retry/retry_test.go +++ b/go/pkg/retry/retry_test.go @@ -136,7 +136,7 @@ func failNTimes(n int) func() error { func TestShouldRetry(t *testing.T) { nonRetryableError := errors.New("non-retryable") retryableError := errors.New("retryable") - + tests := []struct { name string shouldRetry func(error) bool @@ -146,8 +146,8 @@ func TestShouldRetry(t *testing.T) { expectedSleep time.Duration }{ { - name: "should retry all errors by default", - shouldRetry: nil, // default behavior + name: "should retry all errors by default", + shouldRetry: nil, // default behavior errorSequence: []error{retryableError, retryableError, nil}, expectedCalls: 3, expectedError: nil, @@ -184,7 +184,7 @@ func TestShouldRetry(t *testing.T) { expectedSleep: 300 * time.Millisecond, // 100ms + 200ms }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var retrier *retry @@ -193,12 +193,12 @@ func TestShouldRetry(t *testing.T) { } else { retrier = New() } - + totalSleep := time.Duration(0) retrier.sleep = func(d time.Duration) { totalSleep += d } - + calls := 0 err := retrier.Do(func() error { if calls < len(tt.errorSequence) { @@ -209,10 +209,10 @@ func TestShouldRetry(t *testing.T) { calls++ return nil }) - + require.Equal(t, tt.expectedCalls, calls, "unexpected number of calls") require.Equal(t, tt.expectedSleep, totalSleep, "unexpected sleep duration") - + if tt.expectedError != nil { require.Error(t, err) require.Equal(t, tt.expectedError, err) diff --git a/internal/clickhouse/Dockerfile b/internal/clickhouse/Dockerfile deleted file mode 100644 index ceb5092021..0000000000 --- a/internal/clickhouse/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang - - -RUN go install github.com/pressly/goose/v3/cmd/goose@latest - - -COPY ./schema ./schema - -ENV GOOSE_DRIVER=clickhouse -ENV GOOSE_DBSTRING="tcp://default:password@clickhouse:9000" -ENV GOOSE_MIGRATION_DIR=./schema -CMD ["goose", "up"] diff --git a/internal/clickhouse/package.json b/internal/clickhouse/package.json index fe5ff97988..fe9e24d925 100644 --- a/internal/clickhouse/package.json +++ b/internal/clickhouse/package.json @@ -8,7 +8,8 @@ "author": "Andreas Thomas", "license": "AGPL-3.0", "scripts": { - "test": "vitest run" + "build-clickhouse": "docker compose -f ../../deployment/docker-compose.yaml build clickhouse", + "test": "pnpm build-clickhouse && vitest run" }, "devDependencies": { "@dagger.io/dagger": "^0.14.0", diff --git a/internal/clickhouse/schema/000_README.md b/internal/clickhouse/schema/000_README.md deleted file mode 100644 index b1d380abf9..0000000000 --- a/internal/clickhouse/schema/000_README.md +++ /dev/null @@ -1,72 +0,0 @@ - -# ClickHouse Table Naming Conventions - -This document outlines the naming conventions for tables and materialized views in our ClickHouse setup. Adhering to these conventions ensures consistency, clarity, and ease of management across our data infrastructure. - -## General Rules - -1. Use lowercase letters and separate words with underscores. -2. Avoid ClickHouse reserved words and special characters in names. -3. Be descriptive but concise. - -## Table Naming Convention - -Format: `[prefix]_[domain]_[description]_[version]` - -### Prefixes - -- `raw_`: Input data tables -- `tmp_{yourname}_`: Temporary tables for experiments, add your name, so it's easy to identify ownership. - -### Domain/Category - -Include the domain or category of the data when applicable. - -Examples: -- `keys` -- `audit` -- `user` -- `gateway` - -### Versioning - -- Version numbers: `_v1`, `_v2`, etc. - -### Aggregation Suffixes - -For aggregated or summary tables, use suffixes like: -- `_per_day` -- `_per_month` -- `_summary` - -## Materialized View Naming Convention - -Format: `[description]_[aggregation]_mv_[version]` - -- Always suffix with `mv_[version]` -- Include a description of the view's purpose -- Add aggregation level if applicable - -## Examples - -1. Raw Data Table: - `raw_sales_transactions_v1` - -2. Materialized View: - `active_users_per_day_mv_v2` - -3. Temporary Table: - `tmp_andreas_user_analysis_v1` - -4. Aggregated Table: - `sales_summary_per_hour_mv_v1` - -## Consistency Across Related Objects - -Maintain consistent naming across related tables, views, and other objects: - -- `raw_user_activity_v1` -- `user_activity_per_day_v1` -- `user_activity_per_day_mv_v1` - -By following these conventions, we ensure a clear, consistent, and scalable naming structure for our ClickHouse setup. diff --git a/internal/clickhouse/schema/001_create_databases.sql b/internal/clickhouse/schema/001_create_databases.sql deleted file mode 100644 index c8b10934e7..0000000000 --- a/internal/clickhouse/schema/001_create_databases.sql +++ /dev/null @@ -1,17 +0,0 @@ --- +goose up - -CREATE DATABASE verifications; -CREATE DATABASE telemetry; -CREATE DATABASE metrics; -CREATE DATABASE ratelimits; -CREATE DATABASE business; -CREATE DATABASE billing; - - --- +goose down -DROP DATABASE verifications; -DROP DATABASE telemetry; -DROP DATABASE metrics; -DROP DATABASE ratelimits; -DROP DATABASE business; -DROP DATABASE billing; diff --git a/internal/clickhouse/schema/002_create_metrics_raw_api_requests_v1.sql b/internal/clickhouse/schema/002_create_metrics_raw_api_requests_v1.sql deleted file mode 100644 index 829f5f26e2..0000000000 --- a/internal/clickhouse/schema/002_create_metrics_raw_api_requests_v1.sql +++ /dev/null @@ -1,43 +0,0 @@ --- +goose up -CREATE TABLE metrics.raw_api_requests_v1( - request_id String, - -- unix milli - time Int64, - - workspace_id String, - - host String, - - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - path String, - -- "Key: Value" pairs - request_headers Array(String), - request_body String, - - response_status Int, - -- "Key: Value" pairs - response_headers Array(String), - response_body String, - -- internal err.Error() string, empty if no error - error String, - - -- milliseconds - service_latency Int64, - - user_agent String, - ip_address String, - country String, - city String, - colo String, - continent String, - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, time, request_id) -; - --- +goose down -DROP TABLE metrics.raw_api_requests_v1; diff --git a/internal/clickhouse/schema/003_create_verifications_raw_key_verifications_v1.sql b/internal/clickhouse/schema/003_create_verifications_raw_key_verifications_v1.sql deleted file mode 100644 index 9edb56bd49..0000000000 --- a/internal/clickhouse/schema/003_create_verifications_raw_key_verifications_v1.sql +++ /dev/null @@ -1,35 +0,0 @@ --- +goose up -CREATE TABLE verifications.raw_key_verifications_v1( - -- the api request id, so we can correlate the verification with traces and logs - request_id String, - - -- unix milli - time Int64, - - workspace_id String, - key_space_id String, - key_id String, - - -- Right now this is a 3 character airport code, but when we move to aws, - -- this will be the region code such as `us-east-1` - region LowCardinality(String), - - -- Examples: - -- - "VALID" - -- - "RATE_LIMITED" - -- - "EXPIRED" - -- - "DISABLED - outcome LowCardinality(String), - - -- Empty string if the key has no identity - identity_id String, - - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, key_space_id, key_id, time) -; - --- +goose down -DROP TABLE verifications.raw_key_verifications_v1; diff --git a/internal/clickhouse/schema/004_create_verifications_key_verifications_per_hour_v1.sql b/internal/clickhouse/schema/004_create_verifications_key_verifications_per_hour_v1.sql deleted file mode 100644 index bfdf27092a..0000000000 --- a/internal/clickhouse/schema/004_create_verifications_key_verifications_per_hour_v1.sql +++ /dev/null @@ -1,18 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_hour_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_hour_v1; diff --git a/internal/clickhouse/schema/005_create_verifications_key_verifications_per_day_v1.sql b/internal/clickhouse/schema/005_create_verifications_key_verifications_per_day_v1.sql deleted file mode 100644 index d2d8ceb2e6..0000000000 --- a/internal/clickhouse/schema/005_create_verifications_key_verifications_per_day_v1.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_day_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_day_v1; - diff --git a/internal/clickhouse/schema/006_create_verifications_key_verifications_per_month_v1.sql b/internal/clickhouse/schema/006_create_verifications_key_verifications_per_month_v1.sql deleted file mode 100644 index f849f136ed..0000000000 --- a/internal/clickhouse/schema/006_create_verifications_key_verifications_per_month_v1.sql +++ /dev/null @@ -1,17 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_month_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; - --- +goose down -DROP TABLE verifications.key_verifications_per_month_v1; diff --git a/internal/clickhouse/schema/007_create_verifications_key_verifications_per_hour_mv_v1.sql b/internal/clickhouse/schema/007_create_verifications_key_verifications_per_hour_mv_v1.sql deleted file mode 100644 index 4219ab2404..0000000000 --- a/internal/clickhouse/schema/007_create_verifications_key_verifications_per_hour_mv_v1.sql +++ /dev/null @@ -1,25 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_hour_mv_v1 -TO verifications.key_verifications_per_hour_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; - - --- +goose down -DROP VIEW verifications.key_verifications_per_hour_mv_v1; diff --git a/internal/clickhouse/schema/008_create_verifications_key_verifications_per_day_mv_v1.sql b/internal/clickhouse/schema/008_create_verifications_key_verifications_per_day_mv_v1.sql deleted file mode 100644 index 76a050afd2..0000000000 --- a/internal/clickhouse/schema/008_create_verifications_key_verifications_per_day_mv_v1.sql +++ /dev/null @@ -1,25 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_day_mv_v1 -TO verifications.key_verifications_per_day_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; - - --- +goose down -DROP VIEW verifications.key_verifications_per_day_mv_v1; diff --git a/internal/clickhouse/schema/009_create_verifications_key_verifications_per_month_mv_v1.sql b/internal/clickhouse/schema/009_create_verifications_key_verifications_per_month_mv_v1.sql deleted file mode 100644 index e37e73bf4f..0000000000 --- a/internal/clickhouse/schema/009_create_verifications_key_verifications_per_month_mv_v1.sql +++ /dev/null @@ -1,25 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_month_mv_v1 -TO verifications.key_verifications_per_month_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; - - --- +goose down -DROP VIEW verifications.key_verifications_per_month_mv_v1; diff --git a/internal/clickhouse/schema/010_create_ratelimits_raw_ratelimits_table.sql b/internal/clickhouse/schema/010_create_ratelimits_raw_ratelimits_table.sql deleted file mode 100644 index 755dd1d95d..0000000000 --- a/internal/clickhouse/schema/010_create_ratelimits_raw_ratelimits_table.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.raw_ratelimits_v1( - request_id String, - -- unix milli - time Int64, - workspace_id String, - namespace_id String, - identifier String, - passed Bool - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - --- +goose down -DROP TABLE ratelimits.raw_ratelimits_v1; diff --git a/internal/clickhouse/schema/011_create_telemetry_raw_sdks_v1.sql b/internal/clickhouse/schema/011_create_telemetry_raw_sdks_v1.sql deleted file mode 100644 index 3b7a6ad67c..0000000000 --- a/internal/clickhouse/schema/011_create_telemetry_raw_sdks_v1.sql +++ /dev/null @@ -1,24 +0,0 @@ --- +goose up -CREATE TABLE telemetry.raw_sdks_v1( - -- the api request id, so we can correlate the telemetry with traces and logs - request_id String, - - -- unix milli - time Int64, - - -- ie: node@20 - runtime String, - -- ie: vercel - platform String, - - -- ie: [ "@unkey/api@1.2.3", "@unkey/ratelimit@4.5.6" ] - versions Array(String) -) -ENGINE = MergeTree() -ORDER BY (request_id, time) -; - - - --- +goose down -DROP TABLE telemetry.raw_sdks_v1; diff --git a/internal/clickhouse/schema/012_create_billing_billable_verifications_per_month_v1.sql b/internal/clickhouse/schema/012_create_billing_billable_verifications_per_month_v1.sql deleted file mode 100644 index 31dfaf6fe4..0000000000 --- a/internal/clickhouse/schema/012_create_billing_billable_verifications_per_month_v1.sql +++ /dev/null @@ -1,15 +0,0 @@ --- +goose up -CREATE TABLE billing.billable_verifications_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - - --- +goose down -DROP TABLE billing.billable_verifications_per_month_v1; diff --git a/internal/clickhouse/schema/013_create_billing_billable_verifications_per_month_mv_v1.sql b/internal/clickhouse/schema/013_create_billing_billable_verifications_per_month_mv_v1.sql deleted file mode 100644 index 00e8005f1d..0000000000 --- a/internal/clickhouse/schema/013_create_billing_billable_verifications_per_month_mv_v1.sql +++ /dev/null @@ -1,21 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW billing.billable_verifications_per_month_mv_v1 -TO billing.billable_verifications_per_month_v1 -AS -SELECT - workspace_id, - count(*) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v1 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; - - - --- +goose down -DROP VIEW billing.billable_verifications_per_month_mv_v1; diff --git a/internal/clickhouse/schema/014_create_ratelimits_ratelimits_per_minute_v1.sql b/internal/clickhouse/schema/014_create_ratelimits_ratelimits_per_minute_v1.sql deleted file mode 100644 index cfc1b4b6a9..0000000000 --- a/internal/clickhouse/schema/014_create_ratelimits_ratelimits_per_minute_v1.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.ratelimits_per_minute_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - --- +goose down -DROP TABLE ratelimits.ratelimits_per_minute_v1; diff --git a/internal/clickhouse/schema/015_create_ratelimits_ratelimits_per_hour_v1.sql b/internal/clickhouse/schema/015_create_ratelimits_ratelimits_per_hour_v1.sql deleted file mode 100644 index 1cc5f6aca2..0000000000 --- a/internal/clickhouse/schema/015_create_ratelimits_ratelimits_per_hour_v1.sql +++ /dev/null @@ -1,18 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.ratelimits_per_hour_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - --- +goose down -DROP TABLE ratelimits.ratelimits_per_hour_v1; diff --git a/internal/clickhouse/schema/016_create_ratelimits_ratelimits_per_day_v1.sql b/internal/clickhouse/schema/016_create_ratelimits_ratelimits_per_day_v1.sql deleted file mode 100644 index 57b7b9febf..0000000000 --- a/internal/clickhouse/schema/016_create_ratelimits_ratelimits_per_day_v1.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.ratelimits_per_day_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - --- +goose down -DROP TABLE ratelimits.ratelimits_per_day_v1; diff --git a/internal/clickhouse/schema/017_create_ratelimits_ratelimits_per_month_v1.sql b/internal/clickhouse/schema/017_create_ratelimits_ratelimits_per_month_v1.sql deleted file mode 100644 index ffd91a5f63..0000000000 --- a/internal/clickhouse/schema/017_create_ratelimits_ratelimits_per_month_v1.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.ratelimits_per_month_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - --- +goose down -DROP TABLE ratelimits.ratelimits_per_month_v1; diff --git a/internal/clickhouse/schema/018_create_ratelimits_ratelimits_per_minute_mv_v1.sql b/internal/clickhouse/schema/018_create_ratelimits_ratelimits_per_minute_mv_v1.sql deleted file mode 100644 index 93487cd52e..0000000000 --- a/internal/clickhouse/schema/018_create_ratelimits_ratelimits_per_minute_mv_v1.sql +++ /dev/null @@ -1,23 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW ratelimits.ratelimits_per_minute_mv_v1 -TO ratelimits.ratelimits_per_minute_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; - - - --- +goose down -DROP VIEW ratelimits.ratelimits_per_minute_mv_v1; diff --git a/internal/clickhouse/schema/019_create_ratelimits_ratelimits_per_hour_mv_v1.sql b/internal/clickhouse/schema/019_create_ratelimits_ratelimits_per_hour_mv_v1.sql deleted file mode 100644 index fb423d933e..0000000000 --- a/internal/clickhouse/schema/019_create_ratelimits_ratelimits_per_hour_mv_v1.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW ratelimits.ratelimits_per_hour_mv_v1 -TO ratelimits.ratelimits_per_hour_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; - - --- +goose down -DROP VIEW ratelimits.ratelimits_per_hour_mv_v1; diff --git a/internal/clickhouse/schema/020_create_ratelimits_ratelimits_per_day_mv_v1.sql b/internal/clickhouse/schema/020_create_ratelimits_ratelimits_per_day_mv_v1.sql deleted file mode 100644 index 8e76285e3b..0000000000 --- a/internal/clickhouse/schema/020_create_ratelimits_ratelimits_per_day_mv_v1.sql +++ /dev/null @@ -1,23 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW ratelimits.ratelimits_per_day_mv_v1 -TO ratelimits.ratelimits_per_day_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - count(*) as total, - countIf(passed > 0) as passed, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; - - - --- +goose down -DROP VIEW ratelimits.ratelimits_per_day_mv_v1; diff --git a/internal/clickhouse/schema/021_create_ratelimits_ratelimits_per_month_mv_v1.sql b/internal/clickhouse/schema/021_create_ratelimits_ratelimits_per_month_mv_v1.sql deleted file mode 100644 index a97c911186..0000000000 --- a/internal/clickhouse/schema/021_create_ratelimits_ratelimits_per_month_mv_v1.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW ratelimits.ratelimits_per_month_mv_v1 -TO ratelimits.ratelimits_per_month_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; - - --- +goose down -DROP VIEW ratelimits.ratelimits_per_month_mv_v1; diff --git a/internal/clickhouse/schema/022_create_business_active_workspaces_per_month_v1.sql b/internal/clickhouse/schema/022_create_business_active_workspaces_per_month_v1.sql deleted file mode 100644 index 29be93dd73..0000000000 --- a/internal/clickhouse/schema/022_create_business_active_workspaces_per_month_v1.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose up -CREATE TABLE business.active_workspaces_per_month_v1 -( - time Date, - workspace_id String -) -ENGINE = MergeTree() -ORDER BY (time) -; - - --- +goose down -DROP TABLE business.active_workspaces_per_month_v1; diff --git a/internal/clickhouse/schema/023_create_business_active_workspaces_per_month_mv_v1.sql b/internal/clickhouse/schema/023_create_business_active_workspaces_per_month_mv_v1.sql deleted file mode 100644 index a22401696b..0000000000 --- a/internal/clickhouse/schema/023_create_business_active_workspaces_per_month_mv_v1.sql +++ /dev/null @@ -1,21 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_keys_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM verifications.key_verifications_per_month_v1 -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_ratelimits_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM ratelimits.ratelimits_per_month_v1 -; - - --- +goose down -DROP VIEW business.active_workspaces_keys_per_month_mv_v1; -DROP VIEW business.active_workspaces_ratelimits_per_month_mv_v1; diff --git a/internal/clickhouse/schema/024_create_ratelimits_last_used_mv_v1.sql b/internal/clickhouse/schema/024_create_ratelimits_last_used_mv_v1.sql deleted file mode 100644 index 56c2f26e38..0000000000 --- a/internal/clickhouse/schema/024_create_ratelimits_last_used_mv_v1.sql +++ /dev/null @@ -1,34 +0,0 @@ --- +goose up -CREATE TABLE ratelimits.ratelimits_last_used_v1 -( - time Int64, - workspace_id String, - namespace_id String, - identifier String, - -) -ENGINE = AggregatingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - -CREATE MATERIALIZED VIEW ratelimits.ratelimits_last_used_mv_v1 -TO ratelimits.ratelimits_last_used_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - maxSimpleState(time) as time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier -; - - --- +goose down -DROP VIEW IF EXISTS ratelimits.ratelimits_last_used_mv_v1; -DROP TABLE IF EXISTS ratelimits.ratelimits_last_used_v1; diff --git a/internal/clickhouse/schema/025_create_billable_verifications_v2.sql b/internal/clickhouse/schema/025_create_billable_verifications_v2.sql deleted file mode 100644 index d5e89dda4a..0000000000 --- a/internal/clickhouse/schema/025_create_billable_verifications_v2.sql +++ /dev/null @@ -1,30 +0,0 @@ --- +goose up -CREATE TABLE billing.billable_verifications_per_month_v2 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW billing.billable_verifications_per_month_mv_v2 -TO billing.billable_verifications_per_month_v2 -AS SELECT - workspace_id, - sum(count) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v1 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; --- +goose down - -DROP VIEW billing.billable_verifications_per_month_mv_v2; -DROP TABLE billing.billable_verifications_per_month_v2; diff --git a/internal/clickhouse/schema/026_create_billable_ratelimits_v1.sql b/internal/clickhouse/schema/026_create_billable_ratelimits_v1.sql deleted file mode 100644 index 0b0b46d954..0000000000 --- a/internal/clickhouse/schema/026_create_billable_ratelimits_v1.sql +++ /dev/null @@ -1,30 +0,0 @@ --- +goose up - -CREATE TABLE billing.billable_ratelimits_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW billing.billable_ratelimits_per_month_mv_v1 -TO billing.billable_ratelimits_per_month_v1 -AS SELECT - workspace_id, - sum(passed) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM ratelimits.ratelimits_per_month_v1 -WHERE passed > 0 -GROUP BY - workspace_id, - year, - month -; --- +goose down -DROP VIEW billing.billable_ratelimits_per_month_mv_v1; -DROP TABLE billing.billable_ratelimits_per_month_v1; diff --git a/internal/clickhouse/schema/027_add_tags_to_verifications.raw_key_verifications_v1.sql b/internal/clickhouse/schema/027_add_tags_to_verifications.raw_key_verifications_v1.sql deleted file mode 100644 index 06b384707b..0000000000 --- a/internal/clickhouse/schema/027_add_tags_to_verifications.raw_key_verifications_v1.sql +++ /dev/null @@ -1,11 +0,0 @@ --- +goose up -ALTER TABLE verifications.raw_key_verifications_v1 -ADD COLUMN IF NOT EXISTS tags Array(String) DEFAULT []; - - --- +goose down - - - -ALTER TABLE verifications.raw_key_verifications_v1 -DROP COLUMN IF EXISTS tags; diff --git a/internal/clickhouse/schema/028_create_verifications.key_verifications_per_hour_v2.sql b/internal/clickhouse/schema/028_create_verifications.key_verifications_per_hour_v2.sql deleted file mode 100644 index fa83cd0cef..0000000000 --- a/internal/clickhouse/schema/028_create_verifications.key_verifications_per_hour_v2.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_hour_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_hour_v2; diff --git a/internal/clickhouse/schema/030_create_verifications.key_verifications_per_day_v2.sql b/internal/clickhouse/schema/030_create_verifications.key_verifications_per_day_v2.sql deleted file mode 100644 index 48018969b7..0000000000 --- a/internal/clickhouse/schema/030_create_verifications.key_verifications_per_day_v2.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_day_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_day_v2; diff --git a/internal/clickhouse/schema/031_create_verifications.key_verifications_per_day_mv_v2.sql b/internal/clickhouse/schema/031_create_verifications.key_verifications_per_day_mv_v2.sql deleted file mode 100644 index dbb8568e08..0000000000 --- a/internal/clickhouse/schema/031_create_verifications.key_verifications_per_day_mv_v2.sql +++ /dev/null @@ -1,27 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_day_mv_v2 -TO verifications.key_verifications_per_day_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; - - --- +goose down -DROP VIEW verifications.key_verifications_per_day_mv_v2; diff --git a/internal/clickhouse/schema/032_create_verifications.key_verifications_per_month_v2.sql b/internal/clickhouse/schema/032_create_verifications.key_verifications_per_month_v2.sql deleted file mode 100644 index a037c26018..0000000000 --- a/internal/clickhouse/schema/032_create_verifications.key_verifications_per_month_v2.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_month_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_month_v2; diff --git a/internal/clickhouse/schema/033_create_verifications.key_verifications_per_month_mv_v2.sql b/internal/clickhouse/schema/033_create_verifications.key_verifications_per_month_mv_v2.sql deleted file mode 100644 index a045769a07..0000000000 --- a/internal/clickhouse/schema/033_create_verifications.key_verifications_per_month_mv_v2.sql +++ /dev/null @@ -1,27 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW verifications.key_verifications_per_month_mv_v2 -TO verifications.key_verifications_per_month_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; - - --- +goose down -DROP VIEW verifications.key_verifications_per_month_mv_v2; diff --git a/internal/clickhouse/schema/034_billing_read_from_verifications.key_verifications_per_month_v2.sql b/internal/clickhouse/schema/034_billing_read_from_verifications.key_verifications_per_month_v2.sql deleted file mode 100644 index cd5d6fbd3a..0000000000 --- a/internal/clickhouse/schema/034_billing_read_from_verifications.key_verifications_per_month_v2.sql +++ /dev/null @@ -1,33 +0,0 @@ --- +goose up -ALTER TABLE billing.billable_verifications_per_month_mv_v1 -MODIFY QUERY -SELECT - workspace_id, - count(*) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v2 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; - - --- +goose down - -ALTER TABLE billing.billable_verifications_per_month_mv_v1 -MODIFY QUERY -SELECT - workspace_id, - count(*) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v1 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; diff --git a/internal/clickhouse/schema/035_business_update_active_workspaces_keys_per_month_mv_v1_read_from_verifications.key_verifications_per_month_v2.sql b/internal/clickhouse/schema/035_business_update_active_workspaces_keys_per_month_mv_v1_read_from_verifications.key_verifications_per_month_v2.sql deleted file mode 100644 index 1a22f97904..0000000000 --- a/internal/clickhouse/schema/035_business_update_active_workspaces_keys_per_month_mv_v1_read_from_verifications.key_verifications_per_month_v2.sql +++ /dev/null @@ -1,16 +0,0 @@ --- +goose up -ALTER TABLE business.active_workspaces_keys_per_month_mv_v1 -MODIFY QUERY -SELECT - workspace_id, toDate(time) as time -FROM verifications.key_verifications_per_month_v2 -; - --- +goose down - -ALTER TABLE business.active_workspaces_keys_per_month_mv_v1 -MODIFY QUERY -SELECT - workspace_id, toDate(time) as time -FROM verifications.key_verifications_per_month_v1 -; diff --git a/internal/clickhouse/schema/036_create_verifications.key_verifications_per_hour_v3.sql b/internal/clickhouse/schema/036_create_verifications.key_verifications_per_hour_v3.sql deleted file mode 100644 index 6f99ddcefa..0000000000 --- a/internal/clickhouse/schema/036_create_verifications.key_verifications_per_hour_v3.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_hour_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_hour_v3; diff --git a/internal/clickhouse/schema/037_create_verifications.key_verifications_per_hour_mv_v3.sql b/internal/clickhouse/schema/037_create_verifications.key_verifications_per_hour_mv_v3.sql deleted file mode 100644 index 4cf157874c..0000000000 --- a/internal/clickhouse/schema/037_create_verifications.key_verifications_per_hour_mv_v3.sql +++ /dev/null @@ -1,49 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v3 -TO verifications.key_verifications_per_hour_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; - - --- populate from existing data --- INSERT INTO verifications.key_verifications_per_hour_v3 --- SELECT --- toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- tags, --- count(*) as count --- FROM verifications.raw_key_verifications_v1 --- GROUP BY --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- time, --- tags --- ; - --- +goose down -DROP VIEW verifications.key_verifications_per_hour_mv_v3; diff --git a/internal/clickhouse/schema/038_create_verifications.key_verifications_per_day_v3.sql b/internal/clickhouse/schema/038_create_verifications.key_verifications_per_day_v3.sql deleted file mode 100644 index c4cd43aba5..0000000000 --- a/internal/clickhouse/schema/038_create_verifications.key_verifications_per_day_v3.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_day_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_day_v3; diff --git a/internal/clickhouse/schema/039_create_verifications.key_verifications_per_day_mv_v3.sql b/internal/clickhouse/schema/039_create_verifications.key_verifications_per_day_mv_v3.sql deleted file mode 100644 index 05e54a6857..0000000000 --- a/internal/clickhouse/schema/039_create_verifications.key_verifications_per_day_mv_v3.sql +++ /dev/null @@ -1,49 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v3 -TO verifications.key_verifications_per_day_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; - --- populate from existing data --- INSERT INTO verifications.key_verifications_per_day_v3 --- SELECT --- toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- tags, --- count(*) as count --- FROM verifications.raw_key_verifications_v1 --- GROUP BY --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- time, --- tags --- ; - - --- +goose down -DROP VIEW verifications.key_verifications_per_day_mv_v3; diff --git a/internal/clickhouse/schema/040_create_verifications.key_verifications_per_month_v3.sql b/internal/clickhouse/schema/040_create_verifications.key_verifications_per_month_v3.sql deleted file mode 100644 index 9b29bd585a..0000000000 --- a/internal/clickhouse/schema/040_create_verifications.key_verifications_per_month_v3.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_month_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_month_v3; diff --git a/internal/clickhouse/schema/041_create_verifications.key_verifications_per_month_mv_v3.sql b/internal/clickhouse/schema/041_create_verifications.key_verifications_per_month_mv_v3.sql deleted file mode 100644 index b53b3ae6ea..0000000000 --- a/internal/clickhouse/schema/041_create_verifications.key_verifications_per_month_mv_v3.sql +++ /dev/null @@ -1,49 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v3 -TO verifications.key_verifications_per_month_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; - --- populate from existing data --- INSERT INTO verifications.key_verifications_per_month_v3 --- SELECT --- toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- tags, --- count(*) as counts --- FROM verifications.raw_key_verifications_v1 --- GROUP BY --- workspace_id, --- key_space_id, --- identity_id, --- key_id, --- outcome, --- time, --- tags --- ; - - --- +goose down -DROP VIEW verifications.key_verifications_per_month_mv_v3; diff --git a/internal/clickhouse/schema/042_create_api_requests_per_hour_v1.sql b/internal/clickhouse/schema/042_create_api_requests_per_hour_v1.sql deleted file mode 100644 index e2c49a801b..0000000000 --- a/internal/clickhouse/schema/042_create_api_requests_per_hour_v1.sql +++ /dev/null @@ -1,24 +0,0 @@ --- +goose up -CREATE TABLE metrics.api_requests_per_hour_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); - --- +goose down -DROP TABLE metrics.api_requests_per_hour_v1; diff --git a/internal/clickhouse/schema/043_create_api_requests_per_hour_mv_v1.sql b/internal/clickhouse/schema/043_create_api_requests_per_hour_mv_v1.sql deleted file mode 100644 index 1d2f96c113..0000000000 --- a/internal/clickhouse/schema/043_create_api_requests_per_hour_mv_v1.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW metrics.api_requests_per_hour_mv_v1 TO metrics.api_requests_per_hour_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; - --- +goose down -DROP VIEW metrics.api_requests_per_hour_mv_v1; \ No newline at end of file diff --git a/internal/clickhouse/schema/044_create_api_requests_per_minute_v1.sql b/internal/clickhouse/schema/044_create_api_requests_per_minute_v1.sql deleted file mode 100644 index 38651b146b..0000000000 --- a/internal/clickhouse/schema/044_create_api_requests_per_minute_v1.sql +++ /dev/null @@ -1,24 +0,0 @@ --- +goose up -CREATE TABLE metrics.api_requests_per_minute_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); - --- +goose down -DROP TABLE metrics.api_requests_per_minute_v1; diff --git a/internal/clickhouse/schema/045_create_api_requests_per_minute_mv_v1.sql b/internal/clickhouse/schema/045_create_api_requests_per_minute_mv_v1.sql deleted file mode 100644 index 0f58a183bf..0000000000 --- a/internal/clickhouse/schema/045_create_api_requests_per_minute_mv_v1.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW metrics.api_requests_per_minute_mv_v1 TO metrics.api_requests_per_minute_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; - --- +goose down -DROP VIEW metrics.api_requests_per_minute_mv_v1; diff --git a/internal/clickhouse/schema/046_create_api_requests_per_day_v1.sql b/internal/clickhouse/schema/046_create_api_requests_per_day_v1.sql deleted file mode 100644 index 6b6e6fb800..0000000000 --- a/internal/clickhouse/schema/046_create_api_requests_per_day_v1.sql +++ /dev/null @@ -1,24 +0,0 @@ --- +goose up -CREATE TABLE metrics.api_requests_per_day_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); - --- +goose down -DROP TABLE metrics.api_requests_per_day_v1; diff --git a/internal/clickhouse/schema/047_create_api_requests_per_day_mv_v1.sql b/internal/clickhouse/schema/047_create_api_requests_per_day_mv_v1.sql deleted file mode 100644 index a5452eaf48..0000000000 --- a/internal/clickhouse/schema/047_create_api_requests_per_day_mv_v1.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose up -CREATE MATERIALIZED VIEW metrics.api_requests_per_day_mv_v1 TO metrics.api_requests_per_day_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; - --- +goose down -DROP VIEW metrics.api_requests_per_day_mv_v1; diff --git a/internal/clickhouse/schema/048_raw_ratelimits_metrics_indexes_v1.sql b/internal/clickhouse/schema/048_raw_ratelimits_metrics_indexes_v1.sql deleted file mode 100644 index 41d6116ac6..0000000000 --- a/internal/clickhouse/schema/048_raw_ratelimits_metrics_indexes_v1.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose up -ALTER TABLE ratelimits.raw_ratelimits_v1 - ADD INDEX idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE ratelimits.raw_ratelimits_v1 - ADD INDEX idx_request_id (request_id) TYPE minmax GRANULARITY 1; - --- +goose down -ALTER TABLE ratelimits.raw_ratelimits_v1 - DROP INDEX idx_workspace_time; - -ALTER TABLE ratelimits.raw_ratelimits_v1 - DROP INDEX idx_request_id; diff --git a/internal/clickhouse/schema/049_raw_api_metrics_ratelimit_indexes_v1.sql b/internal/clickhouse/schema/049_raw_api_metrics_ratelimit_indexes_v1.sql deleted file mode 100644 index 882d430d74..0000000000 --- a/internal/clickhouse/schema/049_raw_api_metrics_ratelimit_indexes_v1.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose up -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX idx_request_id (request_id) TYPE minmax GRANULARITY 1; - --- +goose down -ALTER TABLE metrics.raw_api_requests_v1 - DROP INDEX idx_workspace_time; - -ALTER TABLE metrics.raw_api_requests_v1 - DROP INDEX idx_request_id; diff --git a/internal/clickhouse/schema/050_create_verifications.key_verifications_per_minute_v1.sql b/internal/clickhouse/schema/050_create_verifications.key_verifications_per_minute_v1.sql deleted file mode 100644 index 2b68530cdc..0000000000 --- a/internal/clickhouse/schema/050_create_verifications.key_verifications_per_minute_v1.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose up -CREATE TABLE verifications.key_verifications_per_minute_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - --- +goose down -DROP TABLE verifications.key_verifications_per_minute_v1; diff --git a/internal/clickhouse/src/testutil.ts b/internal/clickhouse/src/testutil.ts index f6ca8ff85f..a84fbc58e7 100644 --- a/internal/clickhouse/src/testutil.ts +++ b/internal/clickhouse/src/testutil.ts @@ -1,5 +1,4 @@ -import { execa } from "execa"; -import { GenericContainer, Network, type StartedTestContainer } from "testcontainers"; +import { GenericContainer, Network, type StartedTestContainer, Wait } from "testcontainers"; import type { TaskContext } from "vitest"; export class ClickHouseContainer { @@ -31,26 +30,22 @@ export class ClickHouseContainer { ): Promise { const network = await new Network().start(); - const container = await new GenericContainer("bitnami/clickhouse:latest") + const container = await new GenericContainer("unkey-clickhouse:latest") .withEnvironment({ CLICKHOUSE_ADMIN_USER: ClickHouseContainer.username, CLICKHOUSE_ADMIN_PASSWORD: ClickHouseContainer.password, }) .withNetworkMode(network.getName()) .withExposedPorts(8123, 9000) + .withWaitStrategy(Wait.forLogMessage("** Starting ClickHouse **")) .start(); + await new Promise((resolve) => setTimeout(resolve, 5000)); if (!opts?.keepContainer) { t.onTestFinished(async () => { await container.stop(); }); } - const dsn = `tcp://${ClickHouseContainer.username}:${ - ClickHouseContainer.password - }@localhost:${container.getMappedPort(9000)}`; - - await execa("goose", ["-dir=./schema", "clickhouse", dsn, "up"]); - return new ClickHouseContainer(container); } } diff --git a/tools/local/src/main.ts b/tools/local/src/main.ts index d1e8a6867f..e4a038b38a 100644 --- a/tools/local/src/main.ts +++ b/tools/local/src/main.ts @@ -48,7 +48,7 @@ async function main() { switch (app) { case "dashboard": { - await startContainers(["planetscale", "clickhouse", "agent", "clickhouse_migrator"]); + await startContainers(["planetscale", "clickhouse", "agent"]); const resources = await prepareDatabase(); !skipEnv && (await bootstrapDashboard(resources)); @@ -56,14 +56,14 @@ async function main() { } case "api": { - await startContainers(["planetscale", "clickhouse", "agent", "clickhouse_migrator"]); + await startContainers(["planetscale", "clickhouse", "agent"]); const resources = await prepareDatabase(); !skipEnv && (await bootstrapApi(resources)); break; } case "seed": { - await startContainers(["planetscale", "clickhouse", "agent", "clickhouse_migrator"]); + await startContainers(["planetscale", "clickhouse", "agent"]); // Extract workspace ID if provided const workspaceId = passedOptions.ws as string | undefined;