From 511c54a47fbd955e7c2b946327557d07345ac314 Mon Sep 17 00:00:00 2001 From: Flo Date: Fri, 6 Mar 2026 19:23:14 +0100 Subject: [PATCH] fix: drop legacy clickhouse tables and remove v1 schema --- .../migrations/20250903085516_init.sql | 334 ------- pkg/clickhouse/migrations/20260306000000.sql | 84 ++ pkg/clickhouse/schema/000_legacy.sql | 822 ------------------ .../schema/001_key_verifications_raw_v2.sql | 21 - .../schema/006_ratelimits_raw_v2.sql | 19 - .../schema/012_api_requests_raw_v2.sql | 25 - 6 files changed, 84 insertions(+), 1221 deletions(-) delete mode 100644 pkg/clickhouse/migrations/20250903085516_init.sql create mode 100644 pkg/clickhouse/migrations/20260306000000.sql delete mode 100644 pkg/clickhouse/schema/000_legacy.sql diff --git a/pkg/clickhouse/migrations/20250903085516_init.sql b/pkg/clickhouse/migrations/20250903085516_init.sql deleted file mode 100644 index c1ecb2cab7..0000000000 --- a/pkg/clickhouse/migrations/20250903085516_init.sql +++ /dev/null @@ -1,334 +0,0 @@ --- Add new schema named "billing" -CREATE DATABASE `billing` ENGINE Atomic; --- Add new schema named "business" -CREATE DATABASE `business` ENGINE Atomic; --- Add new schema named "metrics" -CREATE DATABASE `metrics` ENGINE Atomic; --- Add new schema named "ratelimits" -CREATE DATABASE `ratelimits` ENGINE Atomic; --- Add new schema named "verifications" -CREATE DATABASE `verifications` ENGINE Atomic; --- Create "billable_ratelimits_per_month_v1" table -CREATE TABLE `billing`.`billable_ratelimits_per_month_v1` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "billable_verifications_per_month_v1" table -CREATE TABLE `billing`.`billable_verifications_per_month_v1` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "billable_verifications_per_month_v2" table -CREATE TABLE `billing`.`billable_verifications_per_month_v2` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "active_workspaces_per_month_v1" table -CREATE TABLE `business`.`active_workspaces_per_month_v1` ( - `time` Date, - `workspace_id` String -) ENGINE = MergeTree -PRIMARY KEY (`time`) ORDER BY (`time`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_day_v1" table -CREATE TABLE `metrics`.`api_requests_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_hour_v1" table -CREATE TABLE `metrics`.`api_requests_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_minute_v1" table -CREATE TABLE `metrics`.`api_requests_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "raw_api_requests_v1" table -CREATE TABLE `metrics`.`raw_api_requests_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `host` String, - `method` LowCardinality(String), - `path` String, - `request_headers` Array(String), - `request_body` String, - `response_status` Int32, - `response_headers` Array(String), - `response_body` String, - `error` String, - `service_latency` Int64, - `user_agent` String, - `ip_address` String, - `country` String, - `city` String, - `colo` String, - `continent` String, - INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, - INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `time`, `request_id`) ORDER BY (`workspace_id`, `time`, `request_id`) SETTINGS index_granularity = 8192; --- Create "ratelimits_last_used_v1" table -CREATE TABLE `ratelimits`.`ratelimits_last_used_v1` ( - `time` Int64, - `workspace_id` String, - `namespace_id` String, - `identifier` String -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_day_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_hour_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_minute_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_month_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_month_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "raw_ratelimits_v1" table -CREATE TABLE `ratelimits`.`raw_ratelimits_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Bool, - INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, - INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v1" table -CREATE TABLE `verifications`.`key_verifications_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v2" table -CREATE TABLE `verifications`.`key_verifications_per_day_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v3" table -CREATE TABLE `verifications`.`key_verifications_per_day_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v1" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v2" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v3" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_minute_v1" table -CREATE TABLE `verifications`.`key_verifications_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v1" table -CREATE TABLE `verifications`.`key_verifications_per_month_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v2" table -CREATE TABLE `verifications`.`key_verifications_per_month_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v3" table -CREATE TABLE `verifications`.`key_verifications_per_month_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "raw_key_verifications_v1" table -CREATE TABLE `verifications`.`raw_key_verifications_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `key_space_id` String, - `key_id` String, - `region` LowCardinality(String), - `outcome` LowCardinality(String), - `identity_id` String, - `tags` Array(String) DEFAULT [] -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `key_id`, `time`) ORDER BY (`workspace_id`, `key_space_id`, `key_id`, `time`) SETTINGS index_granularity = 8192; --- Create "billable_ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `billing`.`billable_ratelimits_per_month_mv_v1` TO `billing`.`billable_ratelimits_per_month_v1` AS SELECT workspace_id, sum(passed) AS count, toYear(time) AS year, toMonth(time) AS month FROM ratelimits.ratelimits_per_month_v1 WHERE passed > 0 GROUP BY workspace_id, year, month; --- Create "billable_verifications_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v1` TO `billing`.`billable_verifications_per_month_v1` AS SELECT workspace_id, count(*) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v2 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; --- Create "billable_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v2` TO `billing`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v1 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; --- Create "active_workspaces_keys_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `business`.`active_workspaces_keys_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM verifications.key_verifications_per_month_v2; --- Create "active_workspaces_ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `business`.`active_workspaces_ratelimits_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM ratelimits.ratelimits_per_month_v1; --- Create "api_requests_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_day_mv_v1` TO `metrics`.`api_requests_per_day_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "api_requests_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_hour_mv_v1` TO `metrics`.`api_requests_per_hour_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "api_requests_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_minute_mv_v1` TO `metrics`.`api_requests_per_minute_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "ratelimits_last_used_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_last_used_mv_v1` TO `ratelimits`.`ratelimits_last_used_v1` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier; --- Create "ratelimits_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_day_mv_v1` TO `ratelimits`.`ratelimits_per_day_v1` AS SELECT workspace_id, namespace_id, identifier, count(*) AS total, countIf(passed > 0) AS passed, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_hour_mv_v1` TO `ratelimits`.`ratelimits_per_hour_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_minute_mv_v1` TO `ratelimits`.`ratelimits_per_minute_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_month_mv_v1` TO `ratelimits`.`ratelimits_per_month_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "key_verifications_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v1` TO `verifications`.`key_verifications_per_day_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_day_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v2` TO `verifications`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_day_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v3` TO `verifications`.`key_verifications_per_day_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v1` TO `verifications`.`key_verifications_per_hour_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_hour_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v2` TO `verifications`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_hour_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v3` TO `verifications`.`key_verifications_per_hour_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_minute_mv_v1` TO `verifications`.`key_verifications_per_minute_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v1` TO `verifications`.`key_verifications_per_month_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v2` TO `verifications`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_month_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v3` TO `verifications`.`key_verifications_per_month_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; diff --git a/pkg/clickhouse/migrations/20260306000000.sql b/pkg/clickhouse/migrations/20260306000000.sql new file mode 100644 index 0000000000..ed889a6ebc --- /dev/null +++ b/pkg/clickhouse/migrations/20260306000000.sql @@ -0,0 +1,84 @@ +-- Drop all legacy ClickHouse tables, materialized views, and databases. +-- The v2/v3 schema in the `default` database fully replaces these. +-- +-- Order: sync MVs → leaf MVs → upstream MVs → tables → databases + +-- 1. Drop temp sync MVs (bridge v1 raw → v2 raw in default db) +DROP VIEW IF EXISTS `default`.`temp_sync_key_verifications_v1_to_v2`; +DROP VIEW IF EXISTS `default`.`temp_sync_ratelimits_raw_v1_to_v2`; +DROP VIEW IF EXISTS `default`.`temp_sync_metrics_v1_to_v2`; + +-- 2. Drop legacy billing MVs +DROP VIEW IF EXISTS `billing`.`billable_verifications_per_month_mv_v1`; +DROP VIEW IF EXISTS `billing`.`billable_verifications_per_month_mv_v2`; +DROP VIEW IF EXISTS `billing`.`billable_ratelimits_per_month_mv_v1`; + +-- 3. Drop legacy business MVs +DROP VIEW IF EXISTS `business`.`active_workspaces_keys_per_month_mv_v1`; +DROP VIEW IF EXISTS `business`.`active_workspaces_ratelimits_per_month_mv_v1`; + +-- 4. Drop legacy verifications MVs +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v3`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v2`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v1`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v3`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v2`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v1`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v3`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v2`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v1`; +DROP VIEW IF EXISTS `verifications`.`key_verifications_per_minute_mv_v1`; + +-- 5. Drop legacy ratelimits MVs +DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_month_mv_v1`; +DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_day_mv_v1`; +DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_hour_mv_v1`; +DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_minute_mv_v1`; +DROP VIEW IF EXISTS `ratelimits`.`ratelimits_last_used_mv_v1`; + +-- 6. Drop legacy metrics MVs +DROP VIEW IF EXISTS `metrics`.`api_requests_per_day_mv_v1`; +DROP VIEW IF EXISTS `metrics`.`api_requests_per_hour_mv_v1`; +DROP VIEW IF EXISTS `metrics`.`api_requests_per_minute_mv_v1`; + +-- 7. Drop legacy verifications tables +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v3`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v2`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v1`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v3`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v2`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v1`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v3`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v2`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v1`; +DROP TABLE IF EXISTS `verifications`.`key_verifications_per_minute_v1`; +DROP TABLE IF EXISTS `verifications`.`raw_key_verifications_v1`; + +-- 8. Drop legacy ratelimits tables +DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_month_v1`; +DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_day_v1`; +DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_hour_v1`; +DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_minute_v1`; +DROP TABLE IF EXISTS `ratelimits`.`ratelimits_last_used_v1`; +DROP TABLE IF EXISTS `ratelimits`.`raw_ratelimits_v1`; + +-- 9. Drop legacy metrics tables +DROP TABLE IF EXISTS `metrics`.`api_requests_per_day_v1`; +DROP TABLE IF EXISTS `metrics`.`api_requests_per_hour_v1`; +DROP TABLE IF EXISTS `metrics`.`api_requests_per_minute_v1`; +DROP TABLE IF EXISTS `metrics`.`raw_api_requests_v1`; + +-- 10. Drop legacy billing tables +DROP TABLE IF EXISTS `billing`.`billable_verifications_per_month_v1`; +DROP TABLE IF EXISTS `billing`.`billable_verifications_per_month_v2`; +DROP TABLE IF EXISTS `billing`.`billable_ratelimits_per_month_v1`; + +-- 11. Drop legacy business tables +DROP TABLE IF EXISTS `business`.`active_workspaces_per_month_v1`; + +-- 12. Drop legacy databases +DROP DATABASE IF EXISTS `verifications`; +DROP DATABASE IF EXISTS `ratelimits`; +DROP DATABASE IF EXISTS `metrics`; +DROP DATABASE IF EXISTS `billing`; +DROP DATABASE IF EXISTS `business`; diff --git a/pkg/clickhouse/schema/000_legacy.sql b/pkg/clickhouse/schema/000_legacy.sql deleted file mode 100644 index 3830eb5eb0..0000000000 --- a/pkg/clickhouse/schema/000_legacy.sql +++ /dev/null @@ -1,822 +0,0 @@ -CREATE DATABASE IF NOT EXISTS verifications; - -CREATE TABLE verifications.raw_key_verifications_v1( - -- the api request id, so we can correlate the verification with traces and logs - request_id String, - - -- unix milli - time Int64, - - workspace_id String, - key_space_id String, - key_id String, - - -- Right now this is a 3 character airport code, but when we move to aws, - -- this will be the region code such as `us-east-1` - region LowCardinality(String), - - -- Examples: - -- - "VALID" - -- - "RATE_LIMITED" - -- - "EXPIRED" - -- - "DISABLED - outcome LowCardinality(String), - - -- Empty string if the key has no identity - identity_id String, - - tags Array(String) DEFAULT [] - - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, key_space_id, key_id, time) -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v1 -TO verifications.key_verifications_per_hour_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v1 -TO verifications.key_verifications_per_day_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v1 -TO verifications.key_verifications_per_month_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v2 -TO verifications.key_verifications_per_hour_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v2 -TO verifications.key_verifications_per_day_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v2 -TO verifications.key_verifications_per_month_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v3 -TO verifications.key_verifications_per_hour_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v3 -TO verifications.key_verifications_per_day_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v3 -TO verifications.key_verifications_per_month_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE verifications.key_verifications_per_minute_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1 -TO verifications.key_verifications_per_minute_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE DATABASE IF NOT EXISTS ratelimits; -CREATE TABLE IF NOT EXISTS ratelimits.raw_ratelimits_v1( - request_id String, - -- unix milli - time Int64, - workspace_id String, - namespace_id String, - identifier String, - passed Bool - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) - -; - -ALTER TABLE ratelimits.raw_ratelimits_v1 ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE ratelimits.raw_ratelimits_v1 -ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_minute_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_minute_mv_v1 -TO ratelimits.ratelimits_per_minute_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_hour_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_hour_mv_v1 -TO ratelimits.ratelimits_per_hour_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_day_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_day_mv_v1 -TO ratelimits.ratelimits_per_day_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - count(*) as total, - countIf(passed > 0) as passed, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_month_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_month_mv_v1 -TO ratelimits.ratelimits_per_month_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_last_used_v1 -( - time Int64, - workspace_id String, - namespace_id String, - identifier String, - -) -ENGINE = AggregatingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_last_used_mv_v1 -TO ratelimits.ratelimits_last_used_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - maxSimpleState(time) as time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier -; -CREATE DATABASE IF NOT EXISTS metrics; -CREATE TABLE IF NOT EXISTS metrics.raw_api_requests_v1( - request_id String, - -- unix milli - time Int64, - - workspace_id String, - - host String, - - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - path String, - -- "Key: Value" pairs - request_headers Array(String), - request_body String, - - response_status Int, - -- "Key: Value" pairs - response_headers Array(String), - response_body String, - -- internal err.Error() string, empty if no error - error String, - - -- milliseconds - service_latency Int64, - - user_agent String, - ip_address String, - country String, - city String, - colo String, - continent String, - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, time, request_id) -; - -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_hour_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_hour_mv_v1 TO metrics.api_requests_per_hour_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_minute_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_minute_mv_v1 TO metrics.api_requests_per_minute_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_day_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_day_mv_v1 TO metrics.api_requests_per_day_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE DATABASE IF NOT EXISTS billing; -CREATE DATABASE IF NOT EXISTS verifications; -CREATE DATABASE IF NOT EXISTS ratelimits; -CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v1 -TO billing.billable_verifications_per_month_v1 -AS -SELECT - workspace_id, - count(*) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v2 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; -CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v2 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v2 -TO billing.billable_verifications_per_month_v2 -AS SELECT - workspace_id, - sum(count) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v1 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; -CREATE TABLE IF NOT EXISTS billing.billable_ratelimits_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_ratelimits_per_month_mv_v1 -TO billing.billable_ratelimits_per_month_v1 -AS SELECT - workspace_id, - sum(passed) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM ratelimits.ratelimits_per_month_v1 -WHERE passed > 0 -GROUP BY - workspace_id, - year, - month -; -CREATE DATABASE IF NOT EXISTS business; -CREATE DATABASE IF NOT EXISTS billing; -CREATE TABLE IF NOT EXISTS business.active_workspaces_per_month_v1 -( - time Date, - workspace_id String -) -ENGINE = MergeTree() -ORDER BY (time) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_keys_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM verifications.key_verifications_per_month_v2 -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_ratelimits_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM ratelimits.ratelimits_per_month_v1 -; diff --git a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql index 145ca97599..a7366df76f 100644 --- a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql +++ b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql @@ -45,24 +45,3 @@ TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 90 DAY DELETE SETTINGS non_replicated_deduplication_window = 10000 ; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 - -CREATE MATERIALIZED VIEW temp_sync_key_verifications_v1_to_v2 -TO key_verifications_raw_v2 -AS -SELECT - request_id, - time, - workspace_id, - key_space_id, - identity_id, - '' as external_id, - key_id, - region, - outcome, - tags, - 0 as spent_credits, -- v1 doesn't have this column, default to 0 - 0.0 as latency -- v1 doesn't have this column, default to 0.0 -FROM verifications.raw_key_verifications_v1; diff --git a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql index 1f9299f1d9..920bdab7e2 100644 --- a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql +++ b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql @@ -26,22 +26,3 @@ ORDER BY TTL toDateTime (fromUnixTimestamp64Milli (time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 -CREATE MATERIALIZED VIEW temp_sync_ratelimits_raw_v1_to_v2 TO ratelimits_raw_v2 AS -SELECT - request_id, - time, - workspace_id, - namespace_id, - identifier, - passed, - -- v1 doesn't have any of those columns - 0.0 as latency, - '' as override_id, - 0 as `limit`, - 0 as remaining, - 0 as reset_at -FROM - ratelimits.raw_ratelimits_v1; diff --git a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql index 2d245e48ad..defe53a4c1 100644 --- a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql +++ b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql @@ -34,28 +34,3 @@ ORDER BY TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 -CREATE MATERIALIZED VIEW temp_sync_metrics_v1_to_v2 TO api_requests_raw_v2 AS -SELECT - request_id, - time, - workspace_id, - host, - method, - path, - '' as query_string, - CAST(mapFromArrays(CAST([],'Array(String)'), CAST([],'Array(Array(String))')), 'Map(String, Array(String))') AS query_params, - request_headers, - request_body, - response_status, - response_headers, - response_body, - error, - service_latency, - user_agent, - ip_address, - '' as region -FROM - metrics.raw_api_requests_v1;