diff --git a/pkg/clickhouse/migrations/20250903085516_init.sql b/pkg/clickhouse/migrations/20250903085516_init.sql new file mode 100644 index 0000000000..c1ecb2cab7 --- /dev/null +++ b/pkg/clickhouse/migrations/20250903085516_init.sql @@ -0,0 +1,334 @@ +-- Add new schema named "billing" +CREATE DATABASE `billing` ENGINE Atomic; +-- Add new schema named "business" +CREATE DATABASE `business` ENGINE Atomic; +-- Add new schema named "metrics" +CREATE DATABASE `metrics` ENGINE Atomic; +-- Add new schema named "ratelimits" +CREATE DATABASE `ratelimits` ENGINE Atomic; +-- Add new schema named "verifications" +CREATE DATABASE `verifications` ENGINE Atomic; +-- Create "billable_ratelimits_per_month_v1" table +CREATE TABLE `billing`.`billable_ratelimits_per_month_v1` ( + `year` Int32, + `month` Int32, + `workspace_id` String, + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; +-- Create "billable_verifications_per_month_v1" table +CREATE TABLE `billing`.`billable_verifications_per_month_v1` ( + `year` Int32, + `month` Int32, + `workspace_id` String, + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; +-- Create "billable_verifications_per_month_v2" table +CREATE TABLE `billing`.`billable_verifications_per_month_v2` ( + `year` Int32, + `month` Int32, + `workspace_id` String, + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; +-- Create "active_workspaces_per_month_v1" table +CREATE TABLE `business`.`active_workspaces_per_month_v1` ( + `time` Date, + `workspace_id` String +) ENGINE = MergeTree +PRIMARY KEY (`time`) ORDER BY (`time`) SETTINGS index_granularity = 8192; +-- Create "api_requests_per_day_v1" table +CREATE TABLE `metrics`.`api_requests_per_day_v1` ( + `time` DateTime, + `workspace_id` String, + `path` String, + `response_status` Int32, + `host` String, + `method` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; +-- Create "api_requests_per_hour_v1" table +CREATE TABLE `metrics`.`api_requests_per_hour_v1` ( + `time` DateTime, + `workspace_id` String, + `path` String, + `response_status` Int32, + `host` String, + `method` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; +-- Create "api_requests_per_minute_v1" table +CREATE TABLE `metrics`.`api_requests_per_minute_v1` ( + `time` DateTime, + `workspace_id` String, + `path` String, + `response_status` Int32, + `host` String, + `method` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; +-- Create "raw_api_requests_v1" table +CREATE TABLE `metrics`.`raw_api_requests_v1` ( + `request_id` String, + `time` Int64, + `workspace_id` String, + `host` String, + `method` LowCardinality(String), + `path` String, + `request_headers` Array(String), + `request_body` String, + `response_status` Int32, + `response_headers` Array(String), + `response_body` String, + `error` String, + `service_latency` Int64, + `user_agent` String, + `ip_address` String, + `country` String, + `city` String, + `colo` String, + `continent` String, + INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, + INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 +) ENGINE = MergeTree +PRIMARY KEY (`workspace_id`, `time`, `request_id`) ORDER BY (`workspace_id`, `time`, `request_id`) SETTINGS index_granularity = 8192; +-- Create "ratelimits_last_used_v1" table +CREATE TABLE `ratelimits`.`ratelimits_last_used_v1` ( + `time` Int64, + `workspace_id` String, + `namespace_id` String, + `identifier` String +) ENGINE = AggregatingMergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "ratelimits_per_day_v1" table +CREATE TABLE `ratelimits`.`ratelimits_per_day_v1` ( + `time` DateTime, + `workspace_id` String, + `namespace_id` String, + `identifier` String, + `passed` Int64, + `total` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "ratelimits_per_hour_v1" table +CREATE TABLE `ratelimits`.`ratelimits_per_hour_v1` ( + `time` DateTime, + `workspace_id` String, + `namespace_id` String, + `identifier` String, + `passed` Int64, + `total` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "ratelimits_per_minute_v1" table +CREATE TABLE `ratelimits`.`ratelimits_per_minute_v1` ( + `time` DateTime, + `workspace_id` String, + `namespace_id` String, + `identifier` String, + `passed` Int64, + `total` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "ratelimits_per_month_v1" table +CREATE TABLE `ratelimits`.`ratelimits_per_month_v1` ( + `time` DateTime, + `workspace_id` String, + `namespace_id` String, + `identifier` String, + `passed` Int64, + `total` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "raw_ratelimits_v1" table +CREATE TABLE `ratelimits`.`raw_ratelimits_v1` ( + `request_id` String, + `time` Int64, + `workspace_id` String, + `namespace_id` String, + `identifier` String, + `passed` Bool, + INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, + INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 +) ENGINE = MergeTree +PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_day_v1" table +CREATE TABLE `verifications`.`key_verifications_per_day_v1` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_day_v2" table +CREATE TABLE `verifications`.`key_verifications_per_day_v2` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_day_v3" table +CREATE TABLE `verifications`.`key_verifications_per_day_v3` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_hour_v1" table +CREATE TABLE `verifications`.`key_verifications_per_hour_v1` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_hour_v2" table +CREATE TABLE `verifications`.`key_verifications_per_hour_v2` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_hour_v3" table +CREATE TABLE `verifications`.`key_verifications_per_hour_v3` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_minute_v1" table +CREATE TABLE `verifications`.`key_verifications_per_minute_v1` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_month_v1" table +CREATE TABLE `verifications`.`key_verifications_per_month_v1` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_month_v2" table +CREATE TABLE `verifications`.`key_verifications_per_month_v2` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; +-- Create "key_verifications_per_month_v3" table +CREATE TABLE `verifications`.`key_verifications_per_month_v3` ( + `time` DateTime, + `workspace_id` String, + `key_space_id` String, + `identity_id` String, + `key_id` String, + `outcome` LowCardinality(String), + `tags` Array(String), + `count` Int64 +) ENGINE = SummingMergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; +-- Create "raw_key_verifications_v1" table +CREATE TABLE `verifications`.`raw_key_verifications_v1` ( + `request_id` String, + `time` Int64, + `workspace_id` String, + `key_space_id` String, + `key_id` String, + `region` LowCardinality(String), + `outcome` LowCardinality(String), + `identity_id` String, + `tags` Array(String) DEFAULT [] +) ENGINE = MergeTree +PRIMARY KEY (`workspace_id`, `key_space_id`, `key_id`, `time`) ORDER BY (`workspace_id`, `key_space_id`, `key_id`, `time`) SETTINGS index_granularity = 8192; +-- Create "billable_ratelimits_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `billing`.`billable_ratelimits_per_month_mv_v1` TO `billing`.`billable_ratelimits_per_month_v1` AS SELECT workspace_id, sum(passed) AS count, toYear(time) AS year, toMonth(time) AS month FROM ratelimits.ratelimits_per_month_v1 WHERE passed > 0 GROUP BY workspace_id, year, month; +-- Create "billable_verifications_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v1` TO `billing`.`billable_verifications_per_month_v1` AS SELECT workspace_id, count(*) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v2 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; +-- Create "billable_verifications_per_month_mv_v2" view +CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v2` TO `billing`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v1 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; +-- Create "active_workspaces_keys_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `business`.`active_workspaces_keys_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM verifications.key_verifications_per_month_v2; +-- Create "active_workspaces_ratelimits_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `business`.`active_workspaces_ratelimits_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM ratelimits.ratelimits_per_month_v1; +-- Create "api_requests_per_day_mv_v1" view +CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_day_mv_v1` TO `metrics`.`api_requests_per_day_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; +-- Create "api_requests_per_hour_mv_v1" view +CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_hour_mv_v1` TO `metrics`.`api_requests_per_hour_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; +-- Create "api_requests_per_minute_mv_v1" view +CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_minute_mv_v1` TO `metrics`.`api_requests_per_minute_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; +-- Create "ratelimits_last_used_mv_v1" view +CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_last_used_mv_v1` TO `ratelimits`.`ratelimits_last_used_v1` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier; +-- Create "ratelimits_per_day_mv_v1" view +CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_day_mv_v1` TO `ratelimits`.`ratelimits_per_day_v1` AS SELECT workspace_id, namespace_id, identifier, count(*) AS total, countIf(passed > 0) AS passed, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; +-- Create "ratelimits_per_hour_mv_v1" view +CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_hour_mv_v1` TO `ratelimits`.`ratelimits_per_hour_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; +-- Create "ratelimits_per_minute_mv_v1" view +CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_minute_mv_v1` TO `ratelimits`.`ratelimits_per_minute_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; +-- Create "ratelimits_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_month_mv_v1` TO `ratelimits`.`ratelimits_per_month_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; +-- Create "key_verifications_per_day_mv_v1" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v1` TO `verifications`.`key_verifications_per_day_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; +-- Create "key_verifications_per_day_mv_v2" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v2` TO `verifications`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_day_mv_v3" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v3` TO `verifications`.`key_verifications_per_day_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_hour_mv_v1" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v1` TO `verifications`.`key_verifications_per_hour_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; +-- Create "key_verifications_per_hour_mv_v2" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v2` TO `verifications`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_hour_mv_v3" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v3` TO `verifications`.`key_verifications_per_hour_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_minute_mv_v1" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_minute_mv_v1` TO `verifications`.`key_verifications_per_minute_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_month_mv_v1" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v1` TO `verifications`.`key_verifications_per_month_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; +-- Create "key_verifications_per_month_mv_v2" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v2` TO `verifications`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; +-- Create "key_verifications_per_month_mv_v3" view +CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v3` TO `verifications`.`key_verifications_per_month_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; diff --git a/pkg/clickhouse/migrations/20260306000000.sql b/pkg/clickhouse/migrations/20260306000000.sql deleted file mode 100644 index ed889a6ebc..0000000000 --- a/pkg/clickhouse/migrations/20260306000000.sql +++ /dev/null @@ -1,84 +0,0 @@ --- Drop all legacy ClickHouse tables, materialized views, and databases. --- The v2/v3 schema in the `default` database fully replaces these. --- --- Order: sync MVs → leaf MVs → upstream MVs → tables → databases - --- 1. Drop temp sync MVs (bridge v1 raw → v2 raw in default db) -DROP VIEW IF EXISTS `default`.`temp_sync_key_verifications_v1_to_v2`; -DROP VIEW IF EXISTS `default`.`temp_sync_ratelimits_raw_v1_to_v2`; -DROP VIEW IF EXISTS `default`.`temp_sync_metrics_v1_to_v2`; - --- 2. Drop legacy billing MVs -DROP VIEW IF EXISTS `billing`.`billable_verifications_per_month_mv_v1`; -DROP VIEW IF EXISTS `billing`.`billable_verifications_per_month_mv_v2`; -DROP VIEW IF EXISTS `billing`.`billable_ratelimits_per_month_mv_v1`; - --- 3. Drop legacy business MVs -DROP VIEW IF EXISTS `business`.`active_workspaces_keys_per_month_mv_v1`; -DROP VIEW IF EXISTS `business`.`active_workspaces_ratelimits_per_month_mv_v1`; - --- 4. Drop legacy verifications MVs -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v3`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v2`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_month_mv_v1`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v3`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v2`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_day_mv_v1`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v3`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v2`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_hour_mv_v1`; -DROP VIEW IF EXISTS `verifications`.`key_verifications_per_minute_mv_v1`; - --- 5. Drop legacy ratelimits MVs -DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_month_mv_v1`; -DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_day_mv_v1`; -DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_hour_mv_v1`; -DROP VIEW IF EXISTS `ratelimits`.`ratelimits_per_minute_mv_v1`; -DROP VIEW IF EXISTS `ratelimits`.`ratelimits_last_used_mv_v1`; - --- 6. Drop legacy metrics MVs -DROP VIEW IF EXISTS `metrics`.`api_requests_per_day_mv_v1`; -DROP VIEW IF EXISTS `metrics`.`api_requests_per_hour_mv_v1`; -DROP VIEW IF EXISTS `metrics`.`api_requests_per_minute_mv_v1`; - --- 7. Drop legacy verifications tables -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v3`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v2`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_month_v1`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v3`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v2`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_day_v1`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v3`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v2`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_hour_v1`; -DROP TABLE IF EXISTS `verifications`.`key_verifications_per_minute_v1`; -DROP TABLE IF EXISTS `verifications`.`raw_key_verifications_v1`; - --- 8. Drop legacy ratelimits tables -DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_month_v1`; -DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_day_v1`; -DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_hour_v1`; -DROP TABLE IF EXISTS `ratelimits`.`ratelimits_per_minute_v1`; -DROP TABLE IF EXISTS `ratelimits`.`ratelimits_last_used_v1`; -DROP TABLE IF EXISTS `ratelimits`.`raw_ratelimits_v1`; - --- 9. Drop legacy metrics tables -DROP TABLE IF EXISTS `metrics`.`api_requests_per_day_v1`; -DROP TABLE IF EXISTS `metrics`.`api_requests_per_hour_v1`; -DROP TABLE IF EXISTS `metrics`.`api_requests_per_minute_v1`; -DROP TABLE IF EXISTS `metrics`.`raw_api_requests_v1`; - --- 10. Drop legacy billing tables -DROP TABLE IF EXISTS `billing`.`billable_verifications_per_month_v1`; -DROP TABLE IF EXISTS `billing`.`billable_verifications_per_month_v2`; -DROP TABLE IF EXISTS `billing`.`billable_ratelimits_per_month_v1`; - --- 11. Drop legacy business tables -DROP TABLE IF EXISTS `business`.`active_workspaces_per_month_v1`; - --- 12. Drop legacy databases -DROP DATABASE IF EXISTS `verifications`; -DROP DATABASE IF EXISTS `ratelimits`; -DROP DATABASE IF EXISTS `metrics`; -DROP DATABASE IF EXISTS `billing`; -DROP DATABASE IF EXISTS `business`; diff --git a/pkg/clickhouse/schema/000_legacy.sql b/pkg/clickhouse/schema/000_legacy.sql new file mode 100644 index 0000000000..3830eb5eb0 --- /dev/null +++ b/pkg/clickhouse/schema/000_legacy.sql @@ -0,0 +1,822 @@ +CREATE DATABASE IF NOT EXISTS verifications; + +CREATE TABLE verifications.raw_key_verifications_v1( + -- the api request id, so we can correlate the verification with traces and logs + request_id String, + + -- unix milli + time Int64, + + workspace_id String, + key_space_id String, + key_id String, + + -- Right now this is a 3 character airport code, but when we move to aws, + -- this will be the region code such as `us-east-1` + region LowCardinality(String), + + -- Examples: + -- - "VALID" + -- - "RATE_LIMITED" + -- - "EXPIRED" + -- - "DISABLED + outcome LowCardinality(String), + + -- Empty string if the key has no identity + identity_id String, + + tags Array(String) DEFAULT [] + + + +) +ENGINE = MergeTree() +ORDER BY (workspace_id, key_space_id, key_id, time) +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v1 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v1 +TO verifications.key_verifications_per_hour_v1 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v1 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v1 +TO verifications.key_verifications_per_day_v1 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v1 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v1 +TO verifications.key_verifications_per_month_v1 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v2 +TO verifications.key_verifications_per_hour_v2 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v2 +TO verifications.key_verifications_per_day_v2 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v2 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v2 +TO verifications.key_verifications_per_month_v2 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v3 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v3 +TO verifications.key_verifications_per_hour_v3 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v3 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v3 +TO verifications.key_verifications_per_day_v3 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v3 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v3 +TO verifications.key_verifications_per_month_v3 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE TABLE verifications.key_verifications_per_minute_v1 +( + time DateTime, + workspace_id String, + key_space_id String, + identity_id String, + key_id String, + outcome LowCardinality(String), + tags Array(String), + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) +; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1 +TO verifications.key_verifications_per_minute_v1 +AS +SELECT + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + count(*) as count, + toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, + tags +FROM verifications.raw_key_verifications_v1 +GROUP BY + workspace_id, + key_space_id, + identity_id, + key_id, + outcome, + time, + tags +; +CREATE DATABASE IF NOT EXISTS ratelimits; +CREATE TABLE IF NOT EXISTS ratelimits.raw_ratelimits_v1( + request_id String, + -- unix milli + time Int64, + workspace_id String, + namespace_id String, + identifier String, + passed Bool + +) +ENGINE = MergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) + +; + +ALTER TABLE ratelimits.raw_ratelimits_v1 ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; + +ALTER TABLE ratelimits.raw_ratelimits_v1 +ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_minute_v1 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + + passed Int64, + total Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_minute_mv_v1 +TO ratelimits.ratelimits_per_minute_v1 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v1 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_hour_v1 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + + passed Int64, + total Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_hour_mv_v1 +TO ratelimits.ratelimits_per_hour_v1 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v1 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_day_v1 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + + passed Int64, + total Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_day_mv_v1 +TO ratelimits.ratelimits_per_day_v1 +AS +SELECT + workspace_id, + namespace_id, + identifier, + count(*) as total, + countIf(passed > 0) as passed, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v1 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_month_v1 +( + time DateTime, + workspace_id String, + namespace_id String, + identifier String, + + passed Int64, + total Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_month_mv_v1 +TO ratelimits.ratelimits_per_month_v1 +AS +SELECT + workspace_id, + namespace_id, + identifier, + countIf(passed > 0) as passed, + count(*) as total, + toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time +FROM ratelimits.raw_ratelimits_v1 +GROUP BY + workspace_id, + namespace_id, + identifier, + time +; +CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_last_used_v1 +( + time Int64, + workspace_id String, + namespace_id String, + identifier String, + +) +ENGINE = AggregatingMergeTree() +ORDER BY (workspace_id, namespace_id, time, identifier) +; + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_last_used_mv_v1 +TO ratelimits.ratelimits_last_used_v1 +AS +SELECT + workspace_id, + namespace_id, + identifier, + maxSimpleState(time) as time +FROM ratelimits.raw_ratelimits_v1 +GROUP BY + workspace_id, + namespace_id, + identifier +; +CREATE DATABASE IF NOT EXISTS metrics; +CREATE TABLE IF NOT EXISTS metrics.raw_api_requests_v1( + request_id String, + -- unix milli + time Int64, + + workspace_id String, + + host String, + + -- Upper case HTTP method + -- Examples: "GET", "POST", "PUT", "DELETE" + method LowCardinality(String), + path String, + -- "Key: Value" pairs + request_headers Array(String), + request_body String, + + response_status Int, + -- "Key: Value" pairs + response_headers Array(String), + response_body String, + -- internal err.Error() string, empty if no error + error String, + + -- milliseconds + service_latency Int64, + + user_agent String, + ip_address String, + country String, + city String, + colo String, + continent String, + + +) +ENGINE = MergeTree() +ORDER BY (workspace_id, time, request_id) +; + +ALTER TABLE metrics.raw_api_requests_v1 + ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; + +ALTER TABLE metrics.raw_api_requests_v1 + ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; +CREATE TABLE IF NOT EXISTS metrics.api_requests_per_hour_v1 ( + time DateTime, + workspace_id String, + path String, + response_status Int, + host String, + -- Upper case HTTP method + -- Examples: "GET", "POST", "PUT", "DELETE" + method LowCardinality(String), + count Int64 +) ENGINE = SummingMergeTree() +ORDER BY + ( + workspace_id, + time, + host, + path, + response_status, + method + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_hour_mv_v1 TO metrics.api_requests_per_hour_v1 AS +SELECT + workspace_id, + path, + response_status, + host, + method, + count(*) as count, + toStartOfHour(fromUnixTimestamp64Milli(time)) AS time +FROM + metrics.raw_api_requests_v1 +GROUP BY + workspace_id, + path, + response_status, + host, + method, + time; +CREATE TABLE IF NOT EXISTS metrics.api_requests_per_minute_v1 ( + time DateTime, + workspace_id String, + path String, + response_status Int, + host String, + -- Upper case HTTP method + -- Examples: "GET", "POST", "PUT", "DELETE" + method LowCardinality(String), + count Int64 +) ENGINE = SummingMergeTree() +ORDER BY + ( + workspace_id, + time, + host, + path, + response_status, + method + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_minute_mv_v1 TO metrics.api_requests_per_minute_v1 AS +SELECT + workspace_id, + path, + response_status, + host, + method, + count(*) as count, + toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time +FROM + metrics.raw_api_requests_v1 +GROUP BY + workspace_id, + path, + response_status, + host, + method, + time; +CREATE TABLE IF NOT EXISTS metrics.api_requests_per_day_v1 ( + time DateTime, + workspace_id String, + path String, + response_status Int, + host String, + -- Upper case HTTP method + -- Examples: "GET", "POST", "PUT", "DELETE" + method LowCardinality(String), + count Int64 +) ENGINE = SummingMergeTree() +ORDER BY + ( + workspace_id, + time, + host, + path, + response_status, + method + ); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_day_mv_v1 TO metrics.api_requests_per_day_v1 AS +SELECT + workspace_id, + path, + response_status, + host, + method, + count(*) as count, + toStartOfDay(fromUnixTimestamp64Milli(time)) AS time +FROM + metrics.raw_api_requests_v1 +GROUP BY + workspace_id, + path, + response_status, + host, + method, + time; +CREATE DATABASE IF NOT EXISTS billing; +CREATE DATABASE IF NOT EXISTS verifications; +CREATE DATABASE IF NOT EXISTS ratelimits; +CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v1 +( + year Int, + month Int, + workspace_id String, + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, year, month) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v1 +TO billing.billable_verifications_per_month_v1 +AS +SELECT + workspace_id, + count(*) AS count, + toYear(time) AS year, + toMonth(time) AS month +FROM verifications.key_verifications_per_month_v2 +WHERE outcome = 'VALID' +GROUP BY + workspace_id, + year, + month +; +CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v2 +( + year Int, + month Int, + workspace_id String, + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, year, month) +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v2 +TO billing.billable_verifications_per_month_v2 +AS SELECT + workspace_id, + sum(count) AS count, + toYear(time) AS year, + toMonth(time) AS month +FROM verifications.key_verifications_per_month_v1 +WHERE outcome = 'VALID' +GROUP BY + workspace_id, + year, + month +; +CREATE TABLE IF NOT EXISTS billing.billable_ratelimits_per_month_v1 +( + year Int, + month Int, + workspace_id String, + count Int64 +) +ENGINE = SummingMergeTree() +ORDER BY (workspace_id, year, month) +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_ratelimits_per_month_mv_v1 +TO billing.billable_ratelimits_per_month_v1 +AS SELECT + workspace_id, + sum(passed) AS count, + toYear(time) AS year, + toMonth(time) AS month +FROM ratelimits.ratelimits_per_month_v1 +WHERE passed > 0 +GROUP BY + workspace_id, + year, + month +; +CREATE DATABASE IF NOT EXISTS business; +CREATE DATABASE IF NOT EXISTS billing; +CREATE TABLE IF NOT EXISTS business.active_workspaces_per_month_v1 +( + time Date, + workspace_id String +) +ENGINE = MergeTree() +ORDER BY (time) +; +CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_keys_per_month_mv_v1 +TO business.active_workspaces_per_month_v1 +AS +SELECT + workspace_id, toDate(time) as time +FROM verifications.key_verifications_per_month_v2 +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_ratelimits_per_month_mv_v1 +TO business.active_workspaces_per_month_v1 +AS +SELECT + workspace_id, toDate(time) as time +FROM ratelimits.ratelimits_per_month_v1 +; diff --git a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql index a7366df76f..145ca97599 100644 --- a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql +++ b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql @@ -45,3 +45,24 @@ TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 90 DAY DELETE SETTINGS non_replicated_deduplication_window = 10000 ; +-- Temporary materialized view to sync new writes from v1 to v2 during migration +-- This ensures zero-downtime migration by duplicating all new inserts +-- DROP this view after migration is complete and application switches to v2 + +CREATE MATERIALIZED VIEW temp_sync_key_verifications_v1_to_v2 +TO key_verifications_raw_v2 +AS +SELECT + request_id, + time, + workspace_id, + key_space_id, + identity_id, + '' as external_id, + key_id, + region, + outcome, + tags, + 0 as spent_credits, -- v1 doesn't have this column, default to 0 + 0.0 as latency -- v1 doesn't have this column, default to 0.0 +FROM verifications.raw_key_verifications_v1; diff --git a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql index 920bdab7e2..1f9299f1d9 100644 --- a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql +++ b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql @@ -26,3 +26,22 @@ ORDER BY TTL toDateTime (fromUnixTimestamp64Milli (time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; +-- Temporary materialized view to sync new writes from v1 to v2 during migration +-- This ensures zero-downtime migration by duplicating all new inserts +-- DROP this view after migration is complete and application switches to v2 +CREATE MATERIALIZED VIEW temp_sync_ratelimits_raw_v1_to_v2 TO ratelimits_raw_v2 AS +SELECT + request_id, + time, + workspace_id, + namespace_id, + identifier, + passed, + -- v1 doesn't have any of those columns + 0.0 as latency, + '' as override_id, + 0 as `limit`, + 0 as remaining, + 0 as reset_at +FROM + ratelimits.raw_ratelimits_v1; diff --git a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql index defe53a4c1..2d245e48ad 100644 --- a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql +++ b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql @@ -34,3 +34,28 @@ ORDER BY TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; +-- Temporary materialized view to sync new writes from v1 to v2 during migration +-- This ensures zero-downtime migration by duplicating all new inserts +-- DROP this view after migration is complete and application switches to v2 +CREATE MATERIALIZED VIEW temp_sync_metrics_v1_to_v2 TO api_requests_raw_v2 AS +SELECT + request_id, + time, + workspace_id, + host, + method, + path, + '' as query_string, + CAST(mapFromArrays(CAST([],'Array(String)'), CAST([],'Array(Array(String))')), 'Map(String, Array(String))') AS query_params, + request_headers, + request_body, + response_status, + response_headers, + response_body, + error, + service_latency, + user_agent, + ip_address, + '' as region +FROM + metrics.raw_api_requests_v1;