diff --git a/pkg/clickhouse/migrations/20250903085516_init.sql b/pkg/clickhouse/migrations/20250903085516_init.sql deleted file mode 100644 index c1ecb2cab7..0000000000 --- a/pkg/clickhouse/migrations/20250903085516_init.sql +++ /dev/null @@ -1,334 +0,0 @@ --- Add new schema named "billing" -CREATE DATABASE `billing` ENGINE Atomic; --- Add new schema named "business" -CREATE DATABASE `business` ENGINE Atomic; --- Add new schema named "metrics" -CREATE DATABASE `metrics` ENGINE Atomic; --- Add new schema named "ratelimits" -CREATE DATABASE `ratelimits` ENGINE Atomic; --- Add new schema named "verifications" -CREATE DATABASE `verifications` ENGINE Atomic; --- Create "billable_ratelimits_per_month_v1" table -CREATE TABLE `billing`.`billable_ratelimits_per_month_v1` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "billable_verifications_per_month_v1" table -CREATE TABLE `billing`.`billable_verifications_per_month_v1` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "billable_verifications_per_month_v2" table -CREATE TABLE `billing`.`billable_verifications_per_month_v2` ( - `year` Int32, - `month` Int32, - `workspace_id` String, - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "active_workspaces_per_month_v1" table -CREATE TABLE `business`.`active_workspaces_per_month_v1` ( - `time` Date, - `workspace_id` String -) ENGINE = MergeTree -PRIMARY KEY (`time`) ORDER BY (`time`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_day_v1" table -CREATE TABLE `metrics`.`api_requests_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_hour_v1" table -CREATE TABLE `metrics`.`api_requests_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "api_requests_per_minute_v1" table -CREATE TABLE `metrics`.`api_requests_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `path` String, - `response_status` Int32, - `host` String, - `method` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) ORDER BY (`workspace_id`, `time`, `host`, `path`, `response_status`, `method`) SETTINGS index_granularity = 8192; --- Create "raw_api_requests_v1" table -CREATE TABLE `metrics`.`raw_api_requests_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `host` String, - `method` LowCardinality(String), - `path` String, - `request_headers` Array(String), - `request_body` String, - `response_status` Int32, - `response_headers` Array(String), - `response_body` String, - `error` String, - `service_latency` Int64, - `user_agent` String, - `ip_address` String, - `country` String, - `city` String, - `colo` String, - `continent` String, - INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, - INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `time`, `request_id`) ORDER BY (`workspace_id`, `time`, `request_id`) SETTINGS index_granularity = 8192; --- Create "ratelimits_last_used_v1" table -CREATE TABLE `ratelimits`.`ratelimits_last_used_v1` ( - `time` Int64, - `workspace_id` String, - `namespace_id` String, - `identifier` String -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_day_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_hour_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_minute_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "ratelimits_per_month_v1" table -CREATE TABLE `ratelimits`.`ratelimits_per_month_v1` ( - `time` DateTime, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Int64, - `total` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "raw_ratelimits_v1" table -CREATE TABLE `ratelimits`.`raw_ratelimits_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `namespace_id` String, - `identifier` String, - `passed` Bool, - INDEX `idx_request_id` ((request_id)) TYPE minmax GRANULARITY 1, - INDEX `idx_workspace_time` ((workspace_id, time)) TYPE minmax GRANULARITY 1 -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `namespace_id`, `time`, `identifier`) ORDER BY (`workspace_id`, `namespace_id`, `time`, `identifier`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v1" table -CREATE TABLE `verifications`.`key_verifications_per_day_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v2" table -CREATE TABLE `verifications`.`key_verifications_per_day_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v3" table -CREATE TABLE `verifications`.`key_verifications_per_day_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v1" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v2" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v3" table -CREATE TABLE `verifications`.`key_verifications_per_hour_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_minute_v1" table -CREATE TABLE `verifications`.`key_verifications_per_minute_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v1" table -CREATE TABLE `verifications`.`key_verifications_per_month_v1` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) ORDER BY (`workspace_id`, `key_space_id`, `time`, `identity_id`, `key_id`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v2" table -CREATE TABLE `verifications`.`key_verifications_per_month_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v3" table -CREATE TABLE `verifications`.`key_verifications_per_month_v3` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` Int64 -) ENGINE = SummingMergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) ORDER BY (`workspace_id`, `key_space_id`, `identity_id`, `key_id`, `time`, `tags`, `outcome`) SETTINGS index_granularity = 8192; --- Create "raw_key_verifications_v1" table -CREATE TABLE `verifications`.`raw_key_verifications_v1` ( - `request_id` String, - `time` Int64, - `workspace_id` String, - `key_space_id` String, - `key_id` String, - `region` LowCardinality(String), - `outcome` LowCardinality(String), - `identity_id` String, - `tags` Array(String) DEFAULT [] -) ENGINE = MergeTree -PRIMARY KEY (`workspace_id`, `key_space_id`, `key_id`, `time`) ORDER BY (`workspace_id`, `key_space_id`, `key_id`, `time`) SETTINGS index_granularity = 8192; --- Create "billable_ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `billing`.`billable_ratelimits_per_month_mv_v1` TO `billing`.`billable_ratelimits_per_month_v1` AS SELECT workspace_id, sum(passed) AS count, toYear(time) AS year, toMonth(time) AS month FROM ratelimits.ratelimits_per_month_v1 WHERE passed > 0 GROUP BY workspace_id, year, month; --- Create "billable_verifications_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v1` TO `billing`.`billable_verifications_per_month_v1` AS SELECT workspace_id, count(*) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v2 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; --- Create "billable_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `billing`.`billable_verifications_per_month_mv_v2` TO `billing`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM verifications.key_verifications_per_month_v1 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; --- Create "active_workspaces_keys_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `business`.`active_workspaces_keys_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM verifications.key_verifications_per_month_v2; --- Create "active_workspaces_ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `business`.`active_workspaces_ratelimits_per_month_mv_v1` TO `business`.`active_workspaces_per_month_v1` AS SELECT workspace_id, toDate(time) AS time FROM ratelimits.ratelimits_per_month_v1; --- Create "api_requests_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_day_mv_v1` TO `metrics`.`api_requests_per_day_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "api_requests_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_hour_mv_v1` TO `metrics`.`api_requests_per_hour_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "api_requests_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `metrics`.`api_requests_per_minute_mv_v1` TO `metrics`.`api_requests_per_minute_v1` AS SELECT workspace_id, path, response_status, host, method, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM metrics.raw_api_requests_v1 GROUP BY workspace_id, path, response_status, host, method, time; --- Create "ratelimits_last_used_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_last_used_mv_v1` TO `ratelimits`.`ratelimits_last_used_v1` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier; --- Create "ratelimits_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_day_mv_v1` TO `ratelimits`.`ratelimits_per_day_v1` AS SELECT workspace_id, namespace_id, identifier, count(*) AS total, countIf(passed > 0) AS passed, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_hour_mv_v1` TO `ratelimits`.`ratelimits_per_hour_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_minute_mv_v1` TO `ratelimits`.`ratelimits_per_minute_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "ratelimits_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `ratelimits`.`ratelimits_per_month_mv_v1` TO `ratelimits`.`ratelimits_per_month_v1` AS SELECT workspace_id, namespace_id, identifier, countIf(passed > 0) AS passed, count(*) AS total, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier, time; --- Create "key_verifications_per_day_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v1` TO `verifications`.`key_verifications_per_day_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_day_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v2` TO `verifications`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_day_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_day_mv_v3` TO `verifications`.`key_verifications_per_day_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_hour_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v1` TO `verifications`.`key_verifications_per_hour_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_hour_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v2` TO `verifications`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_hour_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_hour_mv_v3` TO `verifications`.`key_verifications_per_hour_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_minute_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_minute_mv_v1` TO `verifications`.`key_verifications_per_minute_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_month_mv_v1" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v1` TO `verifications`.`key_verifications_per_month_v1` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time; --- Create "key_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v2` TO `verifications`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; --- Create "key_verifications_per_month_mv_v3" view -CREATE MATERIALIZED VIEW `verifications`.`key_verifications_per_month_mv_v3` TO `verifications`.`key_verifications_per_month_v3` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, count(*) AS count, toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, tags FROM verifications.raw_key_verifications_v1 GROUP BY workspace_id, key_space_id, identity_id, key_id, outcome, time, tags; diff --git a/pkg/clickhouse/migrations/20250911070454.sql b/pkg/clickhouse/migrations/20250911070454.sql index 42c80d9bac..03983d479e 100644 --- a/pkg/clickhouse/migrations/20250911070454.sql +++ b/pkg/clickhouse/migrations/20250911070454.sql @@ -97,82 +97,6 @@ CREATE TABLE `default`.`billable_verifications_per_month_v2` ( `count` Int64 ) ENGINE = SummingMergeTree PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_day_v2" table -CREATE TABLE `default`.`key_verifications_per_day_v2` ( - `time` Date, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` SimpleAggregateFunction(sum, Int64), - `spent_credits` SimpleAggregateFunction(sum, Int64), - `latency_avg` AggregateFunction(avg, Float64), - `latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64), - `latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64), - INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(100) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_hour_v2" table -CREATE TABLE `default`.`key_verifications_per_hour_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` SimpleAggregateFunction(sum, Int64), - `spent_credits` SimpleAggregateFunction(sum, Int64), - `latency_avg` AggregateFunction(avg, Float64), - `latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64), - `latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64), - INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(30) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_minute_v2" table -CREATE TABLE `default`.`key_verifications_per_minute_v2` ( - `time` DateTime, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` SimpleAggregateFunction(sum, Int64), - `spent_credits` SimpleAggregateFunction(sum, Int64), - `latency_avg` AggregateFunction(avg, Float64), - `latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64), - `latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64), - INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(7) SETTINGS index_granularity = 8192; --- Create "key_verifications_per_month_v2" table -CREATE TABLE `default`.`key_verifications_per_month_v2` ( - `time` Date, - `workspace_id` String, - `key_space_id` String, - `identity_id` String, - `key_id` String, - `outcome` LowCardinality(String), - `tags` Array(String), - `count` SimpleAggregateFunction(sum, Int64), - `spent_credits` SimpleAggregateFunction(sum, Int64), - `latency_avg` AggregateFunction(avg, Float64), - `latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64), - `latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64), - INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1, - INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree -PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalYear(3) SETTINGS index_granularity = 8192; -- Create "key_verifications_raw_v2" table CREATE TABLE `default`.`key_verifications_raw_v2` ( `request_id` String, @@ -269,8 +193,6 @@ CREATE TABLE `default`.`ratelimits_raw_v2` ( INDEX `idx_request_id` ((request_id)) TYPE bloom_filter GRANULARITY 1 ) ENGINE = MergeTree PRIMARY KEY (`workspace_id`, `time`, `namespace_id`) ORDER BY (`workspace_id`, `time`, `namespace_id`) TTL toDateTime(fromUnixTimestamp64Milli(time)) + toIntervalMonth(1) SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 10000; --- Create "active_workspaces_keys_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`active_workspaces_keys_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.key_verifications_per_month_v2; -- Create "active_workspaces_ratelimits_per_month_mv_v2" view CREATE MATERIALIZED VIEW `default`.`active_workspaces_ratelimits_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.ratelimits_per_month_v2; -- Create "api_requests_per_day_mv_v2" view @@ -283,18 +205,8 @@ CREATE MATERIALIZED VIEW `default`.`api_requests_per_minute_mv_v2` TO `default`. CREATE MATERIALIZED VIEW `default`.`api_requests_per_month_mv_v2` TO `default`.`api_requests_per_month_v2` AS SELECT workspace_id, path, response_status, host, method, sum(count) AS count, toDate(toStartOfMonth(time)) AS time FROM default.api_requests_per_day_v2 GROUP BY workspace_id, path, response_status, host, method, time; -- Create "billable_ratelimits_per_month_mv_v2" view CREATE MATERIALIZED VIEW `default`.`billable_ratelimits_per_month_mv_v2` TO `default`.`billable_ratelimits_per_month_v2` AS SELECT workspace_id, sum(passed) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.ratelimits_per_month_v2 GROUP BY workspace_id, year, month; --- Create "billable_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`billable_verifications_per_month_mv_v2` TO `default`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.key_verifications_per_month_v2 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; --- Create "key_verifications_per_day_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_day_mv_v2` TO `default`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.key_verifications_per_hour_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags; --- Create "key_verifications_per_hour_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_hour_mv_v2` TO `default`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toStartOfHour(time) AS time FROM default.key_verifications_per_minute_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags; --- Create "key_verifications_per_minute_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_minute_mv_v2` TO `default`.`key_verifications_per_minute_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, count(*) AS count, sum(spent_credits) AS spent_credits, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.key_verifications_raw_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags; --- Create "key_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_month_mv_v2` TO `default`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.key_verifications_per_day_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags; -- Create "ratelimits_last_used_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`ratelimits_last_used_mv_v2` TO `default`.`ratelimits_last_used_v2` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier; +CREATE MATERIALIZED VIEW `default`.`ratelimits_last_used_mv_v2` TO `default`.`ratelimits_last_used_v2` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM default.ratelimits_raw_v2 GROUP BY workspace_id, namespace_id, identifier; -- Create "ratelimits_per_day_mv_v2" view CREATE MATERIALIZED VIEW `default`.`ratelimits_per_day_mv_v2` TO `default`.`ratelimits_per_day_v2` AS SELECT workspace_id, namespace_id, identifier, sum(total) AS total, sum(passed) AS passed, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.ratelimits_per_hour_v2 GROUP BY workspace_id, namespace_id, time, identifier; -- Create "ratelimits_per_hour_mv_v2" view @@ -303,9 +215,3 @@ CREATE MATERIALIZED VIEW `default`.`ratelimits_per_hour_mv_v2` TO `default`.`rat CREATE MATERIALIZED VIEW `default`.`ratelimits_per_minute_mv_v2` TO `default`.`ratelimits_per_minute_v2` AS SELECT workspace_id, namespace_id, identifier, count(*) AS total, countIf(passed > 0) AS passed, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.ratelimits_raw_v2 GROUP BY workspace_id, namespace_id, time, identifier; -- Create "ratelimits_per_month_mv_v2" view CREATE MATERIALIZED VIEW `default`.`ratelimits_per_month_mv_v2` TO `default`.`ratelimits_per_month_v2` AS SELECT workspace_id, namespace_id, identifier, sum(total) AS total, sum(passed) AS passed, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.ratelimits_per_day_v2 GROUP BY workspace_id, namespace_id, time, identifier; --- Create "temp_sync_key_verifications_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_key_verifications_v1_to_v2` TO `default`.`key_verifications_raw_v2` AS SELECT request_id, time, workspace_id, key_space_id, identity_id, key_id, region, outcome, tags, 0 AS spent_credits, 0. AS latency FROM verifications.raw_key_verifications_v1; --- Create "temp_sync_metrics_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_metrics_v1_to_v2` TO `default`.`api_requests_raw_v2` AS SELECT request_id, time, workspace_id, host, method, path, request_headers, request_body, response_status, response_headers, response_body, error, service_latency, user_agent, ip_address, '' AS region FROM metrics.raw_api_requests_v1; --- Create "temp_sync_ratelimits_raw_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2` TO `default`.`ratelimits_raw_v2` AS SELECT request_id, time, workspace_id, namespace_id, identifier, passed, 0. AS latency FROM ratelimits.raw_ratelimits_v1; diff --git a/pkg/clickhouse/migrations/20250925091254.sql b/pkg/clickhouse/migrations/20250925091254.sql index 7fb82f4079..d5f244e0a8 100644 --- a/pkg/clickhouse/migrations/20250925091254.sql +++ b/pkg/clickhouse/migrations/20250925091254.sql @@ -4,11 +4,3 @@ ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `override_id` String; ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `limit` UInt64; ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `remaining` UInt64; ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `reset_at` Int64 CODEC(Delta(8), LZ4); --- Drop "temp_sync_metrics_v1_to_v2" view -DROP VIEW `default`.`temp_sync_metrics_v1_to_v2`; --- Create "temp_sync_metrics_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_metrics_v1_to_v2` TO `default`.`api_requests_raw_v2` AS SELECT request_id, time, workspace_id, host, method, path, '' AS query_string, CAST(mapFromArrays(CAST([], 'Array(String)'), CAST([], 'Array(Array(String))')), 'Map(String, Array(String))') AS query_params, request_headers, request_body, response_status, response_headers, response_body, error, service_latency, user_agent, ip_address, '' AS region FROM metrics.raw_api_requests_v1; --- Drop "temp_sync_ratelimits_raw_v1_to_v2" view -DROP VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2`; --- Create "temp_sync_ratelimits_raw_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2` TO `default`.`ratelimits_raw_v2` AS SELECT request_id, time, workspace_id, namespace_id, identifier, passed, 0. AS latency, '' AS override_id, 0 AS limit, 0 AS remaining, 0 AS reset_at FROM ratelimits.raw_ratelimits_v1; diff --git a/pkg/clickhouse/migrations/20251010160229.sql b/pkg/clickhouse/migrations/20251010160229.sql index a11712c85f..3675a29b75 100644 --- a/pkg/clickhouse/migrations/20251010160229.sql +++ b/pkg/clickhouse/migrations/20251010160229.sql @@ -1,26 +1,2 @@ --- Drop "key_verifications_per_month_mv_v2" view -DROP VIEW `default`.`key_verifications_per_month_mv_v2`; -ALTER TABLE `default`.`key_verifications_per_day_v2` ADD COLUMN `external_id` String; --- Drop "key_verifications_per_day_mv_v2" view -DROP VIEW `default`.`key_verifications_per_day_mv_v2`; -ALTER TABLE `default`.`key_verifications_per_hour_v2` ADD COLUMN `external_id` String; --- Drop "key_verifications_per_hour_mv_v2" view -DROP VIEW `default`.`key_verifications_per_hour_mv_v2`; -ALTER TABLE `default`.`key_verifications_per_minute_v2` ADD COLUMN `external_id` String; -ALTER TABLE `default`.`key_verifications_per_month_v2` ADD COLUMN `external_id` String; --- Drop "key_verifications_per_minute_mv_v2" view -DROP VIEW `default`.`key_verifications_per_minute_mv_v2`; ALTER TABLE `default`.`key_verifications_raw_v2` ADD COLUMN `external_id` String; ALTER TABLE `default`.`key_verifications_raw_v2` ADD INDEX `idx_external_id` ((external_id)) TYPE bloom_filter GRANULARITY 1; --- Create "key_verifications_per_day_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_day_mv_v2` TO `default`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.key_verifications_per_hour_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags; --- Create "key_verifications_per_hour_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_hour_mv_v2` TO `default`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toStartOfHour(time) AS time FROM default.key_verifications_per_minute_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags; --- Create "key_verifications_per_minute_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_minute_mv_v2` TO `default`.`key_verifications_per_minute_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, count(*) AS count, sum(spent_credits) AS spent_credits, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.key_verifications_raw_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags; --- Create "key_verifications_per_month_mv_v2" view -CREATE MATERIALIZED VIEW `default`.`key_verifications_per_month_mv_v2` TO `default`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.key_verifications_per_day_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags; --- Drop "temp_sync_key_verifications_v1_to_v2" view -DROP VIEW `default`.`temp_sync_key_verifications_v1_to_v2`; --- Create "temp_sync_key_verifications_v1_to_v2" view -CREATE MATERIALIZED VIEW `default`.`temp_sync_key_verifications_v1_to_v2` TO `default`.`key_verifications_raw_v2` AS SELECT request_id, time, workspace_id, key_space_id, identity_id, '' AS external_id, key_id, region, outcome, tags, 0 AS spent_credits, 0. AS latency FROM verifications.raw_key_verifications_v1; diff --git a/pkg/clickhouse/migrations/20251125075943.sql b/pkg/clickhouse/migrations/20251125075943.sql index 9a0b4e7a6d..930144b58d 100644 --- a/pkg/clickhouse/migrations/20251125075943.sql +++ b/pkg/clickhouse/migrations/20251125075943.sql @@ -79,12 +79,8 @@ CREATE TABLE `default`.`key_verifications_per_month_v3` ( INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 4 ) ENGINE = AggregatingMergeTree PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `external_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `external_id`, `key_id`, `outcome`, `tags`) PARTITION BY (toStartOfYear(time)) TTL time + toIntervalYear(3) SETTINGS index_granularity = 8192; --- Drop "active_workspaces_keys_per_month_mv_v2" view -DROP VIEW `default`.`active_workspaces_keys_per_month_mv_v2`; -- Create "active_workspaces_keys_per_month_mv_v2" view CREATE MATERIALIZED VIEW `default`.`active_workspaces_keys_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.key_verifications_per_month_v3; --- Drop "billable_verifications_per_month_mv_v2" view -DROP VIEW `default`.`billable_verifications_per_month_mv_v2`; -- Create "billable_verifications_per_month_mv_v2" view CREATE MATERIALIZED VIEW `default`.`billable_verifications_per_month_mv_v2` TO `default`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.key_verifications_per_month_v3 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month; -- Create "key_verifications_per_day_mv_v3" view diff --git a/pkg/clickhouse/schema/000_legacy.sql b/pkg/clickhouse/schema/000_legacy.sql deleted file mode 100644 index 3830eb5eb0..0000000000 --- a/pkg/clickhouse/schema/000_legacy.sql +++ /dev/null @@ -1,822 +0,0 @@ -CREATE DATABASE IF NOT EXISTS verifications; - -CREATE TABLE verifications.raw_key_verifications_v1( - -- the api request id, so we can correlate the verification with traces and logs - request_id String, - - -- unix milli - time Int64, - - workspace_id String, - key_space_id String, - key_id String, - - -- Right now this is a 3 character airport code, but when we move to aws, - -- this will be the region code such as `us-east-1` - region LowCardinality(String), - - -- Examples: - -- - "VALID" - -- - "RATE_LIMITED" - -- - "EXPIRED" - -- - "DISABLED - outcome LowCardinality(String), - - -- Empty string if the key has no identity - identity_id String, - - tags Array(String) DEFAULT [] - - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, key_space_id, key_id, time) -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v1 -TO verifications.key_verifications_per_hour_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v1 -TO verifications.key_verifications_per_day_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, time, identity_id, key_id) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v1 -TO verifications.key_verifications_per_month_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v2 -TO verifications.key_verifications_per_hour_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v2 -TO verifications.key_verifications_per_day_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v2 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v2 -TO verifications.key_verifications_per_month_v2 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_hour_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_hour_mv_v3 -TO verifications.key_verifications_per_hour_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_day_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_day_mv_v3 -TO verifications.key_verifications_per_day_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE IF NOT EXISTS verifications.key_verifications_per_month_v3 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_month_mv_v3 -TO verifications.key_verifications_per_month_v3 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE TABLE verifications.key_verifications_per_minute_v1 -( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - key_id String, - outcome LowCardinality(String), - tags Array(String), - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags, outcome) -; - - -CREATE MATERIALIZED VIEW IF NOT EXISTS verifications.key_verifications_per_minute_mv_v1 -TO verifications.key_verifications_per_minute_v1 -AS -SELECT - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - count(*) as count, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time, - tags -FROM verifications.raw_key_verifications_v1 -GROUP BY - workspace_id, - key_space_id, - identity_id, - key_id, - outcome, - time, - tags -; -CREATE DATABASE IF NOT EXISTS ratelimits; -CREATE TABLE IF NOT EXISTS ratelimits.raw_ratelimits_v1( - request_id String, - -- unix milli - time Int64, - workspace_id String, - namespace_id String, - identifier String, - passed Bool - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) - -; - -ALTER TABLE ratelimits.raw_ratelimits_v1 ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE ratelimits.raw_ratelimits_v1 -ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_minute_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_minute_mv_v1 -TO ratelimits.ratelimits_per_minute_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_hour_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_hour_mv_v1 -TO ratelimits.ratelimits_per_hour_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_day_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_day_mv_v1 -TO ratelimits.ratelimits_per_day_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - count(*) as total, - countIf(passed > 0) as passed, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_per_month_v1 -( - time DateTime, - workspace_id String, - namespace_id String, - identifier String, - - passed Int64, - total Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_per_month_mv_v1 -TO ratelimits.ratelimits_per_month_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - countIf(passed > 0) as passed, - count(*) as total, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier, - time -; -CREATE TABLE IF NOT EXISTS ratelimits.ratelimits_last_used_v1 -( - time Int64, - workspace_id String, - namespace_id String, - identifier String, - -) -ENGINE = AggregatingMergeTree() -ORDER BY (workspace_id, namespace_id, time, identifier) -; - - - -CREATE MATERIALIZED VIEW IF NOT EXISTS ratelimits.ratelimits_last_used_mv_v1 -TO ratelimits.ratelimits_last_used_v1 -AS -SELECT - workspace_id, - namespace_id, - identifier, - maxSimpleState(time) as time -FROM ratelimits.raw_ratelimits_v1 -GROUP BY - workspace_id, - namespace_id, - identifier -; -CREATE DATABASE IF NOT EXISTS metrics; -CREATE TABLE IF NOT EXISTS metrics.raw_api_requests_v1( - request_id String, - -- unix milli - time Int64, - - workspace_id String, - - host String, - - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - path String, - -- "Key: Value" pairs - request_headers Array(String), - request_body String, - - response_status Int, - -- "Key: Value" pairs - response_headers Array(String), - response_body String, - -- internal err.Error() string, empty if no error - error String, - - -- milliseconds - service_latency Int64, - - user_agent String, - ip_address String, - country String, - city String, - colo String, - continent String, - - -) -ENGINE = MergeTree() -ORDER BY (workspace_id, time, request_id) -; - -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX IF NOT EXISTS idx_workspace_time (workspace_id, time) TYPE minmax GRANULARITY 1; - -ALTER TABLE metrics.raw_api_requests_v1 - ADD INDEX IF NOT EXISTS idx_request_id (request_id) TYPE minmax GRANULARITY 1; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_hour_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_hour_mv_v1 TO metrics.api_requests_per_hour_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_minute_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_minute_mv_v1 TO metrics.api_requests_per_minute_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE TABLE IF NOT EXISTS metrics.api_requests_per_day_v1 ( - time DateTime, - workspace_id String, - path String, - response_status Int, - host String, - -- Upper case HTTP method - -- Examples: "GET", "POST", "PUT", "DELETE" - method LowCardinality(String), - count Int64 -) ENGINE = SummingMergeTree() -ORDER BY - ( - workspace_id, - time, - host, - path, - response_status, - method - ); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS metrics.api_requests_per_day_mv_v1 TO metrics.api_requests_per_day_v1 AS -SELECT - workspace_id, - path, - response_status, - host, - method, - count(*) as count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time -FROM - metrics.raw_api_requests_v1 -GROUP BY - workspace_id, - path, - response_status, - host, - method, - time; -CREATE DATABASE IF NOT EXISTS billing; -CREATE DATABASE IF NOT EXISTS verifications; -CREATE DATABASE IF NOT EXISTS ratelimits; -CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v1 -TO billing.billable_verifications_per_month_v1 -AS -SELECT - workspace_id, - count(*) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v2 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; -CREATE TABLE IF NOT EXISTS billing.billable_verifications_per_month_v2 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_verifications_per_month_mv_v2 -TO billing.billable_verifications_per_month_v2 -AS SELECT - workspace_id, - sum(count) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM verifications.key_verifications_per_month_v1 -WHERE outcome = 'VALID' -GROUP BY - workspace_id, - year, - month -; -CREATE TABLE IF NOT EXISTS billing.billable_ratelimits_per_month_v1 -( - year Int, - month Int, - workspace_id String, - count Int64 -) -ENGINE = SummingMergeTree() -ORDER BY (workspace_id, year, month) -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS billing.billable_ratelimits_per_month_mv_v1 -TO billing.billable_ratelimits_per_month_v1 -AS SELECT - workspace_id, - sum(passed) AS count, - toYear(time) AS year, - toMonth(time) AS month -FROM ratelimits.ratelimits_per_month_v1 -WHERE passed > 0 -GROUP BY - workspace_id, - year, - month -; -CREATE DATABASE IF NOT EXISTS business; -CREATE DATABASE IF NOT EXISTS billing; -CREATE TABLE IF NOT EXISTS business.active_workspaces_per_month_v1 -( - time Date, - workspace_id String -) -ENGINE = MergeTree() -ORDER BY (time) -; -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_keys_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM verifications.key_verifications_per_month_v2 -; - -CREATE MATERIALIZED VIEW IF NOT EXISTS business.active_workspaces_ratelimits_per_month_mv_v1 -TO business.active_workspaces_per_month_v1 -AS -SELECT - workspace_id, toDate(time) as time -FROM ratelimits.ratelimits_per_month_v1 -; diff --git a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql index 145ca97599..a7366df76f 100644 --- a/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql +++ b/pkg/clickhouse/schema/001_key_verifications_raw_v2.sql @@ -45,24 +45,3 @@ TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 90 DAY DELETE SETTINGS non_replicated_deduplication_window = 10000 ; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 - -CREATE MATERIALIZED VIEW temp_sync_key_verifications_v1_to_v2 -TO key_verifications_raw_v2 -AS -SELECT - request_id, - time, - workspace_id, - key_space_id, - identity_id, - '' as external_id, - key_id, - region, - outcome, - tags, - 0 as spent_credits, -- v1 doesn't have this column, default to 0 - 0.0 as latency -- v1 doesn't have this column, default to 0.0 -FROM verifications.raw_key_verifications_v1; diff --git a/pkg/clickhouse/schema/002_key_verifications_per_minute_v2.sql b/pkg/clickhouse/schema/002_key_verifications_per_minute_v2.sql deleted file mode 100644 index f036adf9e3..0000000000 --- a/pkg/clickhouse/schema/002_key_verifications_per_minute_v2.sql +++ /dev/null @@ -1,58 +0,0 @@ -CREATE TABLE key_verifications_per_minute_v2 ( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - external_id String, - key_id String, - outcome LowCardinality (String), - tags Array(String), - count SimpleAggregateFunction(sum, Int64), - spent_credits SimpleAggregateFunction(sum, Int64), - latency_avg AggregateFunction (avg, Float64), - latency_p75 AggregateFunction (quantilesTDigest (0.75), Float64), - latency_p99 AggregateFunction (quantilesTDigest (0.99), Float64), - INDEX idx_identity_id (identity_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_key_id (key_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_tags (tags) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree () -ORDER BY - ( - workspace_id, - time, - key_space_id, - identity_id, - key_id, - outcome, - tags - ) -TTL time + INTERVAL 7 DAY DELETE -; - -CREATE MATERIALIZED VIEW key_verifications_per_minute_mv_v2 TO key_verifications_per_minute_v2 AS -SELECT - workspace_id, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags, - count(*) as count, - sum(spent_credits) as spent_credits, - avgState (latency) as latency_avg, - quantilesTDigestState (0.75) (latency) as latency_p75, - quantilesTDigestState (0.99) (latency) as latency_p99, - toStartOfMinute (fromUnixTimestamp64Milli (time)) AS time -FROM - key_verifications_raw_v2 -GROUP BY - workspace_id, - time, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags -; diff --git a/pkg/clickhouse/schema/003_key_verifications_per_hour_v2.sql b/pkg/clickhouse/schema/003_key_verifications_per_hour_v2.sql deleted file mode 100644 index 5119aea761..0000000000 --- a/pkg/clickhouse/schema/003_key_verifications_per_hour_v2.sql +++ /dev/null @@ -1,57 +0,0 @@ -CREATE TABLE key_verifications_per_hour_v2 ( - time DateTime, - workspace_id String, - key_space_id String, - identity_id String, - external_id String, - key_id String, - outcome LowCardinality (String), - tags Array(String), - count SimpleAggregateFunction(sum, Int64), - spent_credits SimpleAggregateFunction(sum, Int64), - latency_avg AggregateFunction (avg, Float64), - latency_p75 AggregateFunction (quantilesTDigest (0.75), Float64), - latency_p99 AggregateFunction (quantilesTDigest (0.99), Float64), - INDEX idx_identity_id (identity_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_key_id (key_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_tags (tags) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree () -ORDER BY - ( - workspace_id, - time, - key_space_id, - identity_id, - key_id, - outcome, - tags - ) -TTL time + INTERVAL 30 DAY DELETE; - -CREATE MATERIALIZED VIEW key_verifications_per_hour_mv_v2 TO key_verifications_per_hour_v2 AS -SELECT - workspace_id, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags, - sum(count) as count, - sum(spent_credits) as spent_credits, - avgMergeState(latency_avg) as latency_avg, - quantilesTDigestMergeState(0.75)(latency_p75) as latency_p75, - quantilesTDigestMergeState(0.99)(latency_p99) as latency_p99, - toStartOfHour(time) AS time -FROM - key_verifications_per_minute_v2 -GROUP BY - workspace_id, - time, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags -; diff --git a/pkg/clickhouse/schema/004_key_verifications_per_day_v2.sql b/pkg/clickhouse/schema/004_key_verifications_per_day_v2.sql deleted file mode 100644 index bd1ce5e7eb..0000000000 --- a/pkg/clickhouse/schema/004_key_verifications_per_day_v2.sql +++ /dev/null @@ -1,58 +0,0 @@ -CREATE TABLE key_verifications_per_day_v2 ( - time Date, - workspace_id String, - key_space_id String, - identity_id String, - external_id String, - key_id String, - outcome LowCardinality (String), - tags Array(String), - count SimpleAggregateFunction(sum, Int64), - spent_credits SimpleAggregateFunction(sum, Int64), - latency_avg AggregateFunction (avg, Float64), - latency_p75 AggregateFunction (quantilesTDigest (0.75), Float64), - latency_p99 AggregateFunction (quantilesTDigest (0.99), Float64), - INDEX idx_identity_id (identity_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_key_id (key_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_tags (tags) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree () -ORDER BY - ( - workspace_id, - time, - key_space_id, - identity_id, - key_id, - outcome, - tags - ) -TTL time + INTERVAL 100 DAY DELETE - ; - -CREATE MATERIALIZED VIEW key_verifications_per_day_mv_v2 TO key_verifications_per_day_v2 AS -SELECT - workspace_id, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags, - sum(count) as count, - sum(spent_credits) as spent_credits, - avgMergeState(latency_avg) as latency_avg, - quantilesTDigestMergeState(0.75)(latency_p75) as latency_p75, - quantilesTDigestMergeState(0.99)(latency_p99) as latency_p99, - toDate(toStartOfDay(time)) AS time -FROM - key_verifications_per_hour_v2 -GROUP BY - workspace_id, - time, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags -; diff --git a/pkg/clickhouse/schema/005_key_verifications_per_month_v2.sql b/pkg/clickhouse/schema/005_key_verifications_per_month_v2.sql deleted file mode 100644 index 8749be7592..0000000000 --- a/pkg/clickhouse/schema/005_key_verifications_per_month_v2.sql +++ /dev/null @@ -1,58 +0,0 @@ -CREATE TABLE key_verifications_per_month_v2 ( - time Date, - workspace_id String, - key_space_id String, - identity_id String, - external_id String, - key_id String, - outcome LowCardinality (String), - tags Array(String), - count SimpleAggregateFunction(sum, Int64), - spent_credits SimpleAggregateFunction(sum, Int64), - latency_avg AggregateFunction (avg, Float64), - latency_p75 AggregateFunction (quantilesTDigest (0.75), Float64), - latency_p99 AggregateFunction (quantilesTDigest (0.99), Float64), - INDEX idx_identity_id (identity_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_key_id (key_id) TYPE bloom_filter GRANULARITY 1, - INDEX idx_tags (tags) TYPE bloom_filter GRANULARITY 1 -) ENGINE = AggregatingMergeTree () -ORDER BY - ( - workspace_id, - time, - key_space_id, - identity_id, - key_id, - outcome, - tags - ) -TTL time + INTERVAL 3 YEAR DELETE -; - -CREATE MATERIALIZED VIEW key_verifications_per_month_mv_v2 TO key_verifications_per_month_v2 AS -SELECT - workspace_id, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags, - sum(count) as count, - sum(spent_credits) as spent_credits, - avgMergeState(latency_avg) as latency_avg, - quantilesTDigestMergeState(0.75)(latency_p75) as latency_p75, - quantilesTDigestMergeState(0.99)(latency_p99) as latency_p99, - toDate(toStartOfMonth(time)) AS time -FROM - key_verifications_per_day_v2 -GROUP BY - workspace_id, - time, - key_space_id, - identity_id, - external_id, - key_id, - outcome, - tags -; diff --git a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql index 1f9299f1d9..920bdab7e2 100644 --- a/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql +++ b/pkg/clickhouse/schema/006_ratelimits_raw_v2.sql @@ -26,22 +26,3 @@ ORDER BY TTL toDateTime (fromUnixTimestamp64Milli (time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 -CREATE MATERIALIZED VIEW temp_sync_ratelimits_raw_v1_to_v2 TO ratelimits_raw_v2 AS -SELECT - request_id, - time, - workspace_id, - namespace_id, - identifier, - passed, - -- v1 doesn't have any of those columns - 0.0 as latency, - '' as override_id, - 0 as `limit`, - 0 as remaining, - 0 as reset_at -FROM - ratelimits.raw_ratelimits_v1; diff --git a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql index 2d245e48ad..defe53a4c1 100644 --- a/pkg/clickhouse/schema/012_api_requests_raw_v2.sql +++ b/pkg/clickhouse/schema/012_api_requests_raw_v2.sql @@ -34,28 +34,3 @@ ORDER BY TTL toDateTime(fromUnixTimestamp64Milli(time)) + INTERVAL 1 MONTH DELETE SETTINGS non_replicated_deduplication_window = 10000; --- Temporary materialized view to sync new writes from v1 to v2 during migration --- This ensures zero-downtime migration by duplicating all new inserts --- DROP this view after migration is complete and application switches to v2 -CREATE MATERIALIZED VIEW temp_sync_metrics_v1_to_v2 TO api_requests_raw_v2 AS -SELECT - request_id, - time, - workspace_id, - host, - method, - path, - '' as query_string, - CAST(mapFromArrays(CAST([],'Array(String)'), CAST([],'Array(Array(String))')), 'Map(String, Array(String))') AS query_params, - request_headers, - request_body, - response_status, - response_headers, - response_body, - error, - service_latency, - user_agent, - ip_address, - '' as region -FROM - metrics.raw_api_requests_v1;