Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
334 changes: 0 additions & 334 deletions pkg/clickhouse/migrations/20250903085516_init.sql

This file was deleted.

96 changes: 1 addition & 95 deletions pkg/clickhouse/migrations/20250911070454.sql
Original file line number Diff line number Diff line change
Expand Up @@ -97,82 +97,6 @@ CREATE TABLE `default`.`billable_verifications_per_month_v2` (
`count` Int64
) ENGINE = SummingMergeTree
PRIMARY KEY (`workspace_id`, `year`, `month`) ORDER BY (`workspace_id`, `year`, `month`) SETTINGS index_granularity = 8192;
-- Create "key_verifications_per_day_v2" table
CREATE TABLE `default`.`key_verifications_per_day_v2` (
`time` Date,
`workspace_id` String,
`key_space_id` String,
`identity_id` String,
`key_id` String,
`outcome` LowCardinality(String),
`tags` Array(String),
`count` SimpleAggregateFunction(sum, Int64),
`spent_credits` SimpleAggregateFunction(sum, Int64),
`latency_avg` AggregateFunction(avg, Float64),
`latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64),
`latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64),
INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1
) ENGINE = AggregatingMergeTree
PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(100) SETTINGS index_granularity = 8192;
-- Create "key_verifications_per_hour_v2" table
CREATE TABLE `default`.`key_verifications_per_hour_v2` (
`time` DateTime,
`workspace_id` String,
`key_space_id` String,
`identity_id` String,
`key_id` String,
`outcome` LowCardinality(String),
`tags` Array(String),
`count` SimpleAggregateFunction(sum, Int64),
`spent_credits` SimpleAggregateFunction(sum, Int64),
`latency_avg` AggregateFunction(avg, Float64),
`latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64),
`latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64),
INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1
) ENGINE = AggregatingMergeTree
PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(30) SETTINGS index_granularity = 8192;
-- Create "key_verifications_per_minute_v2" table
CREATE TABLE `default`.`key_verifications_per_minute_v2` (
`time` DateTime,
`workspace_id` String,
`key_space_id` String,
`identity_id` String,
`key_id` String,
`outcome` LowCardinality(String),
`tags` Array(String),
`count` SimpleAggregateFunction(sum, Int64),
`spent_credits` SimpleAggregateFunction(sum, Int64),
`latency_avg` AggregateFunction(avg, Float64),
`latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64),
`latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64),
INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1
) ENGINE = AggregatingMergeTree
PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalDay(7) SETTINGS index_granularity = 8192;
-- Create "key_verifications_per_month_v2" table
CREATE TABLE `default`.`key_verifications_per_month_v2` (
`time` Date,
`workspace_id` String,
`key_space_id` String,
`identity_id` String,
`key_id` String,
`outcome` LowCardinality(String),
`tags` Array(String),
`count` SimpleAggregateFunction(sum, Int64),
`spent_credits` SimpleAggregateFunction(sum, Int64),
`latency_avg` AggregateFunction(avg, Float64),
`latency_p75` AggregateFunction(quantilesTDigest(0.75), Float64),
`latency_p99` AggregateFunction(quantilesTDigest(0.99), Float64),
INDEX `idx_identity_id` ((identity_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_key_id` ((key_id)) TYPE bloom_filter GRANULARITY 1,
INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 1
) ENGINE = AggregatingMergeTree
PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `key_id`, `outcome`, `tags`) TTL time + toIntervalYear(3) SETTINGS index_granularity = 8192;
-- Create "key_verifications_raw_v2" table
CREATE TABLE `default`.`key_verifications_raw_v2` (
`request_id` String,
Expand Down Expand Up @@ -269,8 +193,6 @@ CREATE TABLE `default`.`ratelimits_raw_v2` (
INDEX `idx_request_id` ((request_id)) TYPE bloom_filter GRANULARITY 1
) ENGINE = MergeTree
PRIMARY KEY (`workspace_id`, `time`, `namespace_id`) ORDER BY (`workspace_id`, `time`, `namespace_id`) TTL toDateTime(fromUnixTimestamp64Milli(time)) + toIntervalMonth(1) SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 10000;
-- Create "active_workspaces_keys_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`active_workspaces_keys_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.key_verifications_per_month_v2;
-- Create "active_workspaces_ratelimits_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`active_workspaces_ratelimits_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.ratelimits_per_month_v2;
-- Create "api_requests_per_day_mv_v2" view
Expand All @@ -283,18 +205,8 @@ CREATE MATERIALIZED VIEW `default`.`api_requests_per_minute_mv_v2` TO `default`.
CREATE MATERIALIZED VIEW `default`.`api_requests_per_month_mv_v2` TO `default`.`api_requests_per_month_v2` AS SELECT workspace_id, path, response_status, host, method, sum(count) AS count, toDate(toStartOfMonth(time)) AS time FROM default.api_requests_per_day_v2 GROUP BY workspace_id, path, response_status, host, method, time;
-- Create "billable_ratelimits_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`billable_ratelimits_per_month_mv_v2` TO `default`.`billable_ratelimits_per_month_v2` AS SELECT workspace_id, sum(passed) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.ratelimits_per_month_v2 GROUP BY workspace_id, year, month;
-- Create "billable_verifications_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`billable_verifications_per_month_mv_v2` TO `default`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.key_verifications_per_month_v2 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month;
-- Create "key_verifications_per_day_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_day_mv_v2` TO `default`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.key_verifications_per_hour_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags;
-- Create "key_verifications_per_hour_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_hour_mv_v2` TO `default`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toStartOfHour(time) AS time FROM default.key_verifications_per_minute_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags;
-- Create "key_verifications_per_minute_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_minute_mv_v2` TO `default`.`key_verifications_per_minute_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, count(*) AS count, sum(spent_credits) AS spent_credits, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.key_verifications_raw_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags;
-- Create "key_verifications_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_month_mv_v2` TO `default`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.key_verifications_per_day_v2 GROUP BY workspace_id, time, key_space_id, identity_id, key_id, outcome, tags;
-- Create "ratelimits_last_used_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`ratelimits_last_used_mv_v2` TO `default`.`ratelimits_last_used_v2` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM ratelimits.raw_ratelimits_v1 GROUP BY workspace_id, namespace_id, identifier;
CREATE MATERIALIZED VIEW `default`.`ratelimits_last_used_mv_v2` TO `default`.`ratelimits_last_used_v2` AS SELECT workspace_id, namespace_id, identifier, maxSimpleState(time) AS time FROM default.ratelimits_raw_v2 GROUP BY workspace_id, namespace_id, identifier;
-- Create "ratelimits_per_day_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`ratelimits_per_day_mv_v2` TO `default`.`ratelimits_per_day_v2` AS SELECT workspace_id, namespace_id, identifier, sum(total) AS total, sum(passed) AS passed, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.ratelimits_per_hour_v2 GROUP BY workspace_id, namespace_id, time, identifier;
-- Create "ratelimits_per_hour_mv_v2" view
Expand All @@ -303,9 +215,3 @@ CREATE MATERIALIZED VIEW `default`.`ratelimits_per_hour_mv_v2` TO `default`.`rat
CREATE MATERIALIZED VIEW `default`.`ratelimits_per_minute_mv_v2` TO `default`.`ratelimits_per_minute_v2` AS SELECT workspace_id, namespace_id, identifier, count(*) AS total, countIf(passed > 0) AS passed, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.ratelimits_raw_v2 GROUP BY workspace_id, namespace_id, time, identifier;
-- Create "ratelimits_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`ratelimits_per_month_mv_v2` TO `default`.`ratelimits_per_month_v2` AS SELECT workspace_id, namespace_id, identifier, sum(total) AS total, sum(passed) AS passed, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.ratelimits_per_day_v2 GROUP BY workspace_id, namespace_id, time, identifier;
-- Create "temp_sync_key_verifications_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_key_verifications_v1_to_v2` TO `default`.`key_verifications_raw_v2` AS SELECT request_id, time, workspace_id, key_space_id, identity_id, key_id, region, outcome, tags, 0 AS spent_credits, 0. AS latency FROM verifications.raw_key_verifications_v1;
-- Create "temp_sync_metrics_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_metrics_v1_to_v2` TO `default`.`api_requests_raw_v2` AS SELECT request_id, time, workspace_id, host, method, path, request_headers, request_body, response_status, response_headers, response_body, error, service_latency, user_agent, ip_address, '' AS region FROM metrics.raw_api_requests_v1;
-- Create "temp_sync_ratelimits_raw_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2` TO `default`.`ratelimits_raw_v2` AS SELECT request_id, time, workspace_id, namespace_id, identifier, passed, 0. AS latency FROM ratelimits.raw_ratelimits_v1;
8 changes: 0 additions & 8 deletions pkg/clickhouse/migrations/20250925091254.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,3 @@ ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `override_id` String;
ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `limit` UInt64;
ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `remaining` UInt64;
ALTER TABLE `default`.`ratelimits_raw_v2` ADD COLUMN `reset_at` Int64 CODEC(Delta(8), LZ4);
-- Drop "temp_sync_metrics_v1_to_v2" view
DROP VIEW `default`.`temp_sync_metrics_v1_to_v2`;
-- Create "temp_sync_metrics_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_metrics_v1_to_v2` TO `default`.`api_requests_raw_v2` AS SELECT request_id, time, workspace_id, host, method, path, '' AS query_string, CAST(mapFromArrays(CAST([], 'Array(String)'), CAST([], 'Array(Array(String))')), 'Map(String, Array(String))') AS query_params, request_headers, request_body, response_status, response_headers, response_body, error, service_latency, user_agent, ip_address, '' AS region FROM metrics.raw_api_requests_v1;
-- Drop "temp_sync_ratelimits_raw_v1_to_v2" view
DROP VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2`;
-- Create "temp_sync_ratelimits_raw_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_ratelimits_raw_v1_to_v2` TO `default`.`ratelimits_raw_v2` AS SELECT request_id, time, workspace_id, namespace_id, identifier, passed, 0. AS latency, '' AS override_id, 0 AS limit, 0 AS remaining, 0 AS reset_at FROM ratelimits.raw_ratelimits_v1;
24 changes: 0 additions & 24 deletions pkg/clickhouse/migrations/20251010160229.sql
Original file line number Diff line number Diff line change
@@ -1,26 +1,2 @@
-- Drop "key_verifications_per_month_mv_v2" view
DROP VIEW `default`.`key_verifications_per_month_mv_v2`;
ALTER TABLE `default`.`key_verifications_per_day_v2` ADD COLUMN `external_id` String;
-- Drop "key_verifications_per_day_mv_v2" view
DROP VIEW `default`.`key_verifications_per_day_mv_v2`;
ALTER TABLE `default`.`key_verifications_per_hour_v2` ADD COLUMN `external_id` String;
-- Drop "key_verifications_per_hour_mv_v2" view
DROP VIEW `default`.`key_verifications_per_hour_mv_v2`;
ALTER TABLE `default`.`key_verifications_per_minute_v2` ADD COLUMN `external_id` String;
ALTER TABLE `default`.`key_verifications_per_month_v2` ADD COLUMN `external_id` String;
-- Drop "key_verifications_per_minute_mv_v2" view
DROP VIEW `default`.`key_verifications_per_minute_mv_v2`;
ALTER TABLE `default`.`key_verifications_raw_v2` ADD COLUMN `external_id` String;
ALTER TABLE `default`.`key_verifications_raw_v2` ADD INDEX `idx_external_id` ((external_id)) TYPE bloom_filter GRANULARITY 1;
-- Create "key_verifications_per_day_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_day_mv_v2` TO `default`.`key_verifications_per_day_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfDay(time)) AS time FROM default.key_verifications_per_hour_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags;
-- Create "key_verifications_per_hour_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_hour_mv_v2` TO `default`.`key_verifications_per_hour_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toStartOfHour(time) AS time FROM default.key_verifications_per_minute_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags;
-- Create "key_verifications_per_minute_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_minute_mv_v2` TO `default`.`key_verifications_per_minute_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, count(*) AS count, sum(spent_credits) AS spent_credits, avgState(latency) AS latency_avg, quantilesTDigestState(0.75)(latency) AS latency_p75, quantilesTDigestState(0.99)(latency) AS latency_p99, toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time FROM default.key_verifications_raw_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags;
-- Create "key_verifications_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`key_verifications_per_month_mv_v2` TO `default`.`key_verifications_per_month_v2` AS SELECT workspace_id, key_space_id, identity_id, external_id, key_id, outcome, tags, sum(count) AS count, sum(spent_credits) AS spent_credits, avgMergeState(latency_avg) AS latency_avg, quantilesTDigestMergeState(0.75)(latency_p75) AS latency_p75, quantilesTDigestMergeState(0.99)(latency_p99) AS latency_p99, toDate(toStartOfMonth(time)) AS time FROM default.key_verifications_per_day_v2 GROUP BY workspace_id, time, key_space_id, identity_id, external_id, key_id, outcome, tags;
-- Drop "temp_sync_key_verifications_v1_to_v2" view
DROP VIEW `default`.`temp_sync_key_verifications_v1_to_v2`;
-- Create "temp_sync_key_verifications_v1_to_v2" view
CREATE MATERIALIZED VIEW `default`.`temp_sync_key_verifications_v1_to_v2` TO `default`.`key_verifications_raw_v2` AS SELECT request_id, time, workspace_id, key_space_id, identity_id, '' AS external_id, key_id, region, outcome, tags, 0 AS spent_credits, 0. AS latency FROM verifications.raw_key_verifications_v1;
4 changes: 0 additions & 4 deletions pkg/clickhouse/migrations/20251125075943.sql
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,8 @@ CREATE TABLE `default`.`key_verifications_per_month_v3` (
INDEX `idx_tags` ((tags)) TYPE bloom_filter GRANULARITY 4
) ENGINE = AggregatingMergeTree
PRIMARY KEY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `external_id`, `key_id`, `outcome`, `tags`) ORDER BY (`workspace_id`, `time`, `key_space_id`, `identity_id`, `external_id`, `key_id`, `outcome`, `tags`) PARTITION BY (toStartOfYear(time)) TTL time + toIntervalYear(3) SETTINGS index_granularity = 8192;
-- Drop "active_workspaces_keys_per_month_mv_v2" view
DROP VIEW `default`.`active_workspaces_keys_per_month_mv_v2`;
-- Create "active_workspaces_keys_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`active_workspaces_keys_per_month_mv_v2` TO `default`.`active_workspaces_per_month_v2` AS SELECT workspace_id, toDate(time) AS time FROM default.key_verifications_per_month_v3;
-- Drop "billable_verifications_per_month_mv_v2" view
DROP VIEW `default`.`billable_verifications_per_month_mv_v2`;
-- Create "billable_verifications_per_month_mv_v2" view
CREATE MATERIALIZED VIEW `default`.`billable_verifications_per_month_mv_v2` TO `default`.`billable_verifications_per_month_v2` AS SELECT workspace_id, sum(count) AS count, toYear(time) AS year, toMonth(time) AS month FROM default.key_verifications_per_month_v3 WHERE outcome = 'VALID' GROUP BY workspace_id, year, month;
-- Create "key_verifications_per_day_mv_v3" view
Expand Down
Loading
Loading