From 9fec126a18b850c67d1325d3cf76a93ef8f0f60d Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Mon, 3 Aug 2026 23:34:28 +0800 Subject: [PATCH 1/6] fix(dbt): bound metric serving table builds Signed-off-by: Aleksandr Barkhatov --- .../dbt/macros/metric_serving_table.sql | 19 +++++++++++++++++++ src/ingestion/gold/ai_metric_evidence.sql | 10 +++------- src/ingestion/gold/ai_metric_observations.sql | 10 +++------- src/ingestion/gold/collab_metric_evidence.sql | 10 +++------- .../gold/collab_metric_observations.sql | 10 +++------- src/ingestion/gold/git_metric_evidence.sql | 11 +++-------- .../gold/git_metric_observations.sql | 10 +++------- src/ingestion/gold/task_metric_evidence.sql | 10 +++------- .../gold/task_metric_observations.sql | 10 +++------- src/ingestion/gold/wiki_metric_evidence.sql | 10 +++------- .../gold/wiki_metric_observations.sql | 11 +++-------- 11 files changed, 49 insertions(+), 72 deletions(-) create mode 100644 src/ingestion/dbt/macros/metric_serving_table.sql diff --git a/src/ingestion/dbt/macros/metric_serving_table.sql b/src/ingestion/dbt/macros/metric_serving_table.sql new file mode 100644 index 000000000..0de126836 --- /dev/null +++ b/src/ingestion/dbt/macros/metric_serving_table.sql @@ -0,0 +1,19 @@ +{% macro metric_serving_query_settings(join_use_nulls=none) %} + {% set settings = { + 'max_memory_usage': 1610612736, + 'max_threads': 2, + 'max_block_size': 32768, + 'max_insert_block_size': 32768, + 'min_insert_block_size_rows': 32768, + 'min_insert_block_size_bytes': 16777216, + 'max_partitions_per_insert_block': 512, + 'max_bytes_before_external_group_by': 268435456, + 'max_bytes_before_external_sort': 268435456, + 'max_bytes_in_join': 268435456, + 'join_algorithm': 'auto' + } %} + {% if join_use_nulls is not none %} + {% do settings.update({'join_use_nulls': join_use_nulls}) %} + {% endif %} + {{ return(settings) }} +{% endmacro %} diff --git a/src/ingestion/gold/ai_metric_evidence.sql b/src/ingestion/gold/ai_metric_evidence.sql index 082d2f560..9f71d3189 100644 --- a/src/ingestion/gold/ai_metric_evidence.sql +++ b/src/ingestion/gold/ai_metric_evidence.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='ai_metric_evidence', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — diff --git a/src/ingestion/gold/ai_metric_observations.sql b/src/ingestion/gold/ai_metric_observations.sql index 6dda7df25..d7f711e70 100644 --- a/src/ingestion/gold/ai_metric_observations.sql +++ b/src/ingestion/gold/ai_metric_observations.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='ai_metric_observations', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} SELECT diff --git a/src/ingestion/gold/collab_metric_evidence.sql b/src/ingestion/gold/collab_metric_evidence.sql index b8c1b7e0f..b869b4e26 100644 --- a/src/ingestion/gold/collab_metric_evidence.sql +++ b/src/ingestion/gold/collab_metric_evidence.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='collab_metric_evidence', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — diff --git a/src/ingestion/gold/collab_metric_observations.sql b/src/ingestion/gold/collab_metric_observations.sql index 44dfb0d1b..39beda1df 100644 --- a/src/ingestion/gold/collab_metric_observations.sql +++ b/src/ingestion/gold/collab_metric_observations.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='collab_metric_observations', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} SELECT diff --git a/src/ingestion/gold/git_metric_evidence.sql b/src/ingestion/gold/git_metric_evidence.sql index 819bf3922..0e293810d 100644 --- a/src/ingestion/gold/git_metric_evidence.sql +++ b/src/ingestion/gold/git_metric_evidence.sql @@ -1,17 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='git_metric_evidence', tags=['gold'], - query_settings={ - 'join_use_nulls': 1, - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings(join_use_nulls=1) ) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — diff --git a/src/ingestion/gold/git_metric_observations.sql b/src/ingestion/gold/git_metric_observations.sql index 182f94d8d..03a0b03e3 100644 --- a/src/ingestion/gold/git_metric_observations.sql +++ b/src/ingestion/gold/git_metric_observations.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='git_metric_observations', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} SELECT diff --git a/src/ingestion/gold/task_metric_evidence.sql b/src/ingestion/gold/task_metric_evidence.sql index 57e8fd23b..6662a2296 100644 --- a/src/ingestion/gold/task_metric_evidence.sql +++ b/src/ingestion/gold/task_metric_evidence.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='task_metric_evidence', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — diff --git a/src/ingestion/gold/task_metric_observations.sql b/src/ingestion/gold/task_metric_observations.sql index 3e7532e95..766faa6ca 100644 --- a/src/ingestion/gold/task_metric_observations.sql +++ b/src/ingestion/gold/task_metric_observations.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='task_metric_observations', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} SELECT diff --git a/src/ingestion/gold/wiki_metric_evidence.sql b/src/ingestion/gold/wiki_metric_evidence.sql index b7b789b70..03d81d80a 100644 --- a/src/ingestion/gold/wiki_metric_evidence.sql +++ b/src/ingestion/gold/wiki_metric_evidence.sql @@ -1,16 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='wiki_metric_evidence', tags=['gold'], - query_settings={ - 'max_memory_usage': 1610612736, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368 - } + query_settings=metric_serving_query_settings() ) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — diff --git a/src/ingestion/gold/wiki_metric_observations.sql b/src/ingestion/gold/wiki_metric_observations.sql index 61ae2a693..afafcc0a4 100644 --- a/src/ingestion/gold/wiki_metric_observations.sql +++ b/src/ingestion/gold/wiki_metric_observations.sql @@ -1,17 +1,12 @@ {{ config( materialized='table', engine='MergeTree', - order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'], + order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], + partition_by='toYYYYMM(metric_date)', schema='insight', alias='wiki_metric_observations', tags=['gold'], - query_settings={ - 'max_memory_usage': 3221225472, - 'max_threads': 4, - 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368, - 'join_algorithm': 'grace_hash,hash' - } + query_settings=metric_serving_query_settings() ) }} SELECT From f3f4f6497be7d9a6df86d0826c25b536d42be215 Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Mon, 3 Aug 2026 23:42:09 +0800 Subject: [PATCH 2/6] refactor(dbt): avoid repeated git evidence scans Signed-off-by: Aleksandr Barkhatov --- src/ingestion/gold/git_metric_evidence.sql | 153 ++++++++++----------- 1 file changed, 76 insertions(+), 77 deletions(-) diff --git a/src/ingestion/gold/git_metric_evidence.sql b/src/ingestion/gold/git_metric_evidence.sql index 0e293810d..41952d6d8 100644 --- a/src/ingestion/gold/git_metric_evidence.sql +++ b/src/ingestion/gold/git_metric_evidence.sql @@ -98,7 +98,6 @@ file_changes_source AS ( file_changes.lines_removed AS lines_removed, commits.repository_value AS repository_value, commits.repository_label AS repository_label, - commits.source_dimensions AS source_dimensions, CAST( [ tuple('file_extension', file_extension, file_extension_label), @@ -234,7 +233,7 @@ pull_requests_source AS ( AND pr_commit_emails.repo_slug = prs.repo_slug AND pr_commit_emails.pr_id = prs.pr_id ), -prs_created_source AS ( +pull_request_measures AS ( SELECT tenant_id, pr_id, @@ -242,52 +241,89 @@ prs_created_source AS ( title, author_name, assumeNotNull(entity_id) AS entity_id, - toDate(created_on) AS metric_date, - created_on AS observed_at, - state, - change_size, + toDate(pr_measure.3) AS metric_date, + pr_measure.3 AS observed_at, + pr_measure.1 AS measure_key, + pr_measure.2 AS contribution, repository_label, repository_value, source_dimensions - FROM pull_requests_source - WHERE entity_id IS NOT NULL - AND entity_id != '' - AND created_on IS NOT NULL + FROM pull_requests_source AS pull_request + ARRAY JOIN CAST(arrayConcat( + if( + created_on IS NOT NULL, + [tuple('pr_created', toFloat64(1), toDateTime64(assumeNotNull(created_on), 3))], + [] + ), + if( + created_on IS NOT NULL AND state = 'MERGED', + [tuple('pr_created_merged', toFloat64(1), toDateTime64(assumeNotNull(created_on), 3))], + [] + ), + if( + created_on IS NOT NULL AND ifNull(change_size, 0) > 0, + [tuple('pr_change_size', toFloat64(ifNull(change_size, 0)), toDateTime64(assumeNotNull(created_on), 3))], + [] + ), + if( + state = 'MERGED' AND closed_on IS NOT NULL, + [tuple('pr_merged', toFloat64(1), toDateTime64(assumeNotNull(closed_on), 3))], + [] + ), + if( + cycle_hours IS NOT NULL AND closed_on IS NOT NULL, + [tuple('pr_cycle_hours', toFloat64(assumeNotNull(cycle_hours)), toDateTime64(assumeNotNull(closed_on), 3))], + [] + ) + ) AS Array(Tuple(measure_key String, contribution Float64, observed_at DateTime64(3)))) AS pr_measure + WHERE pull_request.entity_id IS NOT NULL + AND pull_request.entity_id != '' ), -prs_merged_source AS ( +file_change_measures AS ( SELECT tenant_id, - pr_id, - pr_number, - title, - author_name, - assumeNotNull(entity_id) AS entity_id, - toDate(closed_on) AS metric_date, - closed_on AS observed_at, - cycle_hours, - repository_label, - repository_value, - source_dimensions - FROM pull_requests_source - WHERE entity_id IS NOT NULL - AND entity_id != '' - AND state = 'MERGED' - AND closed_on IS NOT NULL + entity_id, + metric_date, + file_measure.1 AS measure_key, + file_measure.2 AS value, + file_measure.3 AS dimensions + FROM file_changes_source + ARRAY JOIN CAST(arrayConcat( + if( + lines_added IS NOT NULL, + [tuple('lines_added', toFloat64(assumeNotNull(lines_added)), category_source_dimensions)], + [] + ), + if( + lines_removed IS NOT NULL, + [tuple('lines_removed', toFloat64(assumeNotNull(lines_removed)), category_source_dimensions)], + [] + ), + if( + category = 'code' AND lines_added IS NOT NULL, + [tuple('code_lines_added', toFloat64(assumeNotNull(lines_added)), file_source_dimensions)], + [] + ) + ) AS Array(Tuple( + measure_key String, + value Float64, + dimensions Array(Tuple(key String, value String, label Nullable(String))) + ))) AS file_measure ), measure_observations AS ( {{ presence_measure('commit_day', ['commits_source']) }} UNION ALL - {{ sum_measure('code_lines_added', 'file_changes_source', 'lines_added', 'file_source_dimensions', where="category = 'code'") }} - - UNION ALL - - {{ sum_measure('lines_added', 'file_changes_source', 'lines_added', 'category_source_dimensions') }} - - UNION ALL - - {{ sum_measure('lines_removed', 'file_changes_source', 'lines_removed', 'category_source_dimensions') }} + SELECT + tenant_id, + entity_id, + metric_date, + measure_key, + toNullable(sum(value)) AS value, + dimensions + FROM file_change_measures + GROUP BY tenant_id, entity_id, metric_date, measure_key, dimensions ) SELECT assumeNotNull(tenant_id) AS tenant_id, @@ -363,45 +399,12 @@ SELECT assumeNotNull(entity_id) AS entity_id, assumeNotNull(metric_date) AS metric_date, toNullable(toDateTime64(observed_at, 3)) AS observed_at, - pr_measure.1 AS measure_key, - concat(repository_value, ':pr:', toString(pr_id), ':', pr_measure.1) AS record_id, - 'pull_request' AS record_kind, - 'event' AS granularity, - if(title = '', concat('PR #', toString(pr_number)), title) AS record_label, - toNullable(toFloat64(pr_measure.2)) AS contribution, - CAST(NULL AS Nullable(String)) AS subject_key, - source_dimensions AS dimensions, - map( - 'ref', toString(pr_number), - 'title', title, - 'repository', repository_label, - 'author', author_name - ) AS details -FROM prs_created_source -ARRAY JOIN arrayConcat( - [tuple('pr_created', toFloat64(1))], - if(state = 'MERGED', [tuple('pr_created_merged', toFloat64(1))], []), - if(ifNull(change_size, 0) > 0, [tuple('pr_change_size', toFloat64(change_size))], []) -) AS pr_measure -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL - -UNION ALL - -SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'git' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, - toNullable(toDateTime64(observed_at, 3)) AS observed_at, - pr_measure.1 AS measure_key, - concat(repository_value, ':pr:', toString(pr_id), ':', pr_measure.1) AS record_id, + measure_key, + concat(repository_value, ':pr:', toString(pr_id), ':', measure_key) AS record_id, 'pull_request' AS record_kind, 'event' AS granularity, if(title = '', concat('PR #', toString(pr_number)), title) AS record_label, - toNullable(toFloat64(pr_measure.2)) AS contribution, + toNullable(toFloat64(contribution)) AS contribution, CAST(NULL AS Nullable(String)) AS subject_key, source_dimensions AS dimensions, map( @@ -410,11 +413,7 @@ SELECT 'repository', repository_label, 'author', author_name ) AS details -FROM prs_merged_source -ARRAY JOIN arrayConcat( - [tuple('pr_merged', toFloat64(1))], - if(cycle_hours IS NOT NULL, [tuple('pr_cycle_hours', toFloat64(cycle_hours))], []) -) AS pr_measure +FROM pull_request_measures WHERE tenant_id IS NOT NULL AND entity_id IS NOT NULL AND metric_date IS NOT NULL From 711c1584f3c0bf7c1a8085f957f7320131dcc552 Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Mon, 3 Aug 2026 23:43:30 +0800 Subject: [PATCH 3/6] docs(metrics): document metric table storage Signed-off-by: Aleksandr Barkhatov --- docs/domain/metrics/specs/DESIGN.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md index 5ab3f0b70..b2fe03d22 100644 --- a/docs/domain/metrics/specs/DESIGN.md +++ b/docs/domain/metrics/specs/DESIGN.md @@ -139,6 +139,26 @@ evidence relation per source and one granularity per measure: - `source_summary`: the finest summary preserved by silver. - `derived_population`: a source entity participating in a derived metric. +All managed evidence and observation tables use the shared +`metric_serving_query_settings` dbt macro. Builds are limited to 1.5 GiB and +two execution threads, use 32,768-row source and insert blocks, and spill +aggregation, sorting, and joins after 256 MiB. These limits apply uniformly; +model-specific query settings are retained only when required by model +semantics. + +The tables are partitioned by calendar month from `metric_date`. Evidence is +ordered by tenant, source, entity type, entity, measure, date, and record ID; +observations omit the final record ID. Monthly partitioning is physical +storage only: it does not change metric dates, timestamps, row granularity, or +drilldown results. Each insert block may address at most 512 partitions. + +dbt builds a replacement table and exchanges it only after the build +succeeds. A failed or cancelled build therefore leaves the active table in +place, and the next build removes abandoned temporary relations. Replacement +is atomic per table, not across the complete gold DAG. Replacing evidence +invalidates active evidence cursors through the existing snapshot-expired +contract. + Definitions do not declare a separate drilldown strategy. The runtime resolves the definition's existing input roles and source measures, requires every input to use the same evidence relation, and compiles the evidence selection from From beaaec8b23760d29343831b94a9c55ebf7f45b8c Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Tue, 4 Aug 2026 00:18:03 +0800 Subject: [PATCH 4/6] refactor(dbt): centralize metric table configuration Signed-off-by: Aleksandr Barkhatov --- docs/domain/metrics/specs/DESIGN.md | 5 ++-- .../dbt/macros/metric_serving_table.sql | 25 +++++++++++++++++++ src/ingestion/gold/ai_metric_evidence.sql | 11 +------- src/ingestion/gold/ai_metric_observations.sql | 11 +------- src/ingestion/gold/collab_metric_evidence.sql | 11 +------- .../gold/collab_metric_observations.sql | 11 +------- src/ingestion/gold/git_metric_evidence.sql | 11 +------- .../gold/git_metric_observations.sql | 11 +------- src/ingestion/gold/task_metric_evidence.sql | 11 +------- .../gold/task_metric_observations.sql | 11 +------- src/ingestion/gold/wiki_metric_evidence.sql | 11 +------- .../gold/wiki_metric_observations.sql | 11 +------- 12 files changed, 37 insertions(+), 103 deletions(-) diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md index b2fe03d22..c5beedd63 100644 --- a/docs/domain/metrics/specs/DESIGN.md +++ b/docs/domain/metrics/specs/DESIGN.md @@ -140,9 +140,8 @@ evidence relation per source and one granularity per measure: - `derived_population`: a source entity participating in a derived metric. All managed evidence and observation tables use the shared -`metric_serving_query_settings` dbt macro. Builds are limited to 1.5 GiB and -two execution threads, use 32,768-row source and insert blocks, and spill -aggregation, sorting, and joins after 256 MiB. These limits apply uniformly; +`metric_serving_table` dbt macro. It owns their materialization, storage keys, +partitioning, tags, and bounded query settings. These settings apply uniformly; model-specific query settings are retained only when required by model semantics. diff --git a/src/ingestion/dbt/macros/metric_serving_table.sql b/src/ingestion/dbt/macros/metric_serving_table.sql index 0de126836..dc4496109 100644 --- a/src/ingestion/dbt/macros/metric_serving_table.sql +++ b/src/ingestion/dbt/macros/metric_serving_table.sql @@ -17,3 +17,28 @@ {% endif %} {{ return(settings) }} {% endmacro %} + +{% macro metric_serving_table(table_kind, join_use_nulls=none) %} + {% set order_by = [ + 'tenant_id', + 'source_key', + 'entity_type', + 'entity_id', + 'measure_key', + 'metric_date' + ] %} + {% if table_kind == 'evidence' %} + {% do order_by.append('record_id') %} + {% elif table_kind != 'observations' %} + {{ exceptions.raise_compiler_error('Unsupported metric serving table kind: ' ~ table_kind) }} + {% endif %} + {{ config( + materialized='table', + engine='MergeTree', + order_by=order_by, + partition_by='toYYYYMM(metric_date)', + schema='insight', + tags=['gold'], + query_settings=metric_serving_query_settings(join_use_nulls=join_use_nulls) + ) }} +{% endmacro %} diff --git a/src/ingestion/gold/ai_metric_evidence.sql b/src/ingestion/gold/ai_metric_evidence.sql index 9f71d3189..09c23e440 100644 --- a/src/ingestion/gold/ai_metric_evidence.sql +++ b/src/ingestion/gold/ai_metric_evidence.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='ai_metric_evidence', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('evidence') }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/ai_metric_observations.sql b/src/ingestion/gold/ai_metric_observations.sql index d7f711e70..8d0f7b5a5 100644 --- a/src/ingestion/gold/ai_metric_observations.sql +++ b/src/ingestion/gold/ai_metric_observations.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='ai_metric_observations', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('observations') }} SELECT tenant_id, diff --git a/src/ingestion/gold/collab_metric_evidence.sql b/src/ingestion/gold/collab_metric_evidence.sql index b869b4e26..3d564fc04 100644 --- a/src/ingestion/gold/collab_metric_evidence.sql +++ b/src/ingestion/gold/collab_metric_evidence.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='collab_metric_evidence', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('evidence') }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/collab_metric_observations.sql b/src/ingestion/gold/collab_metric_observations.sql index 39beda1df..3e7aa86b4 100644 --- a/src/ingestion/gold/collab_metric_observations.sql +++ b/src/ingestion/gold/collab_metric_observations.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='collab_metric_observations', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('observations') }} SELECT tenant_id, diff --git a/src/ingestion/gold/git_metric_evidence.sql b/src/ingestion/gold/git_metric_evidence.sql index 41952d6d8..9e5c0d3b0 100644 --- a/src/ingestion/gold/git_metric_evidence.sql +++ b/src/ingestion/gold/git_metric_evidence.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='git_metric_evidence', - tags=['gold'], - query_settings=metric_serving_query_settings(join_use_nulls=1) -) }} +{{ metric_serving_table('evidence', join_use_nulls=1) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/git_metric_observations.sql b/src/ingestion/gold/git_metric_observations.sql index 03a0b03e3..811d04616 100644 --- a/src/ingestion/gold/git_metric_observations.sql +++ b/src/ingestion/gold/git_metric_observations.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='git_metric_observations', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('observations') }} SELECT tenant_id, diff --git a/src/ingestion/gold/task_metric_evidence.sql b/src/ingestion/gold/task_metric_evidence.sql index 6662a2296..7f2b16ee3 100644 --- a/src/ingestion/gold/task_metric_evidence.sql +++ b/src/ingestion/gold/task_metric_evidence.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='task_metric_evidence', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('evidence') }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/task_metric_observations.sql b/src/ingestion/gold/task_metric_observations.sql index 766faa6ca..cd2effdd7 100644 --- a/src/ingestion/gold/task_metric_observations.sql +++ b/src/ingestion/gold/task_metric_observations.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='task_metric_observations', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('observations') }} SELECT tenant_id, diff --git a/src/ingestion/gold/wiki_metric_evidence.sql b/src/ingestion/gold/wiki_metric_evidence.sql index 03d81d80a..10f6735fe 100644 --- a/src/ingestion/gold/wiki_metric_evidence.sql +++ b/src/ingestion/gold/wiki_metric_evidence.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date', 'record_id'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='wiki_metric_evidence', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('evidence') }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/wiki_metric_observations.sql b/src/ingestion/gold/wiki_metric_observations.sql index afafcc0a4..80251d0c6 100644 --- a/src/ingestion/gold/wiki_metric_observations.sql +++ b/src/ingestion/gold/wiki_metric_observations.sql @@ -1,13 +1,4 @@ -{{ config( - materialized='table', - engine='MergeTree', - order_by=['tenant_id', 'source_key', 'entity_type', 'entity_id', 'measure_key', 'metric_date'], - partition_by='toYYYYMM(metric_date)', - schema='insight', - alias='wiki_metric_observations', - tags=['gold'], - query_settings=metric_serving_query_settings() -) }} +{{ metric_serving_table('observations') }} SELECT tenant_id, From 83c8e4b7ec76502029508253082dd5c285bcabcf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:27:15 +0000 Subject: [PATCH 5/6] chore(connectors-ddl): regenerate the snapshot for fix/metric-gold-build-memory Produced by the connectors-ddl gate from a full bootstrap of this branch: real connector discover, destination-clickhouse, dbt, migrations, then dump-ddl.sh. Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../scripts/connectors-ddl/insight.sql | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/ingestion/scripts/connectors-ddl/insight.sql b/src/ingestion/scripts/connectors-ddl/insight.sql index 12b9c27de..74544b689 100644 --- a/src/ingestion/scripts/connectors-ddl/insight.sql +++ b/src/ingestion/scripts/connectors-ddl/insight.sql @@ -23,7 +23,8 @@ CREATE TABLE IF NOT EXISTS insight.ai_metric_evidence `details` Map(String, String) ) ENGINE = MergeTree -ORDER BY (tenant_id, source_key, measure_key, entity_id, metric_date, record_id) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date, record_id) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -44,7 +45,8 @@ CREATE TABLE IF NOT EXISTS insight.ai_metric_observations label Nullable(String))) ) ENGINE = MergeTree -ORDER BY (source_key, measure_key, entity_id, metric_date) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -71,7 +73,8 @@ CREATE TABLE IF NOT EXISTS insight.collab_metric_evidence `details` Map(String, String) ) ENGINE = MergeTree -ORDER BY (tenant_id, source_key, measure_key, entity_id, metric_date, record_id) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date, record_id) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -92,7 +95,8 @@ CREATE TABLE IF NOT EXISTS insight.collab_metric_observations label Nullable(String))) ) ENGINE = MergeTree -ORDER BY (source_key, measure_key, entity_id, metric_date) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -119,7 +123,8 @@ CREATE TABLE IF NOT EXISTS insight.git_metric_evidence `details` Map(String, String) ) ENGINE = MergeTree -ORDER BY (tenant_id, source_key, measure_key, entity_id, metric_date, record_id) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date, record_id) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -140,7 +145,8 @@ CREATE TABLE IF NOT EXISTS insight.git_metric_observations label Nullable(String))) ) ENGINE = MergeTree -ORDER BY (source_key, measure_key, entity_id, metric_date) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -213,7 +219,8 @@ CREATE TABLE IF NOT EXISTS insight.task_metric_evidence `details` Map(String, String) ) ENGINE = MergeTree -ORDER BY (tenant_id, source_key, measure_key, entity_id, metric_date, record_id) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date, record_id) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -234,7 +241,8 @@ CREATE TABLE IF NOT EXISTS insight.task_metric_observations label Nullable(String))) ) ENGINE = MergeTree -ORDER BY (source_key, measure_key, entity_id, metric_date) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -288,7 +296,8 @@ CREATE TABLE IF NOT EXISTS insight.wiki_metric_evidence `details` Map(String, String) ) ENGINE = MergeTree -ORDER BY (tenant_id, source_key, measure_key, entity_id, metric_date, record_id) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date, record_id) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; @@ -309,7 +318,8 @@ CREATE TABLE IF NOT EXISTS insight.wiki_metric_observations label Nullable(String))) ) ENGINE = MergeTree -ORDER BY (source_key, measure_key, entity_id, metric_date) +PARTITION BY toYYYYMM(metric_date) +ORDER BY (tenant_id, source_key, entity_type, entity_id, measure_key, metric_date) SETTINGS replicated_deduplication_window = '0', index_granularity = 8192 ; From 32517b53dc477daf0d766aeb238ddf5c8e00cc5b Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Tue, 4 Aug 2026 00:30:00 +0800 Subject: [PATCH 6/6] refactor(dbt): use explicit metric table macros Signed-off-by: Aleksandr Barkhatov --- docs/domain/metrics/specs/DESIGN.md | 8 ++++---- src/ingestion/dbt/macros/metric_serving_table.sql | 14 ++++++++++---- src/ingestion/gold/ai_metric_evidence.sql | 2 +- src/ingestion/gold/ai_metric_observations.sql | 2 +- src/ingestion/gold/collab_metric_evidence.sql | 2 +- src/ingestion/gold/collab_metric_observations.sql | 2 +- src/ingestion/gold/git_metric_evidence.sql | 2 +- src/ingestion/gold/git_metric_observations.sql | 2 +- src/ingestion/gold/task_metric_evidence.sql | 2 +- src/ingestion/gold/task_metric_observations.sql | 2 +- src/ingestion/gold/wiki_metric_evidence.sql | 2 +- src/ingestion/gold/wiki_metric_observations.sql | 2 +- 12 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md index c5beedd63..3be6037f4 100644 --- a/docs/domain/metrics/specs/DESIGN.md +++ b/docs/domain/metrics/specs/DESIGN.md @@ -140,10 +140,10 @@ evidence relation per source and one granularity per measure: - `derived_population`: a source entity participating in a derived metric. All managed evidence and observation tables use the shared -`metric_serving_table` dbt macro. It owns their materialization, storage keys, -partitioning, tags, and bounded query settings. These settings apply uniformly; -model-specific query settings are retained only when required by model -semantics. +`metric_evidence_table` and `metric_observations_table` dbt macros. They own +materialization, storage keys, partitioning, tags, and bounded query settings. +These settings apply uniformly; model-specific query settings are retained +only when required by model semantics. The tables are partitioned by calendar month from `metric_date`. Evidence is ordered by tenant, source, entity type, entity, measure, date, and record ID; diff --git a/src/ingestion/dbt/macros/metric_serving_table.sql b/src/ingestion/dbt/macros/metric_serving_table.sql index dc4496109..dfbb88543 100644 --- a/src/ingestion/dbt/macros/metric_serving_table.sql +++ b/src/ingestion/dbt/macros/metric_serving_table.sql @@ -18,7 +18,7 @@ {{ return(settings) }} {% endmacro %} -{% macro metric_serving_table(table_kind, join_use_nulls=none) %} +{% macro metric_serving_table(include_record_id, join_use_nulls=none) %} {% set order_by = [ 'tenant_id', 'source_key', @@ -27,10 +27,8 @@ 'measure_key', 'metric_date' ] %} - {% if table_kind == 'evidence' %} + {% if include_record_id %} {% do order_by.append('record_id') %} - {% elif table_kind != 'observations' %} - {{ exceptions.raise_compiler_error('Unsupported metric serving table kind: ' ~ table_kind) }} {% endif %} {{ config( materialized='table', @@ -42,3 +40,11 @@ query_settings=metric_serving_query_settings(join_use_nulls=join_use_nulls) ) }} {% endmacro %} + +{% macro metric_evidence_table(join_use_nulls=none) %} + {{ metric_serving_table(true, join_use_nulls=join_use_nulls) }} +{% endmacro %} + +{% macro metric_observations_table() %} + {{ metric_serving_table(false) }} +{% endmacro %} diff --git a/src/ingestion/gold/ai_metric_evidence.sql b/src/ingestion/gold/ai_metric_evidence.sql index 09c23e440..ff13a9c28 100644 --- a/src/ingestion/gold/ai_metric_evidence.sql +++ b/src/ingestion/gold/ai_metric_evidence.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('evidence') }} +{{ metric_evidence_table() }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/ai_metric_observations.sql b/src/ingestion/gold/ai_metric_observations.sql index 8d0f7b5a5..f1765794f 100644 --- a/src/ingestion/gold/ai_metric_observations.sql +++ b/src/ingestion/gold/ai_metric_observations.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('observations') }} +{{ metric_observations_table() }} SELECT tenant_id, diff --git a/src/ingestion/gold/collab_metric_evidence.sql b/src/ingestion/gold/collab_metric_evidence.sql index 3d564fc04..eb303a54d 100644 --- a/src/ingestion/gold/collab_metric_evidence.sql +++ b/src/ingestion/gold/collab_metric_evidence.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('evidence') }} +{{ metric_evidence_table() }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/collab_metric_observations.sql b/src/ingestion/gold/collab_metric_observations.sql index 3e7aa86b4..5c3cecf02 100644 --- a/src/ingestion/gold/collab_metric_observations.sql +++ b/src/ingestion/gold/collab_metric_observations.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('observations') }} +{{ metric_observations_table() }} SELECT tenant_id, diff --git a/src/ingestion/gold/git_metric_evidence.sql b/src/ingestion/gold/git_metric_evidence.sql index 9e5c0d3b0..0d08703c4 100644 --- a/src/ingestion/gold/git_metric_evidence.sql +++ b/src/ingestion/gold/git_metric_evidence.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('evidence', join_use_nulls=1) }} +{{ metric_evidence_table(join_use_nulls=1) }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/git_metric_observations.sql b/src/ingestion/gold/git_metric_observations.sql index 811d04616..e2f62cb2c 100644 --- a/src/ingestion/gold/git_metric_observations.sql +++ b/src/ingestion/gold/git_metric_observations.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('observations') }} +{{ metric_observations_table() }} SELECT tenant_id, diff --git a/src/ingestion/gold/task_metric_evidence.sql b/src/ingestion/gold/task_metric_evidence.sql index 7f2b16ee3..1b783f43f 100644 --- a/src/ingestion/gold/task_metric_evidence.sql +++ b/src/ingestion/gold/task_metric_evidence.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('evidence') }} +{{ metric_evidence_table() }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/task_metric_observations.sql b/src/ingestion/gold/task_metric_observations.sql index cd2effdd7..2f7258664 100644 --- a/src/ingestion/gold/task_metric_observations.sql +++ b/src/ingestion/gold/task_metric_observations.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('observations') }} +{{ metric_observations_table() }} SELECT tenant_id, diff --git a/src/ingestion/gold/wiki_metric_evidence.sql b/src/ingestion/gold/wiki_metric_evidence.sql index 10f6735fe..42c1cca3d 100644 --- a/src/ingestion/gold/wiki_metric_evidence.sql +++ b/src/ingestion/gold/wiki_metric_evidence.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('evidence') }} +{{ metric_evidence_table() }} -- Resolution happens HERE, once per gold build: evidence carries BOTH keys — -- `entity_id` is the canonical person id (or '' when identity does not know diff --git a/src/ingestion/gold/wiki_metric_observations.sql b/src/ingestion/gold/wiki_metric_observations.sql index 80251d0c6..5fd9dd9d9 100644 --- a/src/ingestion/gold/wiki_metric_observations.sql +++ b/src/ingestion/gold/wiki_metric_observations.sql @@ -1,4 +1,4 @@ -{{ metric_serving_table('observations') }} +{{ metric_observations_table() }} SELECT tenant_id,