diff --git a/docs/components/backend/analytics/openapi.json b/docs/components/backend/analytics/openapi.json index 184b572c0..b4d62f745 100644 --- a/docs/components/backend/analytics/openapi.json +++ b/docs/components/backend/analytics/openapi.json @@ -528,6 +528,14 @@ ], "type": "object" }, + "EvidenceGranularity": { + "enum": [ + "event", + "source_summary", + "derived_population" + ], + "type": "string" + }, "GetMetricsRequest": { "additionalProperties": false, "description": "Request body for `POST /v1/catalog/get_metrics`.\n\n`tenant_id` is intentionally NOT accepted here — it is resolved server-side\nfrom the session by `tenant_middleware` (Refs #522 auth-trait). Allowing a\nbody-supplied `tenant_id` would open a cross-tenant disclosure surface.\n`deny_unknown_fields` enforces that defensively at the parser layer: a\ncaller that smuggles `\"tenant_id\": \"...\"` into the body gets a 400 instead\nof a silent ignore.", @@ -685,6 +693,16 @@ "direction": { "$ref": "#/components/schemas/MetricDirection" }, + "drilldown": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/MetricDrilldownCapability" + } + ] + }, "explanation": { "type": [ "string", @@ -771,6 +789,24 @@ ], "type": "object" }, + "MetricDimensionFilterDto": { + "properties": { + "dimension": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "dimension", + "values" + ], + "type": "object" + }, "MetricDimensionFilterRequest": { "properties": { "dimension": { @@ -797,6 +833,260 @@ ], "type": "string" }, + "MetricDrilldownCapability": { + "properties": { + "granularity": { + "items": { + "$ref": "#/components/schemas/EvidenceGranularity" + }, + "type": "array" + } + }, + "required": [ + "granularity" + ], + "type": "object" + }, + "MetricDrilldownColumn": { + "properties": { + "key": { + "type": "string" + }, + "label": { + "type": "string" + }, + "type": { + "$ref": "#/components/schemas/MetricDrilldownColumnType" + } + }, + "required": [ + "key", + "label", + "type" + ], + "type": "object" + }, + "MetricDrilldownColumnType": { + "enum": [ + "string", + "date", + "number" + ], + "type": "string" + }, + "MetricDrilldownEntity": { + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "id" + ], + "type": "object" + }, + "MetricDrilldownExportFormat": { + "enum": [ + "csv", + "xlsx" + ], + "type": "string" + }, + "MetricDrilldownExportRequest": { + "properties": { + "display_dimensions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "entity": { + "$ref": "#/components/schemas/MetricDrilldownEntity" + }, + "filters": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownFilter" + }, + "type": "array" + }, + "format": { + "$ref": "#/components/schemas/MetricDrilldownExportFormat" + }, + "metric_key": { + "type": "string" + }, + "period": { + "$ref": "#/components/schemas/MetricDrilldownPeriod" + } + }, + "required": [ + "metric_key", + "entity", + "period", + "format" + ], + "type": "object" + }, + "MetricDrilldownFilter": { + "properties": { + "dimension": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "dimension", + "values" + ], + "type": "object" + }, + "MetricDrilldownPeriod": { + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "required": [ + "from", + "to" + ], + "type": "object" + }, + "MetricDrilldownRequest": { + "properties": { + "cursor": { + "type": [ + "string", + "null" + ] + }, + "display_dimensions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "entity": { + "$ref": "#/components/schemas/MetricDrilldownEntity" + }, + "filters": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownFilter" + }, + "type": "array" + }, + "limit": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "metric_key": { + "type": "string" + }, + "period": { + "$ref": "#/components/schemas/MetricDrilldownPeriod" + } + }, + "required": [ + "metric_key", + "entity", + "period" + ], + "type": "object" + }, + "MetricDrilldownResponse": { + "properties": { + "columns": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownColumn" + }, + "type": "array" + }, + "next_cursor": { + "type": [ + "string", + "null" + ] + }, + "rows": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownRow" + }, + "type": "array" + }, + "selection": { + "$ref": "#/components/schemas/MetricDrilldownSelection" + } + }, + "required": [ + "selection", + "columns", + "rows" + ], + "type": "object" + }, + "MetricDrilldownRow": { + "properties": { + "values": { + "additionalProperties": {}, + "propertyNames": { + "type": "string" + }, + "type": "object" + } + }, + "required": [ + "values" + ], + "type": "object" + }, + "MetricDrilldownSelection": { + "properties": { + "display_dimensions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "entity": { + "$ref": "#/components/schemas/MetricDrilldownEntity" + }, + "filters": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownFilter" + }, + "type": "array" + }, + "metric_key": { + "type": "string" + }, + "period": { + "$ref": "#/components/schemas/MetricDrilldownPeriod" + } + }, + "required": [ + "metric_key", + "entity", + "period", + "filters", + "display_dimensions" + ], + "type": "object" + }, "MetricFormat": { "enum": [ "integer", @@ -906,6 +1196,16 @@ "direction": { "$ref": "#/components/schemas/MetricDirection" }, + "drilldown": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/MetricDrilldownCapability" + } + ] + }, "explanation": { "type": [ "string", @@ -921,6 +1221,9 @@ "metric_key": { "type": "string" }, + "selection": { + "$ref": "#/components/schemas/MetricResultSelectionDto" + }, "short_label": { "type": [ "string", @@ -945,12 +1248,39 @@ "label", "format", "direction", - "views" + "views", + "selection" ], "type": "object" } ] }, + "MetricResultSelectionDto": { + "properties": { + "entity": { + "$ref": "#/components/schemas/MetricResultsEntityDto" + }, + "filters": { + "items": { + "$ref": "#/components/schemas/MetricDimensionFilterDto" + }, + "type": "array" + }, + "metric_key": { + "type": "string" + }, + "period": { + "$ref": "#/components/schemas/MetricResultsPeriodDto" + } + }, + "required": [ + "metric_key", + "entity", + "period", + "filters" + ], + "type": "object" + }, "MetricResultViewDto": { "oneOf": [ { @@ -1089,6 +1419,24 @@ ], "type": "object" }, + "MetricResultsEntityDto": { + "properties": { + "ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "ids" + ], + "type": "object" + }, "MetricResultsPeriod": { "properties": { "from": { @@ -1104,6 +1452,21 @@ ], "type": "object" }, + "MetricResultsPeriodDto": { + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "required": [ + "from", + "to" + ], + "type": "object" + }, "MetricResultsRequest": { "properties": { "entity": { @@ -2816,6 +3179,229 @@ "summary": "List unified metric definitions" } }, + "/v1/metric-drilldown": { + "post": { + "operationId": "analytics_api.metric_drilldown.create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetricDrilldownRequest" + } + } + }, + "description": "Metric evidence selection", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetricDrilldownResponse" + } + } + }, + "description": "Metric evidence" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "List metric evidence" + } + }, + "/v1/metric-drilldown/export": { + "post": { + "operationId": "analytics_api.metric_drilldown.export", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetricDrilldownExportRequest" + } + } + }, + "description": "Metric evidence export selection", + "required": true + }, + "responses": { + "200": { + "content": { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "schema": { + "format": "binary", + "type": "string" + } + }, + "text/csv": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Complete metric evidence export", + "headers": { + "Content-Disposition": { + "description": "Attachment filename", + "schema": { + "type": "string" + } + } + } + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Export metric evidence" + } + }, "/v1/metric-results": { "post": { "operationId": "analytics_api.metric_results.create", diff --git a/docs/domain/metrics/README.md b/docs/domain/metrics/README.md index 226d6b815..9644116d3 100644 --- a/docs/domain/metrics/README.md +++ b/docs/domain/metrics/README.md @@ -28,6 +28,14 @@ anywhere: at request time the runtime applies a definition to the matching observations — "alice, January: 312 ÷ 405 = 77%" — and returns it labeled and ready to render. +Metric **evidence** is the source-level population behind that answer. Each +managed source exposes a normalized serving table. +Definitions inherit drilldown support when all compatible inputs are backed by +the same validated evidence relation, so adding a metric over an existing +measure requires no drilldown-specific configuration. +The backend applies the same entity, period, dimension, input-role, and +computation semantics used by metric results. + The split exists because one fact serves many meanings and one meaning serves many questions. The same `accepted_edit_actions` observation is the whole value of `ai.accepted_edit_actions` and the numerator of @@ -43,12 +51,12 @@ in that shape, definitions reference facts only by measure key, and the runtime can therefore connect any definition to any matching facts without either side knowing the other exists. -| | Observation | Definition | Metric result | -|---|---|---|---| -| What | a fact | the meaning of facts | the computed answer | -| Lives | ClickHouse views over silver (computed on read, nothing stored) | registry (MariaDB, seeded from Rust) | nowhere — made per request | -| Knows | what happened | what it is called, how to compute, how to show | both, combined | -| Authored by | connector + gold model | one struct per metric | nobody — the runtime derives it | +| | Observation | Evidence | Definition | Metric result | +|---|---|---|---|---| +| What | an aggregate-ready fact | the participating source population | the meaning of facts | the computed answer | +| Lives | ClickHouse views or tables over silver | ClickHouse serving tables | registry | nowhere | +| Knows | what happened numerically | which records participated | how to compute and display | all three, combined | +| Authored by | connector + gold model | connector + gold model | one struct per metric | nobody | ## Documents @@ -64,12 +72,19 @@ either side knowing the other exists. | Definition loading, reconciler, schema validator | [`src/backend/services/analytics/src/domain/metric_definitions/`](../../../src/backend/services/analytics/src/domain/metric_definitions/) | | Result runtime (validation, query compiler, response builder) | [`src/backend/services/analytics/src/domain/metric_results/`](../../../src/backend/services/analytics/src/domain/metric_results/) | | Result endpoint | [`src/backend/services/analytics/src/api/metric_results.rs`](../../../src/backend/services/analytics/src/api/metric_results.rs) | +| Drilldown runtime | [`src/backend/services/analytics/src/domain/metric_drilldown/`](../../../src/backend/services/analytics/src/domain/metric_drilldown/) | +| Drilldown endpoints | [`src/backend/services/analytics/src/api/metric_drilldown.rs`](../../../src/backend/services/analytics/src/api/metric_drilldown.rs) | | Registry schema migration | [`src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs`](../../../src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs) | | Managed observation sources (dbt gold models) | [`src/ingestion/gold/`](../../../src/ingestion/gold/) | | Class-contract data-quality tests | [`src/ingestion/dbt/tests/ai/`](../../../src/ingestion/dbt/tests/ai/) | ## Boundaries +- Current deployments isolate one tenant per instance. Drilldown entity IDs + remain source-derived identifiers, commonly normalized email addresses. + Multi-tenant warehouse predicates, canonical person IDs, cross-source alias + resolution, and subordinate authorization belong to the identity-resolution + epic and are required before a multi-tenant instance enables drilldown. - The AI class contracts feeding the observation models are documented in [`src/ingestion/silver/ai/schema.yml`](../../../src/ingestion/silver/ai/schema.yml) (activity invariant, label and conversation-count semantics). diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md index 0020fc78c..120547e69 100644 --- a/docs/domain/metrics/specs/DESIGN.md +++ b/docs/domain/metrics/specs/DESIGN.md @@ -59,7 +59,8 @@ Rules: ## Managed Source Ownership Managed observation sources and the cohort view are dbt gold models -(`src/ingestion/gold/`), materialized as views in the `insight` database: +(`src/ingestion/gold/`), materialized as views or MergeTree serving tables in +the `insight` database: - `insight.ai_metric_observations` - `insight.metric_entity_cohorts_current` @@ -81,7 +82,7 @@ contract; a source that needs different columns is a different source kind. Gold models are built at deploy time by the ClickHouse migrate hook (`dbt run --select tag:gold`, final step of -`src/ingestion/scripts/apply-ch-migrations.sh`), so the views exist before +`src/ingestion/scripts/apply-ch-migrations.sh`), so the relations exist before any connector sync — bronze/silver placeholders guarantee the DDL type-checks on a fresh cluster. Per-connector scoped dbt runs keep them current afterwards. @@ -90,6 +91,104 @@ The cohort view is unique per `(tenant_id, entity_type, entity_id, cohort_key)`. The peer query relies on this; a dbt build-integrity test asserts it. +## Metric Evidence Contract + +Each managed source may expose `_metric_evidence` in the `insight` +database: + +```sql +tenant_id String, +source_key String, +entity_type String, +entity_id String, +metric_date Date, +observed_at Nullable(DateTime64(3)), +measure_key String, +record_id String, +record_kind String, +granularity String, +record_label String, +contribution Nullable(Float64), +subject_key Nullable(String), +dimensions Array(Tuple(key String, value String, label Nullable(String))), +details Map(String, String) +``` + +Evidence relations are MergeTree serving tables built from silver. Their +ordering follows the drilldown predicate and cursor access pattern, avoiding +repeated silver reconstruction for every page and export. Observation models +derive their values from these evidence tables. The registry stores one +evidence relation per source and one granularity per measure: + +- `event`: one source event, such as a commit. +- `source_summary`: the finest summary preserved by silver. +- `derived_population`: a source entity participating in a derived metric. + +Definitions do not declare a separate drilldown strategy. The runtime resolves +the definition's existing input roles and source measures, requires every input +to use the same evidence relation, and compiles the evidence selection from +that metadata. A new metric over existing evidence-backed measures therefore +inherits drilldown without metric-specific SQL, backend branches, or frontend +configuration. + +The schema validator probes every standard column. Drilldown capability is +absent until the probe is definitively healthy and every metric input has +granularity metadata. Missing, unchecked, or invalid evidence fails closed. +`POST /v1/metric-results` and `GET /v1/metric-definitions` expose that +capability; consumers omit evidence actions when it is absent. + +The evidence runtime owns presentation. It projects the internal contract into +typed human-facing columns rather than exposing `record_kind`, input role, +dimensions, or other storage fields directly: + +- source-summary and derived-population measures default to date plus value. +- event measures declare reusable detail keys by `(source_key, measure_key)`, + such as ref, title, repository, author, or issue type. +- selected chart dimensions are added from the typed `dimensions` array. +- ratio metrics return daily columns named after their numerator and + denominator measures instead of an ambiguous value column. +- unknown detail keys are humanized and treated as strings; fields requiring + another label or type are added to the centralized presentation registry. + +Presentation is source-measure metadata in runtime code today. It is declared +once per reusable measure shape, never once per metric and never in frontend +configuration. + +`POST /v1/metric-drilldown` accepts one metric, one person entity, a period, +declared dimension filters, and an encoded continuation cursor. It returns the +canonical selection, typed server-owned columns, projected evidence rows, and +a next cursor. Ordering is ascending over the complete evidence key. The +cursor is versioned and bound to the normalized selection and request tenant. +It is not an authorization token and modifying its ordering key cannot widen +the server-owned relation or selection. + +`POST /v1/metric-drilldown/export` produces the complete selected population +with the same projected columns as CSV or XLSX. Export is server-side and +rejects results exceeding its row, byte, cell, execution-time, or concurrency +limits. It never silently truncates. + +The evidence contract has these limitations: + +- Summary-grain silver cannot produce event-grain evidence. AI and + collaboration currently expose source summaries or derived populations. + Git exposes commit and pull-request events, task duration metrics expose + issue events, and wiki page creation exposes page events; their remaining + measures use the finest summary grain preserved by silver. +- Metric results and evidence are not transactionally snapshot-isolated from + each other during a dbt rebuild. They reconcile after the complete gold + build because observations derive from evidence. +- Pagination and each export are bound to the evidence table UUID. A rebuild + during the operation fails with `EVIDENCE_SNAPSHOT_EXPIRED`; the client must + restart the selection rather than mix rows from two builds. Previous table + snapshots are not retained. +- Source links are omitted. Hosted services commonly use custom domains, and + the current silver contract does not preserve a canonical web base URL. A + future source registry can add a non-secret `web_base_url` keyed by source + instance and combine it with provider-specific record identifiers. +- Drilldown preserves the existing metric entity and tenant behavior. This + change does not add identity-tree authorization or warehouse tenant + enforcement. + ## Computations The computation vocabulary is closed and fully executable: @@ -277,10 +376,20 @@ type MetricResult = { format: "integer" | "decimal" | "currency" | "percent" direction: "higher_is_better" | "lower_is_better" | "neutral" views: MetricResultView[] + selection: { + metric_key: string + entity: { type: string; ids: string[] } + period: { from: string; to: string } + filters: Array<{ dimension: string; values: string[] }> + } + drilldown?: { + granularity: Array<"event" | "source_summary" | "derived_population"> + } } & ( | { computation: "sum" } | { computation: "ratio"; scale: number } | { computation: "median" } + | { computation: "distinct_count" } ) ``` @@ -424,8 +533,9 @@ one that applies. ### Case 1: metric over an existing measure -The measure already appears in a managed observation source (check the -`measures` list of the source in `builtin.rs` and the emitting gold model). +The measure already appears in a managed source. Check the source's `measures` +list in `builtin.rs`, the emitting evidence model, and the observation model +derived from it. 1. Add one `MetricSeed` to `BUILTIN_METRICS` in `src/backend/services/analytics/src/domain/metric_definitions/builtin.rs`: @@ -435,54 +545,72 @@ The measure already appears in a managed observation source (check the 2. Run `cargo test -p analytics` — the registry invariant tests validate key shapes, input/measure references, and computation field combinations. -The reconciler seeds the definition on the next deploy. No SQL, no migration, -no dbt change. +The reconciler seeds the definition on the next deploy. If every input measure +has healthy evidence metadata, the metric automatically receives drilldown, +table, and CSV/XLSX export support. No drilldown-specific SQL, frontend +configuration, migration, or dbt change is required. ### Case 2: new measure from an existing source The source exists but does not emit the measure yet. -1. Add the measure branch to the source's gold model in `src/ingestion/gold/`: - one `UNION ALL` entry calling a shape macro from - `src/ingestion/dbt/macros/metric_observation_measures.sql` — - `sum_measure(measure_key, relation, value_expr, dimensions_col, - where=none)` for aggregated numerics, `presence_measure(measure_key, - relations)` for row-existence markers, `event_measure(measure_key, - relation, value_expr, dimensions_col, where=none)` for per-event values - feeding median metrics. Every branch is a shape-macro call; a new macro - is added only when a new computation kind becomes executable. - Read only class-contract columns; never vendor-specific ones — if the fact - you need is not in the class contract, extend the class contract first - (staging models declare semantics, see the class `schema.yml`). -2. Add the `measure_key` to the gold model's `schema.yml` `accepted_values` - test. -3. Add the measure key to the source's `measures` list in `builtin.rs`. -4. Add the `MetricSeed` as in case 1. -5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation +1. Add the measure to the source's `_metric_evidence` model in + `src/ingestion/gold/`. Summary measures use the shared shape macros from + `src/ingestion/dbt/macros/metric_observation_measures.sql`; + event measures select stable source records into the evidence contract. + Read only class-contract columns; never vendor-specific ones. If the fact + is absent from the class contract, extend that contract first (staging + models declare semantics in their `schema.yml`). +2. Choose the evidence granularity deliberately: + - emit one stable row per source record for `event`. + - emit the finest source-retained grouping for `source_summary`. + - emit the reconstructed participating population for + `derived_population`. + Event rows need deterministic `record_id` values and should place reusable + grouping fields in `dimensions` and human-facing fields in `details`. +3. Derive the matching observation measure from the evidence relation. The + observation remains the aggregate-ready runtime input; the evidence row is + the population that explains it. +4. Add the `measure_key` to the observation model's `schema.yml` + `accepted_values` test. +5. Add the measure key to the source's `measures` list in `builtin.rs` and + classify its evidence granularity. +6. Add or reuse a source-measure presentation rule when the default + date-plus-value table is insufficient. Declare detail keys there and add + explicit column metadata only for fields that are not humanized strings. +7. Add the `MetricSeed` as in case 1. +8. Validate: `dbt parse` + `cargo test -p analytics` (see Validation commands). ### Case 3: new observation source The metric family reads data no managed source covers. -1. Create a dbt gold model in `src/ingestion/gold/` named - `_metric_observations`, emitting the source measure observation - contract, `schema=insight`, `ref()`-ing silver models (medallion layering - rules: `docs/domain/ingestion-data-flow/specs/DESIGN.md`). Document columns - and measure keys in `src/ingestion/gold/schema.yml`. +1. Create `_metric_evidence` and + `_metric_observations` dbt gold models in + `src/ingestion/gold/`, `schema=insight`, `ref()`-ing silver models + (medallion layering rules: + `docs/domain/ingestion-data-flow/specs/DESIGN.md`). The evidence model emits + the evidence contract; the observation model derives the aggregate-ready + observation contract from it. Document both in + `src/ingestion/gold/schema.yml`. 2. Add a `BuiltinSource` (source + measures + dimensions) to `builtin.rs`, - with `source_ref` set to the relation name. No backend enum or table-name - code changes: the relation name is data, validated on load against the - `_metric_observations` shape (`ObservationRelation`) and probed at - runtime by the schema validator. -3. Add `MetricSeed`s as in case 1. -4. Validate: `dbt parse` + `cargo test -p analytics` (see Validation + with `source_ref` and `evidence_ref` set to their relation names and every + measure assigned an evidence granularity. No backend enum or table-name + code changes are required: relation names are validated data and both + contracts are probed by the runtime schema validator. +3. Add source-measure presentation rules for event shapes that need + human-facing detail columns. +4. Add `MetricSeed`s as in case 1. +5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation commands). The runtime schema validator probes the new relation at startup. ### Rules that hold for every case - No metric-key-specific branches in runtime code. +- Evidence presentation branches may depend on source and measure, never on + final metric key. - No vendor names, vendor columns, or label mappings in gold models — labels and taxonomy come from class-contract columns declared by staging. - Measure filter predicates (`where=` on shape macros) may reference only @@ -540,6 +668,14 @@ Until one exists, custom definitions can be stored but cannot produce new source Frontend collection rendering: - requests metric keys and views. +- treats the optional `drilldown` capability as the only evidence-action + switch; there is no frontend metric allowlist. +- forwards the canonical metric selection returned by `/v1/metric-results`, + narrowing period and dimension filters for chart-point interactions. +- renders server-owned typed evidence columns and rows without interpreting + the internal evidence contract. +- uses the same canonical selection for table pagination and server-side + CSV/XLSX export. - treats configured required views as required. - normalizes response arrays only for local lookup. - renders using returned label, description, explanation, unit, format, diff --git a/src/backend/Cargo.lock b/src/backend/Cargo.lock index 826771880..ba206c994 100644 --- a/src/backend/Cargo.lock +++ b/src/backend/Cargo.lock @@ -76,6 +76,7 @@ dependencies = [ "anyhow", "async-trait", "axum", + "base64 0.22.1", "cf-gears-api-gateway", "cf-gears-authn-resolver", "cf-gears-authz-resolver", @@ -92,15 +93,18 @@ dependencies = [ "chrono", "clap", "clickhouse", + "csv", "futures", "futures-util", "insight-clickhouse", "redis", "reqwest 0.12.28", + "rust_xlsxwriter", "sea-orm", "sea-orm-migration", "serde", "serde_json", + "sha2 0.10.9", "sqlparser", "thiserror 2.0.18", "tokio", @@ -196,6 +200,15 @@ dependencies = [ "object", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.1" @@ -1713,6 +1726,27 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1870,6 +1904,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2272,6 +2317,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -5105,6 +5151,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "rust_xlsxwriter" +version = "0.90.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2be778223b36bb449b2ef2df4856ced2d311680818a7310db5c5dc370170f935" +dependencies = [ + "zip", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -7682,8 +7737,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/src/backend/Cargo.toml b/src/backend/Cargo.toml index 016015dd0..a077844ee 100644 --- a/src/backend/Cargo.toml +++ b/src/backend/Cargo.toml @@ -65,6 +65,10 @@ sea-orm-migration = { version = "1.1", features = ["runtime-tokio", "sqlx-mysql" # IDs and time uuid = { version = "1.19", features = ["serde", "v7"] } chrono = { version = "0.4", features = ["serde"] } +base64 = "0.22" +sha2 = "0.10" +csv = "1.3" +rust_xlsxwriter = "0.90" time = { version = "0.3", features = ["serde", "formatting", "parsing"] } # Logging @@ -126,4 +130,3 @@ authenticator-sdk = { path = "libs/authenticator-sdk" } # CI rebuild marker — bumped to retrigger the analytics # image build on the cf → gears crate migration (2026-06-12). - diff --git a/src/backend/services/analytics/Cargo.toml b/src/backend/services/analytics/Cargo.toml index ad8c2448a..f21738aae 100644 --- a/src/backend/services/analytics/Cargo.toml +++ b/src/backend/services/analytics/Cargo.toml @@ -65,6 +65,10 @@ futures = { workspace = true } futures-util = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } +base64 = { workspace = true } +sha2 = { workspace = true } +csv = { workspace = true } +rust_xlsxwriter = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } clap = { workspace = true } diff --git a/src/backend/services/analytics/src/api/http_live_tests.rs b/src/backend/services/analytics/src/api/http_live_tests.rs index 67d4ef7fc..c615da9dc 100644 --- a/src/backend/services/analytics/src/api/http_live_tests.rs +++ b/src/backend/services/analytics/src/api/http_live_tests.rs @@ -43,6 +43,7 @@ use crate::config::GearConfig; use crate::domain::admin_threshold::AdminThresholdService; use crate::domain::auth::{ConfigTenantAuthorization, TenantAuthorization}; use crate::domain::catalog::{CatalogReader, ThresholdResolver}; +use crate::domain::metric_definitions::test_fixture::DrilldownFixture; use crate::domain::schema_validator::SchemaValidator; use crate::infra::cache::catalog_cache::{CatalogCache, NoopCatalogCache}; use crate::infra::db::entities; @@ -443,3 +444,111 @@ async fn query_metric_without_clickhouse_maps_to_error() -> TestResult { ); Ok(()) } + +#[tokio::test] +#[ignore = "requires live MariaDB (INTEGRATION_TESTS_MARIADB_URL)"] +async fn metric_results_loads_drilldown_capabilities_before_clickhouse_error() -> TestResult { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let fixture = DrilldownFixture::insert(&db, &["git.commits"], &[]).await?; + let result: anyhow::Result<()> = async { + let app = app(db.clone(), fixture.tenant_id); + let resp = app + .oneshot(json_req( + "POST", + "/v1/metric-results", + &json!({ + "entity": {"type": "person", "ids": ["person@example.com"]}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "metrics": [{ + "metric_key": "git.commits", + "views": [{"view": "period"}] + }] + }), + )?) + .await?; + anyhow::ensure!(resp.status().is_server_error()); + Ok(()) + } + .await; + fixture.delete(&db).await?; + result.map_err(Into::into) +} + +#[tokio::test] +#[ignore = "requires live MariaDB (INTEGRATION_TESTS_MARIADB_URL)"] +async fn metric_drilldown_validates_selection_before_clickhouse_error() -> TestResult { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let fixture = DrilldownFixture::insert(&db, &["git.commits"], &["repository"]).await?; + let result: anyhow::Result<()> = async { + let app = app(db.clone(), fixture.tenant_id); + let body = json!({ + "metric_key": "git.commits", + "entity": {"type": "person", "id": "person@example.com"}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "filters": [{"dimension": "repository", "values": ["org/repo"]}], + "display_dimensions": ["repository"], + "limit": 100 + }); + let resp = app + .clone() + .oneshot(json_req("POST", "/v1/metric-drilldown", &body)?) + .await?; + anyhow::ensure!(resp.status() == StatusCode::BAD_REQUEST); + let export = json!({ + "metric_key": "git.commits", + "entity": {"type": "person", "id": "person@example.com"}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "filters": [], + "display_dimensions": [], + "format": "csv" + }); + let resp = app + .oneshot(json_req("POST", "/v1/metric-drilldown/export", &export)?) + .await?; + anyhow::ensure!(resp.status() == StatusCode::BAD_REQUEST); + Ok(()) + } + .await; + fixture.delete(&db).await?; + result.map_err(Into::into) +} + +#[tokio::test] +#[ignore = "requires live MariaDB (INTEGRATION_TESTS_MARIADB_URL)"] +async fn metric_drilldown_rejects_invalid_selection_without_clickhouse() -> TestResult { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let app = app(db, Uuid::now_v7()); + for body in [ + json!({ + "metric_key": "git.commits", + "entity": {"type": "team", "id": "team"}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "limit": 100 + }), + json!({ + "metric_key": "git.commits", + "entity": {"type": "person", "id": ""}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "limit": 100 + }), + json!({ + "metric_key": "git.commits", + "entity": {"type": "person", "id": "person@example.com"}, + "period": {"from": "2026-07-28", "to": "2026-07-01"}, + "limit": 100 + }), + ] { + let resp = app + .clone() + .oneshot(json_req("POST", "/v1/metric-drilldown", &body)?) + .await?; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + Ok(()) +} diff --git a/src/backend/services/analytics/src/api/metric_drilldown.rs b/src/backend/services/analytics/src/api/metric_drilldown.rs new file mode 100644 index 000000000..665b23c8f --- /dev/null +++ b/src/backend/services/analytics/src/api/metric_drilldown.rs @@ -0,0 +1,709 @@ +use std::io::{Cursor, Seek, SeekFrom, Write}; +use std::sync::{Arc, LazyLock}; +use std::time::{Duration, Instant}; + +use axum::Json; +use axum::body::Body; +use axum::extract::Extension; +use axum::http::header::{CONTENT_DISPOSITION, CONTENT_TYPE}; +use axum::http::{HeaderValue, Response}; +use rust_xlsxwriter::{ExcelDateTime, Format, Table, TableStyle, Workbook}; +use tokio::sync::Semaphore; +use toolkit_canonical_errors::CanonicalError; +use toolkit_security::SecurityContext; + +use super::AppState; +use crate::api::error::MetricError; +use crate::domain::metric_drilldown::{ + EVIDENCE_QUERY_MEMORY_BYTES, EVIDENCE_QUERY_READ_BYTES, EVIDENCE_QUERY_RESULT_BYTES, + EVIDENCE_QUERY_TIMEOUT_SECS, EvidenceQueryRow, MAX_EXPORT_ROWS, MetricDrilldownColumn, + MetricDrilldownExportFormat, MetricDrilldownExportRequest, MetricDrilldownRequest, + MetricDrilldownResponse, MetricDrilldownRow, build_response, compile_query, presentation, + validate_export_request, validate_request, verify_evidence_snapshot, +}; + +const QUERY_TIMEOUT: Duration = Duration::from_secs(EVIDENCE_QUERY_TIMEOUT_SECS); +const EXPORT_TIMEOUT: Duration = Duration::from_mins(1); +const EXPORT_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(2); +const MAX_EXPORT_BYTES: usize = 25 * 1024 * 1024; +const MAX_CELL_BYTES: usize = 32 * 1024; +const MAX_CONCURRENT_EXPORTS: usize = 2; +const MAX_CONCURRENT_QUERIES: usize = 8; +static EXPORT_SEMAPHORE: LazyLock = + LazyLock::new(|| Semaphore::new(MAX_CONCURRENT_EXPORTS)); +static QUERY_SEMAPHORE: LazyLock = + LazyLock::new(|| Semaphore::new(MAX_CONCURRENT_QUERIES)); + +pub async fn query_metric_drilldown( + Extension(state): Extension>, + Extension(ctx): Extension, + Json(req): Json, +) -> Result, CanonicalError> { + let started = Instant::now(); + let req = validate_request(&state.db, &state.ch, ctx.subject_tenant_id(), req).await?; + let log_comment = format!("metric-drilldown:page:{}", req.plan.definition.key()); + let rows = fetch_rows(&state, &req, &log_comment).await?; + verify_evidence_snapshot(&state.ch, &req.plan.relation, &req.snapshot_id).await?; + let fetched_rows = rows.len(); + let response = build_response(&req, rows)?; + tracing::info!( + duration_ms = started.elapsed().as_millis(), + rows = response.rows.len(), + fetched_rows, + limit = req.limit, + has_next_page = response.next_cursor.is_some(), + "metric drilldown page completed" + ); + Ok(Json(response)) +} + +pub async fn export_metric_drilldown( + Extension(state): Extension>, + Extension(ctx): Extension, + Json(req): Json, +) -> Result, CanonicalError> { + let started = Instant::now(); + let permit = tokio::time::timeout(EXPORT_ACQUIRE_TIMEOUT, EXPORT_SEMAPHORE.acquire()) + .await + .map_err(|_| { + tracing::warn!( + capacity = MAX_CONCURRENT_EXPORTS, + available = EXPORT_SEMAPHORE.available_permits(), + "metric drilldown export capacity exhausted" + ); + export_busy() + })? + .map_err(|_| export_busy())?; + let deadline = tokio::time::Instant::now() + EXPORT_TIMEOUT; + let validated = validate_export_request( + &state.db, + &state.ch, + ctx.subject_tenant_id(), + &req, + MAX_EXPORT_ROWS + 1, + ) + .await?; + let log_comment = format!( + "metric-drilldown:export:{}", + validated.plan.definition.key() + ); + let result = tokio::time::timeout_at(deadline, fetch_rows(&state, &validated, &log_comment)) + .await + .map_err(|_| export_limit("Export exceeded the execution time limit."))??; + verify_evidence_snapshot(&state.ch, &validated.plan.relation, &validated.snapshot_id).await?; + if result.len() > MAX_EXPORT_ROWS { + return Err(export_limit(format!( + "Export exceeds the {MAX_EXPORT_ROWS} row limit." + ))); + } + let exported_rows = result.len(); + let (columns, rows) = presentation( + &result, + &validated.plan, + &validated.selection.filters, + &validated.selection.display_dimensions, + )?; + drop(result); + let export_format = req.format; + let export = tokio::task::spawn_blocking(move || { + let _permit = permit; + build_export(export_format, &columns, &rows) + }); + let (body, content_type, extension) = tokio::time::timeout_at(deadline, export) + .await + .map_err(|_| export_limit("Export exceeded the execution time limit."))? + .map_err(|_| export_internal())??; + tracing::info!( + duration_ms = started.elapsed().as_millis(), + rows = exported_rows, + bytes = body.len(), + format = export_format.as_str(), + row_limit = MAX_EXPORT_ROWS, + byte_limit = MAX_EXPORT_BYTES, + capacity = MAX_CONCURRENT_EXPORTS, + "metric drilldown export completed" + ); + let filename = export_filename( + &validated.plan.definition.base.label, + &validated.selection.metric_key, + &validated.selection.period.from, + &validated.selection.period.to, + validated + .selection + .filters + .iter() + .any(|filter| !filter.values.is_empty()), + extension, + ); + Response::builder() + .header(CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header( + CONTENT_DISPOSITION, + HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")) + .map_err(|_| export_internal())?, + ) + .body(Body::from(body)) + .map_err(|_| export_internal()) +} + +fn build_export( + format: MetricDrilldownExportFormat, + columns: &[MetricDrilldownColumn], + rows: &[MetricDrilldownRow], +) -> Result<(Vec, &'static str, &'static str), CanonicalError> { + let formatted_rows = rows + .iter() + .map(|row| export_values(columns, row)) + .collect::, _>>()?; + ensure_export_input_bound(columns, &formatted_rows)?; + match format { + MetricDrilldownExportFormat::Csv => Ok(( + build_csv(columns, formatted_rows)?, + "text/csv; charset=utf-8", + "csv", + )), + MetricDrilldownExportFormat::Xlsx => Ok(( + build_xlsx(columns, rows)?, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlsx", + )), + } +} + +async fn fetch_rows( + state: &Arc, + req: &crate::domain::metric_drilldown::ValidatedMetricDrilldown, + log_comment: &str, +) -> Result, CanonicalError> { + let _permit = tokio::time::timeout(EXPORT_ACQUIRE_TIMEOUT, QUERY_SEMAPHORE.acquire()) + .await + .map_err(|_| query_busy())? + .map_err(|_| query_busy())?; + let (sql, params) = compile_query(req)?; + let mut query = state + .ch + .query(&sql) + .with_option("log_comment", log_comment) + .with_option("max_execution_time", QUERY_TIMEOUT.as_secs().to_string()) + .with_option("max_threads", "2") + .with_option("max_memory_usage", EVIDENCE_QUERY_MEMORY_BYTES.to_string()) + .with_option("max_bytes_to_read", EVIDENCE_QUERY_READ_BYTES.to_string()) + .with_option("max_result_bytes", EVIDENCE_QUERY_RESULT_BYTES.to_string()); + for param in params { + query = query.bind(param); + } + let mut cursor = query.fetch_bytes("JSONEachRow").map_err(|error| { + tracing::error!(error = %error, "ClickHouse metric drilldown query failed"); + query_error(&error.to_string()) + })?; + let bytes = tokio::time::timeout(QUERY_TIMEOUT, cursor.collect()) + .await + .map_err(|_| CanonicalError::internal("metric evidence query timed out").create())? + .map_err(|error| { + tracing::error!(error = %error, "ClickHouse metric drilldown fetch failed"); + query_error(&error.to_string()) + })?; + if bytes.is_empty() { + return Ok(Vec::new()); + } + bytes + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(serde_json::from_slice) + .collect::, _>>() + .map_err(|error| { + tracing::error!(error = %error, "metric drilldown row decoding failed"); + CanonicalError::internal("failed to decode metric evidence").create() + }) +} + +fn build_csv( + columns: &[MetricDrilldownColumn], + rows: Vec>, +) -> Result, CanonicalError> { + let mut writer = csv::Writer::from_writer(LimitedBuffer::new(MAX_EXPORT_BYTES)); + let headers = columns + .iter() + .map(|column| column.label.as_str()) + .collect::>(); + writer + .write_record(&headers) + .map_err(|_| export_limit("CSV export exceeds the byte limit."))?; + for row in rows { + let values = row.into_iter().map(csv_safe_cell).collect::>(); + writer + .write_record(values) + .map_err(|_| export_limit("CSV export exceeds the byte limit."))?; + } + writer + .into_inner() + .map(LimitedBuffer::into_inner) + .map_err(|_| export_limit("CSV export exceeds the byte limit.")) +} + +fn build_xlsx( + columns: &[MetricDrilldownColumn], + rows: &[MetricDrilldownRow], +) -> Result, CanonicalError> { + let mut workbook = Workbook::new(); + let worksheet = workbook.add_worksheet(); + let date_format = Format::new().set_num_format("yyyy-mm-dd"); + for (column, header) in columns.iter().enumerate() { + worksheet + .write_string( + 0, + u16::try_from(column).map_err(|_| export_internal())?, + &header.label, + ) + .map_err(|_| export_internal())?; + } + for (row_index, row) in rows.iter().enumerate() { + let row_index = u32::try_from(row_index + 1).map_err(|_| export_internal())?; + for (column_index, column) in columns.iter().enumerate() { + let column_index = u16::try_from(column_index).map_err(|_| export_internal())?; + let value = row + .values + .get(&column.key) + .unwrap_or(&serde_json::Value::Null); + match (column.r#type, value) { + (_, serde_json::Value::Null) => worksheet + .write_blank(row_index, column_index, &Format::new()) + .map_err(|_| export_internal())?, + (crate::domain::metric_drilldown::MetricDrilldownColumnType::Number, value) => { + if let Some(value) = value.as_f64() { + worksheet + .write_number(row_index, column_index, value) + .map_err(|_| export_internal())? + } else { + worksheet + .write_string(row_index, column_index, value.to_string()) + .map_err(|_| export_internal())? + } + } + ( + crate::domain::metric_drilldown::MetricDrilldownColumnType::Date, + serde_json::Value::String(value), + ) => { + let date = + ExcelDateTime::parse_from_str(value).map_err(|_| export_internal())?; + worksheet + .write_datetime_with_format(row_index, column_index, &date, &date_format) + .map_err(|_| export_internal())? + } + (_, serde_json::Value::String(value)) => worksheet + .write_string(row_index, column_index, value) + .map_err(|_| export_internal())?, + (_, serde_json::Value::Bool(value)) => worksheet + .write_boolean(row_index, column_index, *value) + .map_err(|_| export_internal())?, + (_, value) => worksheet + .write_string( + row_index, + column_index, + serde_json::to_string(value).map_err(|_| export_internal())?, + ) + .map_err(|_| export_internal())?, + }; + } + } + if !rows.is_empty() && !columns.is_empty() { + let last_row = u32::try_from(rows.len()).map_err(|_| export_internal())?; + let last_column = u16::try_from(columns.len() - 1).map_err(|_| export_internal())?; + let table = Table::new() + .set_style(TableStyle::None) + .set_autofilter(false) + .set_banded_rows(false); + worksheet + .add_table(0, 0, last_row, last_column, &table) + .map_err(|_| export_internal())?; + } + let mut output = LimitedBuffer::new(MAX_EXPORT_BYTES); + workbook.save_to_writer(&mut output).map_err(|error| { + tracing::warn!(error = %error, "metric drilldown XLSX generation failed"); + export_limit("XLSX export exceeds the byte limit.") + })?; + Ok(output.into_inner()) +} + +fn export_values( + columns: &[MetricDrilldownColumn], + row: &MetricDrilldownRow, +) -> Result, CanonicalError> { + let values = columns + .iter() + .map(|column| { + let value = row + .values + .get(&column.key) + .unwrap_or(&serde_json::Value::Null); + match value { + serde_json::Value::Null => Ok(String::new()), + serde_json::Value::String(value) => Ok(value.clone()), + serde_json::Value::Bool(value) => Ok(value.to_string()), + serde_json::Value::Number(value) => Ok(value.to_string()), + value => serde_json::to_string(value).map_err(|_| export_internal()), + } + }) + .collect::, _>>()?; + if values.iter().any(|value| value.len() > MAX_CELL_BYTES) { + return Err(export_limit(format!( + "Export contains a value exceeding the {MAX_CELL_BYTES} byte limit." + ))); + } + Ok(values) +} + +fn ensure_export_input_bound( + columns: &[MetricDrilldownColumn], + rows: &[Vec], +) -> Result<(), CanonicalError> { + let mut bytes = columns + .iter() + .try_fold(0usize, |total, column| { + total.checked_add(column.label.len() + 1) + }) + .ok_or_else(|| export_limit("Export input exceeds the byte limit."))?; + for row in rows { + for value in row { + bytes = bytes + .checked_add(value.len() + 1) + .ok_or_else(|| export_limit("Export input exceeds the byte limit."))?; + if bytes > MAX_EXPORT_BYTES { + return Err(export_limit("Export input exceeds the byte limit.")); + } + } + } + Ok(()) +} + +fn csv_safe_cell(value: String) -> String { + if value.as_bytes().first().is_some_and(|first| { + matches!( + first, + b'=' | b'+' | b'-' | b'@' | b'\t' | b'\r' | b'\n' | b' ' + ) + }) { + format!("'{value}") + } else { + value + } +} + +struct LimitedBuffer { + inner: Cursor>, + limit: usize, +} + +impl LimitedBuffer { + fn new(limit: usize) -> Self { + Self { + inner: Cursor::new(Vec::new()), + limit, + } + } + + fn into_inner(self) -> Vec { + self.inner.into_inner() + } +} + +impl Write for LimitedBuffer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let end = self + .inner + .position() + .checked_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)) + .ok_or_else(|| std::io::Error::other("export byte limit exceeded"))?; + if end > self.limit as u64 { + return Err(std::io::Error::other("export byte limit exceeded")); + } + self.inner.write(bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +impl Seek for LimitedBuffer { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + let offset = self.inner.seek(position)?; + if offset > self.limit as u64 { + return Err(std::io::Error::other("export byte limit exceeded")); + } + Ok(offset) + } +} + +fn export_filename( + metric_label: &str, + metric_key: &str, + from: &str, + to: &str, + filtered: bool, + extension: &str, +) -> String { + let metric = filename_slug(metric_label); + let metric = if metric.is_empty() { + filename_slug(metric_key) + } else { + metric + }; + let suffix = if filtered { "_filtered" } else { "" }; + format!("{metric}_{from}_{to}{suffix}.{extension}") +} + +fn filename_slug(value: &str) -> String { + const MAX_BYTES: usize = 80; + + let mut slug = String::with_capacity(value.len().min(MAX_BYTES)); + let mut separated = true; + for character in value.chars() { + if character.is_ascii_alphanumeric() { + if slug.len() == MAX_BYTES { + break; + } + slug.push(character.to_ascii_lowercase()); + separated = false; + } else if !separated && slug.len() < MAX_BYTES { + slug.push('-'); + separated = true; + } + } + while slug.ends_with('-') { + slug.pop(); + } + slug +} + +fn query_error(message: &str) -> CanonicalError { + if message.contains("UNKNOWN_TABLE") || message.contains("Code: 60") { + return MetricError::failed_precondition() + .with_precondition_violation( + "metric evidence relation", + "The evidence relation backing this metric is unavailable.", + "EVIDENCE_RELATION_MISSING", + ) + .create(); + } + if is_clickhouse_resource_limit(message) { + return MetricError::resource_exhausted("Metric evidence query exceeded resource limits.") + .with_quota_violation("metric evidence query", "ClickHouse resource limit reached") + .create(); + } + CanonicalError::internal("metric evidence query failed").create() +} + +fn is_clickhouse_resource_limit(message: &str) -> bool { + [ + "MEMORY_LIMIT_EXCEEDED", + "TOO_MANY_SIMULTANEOUS_QUERIES", + "TOO_MANY_ROWS_OR_BYTES", + "QUOTA_EXCEEDED", + "LIMIT_EXCEEDED", + "Code: 198", + "Code: 201", + "Code: 202", + "Code: 241", + ] + .iter() + .any(|marker| message.contains(marker)) +} + +fn export_busy() -> CanonicalError { + MetricError::resource_exhausted("Metric evidence export capacity is busy.") + .with_quota_violation("metric evidence exports", "concurrency limit reached") + .with_quota_violation_retry_after_seconds(2) + .create() +} + +fn query_busy() -> CanonicalError { + MetricError::resource_exhausted("Metric evidence query capacity is busy.") + .with_quota_violation("metric evidence queries", "concurrency limit reached") + .with_quota_violation_retry_after_seconds(2) + .create() +} + +fn export_limit(description: impl Into) -> CanonicalError { + MetricError::resource_exhausted("Metric evidence export exceeded resource limits.") + .with_quota_violation("metric evidence export", description.into()) + .create() +} + +impl MetricDrilldownExportFormat { + fn as_str(self) -> &'static str { + match self { + Self::Csv => "csv", + Self::Xlsx => "xlsx", + } + } +} + +fn export_internal() -> CanonicalError { + CanonicalError::internal("failed to build metric evidence export").create() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::metric_drilldown::MetricDrilldownColumnType; + use serde_json::json; + use std::collections::BTreeMap; + + fn columns() -> Vec { + vec![ + MetricDrilldownColumn { + key: "ref".to_owned(), + label: "Ref".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "date".to_owned(), + label: "Date".to_owned(), + r#type: MetricDrilldownColumnType::Date, + }, + MetricDrilldownColumn { + key: "value".to_owned(), + label: "Value".to_owned(), + r#type: MetricDrilldownColumnType::Number, + }, + MetricDrilldownColumn { + key: "active".to_owned(), + label: "Active".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + ] + } + + fn row() -> MetricDrilldownRow { + MetricDrilldownRow { + values: BTreeMap::from([ + ("ref".to_owned(), json!("=formula")), + ("date".to_owned(), json!("2026-07-28")), + ("value".to_owned(), json!(12.5)), + ("active".to_owned(), json!(true)), + ]), + } + } + + #[test] + fn csv_export_is_bounded_and_formula_safe() { + let (bytes, content_type, extension) = + build_export(MetricDrilldownExportFormat::Csv, &columns(), &[row()]) + .unwrap_or_else(|error| panic!("CSV export must succeed: {error}")); + let csv = String::from_utf8(bytes) + .unwrap_or_else(|error| panic!("CSV export must be UTF-8: {error}")); + assert_eq!(content_type, "text/csv; charset=utf-8"); + assert_eq!(extension, "csv"); + assert!(csv.contains("'=formula")); + assert!(csv.contains("2026-07-28")); + assert!(csv.contains("12.5")); + } + + #[test] + fn xlsx_export_contains_typed_cells() { + let (bytes, content_type, extension) = + build_export(MetricDrilldownExportFormat::Xlsx, &columns(), &[row()]) + .unwrap_or_else(|error| panic!("XLSX export must succeed: {error}")); + assert_eq!( + content_type, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ); + assert_eq!(extension, "xlsx"); + assert!(bytes.starts_with(b"PK")); + assert!(bytes.len() > 1_000); + } + + #[test] + fn export_values_serialize_supported_json_values() { + let columns = vec![ + MetricDrilldownColumn { + key: "missing".to_owned(), + label: "Missing".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "object".to_owned(), + label: "Object".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + ]; + let row = MetricDrilldownRow { + values: BTreeMap::from([("object".to_owned(), json!({"key": "value"}))]), + }; + assert_eq!( + export_values(&columns, &row) + .unwrap_or_else(|error| panic!("export values must serialize: {error}")), + ["", r#"{"key":"value"}"#] + ); + } + + #[test] + fn oversized_export_cells_are_rejected() { + let columns = vec![MetricDrilldownColumn { + key: "value".to_owned(), + label: "Value".to_owned(), + r#type: MetricDrilldownColumnType::String, + }]; + let row = MetricDrilldownRow { + values: BTreeMap::from([("value".to_owned(), json!("x".repeat(MAX_CELL_BYTES + 1)))]), + }; + assert!(export_values(&columns, &row).is_err()); + } + + #[test] + fn limited_buffer_enforces_write_and_seek_bounds() { + let mut buffer = LimitedBuffer::new(4); + assert_eq!( + buffer + .write(b"1234") + .unwrap_or_else(|error| panic!("bounded write must succeed: {error}")), + 4 + ); + assert!(buffer.write(b"5").is_err()); + assert!(buffer.seek(SeekFrom::Start(5)).is_err()); + assert_eq!(buffer.into_inner(), b"1234"); + } + + #[test] + fn filenames_are_human_readable_and_bounded() { + assert_eq!( + export_filename( + "Tasks closed", + "tasks.closed", + "2025-07-28", + "2026-07-27", + true, + "xlsx" + ), + "tasks-closed_2025-07-28_2026-07-27_filtered.xlsx" + ); + assert_eq!(filename_slug("***"), ""); + assert!(filename_slug(&"a".repeat(100)).len() <= 80); + } + + #[test] + fn query_errors_are_classified() { + assert!(is_clickhouse_resource_limit("MEMORY_LIMIT_EXCEEDED")); + assert!(is_clickhouse_resource_limit("Code: 241")); + assert!(!is_clickhouse_resource_limit("syntax error")); + let missing = query_error("UNKNOWN_TABLE"); + let limited = query_error("QUOTA_EXCEEDED"); + let internal = query_error("syntax error"); + assert_eq!(missing.status_code(), axum::http::StatusCode::BAD_REQUEST); + assert_eq!( + limited.status_code(), + axum::http::StatusCode::TOO_MANY_REQUESTS + ); + assert_eq!( + internal.status_code(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + } + + #[test] + fn export_format_strings_are_stable() { + assert_eq!(MetricDrilldownExportFormat::Csv.as_str(), "csv"); + assert_eq!(MetricDrilldownExportFormat::Xlsx.as_str(), "xlsx"); + } +} diff --git a/src/backend/services/analytics/src/api/metric_results.rs b/src/backend/services/analytics/src/api/metric_results.rs index e985e7bfd..82f636720 100644 --- a/src/backend/services/analytics/src/api/metric_results.rs +++ b/src/backend/services/analytics/src/api/metric_results.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; @@ -10,6 +10,7 @@ use toolkit_canonical_errors::CanonicalError; use super::AppState; use super::error::MetricError; +use crate::domain::metric_drilldown::load_capabilities; use crate::domain::metric_results::{ BatchItem, BreakdownQueryRow, CompiledQuery, HistogramQueryRow, MetricResultViewDto, MetricResultsRequest, MetricResultsResponse, PeerWideRow, PeriodWideRow, PlannedQuery, @@ -32,23 +33,36 @@ pub async fn query_metric_results( Extension(ctx): Extension, Json(req): Json, ) -> Result, CanonicalError> { - let req = validate_request(&state.db, ctx.subject_tenant_id(), req).await?; - let mut ranking_results = BTreeMap::new(); - let mut rankings = stream::iter(plan_rankings(&req)) - .map(|ranking| { - let state = Arc::clone(&state); - async move { - let comment = format!("metric-results:ranking:{}", ranking.key.rank_metric_key); - let rows = fetch_rows::(&state, ranking.query, &comment).await?; - let groups = build_ranked_groups(&ranking.dimensions, rows)?; - Ok::<_, CanonicalError>((ranking.key, groups)) - } - }) - .buffer_unordered(QUERY_CONCURRENCY); - while let Some(result) = rankings.next().await { - let (key, groups) = result?; - ranking_results.insert(key, groups); - } + let tenant_id = ctx.subject_tenant_id(); + let req = validate_request(&state.db, tenant_id, req).await?; + let metric_keys = req + .metrics + .iter() + .map(|metric| metric.def.key().to_owned()) + .collect::>(); + let capabilities = load_capabilities(&state.db, tenant_id, &metric_keys); + let rankings = async { + let mut ranking_results = BTreeMap::new(); + let mut rankings = stream::iter(plan_rankings(&req)) + .map(|ranking| { + let state = Arc::clone(&state); + async move { + let comment = format!("metric-results:ranking:{}", ranking.key.rank_metric_key); + let rows = + fetch_rows::(&state, ranking.query, &comment).await?; + let groups = build_ranked_groups(&ranking.dimensions, rows)?; + Ok::<_, CanonicalError>((ranking.key, groups)) + } + }) + .buffer_unordered(QUERY_CONCURRENCY); + while let Some(result) = rankings.next().await { + let (key, groups) = result?; + ranking_results.insert(key, groups); + } + Ok::<_, CanonicalError>(ranking_results) + }; + let (ranking_results, capabilities) = tokio::join!(rankings, capabilities); + let ranking_results = ranking_results?; let planned = plan_queries(&req, &ranking_results)?; let mut views_by_metric: Vec>> = req @@ -68,6 +82,13 @@ pub async fn query_metric_results( } } + let capabilities = match capabilities { + Ok(capabilities) => capabilities, + Err(error) => { + tracing::warn!(error = ?error, "metric drilldown capability load failed"); + HashMap::default() + } + }; let mut metrics = Vec::with_capacity(req.metrics.len()); for (idx, metric) in req.metrics.iter().enumerate() { let mut views = Vec::with_capacity(metric.views.len()); @@ -78,7 +99,30 @@ pub async fn query_metric_results( enforce_view_row_limit(&view, format!("metrics[{idx}].views[{view_index}]"))?; views.push(view); } - metrics.push(build_metric_result(&metric.def, views)); + let mut result = build_metric_result(&metric.def, views); + result.drilldown = capabilities.get(metric.def.key()).cloned(); + result.selection = crate::domain::metric_results::MetricResultSelectionDto { + metric_key: metric.def.key().to_owned(), + entity: crate::domain::metric_results::MetricResultsEntityDto { + r#type: req.entity_type.clone(), + ids: req.entity_ids.clone(), + }, + period: crate::domain::metric_results::MetricResultsPeriodDto { + from: req.from.to_string(), + to: req.to.to_string(), + }, + filters: metric + .filters + .iter() + .map( + |filter| crate::domain::metric_results::MetricDimensionFilterDto { + dimension: filter.dimension.clone(), + values: filter.values.clone(), + }, + ) + .collect(), + }; + metrics.push(result); } let response = MetricResultsResponse { metrics }; diff --git a/src/backend/services/analytics/src/api/mod.rs b/src/backend/services/analytics/src/api/mod.rs index dd746d42b..be44656d4 100644 --- a/src/backend/services/analytics/src/api/mod.rs +++ b/src/backend/services/analytics/src/api/mod.rs @@ -6,6 +6,7 @@ mod catalog; pub(crate) mod error; mod handlers; mod metric_definitions; +mod metric_drilldown; mod metric_results; #[cfg(test)] @@ -18,7 +19,15 @@ use axum::http::StatusCode; use axum::{Extension, Router}; use sea_orm::DatabaseConnection; use std::sync::Arc; -use toolkit::api::{OpenApiInfo, OpenApiRegistry, OpenApiRegistryImpl, OperationBuilder}; +use toolkit::api::{ + OpenApiInfo, OpenApiRegistry, OpenApiRegistryImpl, OperationBuilder, ResponseSpec, +}; +use utoipa::openapi::RefOr; +use utoipa::openapi::content::ContentBuilder; +use utoipa::openapi::header::HeaderBuilder; +use utoipa::openapi::schema::{ + KnownFormat, ObjectBuilder, Schema, SchemaFormat, SchemaType, Type as OpenApiType, +}; use crate::config::GearConfig; use crate::domain::admin_threshold::AdminThresholdService; @@ -230,6 +239,43 @@ fn build_operations(router: Router, openapi: &dyn OpenApiRegistry) -> Router { .handler(metric_results::query_metric_results) .register(router, openapi); + router = OperationBuilder::post("/v1/metric-drilldown") + .operation_id("analytics_api.metric_drilldown.create") + .summary("List metric evidence") + .authenticated() + .no_license_required() + .json_request::( + openapi, + "Metric evidence selection", + ) + .json_response_with_schema::( + openapi, + StatusCode::OK, + "Metric evidence", + ) + .standard_errors(openapi) + .handler(metric_drilldown::query_metric_drilldown) + .register(router, openapi); + + router = OperationBuilder::post("/v1/metric-drilldown/export") + .operation_id("analytics_api.metric_drilldown.export") + .summary("Export metric evidence") + .authenticated() + .no_license_required() + .json_request::( + openapi, + "Metric evidence export selection", + ) + .response(ResponseSpec { + status: StatusCode::OK.as_u16(), + content_type: "text/csv", + description: "Complete metric evidence export".to_owned(), + schema_name: None, + }) + .standard_errors(openapi) + .handler(metric_drilldown::export_metric_drilldown) + .register(router, openapi); + // Thresholds (legacy) router = OperationBuilder::get("/v1/metrics/{id}/thresholds") .operation_id("analytics_api.thresholds.list") @@ -452,9 +498,44 @@ fn build_operations(router: Router, openapi: &dyn OpenApiRegistry) -> Router { pub fn openapi_document() -> anyhow::Result { let openapi = OpenApiRegistryImpl::new(); let _ = build_operations(Router::new(), &openapi); - openapi + let mut document = openapi .build_openapi(&openapi_info()) - .map_err(|e| anyhow::anyhow!("failed to build analytics OpenAPI document: {e}")) + .map_err(|e| anyhow::anyhow!("failed to build analytics OpenAPI document: {e}"))?; + let response = document + .paths + .paths + .get_mut("/v1/metric-drilldown/export") + .and_then(|path| path.post.as_mut()) + .and_then(|operation| operation.responses.responses.get_mut("200")) + .ok_or_else(|| anyhow::anyhow!("metric drilldown export response is missing"))?; + let RefOr::T(response) = response else { + return Err(anyhow::anyhow!( + "metric drilldown export response must be inline" + )); + }; + let schema = Schema::Object( + ObjectBuilder::new() + .schema_type(SchemaType::Type(OpenApiType::String)) + .format(Some(SchemaFormat::KnownFormat(KnownFormat::Binary))) + .build(), + ); + for media_type in [ + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ] { + response.content.insert( + media_type.to_owned(), + ContentBuilder::new().schema(Some(schema.clone())).build(), + ); + } + response.headers.insert( + "Content-Disposition".to_owned(), + HeaderBuilder::new() + .schema(ObjectBuilder::new().schema_type(OpenApiType::String)) + .description(Some("Attachment filename")) + .build(), + ); + Ok(document) } #[cfg(test)] diff --git a/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs b/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs index a087e3052..2c1f57547 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs @@ -1,5 +1,6 @@ use crate::domain::metric_definitions::definition::{ - MetricComputation, MetricDirection, MetricFormat, MetricInputRole, SourceKind, ValueTransform, + EvidenceGranularity, MetricComputation, MetricDirection, MetricFormat, MetricInputRole, + SourceKind, ValueTransform, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -60,6 +61,7 @@ pub struct SourceSeed { /// Managed-observation relation name; must satisfy /// `ObservationRelation::parse` (pinned by a registry test). pub source_ref: &'static str, + pub evidence_ref: &'static str, } pub struct BuiltinSource { @@ -68,6 +70,33 @@ pub struct BuiltinSource { pub dimensions: &'static [&'static str], } +impl BuiltinSource { + pub fn evidence_granularity(&self, measure_key: &str) -> EvidenceGranularity { + match (self.source.key, measure_key) { + ( + "git", + "commit_count" | "commit_change_size" | "pr_created" | "pr_created_merged" + | "pr_merged" | "pr_cycle_hours" | "pr_change_size", + ) + | ( + "task", + "tasks_closed" | "bugs_fixed" | "due_date_on_time" | "due_date_with_due" + | "slip_days_total" | "late_count" | "dev_time_hours" | "resolution_days" + | "pickup_days", + ) + | ("wiki", "pages_created") => EvidenceGranularity::Event, + ("ai_usage", "active_day") + | ( + "collab", + "active_day" | "active_modality" | "meeting_free_day" | "focus_hours" + | "working_hours", + ) + | ("task", _) => EvidenceGranularity::DerivedPopulation, + _ => EvidenceGranularity::SourceSummary, + } + } +} + pub struct MetricSeed { pub metric_key: &'static str, pub source_key: &'static str, @@ -101,6 +130,7 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[ key: "ai_usage", kind: SourceKind::ManagedObservation, source_ref: "ai_metric_observations", + evidence_ref: "ai_metric_evidence", }, measures: &[ "accepted_lines", @@ -121,6 +151,7 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[ key: "git", kind: SourceKind::ManagedObservation, source_ref: "git_metric_observations", + evidence_ref: "git_metric_evidence", }, measures: &[ "commit_count", @@ -150,6 +181,7 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[ key: "collab", kind: SourceKind::ManagedObservation, source_ref: "collab_metric_observations", + evidence_ref: "collab_metric_evidence", }, measures: &[ "total_chat_messages", @@ -181,6 +213,7 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[ key: "task", kind: SourceKind::ManagedObservation, source_ref: "task_metric_observations", + evidence_ref: "task_metric_evidence", }, measures: &[ "tasks_closed", @@ -209,6 +242,7 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[ key: "wiki", kind: SourceKind::ManagedObservation, source_ref: "wiki_metric_observations", + evidence_ref: "wiki_metric_evidence", }, measures: &["pages_created", "edits", "pages_edited", "comments"], dimensions: &[], diff --git a/src/backend/services/analytics/src/domain/metric_definitions/definition.rs b/src/backend/services/analytics/src/domain/metric_definitions/definition.rs index 317cf2e13..e41dd8d22 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/definition.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/definition.rs @@ -17,7 +17,7 @@ pub enum MetricFormat { Percent, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum MetricComputation { Sum, @@ -26,7 +26,7 @@ pub enum MetricComputation { DistinctCount, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum MetricInputRole { Value, @@ -34,6 +34,33 @@ pub enum MetricInputRole { Denominator, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum EvidenceGranularity { + Event, + SourceSummary, + DerivedPopulation, +} + +impl EvidenceGranularity { + pub fn as_db(self) -> &'static str { + match self { + Self::Event => "event", + Self::SourceSummary => "source_summary", + Self::DerivedPopulation => "derived_population", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "event" => Some(Self::Event), + "source_summary" => Some(Self::SourceSummary), + "derived_population" => Some(Self::DerivedPopulation), + _ => None, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SourceKind { ManagedObservation, @@ -66,6 +93,9 @@ impl SourceKind { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ObservationRelation(String); +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EvidenceRelation(String); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CohortSource { MetricEntityCohortsCurrent, @@ -218,20 +248,7 @@ impl ObservationRelation { /// lowercase `snake_case` ending in `_metric_observations`, with a /// non-empty family prefix. Anything else is a configuration error. pub fn parse(value: &str) -> Option { - let family = value.strip_suffix("_metric_observations")?; - if family.is_empty() { - return None; - } - let mut chars = family.chars(); - let starts_alpha = chars.next().is_some_and(|c| c.is_ascii_lowercase()); - let rest_ok = family - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); - if starts_alpha && rest_ok { - Some(Self(value.to_owned())) - } else { - None - } + parse_relation(value, "_metric_observations").map(Self) } pub fn table_ref(&self) -> (&'static str, &str) { @@ -245,6 +262,30 @@ impl ObservationRelation { } } +impl EvidenceRelation { + pub const DATABASE: &'static str = "insight"; + + pub fn parse(value: &str) -> Option { + parse_relation(value, "_metric_evidence").map(Self) + } + + pub fn table_ref(&self) -> (&'static str, &str) { + (Self::DATABASE, &self.0) + } + + pub fn source_ref(&self) -> &str { + &self.0 + } +} + +fn parse_relation(value: &str, suffix: &str) -> Option { + let family = value.strip_suffix(suffix)?; + let mut chars = family.chars(); + let starts_alpha = chars.next().is_some_and(|c| c.is_ascii_lowercase()); + let rest_ok = chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + (starts_alpha && rest_ok).then(|| value.to_owned()) +} + impl CohortSource { pub fn table_ref(self) -> (&'static str, &'static str) { match self { @@ -401,6 +442,17 @@ mod tests { ] { assert_eq!(MetricInputRole::from_db(role.as_db()), Some(role)); } + for granularity in [ + EvidenceGranularity::Event, + EvidenceGranularity::SourceSummary, + EvidenceGranularity::DerivedPopulation, + ] { + assert_eq!( + EvidenceGranularity::from_db(granularity.as_db()), + Some(granularity) + ); + } + assert_eq!(EvidenceGranularity::from_db("unknown"), None); let relation = ObservationRelation::parse("ai_metric_observations") .unwrap_or_else(|| panic!("builtin relation name must parse")); let (_, table) = relation.table_ref(); @@ -415,6 +467,10 @@ mod tests { ] { assert_eq!(SourceKind::from_db(kind.as_db()), Some(kind)); } + let evidence = EvidenceRelation::parse("ai_metric_evidence") + .unwrap_or_else(|| panic!("builtin evidence must parse")); + assert_eq!(evidence.table_ref(), ("insight", "ai_metric_evidence")); + assert_eq!(evidence.source_ref(), "ai_metric_evidence"); } #[test] diff --git a/src/backend/services/analytics/src/domain/metric_definitions/listing.rs b/src/backend/services/analytics/src/domain/metric_definitions/listing.rs index b583a518c..1a26c7fe2 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/listing.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/listing.rs @@ -19,6 +19,7 @@ use uuid::Uuid; use crate::domain::metric_definitions::definition::{MetricDirection, MetricFormat}; use crate::domain::metric_definitions::error_code::{MetricSchemaErrorCode, SchemaStatus}; use crate::domain::metric_definitions::repository::fetch_dimensions; +use crate::domain::metric_drilldown::{MetricDrilldownCapability, load_capabilities}; /// Response body for `GET /v1/metric-definitions`. Metrics are sorted by /// `metric_key` ascending so the payload is byte-stable for caching and @@ -51,6 +52,8 @@ pub struct MetricDefinitionView { /// measures; absent when no observation has ever been seen. Freshness /// signal, orthogonal to `schema_status`. pub last_observed_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub drilldown: Option, } impl toolkit::api::api_dto::ResponseApiDto for MetricDefinitionListResponse {} @@ -81,6 +84,17 @@ pub async fn list_definition_views( .await .map_err(|error| db_error(&error))?; let selected = select_rows(rows); + let metric_keys = selected + .iter() + .map(|row| row.metric_key.clone()) + .collect::>(); + let mut capabilities = match load_capabilities(db, tenant_id, &metric_keys).await { + Ok(capabilities) => capabilities, + Err(error) => { + tracing::warn!(error = ?error, "metric drilldown capability load failed"); + HashMap::new() + } + }; let definition_ids = selected .iter() @@ -90,7 +104,10 @@ pub async fn list_definition_views( .await .map_err(|error| db_error(&error))?; - let metrics = build_views(selected, dimensions)?; + let mut metrics = build_views(selected, dimensions)?; + for metric in &mut metrics { + metric.drilldown = capabilities.remove(&metric.metric_key); + } Ok(MetricDefinitionListResponse { metrics }) } @@ -148,6 +165,7 @@ fn build_views( schema_status, schema_error_code, last_observed_date: row.last_observed_date, + drilldown: None, }); } Ok(metrics) diff --git a/src/backend/services/analytics/src/domain/metric_definitions/live_tests.rs b/src/backend/services/analytics/src/domain/metric_definitions/live_tests.rs index 7eec36eac..c808d6f69 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/live_tests.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/live_tests.rs @@ -13,7 +13,12 @@ use uuid::Uuid; use crate::domain::metric_definitions::error_code::SchemaStatus; use crate::domain::metric_definitions::listing::list_definition_views; -use crate::domain::metric_definitions::repository::update_definition_status; +use crate::domain::metric_definitions::repository::{ + update_definition_status, update_evidence_status, +}; +use crate::domain::metric_definitions::test_fixture::DrilldownFixture; +use crate::domain::metric_definitions::validator::MetricDefinitionValidator; +use crate::domain::metric_drilldown::load_capabilities; const ENV_VAR: &str = "INTEGRATION_TESTS_MARIADB_URL"; @@ -150,3 +155,106 @@ async fn update_definition_status_advances_but_never_regresses_freshness() -> an assert_eq!(stored_last_observed(&db, id).await?, Some(newest)); Ok(()) } + +#[tokio::test] +#[ignore = "requires live MariaDB 11+; set INTEGRATION_TESTS_MARIADB_URL to enable"] +async fn drilldown_capabilities_follow_healthy_evidence_metadata() -> anyhow::Result<()> { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let fixture = DrilldownFixture::insert(&db, &["git.commits", "tasks.closed"], &[]).await?; + let keys = vec![ + "git.commits".to_owned(), + "tasks.closed".to_owned(), + "missing.metric".to_owned(), + ]; + let result = async { + let capabilities = load_capabilities(&db, fixture.tenant_id, &keys).await?; + anyhow::ensure!(capabilities.contains_key("git.commits")); + anyhow::ensure!(capabilities.contains_key("tasks.closed")); + anyhow::ensure!(!capabilities.contains_key("missing.metric")); + anyhow::ensure!( + load_capabilities(&db, fixture.tenant_id, &[]) + .await? + .is_empty() + ); + Ok(()) + } + .await; + fixture.delete(&db).await?; + result +} + +#[tokio::test] +#[ignore = "requires live MariaDB 11+; set INTEGRATION_TESTS_MARIADB_URL to enable"] +async fn evidence_status_writer_is_revision_conditional() -> anyhow::Result<()> { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let fixture = DrilldownFixture::insert(&db, &["git.commits"], &[]).await?; + let result = async { + let revision = fixture.config_revision(&db).await?; + update_evidence_status( + &db, + fixture.source_id, + &revision, + SchemaStatus::Error, + Some(crate::domain::metric_definitions::error_code::MetricSchemaErrorCode::Unknown), + ) + .await?; + let current = fixture.statuses(&db).await?; + anyhow::ensure!( + current + == ( + "ok".to_owned(), + "error".to_owned(), + Some("unknown".to_owned()) + ) + ); + update_evidence_status( + &db, + fixture.source_id, + "1970-01-01 00:00:00.000000", + SchemaStatus::Ok, + None, + ) + .await?; + let stale = fixture.statuses(&db).await?; + anyhow::ensure!( + stale == current, + "stale evidence status write changed {current:?} to {stale:?}" + ); + Ok(()) + } + .await; + fixture.delete(&db).await?; + result +} + +#[tokio::test] +#[ignore = "requires live MariaDB 11+; set INTEGRATION_TESTS_MARIADB_URL to enable"] +async fn metric_definition_validator_handles_unavailable_clickhouse() -> anyhow::Result<()> { + let Some(db) = connect_or_skip().await else { + return Ok(()); + }; + let fixture = DrilldownFixture::insert(&db, &["git.commits"], &[]).await?; + let result = async { + let before = fixture.statuses(&db).await?; + let ch = insight_clickhouse::Client::new(insight_clickhouse::Config::new( + "http://127.0.0.1:1", + "analytics", + )); + MetricDefinitionValidator::new(db.clone(), ch) + .validate_all() + .await; + let after = fixture.statuses(&db).await?; + anyhow::ensure!( + after == before, + "unavailable ClickHouse changed source status from {before:?} to {after:?}" + ); + Ok(()) + } + .await; + fixture.delete(&db).await?; + result +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/mod.rs b/src/backend/services/analytics/src/domain/metric_definitions/mod.rs index cc3ce083f..634c24502 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/mod.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/mod.rs @@ -6,12 +6,15 @@ pub mod listing; mod live_tests; mod repository; mod seeds; +#[cfg(test)] +pub(crate) mod test_fixture; pub mod validator; pub use definition::{ - CohortSource, ComputationSpec, MetricDefinition, MetricDirection, MetricFormat, - ObservationRelation, + CohortSource, ComputationSpec, EvidenceGranularity, EvidenceRelation, MetricDefinition, + MetricDirection, MetricFormat, ObservationRelation, }; pub use repository::load_definitions; +pub(crate) use repository::load_definitions_with_ids; pub use seeds::reconcile_builtin_definitions; pub use validator::MetricDefinitionValidator; diff --git a/src/backend/services/analytics/src/domain/metric_definitions/repository.rs b/src/backend/services/analytics/src/domain/metric_definitions/repository.rs index e65d1f266..de73579c9 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/repository.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/repository.rs @@ -84,6 +84,21 @@ pub async fn load_definitions( tenant_id: Uuid, metric_keys: &[String], ) -> Result, CanonicalError> { + load_definitions_with_ids(db, tenant_id, metric_keys) + .await + .map(|definitions| { + definitions + .into_iter() + .map(|(key, (_, definition))| (key, definition)) + .collect() + }) +} + +pub async fn load_definitions_with_ids( + db: &DatabaseConnection, + tenant_id: Uuid, + metric_keys: &[String], +) -> Result, CanonicalError> { if metric_keys.is_empty() { return Ok(HashMap::new()); } @@ -115,7 +130,7 @@ pub async fn load_definitions( row_inputs, dimensions.get(&definition_id).cloned().unwrap_or_default(), )?; - definitions.insert(metric_key, definition); + definitions.insert(metric_key, (definition_id, definition)); } for key in metric_keys { @@ -529,37 +544,101 @@ fn one_input( } } +#[derive(FromQueryResult)] +pub struct ManagedSourceValidationTarget { + pub id: Uuid, + pub source_key: String, + pub source_kind: String, + pub source_ref: String, + pub evidence_ref: Option, + pub config_revision: String, +} + pub async fn all_managed_sources( db: &DatabaseConnection, -) -> Result, sea_orm::DbErr> { +) -> Result, sea_orm::DbErr> { + ManagedSourceValidationTarget::find_by_statement(Statement::from_string( + db.get_database_backend(), + "SELECT id, source_key, source_kind, source_ref, evidence_ref, \ + DATE_FORMAT(updated_at, '%Y-%m-%d %H:%i:%s.%f') AS config_revision \ + FROM metric_sources \ + WHERE is_enabled = TRUE", + )) + .all(db) + .await +} + +pub async fn source_evidence_granularities( + db: &DatabaseConnection, + source_id: Uuid, +) -> Result)>, sea_orm::DbErr> { #[derive(FromQueryResult)] struct Row { - id: Uuid, - source_kind: String, - source_ref: String, + measure_key: String, + evidence_granularity: Option, } - Row::find_by_statement(Statement::from_string( + Row::find_by_statement(Statement::from_sql_and_values( db.get_database_backend(), - "SELECT id, source_kind, source_ref \ - FROM metric_sources \ - WHERE is_enabled = TRUE", + "SELECT measure_key, evidence_granularity \ + FROM metric_source_measures \ + WHERE source_id = ? AND is_enabled = TRUE \ + ORDER BY measure_key", + [uuid_value(source_id)], )) .all(db) .await .map(|rows| { rows.into_iter() - .map(|row| (row.id, row.source_kind, row.source_ref)) + .map(|row| (row.measure_key, row.evidence_granularity)) .collect() }) } +pub async fn update_evidence_status( + db: &DatabaseConnection, + source_id: Uuid, + config_revision: &str, + status: SchemaStatus, + error_code: Option, +) -> Result<(), sea_orm::DbErr> { + let result = db + .execute(Statement::from_sql_and_values( + db.get_database_backend(), + "UPDATE metric_sources \ + SET evidence_schema_status = ?, \ + evidence_schema_checked_at = CURRENT_TIMESTAMP(3), \ + evidence_schema_error_code = ?, \ + updated_at = updated_at \ + WHERE id = ? AND updated_at = ?", + [ + Value::from(status.as_db()), + match error_code { + Some(code) => Value::from(code.as_db()), + None => Value::String(None), + }, + uuid_value(source_id), + Value::from(config_revision), + ], + )) + .await?; + if result.rows_affected() == 0 { + tracing::trace!( + %source_id, + config_revision, + "metric evidence status update skipped for stale configuration revision" + ); + } + Ok(()) +} + // `updated_at = updated_at` in the status writers below pins the column so // ON UPDATE CURRENT_TIMESTAMP(3) does not fire: updated_at tracks config // edits, not validator sweeps. pub async fn update_source_status( db: &DatabaseConnection, source_id: Uuid, + config_revision: &str, status: SchemaStatus, error_code: Option, ) -> Result<(), sea_orm::DbErr> { @@ -570,7 +649,7 @@ pub async fn update_source_status( schema_checked_at = CURRENT_TIMESTAMP(3), \ schema_error_code = ?, \ updated_at = updated_at \ - WHERE id = ?", + WHERE id = ? AND updated_at = ?", [ Value::from(status.as_db()), match error_code { @@ -578,6 +657,7 @@ pub async fn update_source_status( None => Value::String(None), }, Value::Bytes(Some(Box::new(source_id.as_bytes().to_vec()))), + Value::from(config_revision), ], )) .await?; diff --git a/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs b/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs index a0f8d63af..f0eba0911 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs @@ -33,14 +33,16 @@ async fn reconcile_source( db.execute(Statement::from_sql_and_values( db.get_database_backend(), "INSERT INTO metric_source_measures \ - (id, source_id, measure_key, is_enabled) \ - VALUES (?, ?, ?, TRUE) \ + (id, source_id, measure_key, evidence_granularity, is_enabled) \ + VALUES (?, ?, ?, ?, TRUE) \ ON DUPLICATE KEY UPDATE \ + evidence_granularity = VALUES(evidence_granularity), \ is_enabled = VALUES(is_enabled)", [ uuid_value(Uuid::now_v7()), uuid_value(source_id), Value::from(*measure_key), + Value::from(builtin_source.evidence_granularity(measure_key).as_db()), ], )) .await?; @@ -74,11 +76,12 @@ async fn upsert_source( db.execute(Statement::from_sql_and_values( db.get_database_backend(), "INSERT INTO metric_sources \ - (id, tenant_id, source_key, source_kind, source_ref, origin, is_enabled) \ - VALUES (?, NULL, ?, ?, ?, 'builtin', TRUE) \ + (id, tenant_id, source_key, source_kind, source_ref, evidence_ref, origin, is_enabled) \ + VALUES (?, NULL, ?, ?, ?, ?, 'builtin', TRUE) \ ON DUPLICATE KEY UPDATE \ source_kind = VALUES(source_kind), \ source_ref = VALUES(source_ref), \ + evidence_ref = VALUES(evidence_ref), \ origin = VALUES(origin), \ is_enabled = VALUES(is_enabled)", [ @@ -86,6 +89,7 @@ async fn upsert_source( Value::from(builtin_source.source.key), Value::from(builtin_source.source.kind.as_db()), Value::from(builtin_source.source.source_ref), + Value::from(builtin_source.source.evidence_ref), ], )) .await?; diff --git a/src/backend/services/analytics/src/domain/metric_definitions/test_fixture.rs b/src/backend/services/analytics/src/domain/metric_definitions/test_fixture.rs new file mode 100644 index 000000000..c7e708731 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/test_fixture.rs @@ -0,0 +1,218 @@ +use sea_orm::{ConnectionTrait, DatabaseConnection, Statement, Value}; +use uuid::Uuid; + +pub(crate) struct DrilldownFixture { + pub(crate) tenant_id: Uuid, + pub(crate) source_id: Uuid, +} + +impl DrilldownFixture { + pub(crate) async fn insert( + db: &DatabaseConnection, + metric_keys: &[&str], + dimensions: &[&str], + ) -> Result { + let tenant_id = Uuid::now_v7(); + let source_id = Uuid::now_v7(); + let measure_id = Uuid::now_v7(); + let suffix = tenant_id.simple().to_string(); + let source_key = format!("test_{suffix}"); + let source_ref = format!("test_{suffix}_metric_observations"); + let evidence_ref = format!("test_{suffix}_metric_evidence"); + + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_sources \ + (id, tenant_id, source_key, source_kind, source_ref, evidence_ref, origin, \ + schema_status, evidence_schema_status) \ + VALUES (?, ?, ?, 'managed_observation', ?, ?, 'custom', 'ok', 'ok')", + [ + uuid_value(source_id), + uuid_value(tenant_id), + Value::from(source_key), + Value::from(source_ref), + Value::from(evidence_ref), + ], + )) + .await?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_source_measures \ + (id, source_id, measure_key, evidence_granularity, schema_status) \ + VALUES (?, ?, 'value_count', 'event', 'ok')", + [uuid_value(measure_id), uuid_value(source_id)], + )) + .await?; + + let source_dimensions = insert_dimensions(db, source_id, dimensions).await?; + insert_definitions(db, tenant_id, measure_id, metric_keys, &source_dimensions).await?; + + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "UPDATE metric_sources \ + SET schema_status = 'ok', schema_error_code = NULL, \ + evidence_schema_status = 'ok', evidence_schema_error_code = NULL, \ + updated_at = updated_at \ + WHERE id = ?", + [uuid_value(source_id)], + )) + .await?; + + Ok(Self { + tenant_id, + source_id, + }) + } + + pub(crate) async fn delete(self, db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "DELETE FROM metric_definitions WHERE tenant_id = ?", + [uuid_value(self.tenant_id)], + )) + .await?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "DELETE FROM metric_sources WHERE id = ?", + [uuid_value(self.source_id)], + )) + .await?; + Ok(()) + } + + pub(crate) async fn config_revision( + &self, + db: &DatabaseConnection, + ) -> Result { + let row = db + .query_one(Statement::from_sql_and_values( + db.get_database_backend(), + "SELECT DATE_FORMAT(updated_at, '%Y-%m-%d %H:%i:%s.%f') AS config_revision \ + FROM metric_sources WHERE id = ?", + [uuid_value(self.source_id)], + )) + .await? + .ok_or_else(|| sea_orm::DbErr::Custom("test source disappeared".to_owned()))?; + row.try_get("", "config_revision") + } + + pub(crate) async fn statuses( + &self, + db: &DatabaseConnection, + ) -> Result<(String, String, Option), sea_orm::DbErr> { + let row = db + .query_one(Statement::from_sql_and_values( + db.get_database_backend(), + "SELECT schema_status, evidence_schema_status, evidence_schema_error_code \ + FROM metric_sources WHERE id = ?", + [uuid_value(self.source_id)], + )) + .await? + .ok_or_else(|| sea_orm::DbErr::Custom("test source disappeared".to_owned()))?; + Ok(( + row.try_get("", "schema_status")?, + row.try_get("", "evidence_schema_status")?, + row.try_get("", "evidence_schema_error_code")?, + )) + } +} + +async fn insert_dimensions( + db: &DatabaseConnection, + source_id: Uuid, + dimensions: &[&str], +) -> Result, sea_orm::DbErr> { + let mut source_dimensions = Vec::with_capacity(dimensions.len()); + for (display_order, dimension) in dimensions.iter().enumerate() { + let display_order = checked_display_order(display_order)?; + let dimension_id = Uuid::now_v7(); + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_source_dimensions \ + (id, source_id, dimension_key, display_order) \ + VALUES (?, ?, ?, ?)", + [ + uuid_value(dimension_id), + uuid_value(source_id), + Value::from(*dimension), + Value::from(display_order), + ], + )) + .await?; + source_dimensions.push(dimension_id); + } + Ok(source_dimensions) +} + +async fn insert_definitions( + db: &DatabaseConnection, + tenant_id: Uuid, + measure_id: Uuid, + metric_keys: &[&str], + source_dimensions: &[Uuid], +) -> Result<(), sea_orm::DbErr> { + for metric_key in metric_keys { + let definition_id = Uuid::now_v7(); + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definitions \ + (id, tenant_id, metric_key, label, format, direction, entity_type, \ + computation_type, origin, schema_status) \ + VALUES (?, ?, ?, ?, 'integer', 'higher_is_better', 'person', \ + 'sum', 'custom', 'ok')", + [ + uuid_value(definition_id), + uuid_value(tenant_id), + Value::from(*metric_key), + Value::from(format!("Test {metric_key}")), + ], + )) + .await?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definition_inputs \ + (id, metric_definition_id, input_role, source_measure_id) \ + VALUES (?, ?, 'value', ?)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(definition_id), + uuid_value(measure_id), + ], + )) + .await?; + insert_definition_dimensions(db, definition_id, source_dimensions).await?; + } + Ok(()) +} + +async fn insert_definition_dimensions( + db: &DatabaseConnection, + definition_id: Uuid, + source_dimensions: &[Uuid], +) -> Result<(), sea_orm::DbErr> { + for (display_order, dimension_id) in source_dimensions.iter().enumerate() { + let display_order = checked_display_order(display_order)?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definition_dimensions \ + (id, metric_definition_id, source_dimension_id, display_order) \ + VALUES (?, ?, ?, ?)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(definition_id), + uuid_value(*dimension_id), + Value::from(display_order), + ], + )) + .await?; + } + Ok(()) +} + +fn checked_display_order(value: usize) -> Result { + i32::try_from(value).map_err(|_| sea_orm::DbErr::Custom("too many test dimensions".to_owned())) +} + +fn uuid_value(value: Uuid) -> Value { + Value::Bytes(Some(Box::new(value.as_bytes().to_vec()))) +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/validator.rs b/src/backend/services/analytics/src/domain/metric_definitions/validator.rs index bad7df5ce..a7f84cee3 100644 --- a/src/backend/services/analytics/src/domain/metric_definitions/validator.rs +++ b/src/backend/services/analytics/src/domain/metric_definitions/validator.rs @@ -6,12 +6,18 @@ use sea_orm::DatabaseConnection; use serde::Deserialize; use crate::domain::metric_definitions::definition::{ - CohortSource, MetricInput, ObservationRelation, SourceKind, + CohortSource, EvidenceGranularity, EvidenceRelation, MetricInput, ObservationRelation, + SourceKind, }; use crate::domain::metric_definitions::error_code::{MetricSchemaErrorCode, SchemaStatus}; use crate::domain::metric_definitions::repository::{ MetricDefinitionValidationSpec, all_managed_sources, managed_definition_validation_specs, - update_definition_status, update_definitions_for_source_status, update_source_status, + source_evidence_granularities, update_definition_status, update_definitions_for_source_status, + update_evidence_status, update_source_status, +}; +use crate::domain::metric_drilldown::{ + EVIDENCE_QUERY_MEMORY_BYTES, EVIDENCE_QUERY_READ_BYTES, EVIDENCE_QUERY_RESULT_BYTES, + EVIDENCE_QUERY_TIMEOUT_SECS, }; // Dimension coverage is checked over a trailing window anchored at the @@ -59,25 +65,40 @@ impl MetricDefinitionValidator { } }; - for (source_id, source_kind, source_ref) in sources { + for source in sources { + self.validate_evidence( + source.id, + &source.source_key, + &source.source_kind, + &source.source_ref, + source.evidence_ref.as_deref(), + &source.config_revision, + ) + .await; let outcome = self - .validate_source(source_kind.as_str(), source_ref.as_str()) + .validate_source(source.source_kind.as_str(), source.source_ref.as_str()) .await; match outcome { ProbeOutcome::Definitive(state) => { let (status, error_code) = state.as_db(); - if let Err(error) = - update_source_status(&self.db, source_id, status, error_code).await + if let Err(error) = update_source_status( + &self.db, + source.id, + &source.config_revision, + status, + error_code, + ) + .await { tracing::warn!(error = %error, "metric definition source status update failed"); continue; } if state.is_ok() { - self.validate_definitions_for_source(source_id, source_ref.as_str()) + self.validate_definitions_for_source(source.id, source.source_ref.as_str()) .await; } else if let Err(error) = update_definitions_for_source_status( - &self.db, source_id, status, error_code, + &self.db, source.id, status, error_code, ) .await { @@ -86,7 +107,7 @@ impl MetricDefinitionValidator { } ProbeOutcome::Inconclusive => { tracing::warn!( - source_ref = %source_ref, + source_ref = %source.source_ref, "metric source validation inconclusive; keeping previous status" ); } @@ -94,6 +115,78 @@ impl MetricDefinitionValidator { } } + async fn validate_evidence( + &self, + source_id: uuid::Uuid, + source_key: &str, + source_kind: &str, + source_ref: &str, + evidence_ref: Option<&str>, + config_revision: &str, + ) { + if SourceKind::from_db(source_kind) == Some(SourceKind::CustomObservationSql) { + return; + } + let state = match ( + evidence_ref.and_then(EvidenceRelation::parse), + ObservationRelation::parse(source_ref), + ) { + (Some(relation), Some(observation_relation)) => match self + .has_exact_columns(relation.table_ref(), EVIDENCE_COLUMN_TYPES) + .await + { + Ok(ColumnCheck::Present) => { + let expected = match source_evidence_granularities(&self.db, source_id).await { + Ok(expected) => expected, + Err(error) => { + tracing::warn!(error = %error, "metric evidence granularity metadata load failed"); + return; + } + }; + match self + .evidence_granularities_match( + &relation, + &observation_relation, + source_key, + &expected, + ) + .await + { + Ok(true) => Some(ValidationState::Ok), + Ok(false) => { + tracing::warn!( + source_key, + evidence_ref, + expected = ?expected, + "metric evidence granularity does not match configured measures" + ); + Some(ValidationState::Error(MetricSchemaErrorCode::Unknown)) + } + Err(error) => { + tracing::warn!(error = %error, "metric evidence granularity validation failed"); + None + } + } + } + Ok(missing) => Some(ValidationState::Error(missing.error_code())), + Err(error) => { + tracing::warn!(error = %error, "metric evidence validation failed"); + None + } + }, + _ => Some(ValidationState::Unchecked), + }; + let Some(state) = state else { + return; + }; + let (status, error_code) = state.as_db(); + if let Err(error) = + update_evidence_status(&self.db, source_id, config_revision, status, error_code).await + { + tracing::warn!(error = %error, "metric evidence status update failed"); + } + } + async fn validate_source(&self, source_kind: &str, source_ref: &str) -> ProbeOutcome { match SourceKind::from_db(source_kind) { Some(SourceKind::ManagedObservation) => {} @@ -334,6 +427,140 @@ impl MetricDefinitionValidator { Ok(ColumnCheck::Present) } + async fn has_exact_columns( + &self, + table: (&str, &str), + columns: &[(&str, &str)], + ) -> Result { + let (database, table) = table; + let exact_columns = columns + .iter() + .map(|(name, r#type)| format!("(name = '{name}' AND type = '{type}')")) + .collect::>() + .join(" OR "); + let sql = format!( + "SELECT \ + count() AS total_columns, \ + countIf({exact_columns}) AS matching_columns \ + FROM system.columns \ + WHERE database = ? AND table = ?" + ); + let row: ColumnProbeRow = self + .ch + .query(&sql) + .bind(database) + .bind(table) + .fetch_one() + .await?; + if row.total_columns == 0 { + return Ok(ColumnCheck::TableMissing); + } + if row.matching_columns < columns.len() as u64 { + return Ok(ColumnCheck::ColumnsMissing); + } + Ok(ColumnCheck::Present) + } + + async fn evidence_granularities_match( + &self, + relation: &EvidenceRelation, + observation_relation: &ObservationRelation, + source_key: &str, + expected: &[(String, Option)], + ) -> Result { + if expected.is_empty() + || expected.iter().any(|(_, value)| { + value + .as_deref() + .and_then(EvidenceGranularity::from_db) + .is_none() + }) + { + return Ok(false); + } + let placeholders = vec!["?"; expected.len()].join(", "); + let (observation_database, observation_table) = observation_relation.table_ref(); + let observation_sql = format!( + "SELECT measure_key, toString(max(metric_date)) AS last_date \ + FROM {observation_database}.{observation_table} \ + WHERE source_key = ? AND measure_key IN ({placeholders}) \ + GROUP BY measure_key" + ); + let mut observation_query = self + .ch + .query(&observation_sql) + .with_option( + "max_execution_time", + EVIDENCE_QUERY_TIMEOUT_SECS.to_string(), + ) + .with_option("max_memory_usage", EVIDENCE_QUERY_MEMORY_BYTES.to_string()) + .with_option("max_bytes_to_read", EVIDENCE_QUERY_READ_BYTES.to_string()) + .with_option("max_result_bytes", EVIDENCE_QUERY_RESULT_BYTES.to_string()) + .bind(source_key); + for (measure_key, _) in expected { + observation_query = observation_query.bind(measure_key); + } + let observed_dates = parse_measure_last_dates(observation_query.fetch_all().await?)?; + let observed_measures = observed_dates.keys().cloned().collect::>(); + let window_start = observed_dates + .values() + .min() + .map(|date| *date - chrono::Duration::days(i64::from(PROBE_WINDOW_DAYS))); + + let (database, table) = relation.table_ref(); + let window_sql = window_start + .map(|_| " AND metric_date >= toDate(?)") + .unwrap_or_default(); + let sql = format!( + "SELECT measure_key, groupUniqArray(granularity) AS granularities \ + FROM {database}.{table} \ + WHERE source_key = ? AND measure_key IN ({placeholders}){window_sql} \ + GROUP BY measure_key" + ); + let mut query = self + .ch + .query(&sql) + .with_option( + "max_execution_time", + EVIDENCE_QUERY_TIMEOUT_SECS.to_string(), + ) + .with_option("max_memory_usage", EVIDENCE_QUERY_MEMORY_BYTES.to_string()) + .with_option("max_bytes_to_read", EVIDENCE_QUERY_READ_BYTES.to_string()) + .with_option("max_result_bytes", EVIDENCE_QUERY_RESULT_BYTES.to_string()) + .bind(source_key); + for (measure_key, _) in expected { + query = query.bind(measure_key); + } + if let Some(window_start) = window_start { + query = query.bind(window_start.to_string()); + } + let rows = query + .fetch_all::() + .await? + .into_iter() + .map(|row| (row.measure_key, row.granularities)) + .collect::>(); + Ok(expected.iter().all(|(measure_key, granularity)| { + let Some(granularity) = granularity.as_deref() else { + return false; + }; + let matches = match rows.get(measure_key) { + Some(values) => values.len() == 1 && values[0] == granularity, + None => !observed_measures.contains(measure_key), + }; + if !matches { + tracing::warn!( + measure_key, + expected_granularity = granularity, + actual_granularities = ?rows.get(measure_key), + observed = observed_measures.contains(measure_key), + "metric evidence measure granularity mismatch" + ); + } + matches + })) + } + async fn measure_last_dates( &self, relation: &ObservationRelation, @@ -407,6 +634,27 @@ const OBSERVATION_COLUMNS: &[&str] = &[ "dimensions", ]; +const EVIDENCE_COLUMN_TYPES: &[(&str, &str)] = &[ + ("tenant_id", "String"), + ("source_key", "String"), + ("entity_type", "String"), + ("entity_id", "String"), + ("metric_date", "Date"), + ("observed_at", "Nullable(DateTime64(3))"), + ("measure_key", "String"), + ("record_id", "String"), + ("record_kind", "String"), + ("granularity", "String"), + ("record_label", "String"), + ("contribution", "Nullable(Float64)"), + ("subject_key", "Nullable(String)"), + ( + "dimensions", + "Array(Tuple(key String, value String, label Nullable(String)))", + ), + ("details", "Map(String, String)"), +]; + const COHORT_COLUMNS: &[&str] = &[ "tenant_id", "entity_type", @@ -434,6 +682,12 @@ struct DimensionCoverageProbeRow { matching_rows: u64, } +#[derive(Row, Deserialize)] +struct EvidenceGranularityProbeRow { + measure_key: String, + granularities: Vec, +} + #[derive(Debug, Clone, Copy)] enum ColumnCheck { Present, diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs b/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs new file mode 100644 index 000000000..96b0cc63e --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs @@ -0,0 +1,1534 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fmt::Write; + +use base64::Engine; +use chrono::NaiveDate; +use sea_orm::{ConnectionTrait, DatabaseConnection, FromQueryResult, Statement, Value}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use toolkit_canonical_errors::CanonicalError; +use uuid::Uuid; + +use crate::api::error::MetricError; +use crate::domain::metric_definitions::definition::MetricInputRole; +use crate::domain::metric_definitions::{ + ComputationSpec, EvidenceGranularity, EvidenceRelation, MetricDefinition, + load_definitions_with_ids, +}; +use crate::domain::metric_results::{ + normalize_entity_id, normalize_entity_type, normalize_metric_key, +}; + +const DEFAULT_PAGE_LIMIT: usize = 100; +const MAX_PAGE_LIMIT: usize = 250; +const MAX_PERIOD_DAYS: i64 = 400; +const MAX_FILTERS: usize = 10; +const MAX_DISPLAY_DIMENSIONS: usize = 10; +const MAX_FILTER_VALUES: usize = 100; +const MAX_FILTER_VALUE_BYTES: usize = 512; +pub const MAX_EXPORT_ROWS: usize = 50_000; +pub const EVIDENCE_QUERY_TIMEOUT_SECS: u64 = 45; +pub const EVIDENCE_QUERY_MEMORY_BYTES: usize = 256 * 1024 * 1024; +pub const EVIDENCE_QUERY_READ_BYTES: usize = 512 * 1024 * 1024; +pub const EVIDENCE_QUERY_RESULT_BYTES: usize = 32 * 1024 * 1024; + +#[derive(Debug, Clone, Deserialize, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownEntity { + pub r#type: String, + pub id: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownPeriod { + pub from: String, + pub to: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownFilter { + pub dimension: String, + pub values: Vec, +} + +#[derive(Debug, Clone, Deserialize, utoipa::ToSchema)] +pub struct MetricDrilldownRequest { + pub metric_key: String, + pub entity: MetricDrilldownEntity, + pub period: MetricDrilldownPeriod, + #[serde(default)] + pub filters: Vec, + #[serde(default)] + pub display_dimensions: Vec, + pub limit: Option, + pub cursor: Option, +} + +#[derive(Debug, Clone, Copy, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum MetricDrilldownExportFormat { + Csv, + Xlsx, +} + +#[derive(Debug, Clone, Deserialize, utoipa::ToSchema)] +pub struct MetricDrilldownExportRequest { + pub metric_key: String, + pub entity: MetricDrilldownEntity, + pub period: MetricDrilldownPeriod, + #[serde(default)] + pub filters: Vec, + #[serde(default)] + pub display_dimensions: Vec, + pub format: MetricDrilldownExportFormat, +} + +#[derive(Debug, Clone, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownSelection { + pub metric_key: String, + pub entity: MetricDrilldownEntity, + pub period: MetricDrilldownPeriod, + pub filters: Vec, + pub display_dimensions: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum MetricDrilldownColumnType { + String, + Date, + Number, +} + +#[derive(Debug, Clone, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownColumn { + pub key: String, + pub label: String, + pub r#type: MetricDrilldownColumnType, +} + +#[derive(Debug, Clone, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownRow { + pub values: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownCapability { + pub granularity: Vec, +} + +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct MetricDrilldownResponse { + pub selection: MetricDrilldownSelection, + pub columns: Vec, + pub rows: Vec, + pub next_cursor: Option, +} + +impl toolkit::api::api_dto::RequestApiDto for MetricDrilldownRequest {} +impl toolkit::api::api_dto::RequestApiDto for MetricDrilldownExportRequest {} +impl toolkit::api::api_dto::ResponseApiDto for MetricDrilldownResponse {} + +#[derive(Debug)] +pub struct ValidatedMetricDrilldown { + pub selection: MetricDrilldownSelection, + pub from: NaiveDate, + pub to: NaiveDate, + pub limit: usize, + pub cursor: Option, + pub plan: EvidencePlan, + pub snapshot_id: String, + pub fingerprint: String, +} + +#[derive(Debug, Clone)] +pub struct EvidencePlan { + pub definition: MetricDefinition, + pub relation: EvidenceRelation, + pub source_key: String, + pub inputs: Vec, +} + +#[derive(Debug, Clone)] +pub struct EvidenceInput { + pub role: MetricInputRole, + pub measure_key: String, + pub presentation: EvidencePresentation, +} + +#[derive(Debug, Clone)] +pub struct EvidencePresentation { + pub detail_keys: &'static [&'static str], + pub show_value: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct EvidenceQueryRow { + pub role: String, + pub metric_date: String, + pub observed_at: String, + pub source_key: String, + pub measure_key: String, + pub record_id: String, + pub record_kind: String, + pub contribution: Option, + pub numerator: Option, + pub denominator: Option, + pub subject_key: String, + pub dimensions_json: String, + pub details: serde_json::Value, +} + +#[derive(Debug, Deserialize)] +struct EvidenceDimension { + key: String, + value: String, + label: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CursorKey { + role: String, + metric_date: String, + observed_at: String, + source_key: String, + measure_key: String, + record_id: String, + record_kind: String, + subject_key: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct CursorEnvelope { + version: u8, + fingerprint: String, + snapshot_id: String, + key: CursorKey, +} + +struct CommonRequest { + metric_key: String, + entity: MetricDrilldownEntity, + period: MetricDrilldownPeriod, + filters: Vec, + display_dimensions: Vec, + limit: usize, + max_limit: usize, + cursor: Option, +} + +#[derive(Debug, FromQueryResult)] +struct EvidenceInputRow { + input_role: String, + measure_key: String, + evidence_granularity: Option, + source_key: String, + evidence_ref: Option, + evidence_schema_status: String, +} + +#[derive(Debug, FromQueryResult)] +struct CapabilityRow { + metric_key: String, + input_role: String, + evidence_granularity: Option, + source_key: String, + evidence_ref: Option, + evidence_schema_status: String, +} + +#[derive(Debug, Deserialize, clickhouse::Row)] +struct EvidenceSnapshotRow { + snapshot_id: String, +} + +pub async fn load_capabilities( + db: &DatabaseConnection, + tenant_id: Uuid, + metric_keys: &[String], +) -> Result, CanonicalError> { + if metric_keys.is_empty() { + return Ok(HashMap::new()); + } + let placeholders = vec!["?"; metric_keys.len()].join(", "); + let sql = format!( + "SELECT d.metric_key, i.input_role, m.evidence_granularity, s.source_key, \ + s.evidence_ref, s.evidence_schema_status \ + FROM metric_definitions d \ + INNER JOIN metric_definition_inputs i ON i.metric_definition_id = d.id \ + INNER JOIN metric_source_measures m ON m.id = i.source_measure_id \ + INNER JOIN metric_sources s ON s.id = m.source_id \ + WHERE d.metric_key IN ({placeholders}) \ + AND d.id = COALESCE( \ + (SELECT td.id FROM metric_definitions td WHERE td.metric_key = d.metric_key AND td.tenant_id = ? LIMIT 1), \ + (SELECT pd.id FROM metric_definitions pd WHERE pd.metric_key = d.metric_key AND pd.tenant_id IS NULL LIMIT 1) \ + ) \ + AND d.is_enabled = TRUE AND d.schema_status = 'ok' \ + AND m.is_enabled = TRUE AND s.is_enabled = TRUE \ + ORDER BY d.metric_key, i.input_role, m.measure_key" + ); + let mut values = metric_keys.iter().map(Value::from).collect::>(); + values.push(Value::Bytes(Some(Box::new(tenant_id.as_bytes().to_vec())))); + let rows = CapabilityRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .all(db) + .await + .map_err(|error| db_error(&error))?; + let mut grouped: BTreeMap> = BTreeMap::new(); + for row in rows { + grouped.entry(row.metric_key.clone()).or_default().push(row); + } + let mut capabilities = HashMap::new(); + for (metric_key, rows) in grouped { + let relation = rows + .first() + .and_then(|row| row.evidence_ref.as_deref()) + .and_then(EvidenceRelation::parse); + let source_key = rows.first().map(|row| row.source_key.as_str()); + let healthy = !rows.is_empty() + && relation.is_some() + && source_key.is_some() + && rows.iter().all(|row| { + MetricInputRole::from_db(&row.input_role).is_some() + && row.evidence_schema_status == "ok" + && Some(row.source_key.as_str()) == source_key + && row.evidence_ref.as_deref().is_some_and(|value| { + relation + .as_ref() + .is_some_and(|relation| value == relation.source_ref()) + }) + && row + .evidence_granularity + .as_deref() + .and_then(EvidenceGranularity::from_db) + .is_some() + }); + if healthy { + let mut granularity = rows + .iter() + .filter_map(|row| { + row.evidence_granularity + .as_deref() + .and_then(EvidenceGranularity::from_db) + }) + .collect::>(); + granularity.sort_by_key(|value| value.as_db()); + granularity.dedup(); + capabilities.insert(metric_key, MetricDrilldownCapability { granularity }); + } + } + Ok(capabilities) +} + +pub async fn validate_request( + db: &DatabaseConnection, + ch: &insight_clickhouse::Client, + tenant_id: Uuid, + req: MetricDrilldownRequest, +) -> Result { + validate_common( + db, + ch, + tenant_id, + CommonRequest { + metric_key: req.metric_key, + entity: req.entity, + period: req.period, + filters: req.filters, + display_dimensions: req.display_dimensions, + limit: req.limit.unwrap_or(DEFAULT_PAGE_LIMIT), + max_limit: MAX_PAGE_LIMIT, + cursor: req.cursor, + }, + ) + .await +} + +pub async fn validate_export_request( + db: &DatabaseConnection, + ch: &insight_clickhouse::Client, + tenant_id: Uuid, + req: &MetricDrilldownExportRequest, + limit: usize, +) -> Result { + validate_common( + db, + ch, + tenant_id, + CommonRequest { + metric_key: req.metric_key.clone(), + entity: req.entity.clone(), + period: req.period.clone(), + filters: req.filters.clone(), + display_dimensions: req.display_dimensions.clone(), + limit, + max_limit: MAX_EXPORT_ROWS + 1, + cursor: None, + }, + ) + .await +} + +async fn validate_common( + db: &DatabaseConnection, + ch: &insight_clickhouse::Client, + tenant_id: Uuid, + request: CommonRequest, +) -> Result { + let CommonRequest { + metric_key, + entity, + period, + filters, + display_dimensions, + limit, + max_limit, + cursor, + } = request; + let metric_key = normalize_metric_key("metric_key", &metric_key)?; + let entity_type = normalize_entity_type(&entity.r#type)?; + if entity_type != "person" { + return invalid("entity.type", "only person entities are supported"); + } + let entity_id = normalize_entity_id(&entity_type, &entity.id); + if entity_id.is_empty() { + return invalid("entity.id", "person entity id must not be empty"); + } + if limit == 0 || limit > max_limit { + return invalid("limit", format!("limit must be between 1 and {max_limit}")); + } + let from = parse_date("period.from", &period.from)?; + let to = parse_date("period.to", &period.to)?; + if from > to || (to - from).num_days() >= MAX_PERIOD_DAYS { + return invalid( + "period", + format!("period must be ordered and shorter than {MAX_PERIOD_DAYS} days"), + ); + } + let definitions = + load_definitions_with_ids(db, tenant_id, std::slice::from_ref(&metric_key)).await?; + let (definition_id, definition) = definitions.get(&metric_key).cloned().ok_or_else(|| { + MetricError::not_found("metric definition not found") + .with_resource(&metric_key) + .create() + })?; + if definition.base.entity_type != entity_type { + return invalid( + "entity.type", + "entity type does not match metric definition", + ); + } + let filters = normalize_filters(&definition, filters)?; + let display_dimensions = normalize_display_dimensions(&definition, display_dimensions)?; + let plan = load_evidence_plan(db, definition_id, definition).await?; + let snapshot_id = evidence_snapshot_id(ch, &plan.relation).await?; + let selection = MetricDrilldownSelection { + metric_key, + entity: MetricDrilldownEntity { + r#type: entity_type, + id: entity_id, + }, + period: MetricDrilldownPeriod { + from: from.to_string(), + to: to.to_string(), + }, + filters, + display_dimensions, + }; + let fingerprint = selection_fingerprint(tenant_id, &selection)?; + let cursor = match cursor { + Some(value) => { + let envelope = decode_cursor(&value)?; + verify_evidence_snapshot(ch, &plan.relation, &envelope.snapshot_id).await?; + if envelope.fingerprint != fingerprint { + return invalid("cursor", "cursor does not match the metric selection"); + } + Some(envelope.key) + } + None => None, + }; + Ok(ValidatedMetricDrilldown { + selection, + from, + to, + limit, + cursor, + plan, + snapshot_id, + fingerprint, + }) +} + +async fn load_evidence_plan( + db: &DatabaseConnection, + definition_id: Uuid, + definition: MetricDefinition, +) -> Result { + let rows = EvidenceInputRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + "SELECT i.input_role, m.measure_key, m.evidence_granularity, s.source_key, \ + s.evidence_ref, s.evidence_schema_status \ + FROM metric_definition_inputs i \ + INNER JOIN metric_source_measures m ON m.id = i.source_measure_id \ + INNER JOIN metric_sources s ON s.id = m.source_id \ + WHERE i.metric_definition_id = ? AND m.is_enabled = TRUE AND s.is_enabled = TRUE \ + ORDER BY i.input_role, m.measure_key", + [Value::Bytes(Some(Box::new( + definition_id.as_bytes().to_vec(), + )))], + )) + .all(db) + .await + .map_err(|error| db_error(&error))?; + if rows.is_empty() || rows.iter().any(|row| row.evidence_schema_status != "ok") { + return Err(evidence_unavailable()); + } + let evidence_ref = rows[0] + .evidence_ref + .as_deref() + .and_then(EvidenceRelation::parse); + let Some(relation) = evidence_ref else { + return Err(evidence_unavailable()); + }; + let source_key = rows[0].source_key.clone(); + if rows.iter().any(|row| { + row.source_key != source_key + || row.evidence_ref.as_deref() != Some(relation.source_ref()) + || row.evidence_granularity.is_none() + }) { + return Err(evidence_unavailable()); + } + let inputs = rows + .into_iter() + .map(|row| { + let role = MetricInputRole::from_db(&row.input_role).ok_or_else(config_error)?; + let granularity = row + .evidence_granularity + .as_deref() + .and_then(EvidenceGranularity::from_db) + .ok_or_else(config_error)?; + Ok(EvidenceInput { + role, + presentation: evidence_presentation(&source_key, &row.measure_key, granularity), + measure_key: row.measure_key, + }) + }) + .collect::, CanonicalError>>()?; + Ok(EvidencePlan { + definition, + relation, + source_key, + inputs, + }) +} + +pub fn compile_query( + req: &ValidatedMetricDrilldown, +) -> Result<(String, Vec), CanonicalError> { + if matches!(req.plan.definition.spec, ComputationSpec::Ratio { .. }) { + return compile_ratio_query(req); + } + Ok(compile_value_query(req)) +} + +fn compile_value_query(req: &ValidatedMetricDrilldown) -> (String, Vec) { + let (database, table) = req.plan.relation.table_ref(); + let mut params = Vec::new(); + let measures = req + .plan + .inputs + .iter() + .map(|_| "?") + .collect::>() + .join(", "); + let role_expr = role_expression(&req.plan.inputs); + for input in &req.plan.inputs { + params.push(input.measure_key.clone()); + params.push(input.role.as_db().to_owned()); + } + params.extend([ + req.plan.source_key.clone(), + req.selection.entity.r#type.clone(), + req.selection.entity.id.clone(), + req.from.to_string(), + req.to.to_string(), + ]); + params.extend( + req.plan + .inputs + .iter() + .map(|input| input.measure_key.clone()), + ); + let mut filter_sql = String::new(); + for filter in &req.selection.filters { + let placeholders = vec!["?"; filter.values.len()].join(", "); + let _ = write!( + filter_sql, + " AND indexOf(evidence.dimensions.1, ?) > 0 AND evidence.dimensions.2[indexOf(evidence.dimensions.1, ?)] IN ({placeholders})" + ); + params.push(filter.dimension.clone()); + params.push(filter.dimension.clone()); + params.extend(filter.values.iter().cloned()); + } + let mut cursor_sql = String::new(); + if let Some(cursor) = &req.cursor { + cursor_sql.push_str( + " AND tuple(role, toString(evidence.metric_date), ifNull(toString(evidence.observed_at), ''), evidence.source_key, evidence.measure_key, evidence.record_id, evidence.record_kind, ifNull(evidence.subject_key, '')) > tuple(?, ?, ?, ?, ?, ?, ?, ?)", + ); + params.extend([ + cursor.role.clone(), + cursor.metric_date.clone(), + cursor.observed_at.clone(), + cursor.source_key.clone(), + cursor.measure_key.clone(), + cursor.record_id.clone(), + cursor.record_kind.clone(), + cursor.subject_key.clone(), + ]); + } + let limit = req.limit + 1; + let sql = format!( + "WITH {role_expr} AS role \ + SELECT role, toString(evidence.metric_date) AS metric_date, ifNull(toString(evidence.observed_at), '') AS observed_at, \ + evidence.source_key, evidence.measure_key, evidence.record_id, evidence.record_kind, \ + evidence.contribution, CAST(NULL AS Nullable(Float64)) AS numerator, \ + CAST(NULL AS Nullable(Float64)) AS denominator, \ + ifNull(evidence.subject_key, '') AS subject_key, \ + toJSONString(evidence.dimensions) AS dimensions_json, evidence.details \ + FROM {database}.{table} AS evidence \ + WHERE evidence.source_key = ? AND evidence.entity_type = ? AND evidence.entity_id = ? \ + AND evidence.metric_date >= toDate(?) AND evidence.metric_date <= toDate(?) \ + AND evidence.measure_key IN ({measures}){filter_sql}{cursor_sql} \ + ORDER BY role, metric_date, ifNull(toString(observed_at), ''), source_key, measure_key, record_id, record_kind, ifNull(subject_key, '') \ + LIMIT {limit}" + ); + (sql, params) +} + +fn compile_ratio_query( + req: &ValidatedMetricDrilldown, +) -> Result<(String, Vec), CanonicalError> { + let (database, table) = req.plan.relation.table_ref(); + let numerator = req + .plan + .inputs + .iter() + .find(|input| input.role == MetricInputRole::Numerator) + .ok_or_else(config_error)?; + let denominator = req + .plan + .inputs + .iter() + .find(|input| input.role == MetricInputRole::Denominator) + .ok_or_else(config_error)?; + let mut params = vec![ + numerator.measure_key.clone(), + denominator.measure_key.clone(), + req.plan.source_key.clone(), + req.selection.entity.r#type.clone(), + req.selection.entity.id.clone(), + req.from.to_string(), + req.to.to_string(), + numerator.measure_key.clone(), + denominator.measure_key.clone(), + ]; + let mut filter_sql = String::new(); + for filter in &req.selection.filters { + let placeholders = vec!["?"; filter.values.len()].join(", "); + let _ = write!( + filter_sql, + " AND indexOf(evidence.dimensions.1, ?) > 0 AND evidence.dimensions.2[indexOf(evidence.dimensions.1, ?)] IN ({placeholders})" + ); + params.push(filter.dimension.clone()); + params.push(filter.dimension.clone()); + params.extend(filter.values.iter().cloned()); + } + let mut cursor_sql = String::new(); + if let Some(cursor) = &req.cursor { + cursor_sql.push_str( + " WHERE tuple(role, metric_date, observed_at, source_key, measure_key, record_id, record_kind, subject_key) > tuple(?, ?, ?, ?, ?, ?, ?, ?)", + ); + params.extend([ + cursor.role.clone(), + cursor.metric_date.clone(), + cursor.observed_at.clone(), + cursor.source_key.clone(), + cursor.measure_key.clone(), + cursor.record_id.clone(), + cursor.record_kind.clone(), + cursor.subject_key.clone(), + ]); + } + let limit = req.limit + 1; + let sql = format!( + "SELECT * FROM (\ + SELECT 'value' AS role, toString(evidence.metric_date) AS metric_date, \ + '' AS observed_at, \ + any(evidence.source_key) AS source_key, '' AS measure_key, \ + toString(evidence.metric_date) AS record_id, 'daily_ratio' AS record_kind, \ + CAST(NULL AS Nullable(Float64)) AS contribution, \ + sumIf(ifNull(evidence.contribution, 0), evidence.measure_key = ?) AS numerator, \ + sumIf(ifNull(evidence.contribution, 0), evidence.measure_key = ?) AS denominator, \ + '' AS subject_key, any(toJSONString(evidence.dimensions)) AS dimensions_json, \ + CAST(map() AS Map(String, String)) AS details \ + FROM {database}.{table} AS evidence \ + WHERE evidence.source_key = ? AND evidence.entity_type = ? AND evidence.entity_id = ? \ + AND evidence.metric_date >= toDate(?) AND evidence.metric_date <= toDate(?) \ + AND evidence.measure_key IN (?, ?){filter_sql} \ + GROUP BY evidence.metric_date\ + ){cursor_sql} \ + ORDER BY role, metric_date, observed_at, source_key, measure_key, record_id, record_kind, subject_key \ + LIMIT {limit}" + ); + Ok((sql, params)) +} + +fn role_expression(inputs: &[EvidenceInput]) -> String { + let branches = inputs + .iter() + .map(|_| "evidence.measure_key = ?, ?") + .collect::>() + .join(", "); + format!("multiIf({branches}, 'value')") +} + +pub fn build_response( + req: &ValidatedMetricDrilldown, + mut rows: Vec, +) -> Result { + let next_cursor = if rows.len() > req.limit { + rows.truncate(req.limit); + rows.last() + .map(|row| encode_cursor(&req.fingerprint, &req.snapshot_id, row)) + .transpose()? + } else { + None + }; + let (columns, rows) = presentation( + &rows, + &req.plan, + &req.selection.filters, + &req.selection.display_dimensions, + )?; + Ok(MetricDrilldownResponse { + selection: req.selection.clone(), + columns, + rows, + next_cursor, + }) +} + +pub fn presentation( + rows: &[EvidenceQueryRow], + plan: &EvidencePlan, + filters: &[MetricDrilldownFilter], + display_dimensions: &[String], +) -> Result<(Vec, Vec), CanonicalError> { + let details = rows + .iter() + .map(|row| row.details.as_object().ok_or_else(config_error)) + .collect::, _>>()?; + let ratio = matches!(plan.definition.spec, ComputationSpec::Ratio { .. }); + let display_dimensions = if ratio { &[] } else { display_dimensions }; + let dimensions = presentation_dimensions(rows)?; + let mut detail_keys = if ratio { + BTreeSet::new() + } else { + plan.inputs + .iter() + .flat_map(|input| input.presentation.detail_keys) + .map(|key| (*key).to_owned()) + .collect::>() + }; + let dimension_keys = filters + .iter() + .filter(|filter| filter.values.len() == 1) + .map(|filter| filter.dimension.clone()) + .chain(display_dimensions.iter().cloned()) + .collect::>(); + detail_keys.extend(dimension_keys); + let include_value = !ratio + && plan + .inputs + .iter() + .any(|input| input.presentation.show_value); + let mut ordered_keys = Vec::new(); + if detail_keys.remove("ref") { + ordered_keys.push("ref".to_owned()); + } + if detail_keys.remove("title") { + ordered_keys.push("title".to_owned()); + } + for key in ["repository", "author"] { + if detail_keys.remove(key) { + ordered_keys.push(key.to_owned()); + } + } + ordered_keys.extend(detail_keys); + ordered_keys.push("date".to_owned()); + if ratio { + ordered_keys.push("numerator".to_owned()); + ordered_keys.push("denominator".to_owned()); + } else if include_value { + ordered_keys.push("value".to_owned()); + } + + let columns = ordered_keys + .iter() + .map(|key| presentation_column(key, plan)) + .collect(); + let projected_rows = rows + .iter() + .zip(details) + .zip(dimensions) + .map(|((row, details), dimensions)| { + let mut values = BTreeMap::new(); + for key in &ordered_keys { + let value = match key.as_str() { + "date" => row.metric_date.clone().into(), + "value" => { + serde_json::to_value(row.contribution).map_err(|_| config_error())? + } + "numerator" => { + serde_json::to_value(row.numerator).map_err(|_| config_error())? + } + "denominator" => { + serde_json::to_value(row.denominator).map_err(|_| config_error())? + } + _ => details + .get(key) + .filter(|value| visible_value(value)) + .cloned() + .or_else(|| { + dimensions + .iter() + .find(|dimension| dimension.key == *key) + .map(|dimension| { + serde_json::Value::from( + dimension + .label + .as_deref() + .filter(|label| !label.trim().is_empty()) + .unwrap_or(&dimension.value), + ) + }) + }) + .unwrap_or(serde_json::Value::Null), + }; + values.insert(key.clone(), normalize_presentation_value(key, value)); + } + Ok(MetricDrilldownRow { values }) + }) + .collect::, CanonicalError>>()?; + + Ok((columns, projected_rows)) +} + +fn presentation_dimensions( + rows: &[EvidenceQueryRow], +) -> Result>, CanonicalError> { + rows.iter() + .map(|row| { + serde_json::from_str::>(&row.dimensions_json) + .map_err(|_| config_error()) + }) + .collect() +} + +fn presentation_column(key: &str, plan: &EvidencePlan) -> MetricDrilldownColumn { + let (label, r#type) = match key { + "ref" => ("Ref".to_owned(), MetricDrilldownColumnType::String), + "title" => ("Title".to_owned(), MetricDrilldownColumnType::String), + "repository" => ("Repository".to_owned(), MetricDrilldownColumnType::String), + "author" => ("Author".to_owned(), MetricDrilldownColumnType::String), + "date" => ("Date".to_owned(), MetricDrilldownColumnType::Date), + "value" => ("Value".to_owned(), MetricDrilldownColumnType::Number), + "numerator" => ( + input_label(plan, MetricInputRole::Numerator), + MetricDrilldownColumnType::Number, + ), + "denominator" => ( + input_label(plan, MetricInputRole::Denominator), + MetricDrilldownColumnType::Number, + ), + "lines_added" => ("Lines added".to_owned(), MetricDrilldownColumnType::Number), + "lines_removed" => ( + "Lines removed".to_owned(), + MetricDrilldownColumnType::Number, + ), + "issue_type" => ("Issue type".to_owned(), MetricDrilldownColumnType::String), + _ => (humanize_field_name(key), MetricDrilldownColumnType::String), + }; + MetricDrilldownColumn { + key: key.to_owned(), + label, + r#type, + } +} + +fn evidence_presentation( + source_key: &str, + measure_key: &str, + granularity: EvidenceGranularity, +) -> EvidencePresentation { + match (source_key, measure_key) { + ("git", "commit_count" | "commit_change_size") => EvidencePresentation { + detail_keys: &[ + "ref", + "title", + "repository", + "author", + "lines_added", + "lines_removed", + ], + show_value: false, + }, + ("git", "pr_created" | "pr_created_merged" | "pr_merged") => EvidencePresentation { + detail_keys: &["ref", "title", "repository", "author"], + show_value: false, + }, + ("git", "pr_cycle_hours" | "pr_change_size") => EvidencePresentation { + detail_keys: &["ref", "title", "repository", "author"], + show_value: true, + }, + ( + "task", + "tasks_closed" | "bugs_fixed" | "due_date_on_time" | "due_date_with_due" | "late_count", + ) => EvidencePresentation { + detail_keys: &["ref", "issue_type"], + show_value: false, + }, + ("task", _) if granularity == EvidenceGranularity::Event => EvidencePresentation { + detail_keys: &["ref", "issue_type"], + show_value: true, + }, + ("wiki", "pages_created") => EvidencePresentation { + detail_keys: &["ref", "title"], + show_value: false, + }, + _ => EvidencePresentation { + detail_keys: &[], + show_value: granularity != EvidenceGranularity::Event, + }, + } +} + +fn input_label(plan: &EvidencePlan, role: MetricInputRole) -> String { + plan.inputs + .iter() + .find(|input| input.role == role) + .map_or_else( + || humanize_field_name(role.as_db()), + |input| humanize_field_name(&input.measure_key), + ) +} + +fn visible_value(value: &serde_json::Value) -> bool { + !value.is_null() && value.as_str().is_none_or(|value| !value.trim().is_empty()) +} + +fn normalize_presentation_value(key: &str, value: serde_json::Value) -> serde_json::Value { + if matches!(key, "lines_added" | "lines_removed") + && let Some(value) = value.as_str().and_then(|value| value.parse::().ok()) + { + return serde_json::Value::from(value); + } + value +} + +fn humanize_field_name(key: &str) -> String { + let label = key.replace('_', " "); + let mut characters = label.chars(); + match characters.next() { + Some(first) => first.to_uppercase().chain(characters).collect(), + None => label, + } +} + +fn normalize_filters( + definition: &MetricDefinition, + filters: Vec, +) -> Result, CanonicalError> { + if filters.len() > MAX_FILTERS { + return invalid("filters", format!("at most {MAX_FILTERS} filters")); + } + let mut normalized = Vec::with_capacity(filters.len()); + for filter in filters { + let dimension = filter.dimension.trim(); + if definition.allowed_dimension(dimension).is_none() { + return invalid( + "filters.dimension", + format!("dimension {dimension} is not declared by the metric"), + ); + } + if filter.values.is_empty() || filter.values.len() > MAX_FILTER_VALUES { + return invalid( + "filters.values", + format!("between 1 and {MAX_FILTER_VALUES} values are required"), + ); + } + let mut values = filter + .values + .into_iter() + .map(|value| value.trim().to_owned()) + .collect::>(); + if values + .iter() + .any(|value| value.is_empty() || value.len() > MAX_FILTER_VALUE_BYTES) + { + return invalid("filters.values", "filter value is empty or too long"); + } + values.sort(); + values.dedup(); + normalized.push(MetricDrilldownFilter { + dimension: dimension.to_owned(), + values, + }); + } + normalized.sort_by(|left, right| left.dimension.cmp(&right.dimension)); + if normalized + .windows(2) + .any(|pair| pair[0].dimension == pair[1].dimension) + { + return invalid("filters", "duplicate dimension filter"); + } + Ok(normalized) +} + +fn normalize_display_dimensions( + definition: &MetricDefinition, + dimensions: Vec, +) -> Result, CanonicalError> { + if dimensions.len() > MAX_DISPLAY_DIMENSIONS { + return invalid( + "display_dimensions", + format!("at most {MAX_DISPLAY_DIMENSIONS} display dimensions"), + ); + } + let mut normalized = dimensions + .into_iter() + .map(|dimension| dimension.trim().to_owned()) + .collect::>(); + if normalized.iter().any(String::is_empty) { + return invalid("display_dimensions", "display dimension is empty"); + } + for dimension in &normalized { + if definition.allowed_dimension(dimension).is_none() { + return invalid( + "display_dimensions", + format!("dimension {dimension} is not declared by the metric"), + ); + } + } + normalized.sort(); + normalized.dedup(); + Ok(normalized) +} + +fn selection_fingerprint( + tenant_id: Uuid, + selection: &MetricDrilldownSelection, +) -> Result { + let bytes = serde_json::to_vec(&(tenant_id, selection)).map_err(|_| config_error())?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} + +pub async fn verify_evidence_snapshot( + ch: &insight_clickhouse::Client, + relation: &EvidenceRelation, + expected: &str, +) -> Result<(), CanonicalError> { + let current = evidence_snapshot_id(ch, relation).await?; + if current != expected { + return Err(MetricError::failed_precondition() + .with_precondition_violation( + "metric evidence snapshot", + "Metric evidence was rebuilt while the request was running.", + "EVIDENCE_SNAPSHOT_EXPIRED", + ) + .create()); + } + Ok(()) +} + +async fn evidence_snapshot_id( + ch: &insight_clickhouse::Client, + relation: &EvidenceRelation, +) -> Result { + let (database, table) = relation.table_ref(); + ch.query( + "SELECT toString(uuid) AS snapshot_id \ + FROM system.tables WHERE database = ? AND name = ?", + ) + .bind(database) + .bind(table) + .fetch_one::() + .await + .map(|row| row.snapshot_id) + .map_err(|error| { + tracing::error!( + error = %error, + database, + table, + "metric evidence snapshot lookup failed" + ); + evidence_unavailable() + }) +} + +fn encode_cursor( + fingerprint: &str, + snapshot_id: &str, + row: &EvidenceQueryRow, +) -> Result { + let envelope = CursorEnvelope { + version: 1, + fingerprint: fingerprint.to_owned(), + snapshot_id: snapshot_id.to_owned(), + key: CursorKey { + role: row.role.clone(), + metric_date: row.metric_date.clone(), + observed_at: row.observed_at.clone(), + source_key: row.source_key.clone(), + measure_key: row.measure_key.clone(), + record_id: row.record_id.clone(), + record_kind: row.record_kind.clone(), + subject_key: row.subject_key.clone(), + }, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| config_error())?; + Ok(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes)) +} + +fn decode_cursor(value: &str) -> Result { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| invalid_error("cursor", "cursor is malformed"))?; + let envelope: CursorEnvelope = serde_json::from_slice(&bytes) + .map_err(|_| invalid_error("cursor", "cursor is malformed"))?; + if envelope.version != 1 { + return invalid("cursor", "cursor version is unsupported"); + } + Ok(envelope) +} + +fn parse_date(field: &str, value: &str) -> Result { + NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map_err(|_| invalid_error(field, "date must use YYYY-MM-DD")) +} + +fn invalid(field: &str, description: impl Into) -> Result { + Err(invalid_error(field, description)) +} + +fn invalid_error(field: &str, description: impl Into) -> CanonicalError { + MetricError::invalid_argument() + .with_field_violation(field, description.into(), "INVALID") + .create() +} + +fn evidence_unavailable() -> CanonicalError { + MetricError::failed_precondition() + .with_precondition_violation( + "metric evidence", + "Evidence is not available for this metric.", + "EVIDENCE_UNAVAILABLE", + ) + .create() +} + +fn db_error(error: &sea_orm::DbErr) -> CanonicalError { + tracing::error!(error = %error, "metric drilldown metadata query failed"); + CanonicalError::internal("failed to load metric evidence metadata").create() +} + +fn config_error() -> CanonicalError { + CanonicalError::internal("corrupt metric evidence configuration").create() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::metric_definitions::definition::{ + MetricBase, MetricDirection, MetricFormat, MetricInput, ObservationRelation, + }; + + fn input(role: MetricInputRole, measure_key: &str) -> MetricInput { + MetricInput { + role, + observation_relation: ObservationRelation::parse("git_metric_observations") + .unwrap_or_else(|| panic!("observation relation must parse")), + source_key: "git".to_owned(), + measure_key: measure_key.to_owned(), + } + } + + fn definition(spec: ComputationSpec, dimensions: &[&str]) -> MetricDefinition { + MetricDefinition { + base: MetricBase { + key: "git.example".to_owned(), + label: "Example".to_owned(), + short_label: None, + description: None, + explanation: None, + entity_type: "person".to_owned(), + format: MetricFormat::Integer, + unit: None, + direction: MetricDirection::Neutral, + peer_cohort_key: None, + allowed_dimensions: dimensions.iter().map(|value| (*value).to_owned()).collect(), + }, + spec, + transform: None, + } + } + + fn plan(spec: ComputationSpec, inputs: Vec) -> EvidencePlan { + EvidencePlan { + definition: definition(spec, &["repository", "category"]), + relation: EvidenceRelation::parse("git_metric_evidence") + .unwrap_or_else(|| panic!("evidence relation must parse")), + source_key: "git".to_owned(), + inputs, + } + } + + fn row() -> EvidenceQueryRow { + EvidenceQueryRow { + role: "value".to_owned(), + metric_date: "2026-07-01".to_owned(), + observed_at: "2026-07-01 10:00:00".to_owned(), + source_key: "git".to_owned(), + measure_key: "commit_count".to_owned(), + record_id: "abc123".to_owned(), + record_kind: "commit".to_owned(), + contribution: Some(1.0), + numerator: None, + denominator: None, + subject_key: String::new(), + dimensions_json: r#"[{"key":"repository","value":"repo","label":"Repository"},{"key":"category","value":"code","label":null}]"#.to_owned(), + details: serde_json::json!({ + "ref": "abc123", + "title": "Change", + "repository": "org/repo", + "author": "Developer", + "lines_added": "12", + "lines_removed": "3" + }), + } + } + + fn validated(plan: EvidencePlan) -> ValidatedMetricDrilldown { + let selection = MetricDrilldownSelection { + metric_key: plan.definition.key().to_owned(), + entity: MetricDrilldownEntity { + r#type: "person".to_owned(), + id: "person@example.com".to_owned(), + }, + period: MetricDrilldownPeriod { + from: "2026-07-01".to_owned(), + to: "2026-07-31".to_owned(), + }, + filters: vec![MetricDrilldownFilter { + dimension: "repository".to_owned(), + values: vec!["org/repo".to_owned()], + }], + display_dimensions: vec!["category".to_owned()], + }; + ValidatedMetricDrilldown { + fingerprint: selection_fingerprint(Uuid::nil(), &selection) + .unwrap_or_else(|error| panic!("selection fingerprint must build: {error}")), + selection, + from: NaiveDate::from_ymd_opt(2026, 7, 1) + .unwrap_or_else(|| panic!("valid test start date")), + to: NaiveDate::from_ymd_opt(2026, 7, 31) + .unwrap_or_else(|| panic!("valid test end date")), + limit: 1, + cursor: None, + plan, + snapshot_id: "snapshot".to_owned(), + } + } + + #[test] + fn value_query_binds_filters_and_cursor() { + let value = input(MetricInputRole::Value, "commit_count"); + let plan = plan( + ComputationSpec::Sum { + value: value.clone(), + }, + vec![EvidenceInput { + role: MetricInputRole::Value, + measure_key: value.measure_key, + presentation: evidence_presentation( + "git", + "commit_count", + EvidenceGranularity::Event, + ), + }], + ); + let mut request = validated(plan); + request.cursor = Some(CursorKey { + role: "value".to_owned(), + metric_date: "2026-07-01".to_owned(), + observed_at: String::new(), + source_key: "git".to_owned(), + measure_key: "commit_count".to_owned(), + record_id: "abc".to_owned(), + record_kind: "commit".to_owned(), + subject_key: String::new(), + }); + let (sql, params) = + compile_query(&request).unwrap_or_else(|error| panic!("query must compile: {error}")); + assert!(sql.contains("insight.git_metric_evidence")); + assert!(sql.contains("indexOf(evidence.dimensions.1, ?)")); + assert!(sql.contains("LIMIT 2")); + assert_eq!( + params + .iter() + .filter(|value| value.as_str() == "repository") + .count(), + 2 + ); + assert!(params.iter().any(|value| value == "abc")); + } + + #[test] + fn ratio_query_uses_named_inputs() { + let numerator = input(MetricInputRole::Numerator, "focus_hours"); + let denominator = input(MetricInputRole::Denominator, "work_hours"); + let plan = plan( + ComputationSpec::Ratio { + numerator: numerator.clone(), + denominator: denominator.clone(), + scale: 100.0, + }, + vec![ + EvidenceInput { + role: MetricInputRole::Numerator, + measure_key: numerator.measure_key, + presentation: EvidencePresentation { + detail_keys: &[], + show_value: true, + }, + }, + EvidenceInput { + role: MetricInputRole::Denominator, + measure_key: denominator.measure_key, + presentation: EvidencePresentation { + detail_keys: &[], + show_value: true, + }, + }, + ], + ); + let request = validated(plan); + let (sql, params) = + compile_query(&request).unwrap_or_else(|error| panic!("query must compile: {error}")); + assert!(sql.contains("sumIf")); + assert!(sql.contains("daily_ratio")); + assert!(params.iter().any(|value| value == "focus_hours")); + assert!(params.iter().any(|value| value == "work_hours")); + } + + #[test] + fn event_presentation_projects_human_fields_and_dimensions() { + let value = input(MetricInputRole::Value, "commit_count"); + let plan = plan( + ComputationSpec::Sum { + value: value.clone(), + }, + vec![EvidenceInput { + role: MetricInputRole::Value, + measure_key: value.measure_key, + presentation: evidence_presentation( + "git", + "commit_count", + EvidenceGranularity::Event, + ), + }], + ); + let (columns, rows) = presentation(&[row()], &plan, &[], &["category".to_owned()]) + .unwrap_or_else(|error| panic!("presentation must build: {error}")); + assert_eq!( + columns + .iter() + .map(|column| column.key.as_str()) + .collect::>(), + [ + "ref", + "title", + "repository", + "author", + "category", + "lines_added", + "lines_removed", + "date" + ] + ); + assert_eq!(rows[0].values["category"], "code"); + assert_eq!(rows[0].values["lines_added"], 12.0); + } + + #[test] + fn ratio_presentation_names_numerator_and_denominator() { + let numerator = input(MetricInputRole::Numerator, "focus_hours"); + let denominator = input(MetricInputRole::Denominator, "work_hours"); + let plan = plan( + ComputationSpec::Ratio { + numerator: numerator.clone(), + denominator: denominator.clone(), + scale: 100.0, + }, + vec![ + EvidenceInput { + role: MetricInputRole::Numerator, + measure_key: numerator.measure_key, + presentation: EvidencePresentation { + detail_keys: &[], + show_value: true, + }, + }, + EvidenceInput { + role: MetricInputRole::Denominator, + measure_key: denominator.measure_key, + presentation: EvidencePresentation { + detail_keys: &[], + show_value: true, + }, + }, + ], + ); + let mut ratio_row = row(); + ratio_row.numerator = Some(6.0); + ratio_row.denominator = Some(8.0); + ratio_row.details = serde_json::json!({}); + let (columns, rows) = presentation(&[ratio_row], &plan, &[], &[]) + .unwrap_or_else(|error| panic!("ratio presentation must build: {error}")); + assert_eq!(columns[1].label, "Focus hours"); + assert_eq!(columns[2].label, "Work hours"); + assert_eq!(rows[0].values["numerator"], 6.0); + assert_eq!(rows[0].values["denominator"], 8.0); + } + + #[test] + fn response_pages_with_snapshot_bound_cursor() { + let value = input(MetricInputRole::Value, "commit_count"); + let plan = plan( + ComputationSpec::Sum { + value: value.clone(), + }, + vec![EvidenceInput { + role: MetricInputRole::Value, + measure_key: value.measure_key, + presentation: evidence_presentation( + "git", + "commit_count", + EvidenceGranularity::Event, + ), + }], + ); + let request = validated(plan); + let response = build_response(&request, vec![row(), row()]) + .unwrap_or_else(|error| panic!("response must build: {error}")); + let cursor = response + .next_cursor + .unwrap_or_else(|| panic!("response must include a next cursor")); + let envelope = + decode_cursor(&cursor).unwrap_or_else(|error| panic!("cursor must decode: {error}")); + assert_eq!(response.rows.len(), 1); + assert_eq!(envelope.snapshot_id, "snapshot"); + assert_eq!(envelope.fingerprint, request.fingerprint); + assert!(decode_cursor("invalid").is_err()); + } + + #[test] + fn filters_and_display_dimensions_are_normalized() { + let definition = definition( + ComputationSpec::Sum { + value: input(MetricInputRole::Value, "commit_count"), + }, + &["repository", "category"], + ); + let filters = normalize_filters( + &definition, + vec![MetricDrilldownFilter { + dimension: " repository ".to_owned(), + values: vec!["b".to_owned(), "a".to_owned(), "a".to_owned()], + }], + ) + .unwrap_or_else(|error| panic!("filter value must normalize: {error}")); + assert_eq!(filters[0].values, ["a", "b"]); + assert_eq!( + normalize_display_dimensions( + &definition, + vec!["category".to_owned(), "category".to_owned()] + ) + .unwrap_or_else(|error| panic!("display dimensions must normalize: {error}")), + ["category"] + ); + assert!( + normalize_filters( + &definition, + vec![MetricDrilldownFilter { + dimension: "unknown".to_owned(), + values: vec!["value".to_owned()], + }] + ) + .is_err() + ); + assert!(normalize_display_dimensions(&definition, vec!["unknown".to_owned()]).is_err()); + } + + #[test] + fn presentation_rejects_invalid_warehouse_json() { + let value = input(MetricInputRole::Value, "commit_count"); + let plan = plan( + ComputationSpec::Sum { + value: value.clone(), + }, + vec![EvidenceInput { + role: MetricInputRole::Value, + measure_key: value.measure_key, + presentation: evidence_presentation( + "git", + "commit_count", + EvidenceGranularity::Event, + ), + }], + ); + let mut invalid_details = row(); + invalid_details.details = serde_json::json!("invalid"); + assert!(presentation(&[invalid_details], &plan, &[], &[]).is_err()); + let mut invalid_dimensions = row(); + invalid_dimensions.dimensions_json = "invalid".to_owned(); + assert!(presentation(&[invalid_dimensions], &plan, &[], &[]).is_err()); + } + + #[test] + fn evidence_presentations_cover_domain_shapes() { + assert!(!evidence_presentation("git", "pr_merged", EvidenceGranularity::Event).show_value); + assert!( + evidence_presentation("git", "pr_cycle_hours", EvidenceGranularity::Event).show_value + ); + assert!( + evidence_presentation( + "task", + "average_slip", + EvidenceGranularity::DerivedPopulation + ) + .detail_keys + .is_empty() + ); + assert!(evidence_presentation("task", "custom", EvidenceGranularity::Event).show_value); + assert!( + !evidence_presentation("wiki", "pages_created", EvidenceGranularity::Event).show_value + ); + assert!( + evidence_presentation("collab", "messages", EvidenceGranularity::SourceSummary) + .show_value + ); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_results/builder.rs b/src/backend/services/analytics/src/domain/metric_results/builder.rs index 6076e86fc..28c3213ed 100644 --- a/src/backend/services/analytics/src/domain/metric_results/builder.rs +++ b/src/backend/services/analytics/src/domain/metric_results/builder.rs @@ -283,6 +283,19 @@ pub fn build_metric_result( direction: def.base.direction, computation, views, + drilldown: None, + selection: super::dto::MetricResultSelectionDto { + metric_key: def.key().to_owned(), + entity: super::dto::MetricResultsEntityDto { + r#type: String::new(), + ids: Vec::new(), + }, + period: super::dto::MetricResultsPeriodDto { + from: String::new(), + to: String::new(), + }, + filters: Vec::new(), + }, } } diff --git a/src/backend/services/analytics/src/domain/metric_results/dto.rs b/src/backend/services/analytics/src/domain/metric_results/dto.rs index 9ba6bb7f9..3ffa9d890 100644 --- a/src/backend/services/analytics/src/domain/metric_results/dto.rs +++ b/src/backend/services/analytics/src/domain/metric_results/dto.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use super::view::{Bucket, MetricResultViewKind}; use crate::domain::metric_definitions::{MetricDirection, MetricFormat}; +use crate::domain::metric_drilldown::MetricDrilldownCapability; #[derive(Debug, Deserialize, utoipa::ToSchema)] pub struct MetricResultsRequest { @@ -95,6 +96,35 @@ pub struct MetricResultDto { #[serde(flatten)] pub computation: ComputationDto, pub views: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub drilldown: Option, + pub selection: MetricResultSelectionDto, +} + +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct MetricResultSelectionDto { + pub metric_key: String, + pub entity: MetricResultsEntityDto, + pub period: MetricResultsPeriodDto, + pub filters: Vec, +} + +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct MetricResultsEntityDto { + pub r#type: String, + pub ids: Vec, +} + +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct MetricResultsPeriodDto { + pub from: String, + pub to: String, +} + +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct MetricDimensionFilterDto { + pub dimension: String, + pub values: Vec, } #[derive(Debug, Serialize, utoipa::ToSchema)] diff --git a/src/backend/services/analytics/src/domain/metric_results/mod.rs b/src/backend/services/analytics/src/domain/metric_results/mod.rs index a15842078..05c6b7d5f 100644 --- a/src/backend/services/analytics/src/domain/metric_results/mod.rs +++ b/src/backend/services/analytics/src/domain/metric_results/mod.rs @@ -16,5 +16,9 @@ pub use builder::{ pub use compiler::{ BreakdownQueryRow, CompiledQuery, HistogramQueryRow, RankingQueryRow, TimeseriesQueryRow, }; -pub use dto::{MetricResultViewDto, MetricResultsRequest, MetricResultsResponse}; +pub use dto::{ + MetricDimensionFilterDto, MetricResultSelectionDto, MetricResultViewDto, + MetricResultsEntityDto, MetricResultsPeriodDto, MetricResultsRequest, MetricResultsResponse, +}; pub use validation::{ValidatedMetricResultsRequest, validate_request}; +pub(crate) use validation::{normalize_entity_id, normalize_entity_type, normalize_metric_key}; diff --git a/src/backend/services/analytics/src/domain/metric_results/validation.rs b/src/backend/services/analytics/src/domain/metric_results/validation.rs index e2c40f9d7..2007569c5 100644 --- a/src/backend/services/analytics/src/domain/metric_results/validation.rs +++ b/src/backend/services/analytics/src/domain/metric_results/validation.rs @@ -499,7 +499,7 @@ fn validate_filters( Ok(out) } -fn normalize_entity_type(entity_type: &str) -> Result { +pub(crate) fn normalize_entity_type(entity_type: &str) -> Result { normalize_key("entity.type", entity_type) } @@ -527,7 +527,7 @@ fn normalize_entity_ids( // Id normalization is a property of the entity type: person ids are emails // and the observation sources emit them lowercased, so equality requires // lowercasing here too. Other entity types keep their casing. -fn normalize_entity_id(entity_type: &str, entity_id: &str) -> String { +pub(crate) fn normalize_entity_id(entity_type: &str, entity_id: &str) -> String { let trimmed = entity_id.trim(); match entity_type { "person" => trimmed.to_ascii_lowercase(), @@ -553,7 +553,10 @@ fn normalize_key(field: &'static str, value: &str) -> Result Result { +pub(crate) fn normalize_metric_key( + field: &'static str, + value: &str, +) -> Result { let value = value.trim().to_ascii_lowercase(); if parse_metric_key(&value).is_err() { return invalid(field, "expected a metric key"); diff --git a/src/backend/services/analytics/src/domain/mod.rs b/src/backend/services/analytics/src/domain/mod.rs index c04de070d..d81188f19 100644 --- a/src/backend/services/analytics/src/domain/mod.rs +++ b/src/backend/services/analytics/src/domain/mod.rs @@ -3,6 +3,7 @@ pub mod auth; pub mod catalog; pub mod metric; pub mod metric_definitions; +pub mod metric_drilldown; pub mod metric_results; pub mod query; pub mod query_gate; diff --git a/src/backend/services/analytics/src/migration/m20260727_000001_metric_evidence.rs b/src/backend/services/analytics/src/migration/m20260727_000001_metric_evidence.rs new file mode 100644 index 000000000..e075dc4c0 --- /dev/null +++ b/src/backend/services/analytics/src/migration/m20260727_000001_metric_evidence.rs @@ -0,0 +1,153 @@ +use sea_orm::{ConnectionTrait, DbBackend, Statement}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + add_evidence_columns(manager).await?; + replace_evidence_constraints(manager).await?; + replace_evidence_triggers(manager).await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Err(DbErr::Custom("we have only forward migrations".to_owned())) + } +} + +async fn add_evidence_columns(manager: &SchemaManager<'_>) -> Result<(), DbErr> { + let conn = manager.get_connection(); + let columns = [ + ( + "metric_sources", + "evidence_ref", + "ALTER TABLE metric_sources ADD COLUMN evidence_ref VARCHAR(256) NULL AFTER source_ref", + ), + ( + "metric_sources", + "evidence_schema_status", + "ALTER TABLE metric_sources ADD COLUMN evidence_schema_status ENUM('ok','error','unchecked') NOT NULL DEFAULT 'unchecked' AFTER schema_error_code", + ), + ( + "metric_sources", + "evidence_schema_checked_at", + "ALTER TABLE metric_sources ADD COLUMN evidence_schema_checked_at DATETIME(3) NULL AFTER evidence_schema_status", + ), + ( + "metric_sources", + "evidence_schema_error_code", + "ALTER TABLE metric_sources ADD COLUMN evidence_schema_error_code VARCHAR(64) NULL AFTER evidence_schema_checked_at", + ), + ( + "metric_source_measures", + "evidence_granularity", + "ALTER TABLE metric_source_measures ADD COLUMN evidence_granularity ENUM('event','source_summary','derived_population') NULL AFTER measure_key", + ), + ]; + for (table, column, ddl) in columns { + if !manager.has_column(table, column).await? { + conn.execute_unprepared(ddl).await?; + } + } + Ok(()) +} + +async fn replace_evidence_constraints(manager: &SchemaManager<'_>) -> Result<(), DbErr> { + let conn = manager.get_connection(); + let constraints = [ + ( + "chk_metric_sources_evidence_error_biconditional", + "ALTER TABLE metric_sources ADD CONSTRAINT chk_metric_sources_evidence_error_biconditional CHECK ((evidence_schema_status = 'error') = (evidence_schema_error_code IS NOT NULL))", + ), + ( + "chk_metric_sources_evidence_error_enum", + "ALTER TABLE metric_sources ADD CONSTRAINT chk_metric_sources_evidence_error_enum CHECK (evidence_schema_error_code IS NULL OR evidence_schema_error_code IN ('table_not_found','column_not_found','unknown'))", + ), + ]; + for (name, ddl) in constraints { + if check_constraint_exists(manager, "metric_sources", name).await? { + conn.execute_unprepared(&format!("ALTER TABLE metric_sources DROP CHECK {name}")) + .await?; + } + conn.execute_unprepared(ddl).await?; + } + Ok(()) +} + +async fn replace_evidence_triggers(manager: &SchemaManager<'_>) -> Result<(), DbErr> { + let conn = manager.get_connection(); + let triggers = [ + ( + "trg_metric_sources_evidence_ref_invalidate", + "CREATE TRIGGER trg_metric_sources_evidence_ref_invalidate + BEFORE UPDATE ON metric_sources + FOR EACH ROW + BEGIN + IF NOT (OLD.evidence_ref <=> NEW.evidence_ref) THEN + SET NEW.evidence_schema_status = 'unchecked'; + SET NEW.evidence_schema_checked_at = NULL; + SET NEW.evidence_schema_error_code = NULL; + END IF; + END", + ), + ( + "trg_metric_source_measures_evidence_insert", + "CREATE TRIGGER trg_metric_source_measures_evidence_insert + AFTER INSERT ON metric_source_measures + FOR EACH ROW + UPDATE metric_sources + SET evidence_schema_status = 'unchecked', + evidence_schema_checked_at = NULL, + evidence_schema_error_code = NULL + WHERE id = NEW.source_id", + ), + ( + "trg_metric_source_measures_evidence_update", + "CREATE TRIGGER trg_metric_source_measures_evidence_update + AFTER UPDATE ON metric_source_measures + FOR EACH ROW + BEGIN + IF NOT (OLD.evidence_granularity <=> NEW.evidence_granularity) THEN + UPDATE metric_sources + SET evidence_schema_status = 'unchecked', + evidence_schema_checked_at = NULL, + evidence_schema_error_code = NULL + WHERE id = NEW.source_id; + END IF; + END", + ), + ]; + for (name, ddl) in triggers { + conn.execute_unprepared(&format!("DROP TRIGGER IF EXISTS {name}")) + .await?; + conn.execute_unprepared(ddl).await?; + } + Ok(()) +} + +async fn check_constraint_exists( + manager: &SchemaManager<'_>, + table: &str, + constraint: &str, +) -> Result { + let row = manager + .get_connection() + .query_one(Statement::from_sql_and_values( + DbBackend::MySql, + "SELECT COUNT(*) AS constraint_count + FROM information_schema.TABLE_CONSTRAINTS + WHERE CONSTRAINT_SCHEMA = DATABASE() + AND TABLE_NAME = ? + AND CONSTRAINT_NAME = ? + AND CONSTRAINT_TYPE = 'CHECK'", + [table.into(), constraint.into()], + )) + .await?; + Ok(row + .and_then(|row| row.try_get::("", "constraint_count").ok()) + .unwrap_or_default() + > 0) +} diff --git a/src/backend/services/analytics/src/migration/mod.rs b/src/backend/services/analytics/src/migration/mod.rs index c69462f83..5472f073a 100644 --- a/src/backend/services/analytics/src/migration/mod.rs +++ b/src/backend/services/analytics/src/migration/mod.rs @@ -54,6 +54,7 @@ mod m20260710_000001_metric_distinct_count_computation; mod m20260714_000001_metric_value_transform; mod m20260721_000001_metric_definition_short_label; mod m20260722_000001_metric_definition_last_observed; +mod m20260727_000001_metric_evidence; use sea_orm_migration::prelude::*; @@ -117,6 +118,7 @@ impl MigratorTrait for Migrator { Box::new(m20260714_000001_metric_value_transform::Migration), Box::new(m20260721_000001_metric_definition_short_label::Migration), Box::new(m20260722_000001_metric_definition_last_observed::Migration), + Box::new(m20260727_000001_metric_evidence::Migration), ] } } diff --git a/src/ingestion/dbt/tests/gold/assert_metric_evidence_unique.sql b/src/ingestion/dbt/tests/gold/assert_metric_evidence_unique.sql new file mode 100644 index 000000000..f10970fe7 --- /dev/null +++ b/src/ingestion/dbt/tests/gold/assert_metric_evidence_unique.sql @@ -0,0 +1,25 @@ +{% set evidence_models = [ + 'ai_metric_evidence', + 'collab_metric_evidence', + 'git_metric_evidence', + 'task_metric_evidence', + 'wiki_metric_evidence' +] %} + +{% for model in evidence_models %} +SELECT + '{{ model }}' AS evidence_model, + tenant_id, + source_key, + measure_key, + entity_id, + metric_date, + record_id, + count() AS row_count +FROM {{ ref(model) }} +GROUP BY tenant_id, source_key, measure_key, entity_id, metric_date, record_id +HAVING count() > 1 +{% if not loop.last %} +UNION ALL +{% endif %} +{% endfor %} diff --git a/src/ingestion/dbt/tests/gold/assert_task_issue_state_unique.sql b/src/ingestion/dbt/tests/gold/assert_task_issue_state_unique.sql new file mode 100644 index 000000000..6c278bfe5 --- /dev/null +++ b/src/ingestion/dbt/tests/gold/assert_task_issue_state_unique.sql @@ -0,0 +1,7 @@ +SELECT + insight_source_id, + issue_id, + count() AS row_count +FROM {{ ref('task_issue_state') }} +GROUP BY insight_source_id, issue_id +HAVING count() > 1 diff --git a/src/ingestion/gold/ai_metric_evidence.sql b/src/ingestion/gold/ai_metric_evidence.sql new file mode 100644 index 000000000..0f4d4b6be --- /dev/null +++ b/src/ingestion/gold/ai_metric_evidence.sql @@ -0,0 +1,140 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + schema='insight', + alias='ai_metric_evidence', + tags=['gold'], + query_settings={ + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +ai_dev_usage_source AS ( + SELECT + insight_tenant_id AS tenant_id, + lower(email) AS entity_id, + day AS metric_date, + CAST( + [tuple('tool', tool, {{ ai_tool_label('tool') }})] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + conversation_count, + lines_added, + lines_removed, + tool_use_offered, + tool_use_accepted, + cost_cents + FROM {{ ref('class_ai_dev_usage') }} + WHERE email IS NOT NULL + AND email != '' +), +ai_assistant_usage_source AS ( + SELECT + insight_tenant_id AS tenant_id, + lower(email) AS entity_id, + day AS metric_date, + surface, + CAST( + [tuple('tool', tool, {{ ai_tool_label('tool') }})] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + CAST( + [ + tuple('tool', tool, {{ ai_tool_label('tool') }}), + tuple('surface', surface, {{ ai_surface_label('surface') }}) + ] AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_surface_dimensions, + conversation_count, + message_count, + action_count, + cost_cents + FROM {{ ref('class_ai_assistant_usage') }} + WHERE email IS NOT NULL + AND email != '' +), +measure_observations AS ( + {{ sum_measure('accepted_lines', 'ai_dev_usage_source', 'lines_added', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('removed_lines', 'ai_dev_usage_source', 'lines_removed', 'tool_dimensions') }} + + UNION ALL + + {{ presence_measure('active_day', ['ai_dev_usage_source', 'ai_assistant_usage_source']) }} + + UNION ALL + + {{ sum_measure('cost_usd', 'ai_dev_usage_source', 'cost_cents / 100', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('cost_usd', 'ai_assistant_usage_source', 'cost_cents / 100', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('accepted_edit_actions', 'ai_dev_usage_source', 'tool_use_accepted', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('tool_use_offered', 'ai_dev_usage_source', 'tool_use_offered', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('dev_conversations', 'ai_dev_usage_source', 'conversation_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('assistant_messages', 'ai_assistant_usage_source', 'message_count', 'tool_surface_dimensions') }} + + UNION ALL + + {{ sum_measure('assistant_actions', 'ai_assistant_usage_source', 'action_count', 'tool_surface_dimensions') }} + + UNION ALL + + {{ sum_measure('chat_assistant_conversations', 'ai_assistant_usage_source', 'conversation_count', 'tool_surface_dimensions', where="surface = 'chat'") }} +), +evidence_summaries AS ( + SELECT + tenant_id, + entity_id, + metric_date, + measure_key, + toNullable(sum(value)) AS value, + dimensions + FROM measure_observations + GROUP BY tenant_id, entity_id, metric_date, measure_key, dimensions +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'ai_usage' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(arrayMap(d -> tuple(d.1, d.2), dimensions)))) + ) AS record_id, + measure_key AS record_kind, + if(measure_key = 'active_day', 'derived_population', 'source_summary') AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM evidence_summaries +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL diff --git a/src/ingestion/gold/ai_metric_observations.sql b/src/ingestion/gold/ai_metric_observations.sql index dbd4d7928..4d61ab4b6 100644 --- a/src/ingestion/gold/ai_metric_observations.sql +++ b/src/ingestion/gold/ai_metric_observations.sql @@ -6,127 +6,22 @@ alias='ai_metric_observations', tags=['gold'], query_settings={ - 'max_memory_usage': 3221225472, + 'max_memory_usage': 1610612736, 'max_threads': 4, 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368, - 'join_algorithm': 'grace_hash,hash' + 'max_bytes_before_external_sort': 805306368 } ) }} --- Source measure observations for the unified metrics runtime. Reads only --- class-contract fields: activity is row existence (the class contract --- guarantees rows exist only for real activity — see silver/ai/schema.yml), --- display labels derive from the tool / surface discriminator codes via the --- shared vocabulary macros (macros/ai_labels.sql — static product --- vocabulary, never denormalized into silver rows), and conversation --- semantics come from data presence (conversation_count is NULL for sources --- without a conversation concept). Discriminator columns are non-null --- non-empty by the class contract (enforced by silver schema tests); this --- model consumes them as-is. No vendor mapping is inlined in this model — --- labels go through the macros only. Every measure is emitted through the --- shape macros in macros/metric_observation_measures.sql; filter predicates --- may reference only class-contract dimension values. - -WITH -ai_dev_usage_source AS ( - SELECT - insight_tenant_id AS tenant_id, - lower(email) AS entity_id, - day AS metric_date, - CAST( - [tuple('tool', tool, {{ ai_tool_label('tool') }})] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions, - conversation_count, - lines_added, - lines_removed, - tool_use_offered, - tool_use_accepted, - cost_cents - FROM {{ ref('class_ai_dev_usage') }} - WHERE email IS NOT NULL - AND email != '' -), -ai_assistant_usage_source AS ( - SELECT - insight_tenant_id AS tenant_id, - lower(email) AS entity_id, - day AS metric_date, - surface, - CAST( - [tuple('tool', tool, {{ ai_tool_label('tool') }})] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions, - CAST( - [ - tuple('tool', tool, {{ ai_tool_label('tool') }}), - tuple('surface', surface, {{ ai_surface_label('surface') }}) - ] AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_surface_dimensions, - conversation_count, - message_count, - action_count, - cost_cents - FROM {{ ref('class_ai_assistant_usage') }} - WHERE email IS NOT NULL - AND email != '' -), -measure_observations AS ( - {{ sum_measure('accepted_lines', 'ai_dev_usage_source', 'lines_added', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('removed_lines', 'ai_dev_usage_source', 'lines_removed', 'tool_dimensions') }} - - UNION ALL - - {{ presence_measure('active_day', ['ai_dev_usage_source', 'ai_assistant_usage_source']) }} - - UNION ALL - - {{ sum_measure('cost_usd', 'ai_dev_usage_source', 'cost_cents / 100', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('cost_usd', 'ai_assistant_usage_source', 'cost_cents / 100', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('accepted_edit_actions', 'ai_dev_usage_source', 'tool_use_accepted', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('tool_use_offered', 'ai_dev_usage_source', 'tool_use_offered', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('dev_conversations', 'ai_dev_usage_source', 'conversation_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('assistant_messages', 'ai_assistant_usage_source', 'message_count', 'tool_surface_dimensions') }} - - UNION ALL - - {{ sum_measure('assistant_actions', 'ai_assistant_usage_source', 'action_count', 'tool_surface_dimensions') }} - - UNION ALL - - {{ sum_measure('chat_assistant_conversations', 'ai_assistant_usage_source', 'conversation_count', 'tool_surface_dimensions', where="surface = 'chat'") }} -) SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'ai_usage' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, - CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, + observed_at, measure_key, - value, - CAST(NULL AS Nullable(String)) AS subject_key, + contribution AS value, + subject_key, dimensions -FROM measure_observations -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL +FROM {{ ref('ai_metric_evidence') }} diff --git a/src/ingestion/gold/collab_metric_evidence.sql b/src/ingestion/gold/collab_metric_evidence.sql new file mode 100644 index 000000000..c59546bb2 --- /dev/null +++ b/src/ingestion/gold/collab_metric_evidence.sql @@ -0,0 +1,340 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + schema='insight', + alias='collab_metric_evidence', + tags=['gold'], + query_settings={ + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +chat_source AS ( + SELECT + tenant_id, + person_key AS entity_id, + date AS metric_date, + total_chat_messages, + channel_posts, + channel_replies, + direct_and_group_messages, + replaceOne(data_source, 'insight_', '') AS tool_value, + {{ collab_tool_label('tool_value', m365_label='Microsoft Teams') }} AS tool_label, + CAST( + [tuple('tool', tool_value, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions + FROM {{ ref('class_collab_chat_activity') }} FINAL + WHERE person_key LIKE '%@%' + AND date IS NOT NULL +), +meeting_source AS ( + SELECT + tenant_id, + person_key AS entity_id, + date AS metric_date, + meetings_attended, + meetings_organized, + adhoc_meetings_attended, + scheduled_meetings_attended, + audio_duration_seconds, + video_duration_seconds, + screen_share_duration_seconds, + replaceOne(data_source, 'insight_', '') AS tool_value, + {{ collab_tool_label('tool_value', m365_label='Microsoft Teams') }} AS tool_label, + CAST( + [tuple('tool', tool_value, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions + FROM {{ ref('class_collab_meeting_activity') }} FINAL + WHERE person_key LIKE '%@%' + AND date IS NOT NULL +), +email_source AS ( + SELECT + tenant_id, + person_key AS entity_id, + date AS metric_date, + sent_count, + received_count, + read_count, + replaceOne(data_source, 'insight_', '') AS tool_value, + {{ collab_tool_label('tool_value') }} AS tool_label, + CAST( + [tuple('tool', tool_value, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions + FROM {{ ref('class_collab_email_activity') }} FINAL + WHERE person_key LIKE '%@%' + AND date IS NOT NULL +), +document_source AS ( + SELECT + tenant_id, + person_key AS entity_id, + date AS metric_date, + viewed_or_edited_count, + shared_internally_count, + shared_externally_count, + replaceOne(data_source, 'insight_', '') AS tool_value, + {{ collab_tool_label('tool_value') }} AS tool_label, + CAST( + [tuple('tool', tool_value, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + CAST( + [tuple('scope', 'internal', 'Internal')] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS internal_scope_dimensions, + CAST( + [tuple('scope', 'external', 'External')] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS external_scope_dimensions + FROM {{ ref('class_collab_document_activity') }} FINAL + WHERE person_key LIKE '%@%' + AND date IS NOT NULL +), +focus_source AS ( + SELECT + insight_tenant_id AS tenant_id, + email AS entity_id, + day AS metric_date, + dev_time_h, + working_hours_per_day, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM {{ ref('class_focus_metrics') }} FINAL + WHERE email LIKE '%@%' + AND day IS NOT NULL +), +deliberate_activity AS ( + SELECT + tenant_id, + entity_id, + metric_date, + tool_value, + modality, + CAST( + [tuple('tool', tool_value, {{ collab_tool_label('tool_value') }})] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM ( + SELECT DISTINCT tenant_id, entity_id, metric_date, tool_value, modality + FROM ( + SELECT tenant_id, entity_id, metric_date, tool_value, 'chat' AS modality + FROM chat_source + WHERE total_chat_messages > 0 + UNION ALL + SELECT tenant_id, entity_id, metric_date, tool_value, 'email' AS modality + FROM email_source + WHERE sent_count > 0 + UNION ALL + SELECT tenant_id, entity_id, metric_date, tool_value, 'documents' AS modality + FROM document_source + WHERE viewed_or_edited_count > 0 + OR shared_internally_count > 0 + OR shared_externally_count > 0 + UNION ALL + SELECT tenant_id, entity_id, metric_date, tool_value, 'meetings' AS modality + FROM meeting_source + WHERE meetings_attended > 0 + ) + ) +), +meeting_free_source AS ( + SELECT + tenant_id, + entity_id, + metric_date, + if(sum(meeting_seconds) = 0, 1, 0) AS meeting_free_flag, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM ( + SELECT DISTINCT + tenant_id, + entity_id, + metric_date, + 0 AS meeting_seconds, + 1 AS is_active + FROM deliberate_activity + UNION ALL + SELECT + tenant_id, + entity_id, + metric_date, + ifNull(audio_duration_seconds, 0) + + ifNull(video_duration_seconds, 0) + + ifNull(screen_share_duration_seconds, 0) AS meeting_seconds, + 0 AS is_active + FROM meeting_source + ) + GROUP BY tenant_id, entity_id, metric_date + HAVING max(is_active) = 1 +), +value_measures AS ( + {{ sum_measure('total_chat_messages', 'chat_source', 'total_chat_messages', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('channel_posts', 'chat_source', 'channel_posts + ifNull(channel_replies, 0)', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('direct_and_group_messages', 'chat_source', 'direct_and_group_messages', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('emails_sent', 'email_source', 'sent_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('emails_received', 'email_source', 'received_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('emails_read', 'email_source', 'read_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('files_engaged', 'document_source', 'viewed_or_edited_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('files_shared_internal', 'document_source', 'shared_internally_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('files_shared_external', 'document_source', 'shared_externally_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('files_shared', 'document_source', 'shared_internally_count', 'internal_scope_dimensions') }} + + UNION ALL + + {{ sum_measure('files_shared', 'document_source', 'shared_externally_count', 'external_scope_dimensions') }} + + UNION ALL + + {{ sum_measure('meeting_hours', 'meeting_source', 'greatest(ifNull(audio_duration_seconds, 0), ifNull(video_duration_seconds, 0), ifNull(screen_share_duration_seconds, 0)) / 3600.0', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('meetings_attended', 'meeting_source', 'meetings_attended', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('meetings_organized', 'meeting_source', 'meetings_organized', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('adhoc_meetings_attended', 'meeting_source', 'adhoc_meetings_attended', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('scheduled_meetings_attended', 'meeting_source', 'scheduled_meetings_attended', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('focus_hours', 'focus_source', 'dev_time_h', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('working_hours', 'focus_source', 'working_hours_per_day', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('chat_active_day', 'chat_source', 'if(total_chat_messages > 0, 1, NULL)', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('meeting_free_day', 'meeting_free_source', 'meeting_free_flag', 'no_dimensions') }} +), +active_day_grain AS ( + SELECT DISTINCT + tenant_id, + entity_id, + metric_date, + tool_dimensions + FROM deliberate_activity +), +active_modality_grain AS ( + SELECT DISTINCT + tenant_id, + entity_id, + metric_date, + modality, + no_dimensions + FROM deliberate_activity +), +subject_measures AS ( + {{ distinct_measure('active_day', 'active_day_grain', 'metric_date', 'tool_dimensions') }} + + UNION ALL + + {{ distinct_measure('active_modality', 'active_modality_grain', 'modality', 'no_dimensions') }} +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'collab' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(arrayMap(d -> tuple(d.1, d.2), dimensions)))) + ) AS record_id, + measure_key AS record_kind, + if( + measure_key IN ('focus_hours', 'working_hours', 'meeting_free_day'), + 'derived_population', + 'source_summary' + ) AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM value_measures +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'collab' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(tuple(arrayMap(d -> tuple(d.1, d.2), dimensions), subject_key)))) + ) AS record_id, + measure_key AS record_kind, + 'derived_population' AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM subject_measures +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL diff --git a/src/ingestion/gold/collab_metric_observations.sql b/src/ingestion/gold/collab_metric_observations.sql index bcb974215..f0deaf480 100644 --- a/src/ingestion/gold/collab_metric_observations.sql +++ b/src/ingestion/gold/collab_metric_observations.sql @@ -6,437 +6,22 @@ alias='collab_metric_observations', tags=['gold'], query_settings={ - 'max_memory_usage': 3221225472, + 'max_memory_usage': 1610612736, 'max_threads': 4, 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368, - 'join_algorithm': 'grace_hash,hash' + 'max_bytes_before_external_sort': 805306368 } ) }} --- Source measure observations for the unified metrics runtime, collaboration --- family. Reads class contracts only; no vendor-specific columns or tool --- names appear inline. Every measure is emitted through the shape macros in --- macros/metric_observation_measures.sql; the `tool` dimension display label --- comes from macros/collab_tool_label.sql (static product vocabulary, --- computed here rather than in silver so label changes apply retroactively on --- the next build). --- --- Materialized as a sorted table: the observation pipeline below (FINAL --- dedup, nineteen measure branches) runs once per dbt build — which is --- also the only time the silver inputs can have changed — instead of once --- per metric query. The ordering key mirrors the runtime's filter shape --- (source_key, measure_key, entity_id, metric_date), so single-measure --- queries read index-pruned ranges rather than the whole relation. --- --- query_settings bound the CREATE-AS-SELECT for every runner: an --- over-limit build spills aggregation/sort/join state to disk instead of --- failing on the server memory tracker. --- --- The `tool` dimension value is the `data_source` discriminator with the --- `insight_` prefix stripped (m365, slack, zoom, zulip_proxy). All Microsoft --- 365 surfaces (Teams, Outlook, OneDrive, SharePoint) share the m365 tool. --- --- Grain per measure: --- day-grain sums (per tool): total_chat_messages, channel_posts, --- direct_and_group_messages, emails_sent/received/read, --- files_engaged, files_shared_internal/external, --- meeting_hours, meetings_attended/organized, --- adhoc/scheduled_meetings_attended --- day-grain sums (per scope): files_shared (recipient scope --- internal/external — the combined files-shared total --- with a scope breakdown) --- day-grain presence (per tool): chat_active_day (days with chat --- messages — the messages-per-active-day denominator, --- matching the day set its numerator draws from) --- day-grain sums (no tool): focus_hours, working_hours (HR-derived); --- meeting_free_day (0/1 on every deliberately-active --- day — a measured 0 for people in meetings daily, --- never conflated with missing coverage) --- distinct-count subjects: active_day (subject = date, per tool), --- active_modality (subject = modality) --- --- Distinct-count measures carry a `subject_key`; the other macros do not, so --- they are unioned in a separate branch that stamps subject_key = NULL on the --- value/presence measures at the final projection. --- --- Attribution: every measure keys on the class `person_key`, and only --- email-shaped keys pass. The class contract falls back to lower(user_id) --- where a source has no email (Slack), but cohorts and API requests address --- people by email — a user-id key can never match either, so those rows are --- excluded as unmatchable rather than carried as dead entities. --- --- `focus_hours` / `working_hours` come from class_focus_metrics, which joins --- HR scheduled hours. focus_time_pct is their ratio (× 100). Working hours --- default to a nominal eight-hour day where the HR source omits them, and --- focus is defined only on days a person has meeting records. --- --- Memory shape (the measure branches run as concurrent pipelines within --- the build query): every read keeps FINAL — the cheapest dedup, a --- streaming merge of sorted parts — because no branch is duplicate-immune: --- sum and presence measures inflate on duplicate row versions, and the --- deliberate-activity gates (`> 0`) would pass a stale version's value. --- The class tables are person x day x source grain (no per-event or --- per-file rows), and the model contains no joins — the HR join lives in --- silver (class_focus_metrics, materialized) — so no wide rows or strings --- cross a join boundary. --- --- Peer measurability (who enters a metric's peer pool) is decided HERE, by --- row emission — the runtime never fabricates a zero for an entity with no --- rows (see metrics DESIGN, "Peer measurability"). Each measure's emission --- gate is therefore a deliberate semantic choice: --- * value-gated (rows whenever the source reports the person, zeros --- included): all volume counts (messages, emails, files, meetings). --- A reported zero is a real behavioral observation — a quiet email --- week — and belongs in peer pools. --- * engagement-gated (rows only on deliberate activity): active_day, --- active_modality, chat_active_day, and meeting_free_day (0/1 per --- active day — "worked uninterrupted" is only defined on days the --- person worked). A zero here would mean non-engagement (rostered --- accounts with no activity: leavers, leave, service accounts), which --- would drag peer medians toward zero and rank absent people; pools --- compare engaged users among engaged users, matching the ai/git --- activity metrics on the same dashboard. --- Changing a gate re-ranks every peer standing for that metric — make it --- an explicit decision, never a side effect of a connector reshaping its --- emission. - -WITH -chat_source AS ( - SELECT - tenant_id, - person_key AS entity_id, - date AS metric_date, - total_chat_messages, - channel_posts, - channel_replies, - direct_and_group_messages, - replaceOne(data_source, 'insight_', '') AS tool_value, - {{ collab_tool_label('tool_value', m365_label='Microsoft Teams') }} AS tool_label, - CAST( - [tuple('tool', tool_value, tool_label)] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions - FROM {{ ref('class_collab_chat_activity') }} FINAL - WHERE person_key LIKE '%@%' - AND date IS NOT NULL -), -meeting_source AS ( - SELECT - tenant_id, - person_key AS entity_id, - date AS metric_date, - meetings_attended, - meetings_organized, - adhoc_meetings_attended, - scheduled_meetings_attended, - audio_duration_seconds, - video_duration_seconds, - screen_share_duration_seconds, - replaceOne(data_source, 'insight_', '') AS tool_value, - {{ collab_tool_label('tool_value', m365_label='Microsoft Teams') }} AS tool_label, - CAST( - [tuple('tool', tool_value, tool_label)] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions - FROM {{ ref('class_collab_meeting_activity') }} FINAL - WHERE person_key LIKE '%@%' - AND date IS NOT NULL -), -email_source AS ( - SELECT - tenant_id, - person_key AS entity_id, - date AS metric_date, - sent_count, - received_count, - read_count, - replaceOne(data_source, 'insight_', '') AS tool_value, - {{ collab_tool_label('tool_value') }} AS tool_label, - CAST( - [tuple('tool', tool_value, tool_label)] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions - FROM {{ ref('class_collab_email_activity') }} FINAL - WHERE person_key LIKE '%@%' - AND date IS NOT NULL -), -document_source AS ( - SELECT - tenant_id, - person_key AS entity_id, - date AS metric_date, - viewed_or_edited_count, - shared_internally_count, - shared_externally_count, - replaceOne(data_source, 'insight_', '') AS tool_value, - {{ collab_tool_label('tool_value') }} AS tool_label, - CAST( - [tuple('tool', tool_value, tool_label)] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions, - -- Recipient-scope dimension for the combined files_shared measure - -- (static product vocabulary, like the tool labels). - CAST( - [tuple('scope', 'internal', 'Internal')] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS internal_scope_dimensions, - CAST( - [tuple('scope', 'external', 'External')] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS external_scope_dimensions - FROM {{ ref('class_collab_document_activity') }} FINAL - WHERE person_key LIKE '%@%' - AND date IS NOT NULL -), -focus_source AS ( - SELECT - insight_tenant_id AS tenant_id, - email AS entity_id, - day AS metric_date, - dev_time_h, - working_hours_per_day, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM {{ ref('class_focus_metrics') }} FINAL - WHERE email LIKE '%@%' - AND day IS NOT NULL -), --- A day/tool is active on a deliberate signal only (a message or email sent, --- a file engaged or shared, a meeting attended); passive email received/read --- is excluded. Deduped to one row per (tenant, entity, date, tool) and --- re-labelled with the canonical suite label: the runtime groups breakdowns --- by (value, label), so carrying the per-surface labels (Teams vs --- Microsoft 365) across this union would split one m365 platform into two --- active_day groups. -deliberate_activity AS ( - SELECT - tenant_id, - entity_id, - metric_date, - tool_value, - modality, - CAST( - [tuple('tool', tool_value, {{ collab_tool_label('tool_value') }})] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS tool_dimensions, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM ( - SELECT DISTINCT tenant_id, entity_id, metric_date, tool_value, modality - FROM ( - SELECT tenant_id, entity_id, metric_date, tool_value, 'chat' AS modality - FROM chat_source - WHERE total_chat_messages > 0 - UNION ALL - SELECT tenant_id, entity_id, metric_date, tool_value, 'email' AS modality - FROM email_source - WHERE sent_count > 0 - UNION ALL - SELECT tenant_id, entity_id, metric_date, tool_value, 'documents' AS modality - FROM document_source - WHERE viewed_or_edited_count > 0 - OR shared_internally_count > 0 - OR shared_externally_count > 0 - UNION ALL - SELECT tenant_id, entity_id, metric_date, tool_value, 'meetings' AS modality - FROM meeting_source - WHERE meetings_attended > 0 - ) - ) -), --- A meeting-free day: a DELIBERATELY-ACTIVE day (any modality) with zero --- meeting time across every meeting tool. Anchoring on activity rather than --- meeting records keeps the metric defined for people whose meeting tool --- only writes rows on attendance (Zoom), and reads as "worked uninterrupted" --- rather than "had an empty meeting report". Emitted as a 0/1 value on every --- active day: a person in meetings every working day is a measured 0 — in --- peer pools, scored — never conflated with a person who wasn't active at --- all. Join-free: active-day markers and meeting seconds union into one --- day-grain aggregate. -meeting_free_source AS ( - SELECT - tenant_id, - entity_id, - metric_date, - if(sum(meeting_seconds) = 0, 1, 0) AS meeting_free_flag, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM ( - SELECT DISTINCT - tenant_id, - entity_id, - metric_date, - 0 AS meeting_seconds, - 1 AS is_active - FROM deliberate_activity - UNION ALL - SELECT - tenant_id, - entity_id, - metric_date, - ifNull(audio_duration_seconds, 0) - + ifNull(video_duration_seconds, 0) - + ifNull(screen_share_duration_seconds, 0) AS meeting_seconds, - 0 AS is_active - FROM meeting_source - ) - GROUP BY tenant_id, entity_id, metric_date - HAVING max(is_active) = 1 -), -value_measures AS ( - {{ sum_measure('total_chat_messages', 'chat_source', 'total_chat_messages', 'tool_dimensions') }} - - UNION ALL - - -- Posts + thread replies: Slack folds replies into channel_posts (it - -- cannot split them; channel_replies is NULL), M365 reports them apart — - -- adding both keeps the count comparable across tools. NULL channel_posts - -- (Zulip) stays NULL: the addition never resurrects an absent source. - {{ sum_measure('channel_posts', 'chat_source', 'channel_posts + ifNull(channel_replies, 0)', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('direct_and_group_messages', 'chat_source', 'direct_and_group_messages', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('emails_sent', 'email_source', 'sent_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('emails_received', 'email_source', 'received_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('emails_read', 'email_source', 'read_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('files_engaged', 'document_source', 'viewed_or_edited_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('files_shared_internal', 'document_source', 'shared_internally_count', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('files_shared_external', 'document_source', 'shared_externally_count', 'tool_dimensions') }} - - UNION ALL - - -- Combined files-shared total with a recipient-scope breakdown: the same - -- measure emitted once per scope (mirrors ai's cost_usd emitted from two - -- relations). Unfiltered queries sum both scopes; a scope breakdown - -- splits them. - {{ sum_measure('files_shared', 'document_source', 'shared_internally_count', 'internal_scope_dimensions') }} - - UNION ALL - - {{ sum_measure('files_shared', 'document_source', 'shared_externally_count', 'external_scope_dimensions') }} - - UNION ALL - - -- ifNull per modality, not a bare greatest(): Zoom rows carry NULL for - -- modalities it does not report, and greatest() over a NULL argument is - -- version-dependent in ClickHouse (NULL before 24.12, ignored after) — - -- a bare form silently drops those rows' real audio time on older - -- servers. Mirrors the modality handling the silver focus model uses. - {{ sum_measure('meeting_hours', 'meeting_source', 'greatest(ifNull(audio_duration_seconds, 0), ifNull(video_duration_seconds, 0), ifNull(screen_share_duration_seconds, 0)) / 3600.0', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('meetings_attended', 'meeting_source', 'meetings_attended', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('meetings_organized', 'meeting_source', 'meetings_organized', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('adhoc_meetings_attended', 'meeting_source', 'adhoc_meetings_attended', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('scheduled_meetings_attended', 'meeting_source', 'scheduled_meetings_attended', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('focus_hours', 'focus_source', 'dev_time_h', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('working_hours', 'focus_source', 'working_hours_per_day', 'no_dimensions') }} - - UNION ALL - - -- One row of 1 per (entity, date, tool) with chat messages: the - -- messages-per-active-day denominator. Deliberately chat-gated, not the - -- all-modality active_day — the ratio's numerator is chat messages, so - -- its denominator must count only chat-active days. - {{ sum_measure('chat_active_day', 'chat_source', 'if(total_chat_messages > 0, 1, NULL)', 'tool_dimensions') }} - - UNION ALL - - {{ sum_measure('meeting_free_day', 'meeting_free_source', 'meeting_free_flag', 'no_dimensions') }} -), --- distinct_measure emits one row per source row, so each emission dedups --- deliberate_activity (tool x modality grain) to its own grain first: --- active_day to (entity, date, tool), active_modality to (entity, date, --- modality) — otherwise a chat+email day duplicates active_day rows and a --- two-tool modality duplicates active_modality rows, violating the --- published unique-grain contract. -active_day_grain AS ( - SELECT DISTINCT - tenant_id, - entity_id, - metric_date, - tool_dimensions - FROM deliberate_activity -), -active_modality_grain AS ( - SELECT DISTINCT - tenant_id, - entity_id, - metric_date, - modality, - no_dimensions - FROM deliberate_activity -), -subject_measures AS ( - {{ distinct_measure('active_day', 'active_day_grain', 'metric_date', 'tool_dimensions') }} - - UNION ALL - - {{ distinct_measure('active_modality', 'active_modality_grain', 'modality', 'no_dimensions') }} -) -SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'collab' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, - CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, - measure_key, - value, - CAST(NULL AS Nullable(String)) AS subject_key, - dimensions -FROM value_measures -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL - -UNION ALL - SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'collab' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, - CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, + observed_at, measure_key, - value, + contribution AS value, subject_key, dimensions -FROM subject_measures -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL +FROM {{ ref('collab_metric_evidence') }} diff --git a/src/ingestion/gold/git_metric_evidence.sql b/src/ingestion/gold/git_metric_evidence.sql new file mode 100644 index 000000000..95cc02d6a --- /dev/null +++ b/src/ingestion/gold/git_metric_evidence.sql @@ -0,0 +1,388 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + schema='insight', + alias='git_metric_evidence', + tags=['gold'], + query_settings={ + 'join_use_nulls': 1, + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +commits_source AS ( + SELECT + tenant_id, + source_id, + project_key, + repo_slug, + commit_hash, + author_name, + message, + date AS observed_at, + lower(trimBoth(author_email)) AS entity_id, + toDate(date) AS metric_date, + lines_added, + lines_removed, + if(coalesce(project_key, '') = '', '__unknown__', concat(coalesce(toString(source_id), ''), ':', project_key)) AS project_value, + if(coalesce(project_key, '') = '', 'Unknown', project_key) AS project_label, + concat(coalesce(toString(source_id), ''), ':', coalesce(project_key, ''), '/', coalesce(repo_slug, '')) AS repository_value, + if(coalesce(project_key, '') = '', coalesce(repo_slug, ''), concat(project_key, '/', repo_slug)) AS repository_label, + replaceOne(data_source, 'insight_', '') AS source_value, + {{ git_source_label('source_value') }} AS source_label, + CAST( + [ + tuple('repository', repository_value, repository_label), + tuple('project', project_value, project_label), + tuple('source', source_value, source_label) + ] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS source_dimensions + FROM {{ ref('class_git_commits') }} FINAL + WHERE trimBoth(author_email) != '' + AND date IS NOT NULL + AND is_merge_commit = 0 + ORDER BY tenant_id, data_source, commit_hash, source_id, project_key, repo_slug + LIMIT 1 BY tenant_id, data_source, commit_hash +), +file_changes_source AS ( + SELECT + commits.tenant_id AS tenant_id, + commits.entity_id AS entity_id, + commits.metric_date AS metric_date, + file_changes.category AS category, + {{ git_file_category_label('file_changes.category') }} AS category_label, + file_changes.file_extension_value AS file_extension, + file_changes.file_extension_label AS file_extension_label, + file_changes.change_type_value AS change_type, + file_changes.change_type_label AS change_type_label, + file_changes.lines_added AS lines_added, + file_changes.lines_removed AS lines_removed, + commits.repository_value AS repository_value, + commits.repository_label AS repository_label, + commits.source_dimensions AS source_dimensions, + CAST( + [ + tuple('file_extension', file_extension, file_extension_label), + tuple('change_type', change_type, change_type_label), + tuple('repository', repository_value, repository_label), + tuple('project', commits.project_value, commits.project_label), + tuple('source', commits.source_value, commits.source_label) + ] AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS file_source_dimensions, + CAST( + [ + tuple('category', category, category_label), + tuple('file_extension', file_extension, file_extension_label), + tuple('change_type', change_type, change_type_label), + tuple('repository', repository_value, repository_label), + tuple('project', commits.project_value, commits.project_label), + tuple('source', commits.source_value, commits.source_label) + ] AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS category_source_dimensions + FROM ( + SELECT + tenant_id, + source_id, + project_key, + repo_slug, + commit_hash, + {{ git_file_category('file_path') }} AS category, + if(raw_file_change.file_extension = '', '__unknown__', lower(raw_file_change.file_extension)) AS file_extension_value, + if(raw_file_change.file_extension = '', 'Unknown', lower(raw_file_change.file_extension)) AS file_extension_label, + if(raw_file_change.change_type = '', '__unknown__', lower(raw_file_change.change_type)) AS change_type_value, + multiIf( + raw_file_change.change_type = '', 'Unknown', + lower(raw_file_change.change_type) = 'added', 'Added', + lower(raw_file_change.change_type) = 'modified', 'Modified', + lower(raw_file_change.change_type) = 'renamed', 'Renamed', + lower(raw_file_change.change_type) = 'deleted', 'Deleted', + raw_file_change.change_type + ) AS change_type_label, + sum(lines_added) AS lines_added, + sum(lines_removed) AS lines_removed + FROM {{ ref('class_git_file_changes') }} AS raw_file_change FINAL + GROUP BY tenant_id, source_id, project_key, repo_slug, commit_hash, category, file_extension_value, file_extension_label, change_type_value, change_type_label + ) AS file_changes + INNER JOIN commits_source AS commits + ON commits.tenant_id = file_changes.tenant_id + AND commits.source_id = file_changes.source_id + AND commits.project_key = file_changes.project_key + AND commits.repo_slug = file_changes.repo_slug + AND commits.commit_hash = file_changes.commit_hash +), +pr_commit_emails AS ( + SELECT + tenant_id, + source_id, + project_key, + repo_slug, + pr_id, + if(uniqExact(email) = 1, any(email), CAST(NULL AS Nullable(String))) AS email + FROM ( + SELECT + links.tenant_id AS tenant_id, + links.source_id AS source_id, + links.project_key AS project_key, + links.repo_slug AS repo_slug, + links.pr_id AS pr_id, + lower(trimBoth(commits.author_email)) AS email, + uniqExact(commits.commit_hash) AS email_count, + max(uniqExact(commits.commit_hash)) OVER ( + PARTITION BY links.tenant_id, links.source_id, + links.project_key, links.repo_slug, links.pr_id + ) AS max_count + FROM {{ ref('class_git_pull_requests_commits') }} AS links + INNER JOIN {{ ref('class_git_commits') }} AS commits + ON commits.tenant_id = links.tenant_id + AND commits.source_id = links.source_id + AND commits.project_key = links.project_key + AND commits.repo_slug = links.repo_slug + AND commits.commit_hash = links.commit_hash + WHERE trimBoth(commits.author_email) != '' + AND commits.is_merge_commit = 0 + GROUP BY tenant_id, source_id, project_key, repo_slug, pr_id, email + ) + WHERE email_count = max_count + GROUP BY tenant_id, source_id, project_key, repo_slug, pr_id +), +pull_requests_source AS ( + SELECT + prs.tenant_id AS tenant_id, + prs.source_id AS source_id, + prs.pr_id AS pr_id, + prs.pr_number AS pr_number, + prs.title AS title, + prs.author_name AS author_name, + multiIf( + trimBoth(prs.author_email) != '', lower(trimBoth(prs.author_email)), + pr_commit_emails.email IS NOT NULL AND pr_commit_emails.email != '', pr_commit_emails.email, + CAST(NULL AS Nullable(String)) + ) AS entity_id, + prs.state AS state, + prs.created_on AS created_on, + prs.closed_on AS closed_on, + prs.lines_added + prs.lines_removed AS change_size, + if( + prs.state = 'MERGED' + AND prs.closed_on IS NOT NULL + AND prs.created_on IS NOT NULL + AND prs.closed_on >= prs.created_on, + dateDiff('second', prs.created_on, prs.closed_on) / 3600.0, + CAST(NULL AS Nullable(Float64)) + ) AS cycle_hours, + if(coalesce(prs.project_key, '') = '', '__unknown__', concat(coalesce(toString(prs.source_id), ''), ':', prs.project_key)) AS project_value, + if(coalesce(prs.project_key, '') = '', 'Unknown', prs.project_key) AS project_label, + concat(coalesce(toString(prs.source_id), ''), ':', coalesce(prs.project_key, ''), '/', coalesce(prs.repo_slug, '')) AS repository_value, + if(coalesce(prs.project_key, '') = '', coalesce(prs.repo_slug, ''), concat(prs.project_key, '/', prs.repo_slug)) AS repository_label, + if(prs.destination_branch = '', '__unknown__', prs.destination_branch) AS destination_branch_value, + if(prs.destination_branch = '', 'Unknown', prs.destination_branch) AS destination_branch_label, + replaceOne(prs.data_source, 'insight_', '') AS source_value, + {{ git_source_label('source_value') }} AS source_label, + CAST( + [ + tuple('destination_branch', destination_branch_value, destination_branch_label), + tuple('repository', repository_value, repository_label), + tuple('project', project_value, project_label), + tuple('source', source_value, source_label) + ] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS source_dimensions + FROM {{ ref('class_git_pull_requests') }} AS prs FINAL + LEFT JOIN pr_commit_emails + ON pr_commit_emails.tenant_id = prs.tenant_id + AND pr_commit_emails.source_id = prs.source_id + AND pr_commit_emails.project_key = prs.project_key + AND pr_commit_emails.repo_slug = prs.repo_slug + AND pr_commit_emails.pr_id = prs.pr_id +), +prs_created_source AS ( + SELECT + tenant_id, + pr_id, + pr_number, + title, + author_name, + assumeNotNull(entity_id) AS entity_id, + toDate(created_on) AS metric_date, + created_on AS observed_at, + state, + change_size, + repository_label, + repository_value, + source_dimensions + FROM pull_requests_source + WHERE entity_id IS NOT NULL + AND entity_id != '' + AND created_on IS NOT NULL +), +prs_merged_source AS ( + SELECT + tenant_id, + pr_id, + pr_number, + title, + author_name, + assumeNotNull(entity_id) AS entity_id, + toDate(closed_on) AS metric_date, + closed_on AS observed_at, + cycle_hours, + repository_label, + repository_value, + source_dimensions + FROM pull_requests_source + WHERE entity_id IS NOT NULL + AND entity_id != '' + AND state = 'MERGED' + AND closed_on IS NOT NULL +), +measure_observations AS ( + {{ presence_measure('commit_day', ['commits_source']) }} + + UNION ALL + + {{ sum_measure('code_lines_added', 'file_changes_source', 'lines_added', 'file_source_dimensions', where="category = 'code'") }} + + UNION ALL + + {{ sum_measure('lines_added', 'file_changes_source', 'lines_added', 'category_source_dimensions') }} + + UNION ALL + + {{ sum_measure('lines_removed', 'file_changes_source', 'lines_removed', 'category_source_dimensions') }} +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'git' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(arrayMap(d -> tuple(d.1, d.2), dimensions)))) + ) AS record_id, + measure_key AS record_kind, + 'source_summary' AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM measure_observations +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'git' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + commit_measure.1 AS measure_key, + concat(source_value, ':', commit_hash, ':', commit_measure.1) AS record_id, + 'commit' AS record_kind, + 'event' AS granularity, + if(message = '', commit_hash, message) AS record_label, + toNullable(toFloat64(commit_measure.2)) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + source_dimensions AS dimensions, + map( + 'ref', commit_hash, + 'title', message, + 'repository', repository_label, + 'author', author_name, + 'lines_added', coalesce(toString(lines_added), ''), + 'lines_removed', coalesce(toString(lines_removed), '') + ) AS details +FROM commits_source +ARRAY JOIN arrayConcat( + [tuple('commit_count', toFloat64(1))], + if( + lines_added IS NOT NULL AND lines_removed IS NOT NULL, + [tuple('commit_change_size', toFloat64(lines_added + lines_removed))], + [] + ) +) AS commit_measure +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'git' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + pr_measure.1 AS measure_key, + concat(repository_value, ':pr:', toString(pr_id), ':', pr_measure.1) AS record_id, + 'pull_request' AS record_kind, + 'event' AS granularity, + if(title = '', concat('PR #', toString(pr_number)), title) AS record_label, + toNullable(toFloat64(pr_measure.2)) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + source_dimensions AS dimensions, + map( + 'ref', toString(pr_number), + 'title', title, + 'repository', repository_label, + 'author', author_name + ) AS details +FROM prs_created_source +ARRAY JOIN arrayConcat( + [tuple('pr_created', toFloat64(1))], + if(state = 'MERGED', [tuple('pr_created_merged', toFloat64(1))], []), + if(ifNull(change_size, 0) > 0, [tuple('pr_change_size', toFloat64(change_size))], []) +) AS pr_measure +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'git' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + pr_measure.1 AS measure_key, + concat(repository_value, ':pr:', toString(pr_id), ':', pr_measure.1) AS record_id, + 'pull_request' AS record_kind, + 'event' AS granularity, + if(title = '', concat('PR #', toString(pr_number)), title) AS record_label, + toNullable(toFloat64(pr_measure.2)) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + source_dimensions AS dimensions, + map( + 'ref', toString(pr_number), + 'title', title, + 'repository', repository_label, + 'author', author_name + ) AS details +FROM prs_merged_source +ARRAY JOIN arrayConcat( + [tuple('pr_merged', toFloat64(1))], + if(cycle_hours IS NOT NULL, [tuple('pr_cycle_hours', toFloat64(cycle_hours))], []) +) AS pr_measure +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL diff --git a/src/ingestion/gold/git_metric_observations.sql b/src/ingestion/gold/git_metric_observations.sql index 04ce56ded..fdea562e0 100644 --- a/src/ingestion/gold/git_metric_observations.sql +++ b/src/ingestion/gold/git_metric_observations.sql @@ -6,358 +6,40 @@ alias='git_metric_observations', tags=['gold'], query_settings={ - 'max_memory_usage': 3221225472, + 'max_memory_usage': 1610612736, 'max_threads': 4, 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368, - 'join_algorithm': 'grace_hash,hash' + 'max_bytes_before_external_sort': 805306368 } ) }} --- Source measure observations for the unified metrics runtime, git family. --- Reads class contracts only; no vendor-specific columns or tool names may --- appear inline. Every measure is emitted through the shape macros in --- macros/metric_observation_measures.sql; file classification and the --- source-dimension display label come from macros/git_file_category.sql --- (static product vocabulary, computed here rather than in silver so --- taxonomy/label changes apply retroactively on the next build). --- --- Materialized as a sorted table: the observation pipeline below (FINAL --- dedup, joins, measure branches) runs once per dbt build — which is --- also the only time the silver inputs can have changed — instead of once --- per metric query. The ordering key mirrors the runtime's filter shape --- (source_key, measure_key, entity_id, metric_date), so single-measure --- queries read index-pruned ranges rather than the whole relation. --- --- query_settings bound the CREATE-AS-SELECT for every runner: an --- over-limit build spills aggregation/sort/join state to disk instead of --- failing on the server memory tracker. --- --- Grain per measure: --- day-grain sums: commit_count, code_lines_added, lines_added, --- lines_removed, pr_created, pr_merged --- day-grain presence: commit_day --- event-grain (one row per source event, feeding median metrics): --- commit_change_size (per non-merge commit), --- pr_cycle_hours (per merged pull request), --- pr_change_size (per pull request) --- --- Attribution: --- Commits and file changes attribute by the commit author_email. --- Pull requests resolve in tiers, never guessing: the PR's own --- author_email when present; else the dominant author_email among the --- PR's linked commits (tie -> unresolved). Rows that resolve to no --- email are excluded — honest absence, never a name-matched guess. --- --- Dating: --- pr_created anchors at created_on. pr_merged and pr_cycle_hours anchor --- at closed_on gated on state MERGED: the sources set the close --- timestamp to (or coalesce it with) the merge timestamp for merged --- pull requests, and a merge-commit join would risk fan-out for --- marginal precision. Negative durations (dirty close timestamps) are --- excluded from cycle hours. --- --- Merge commits are excluded once in commits_source; the exclusion --- propagates to file-change measures through the authorship join. --- `LIMIT 1 BY` collapses the same commit hash appearing in more than one --- repo of a source (forks), keeping commit_count a distinct-hash count. --- --- Memory shape (the measure branches run as concurrent pipelines within --- the build query): FINAL is the cheapest dedup here — a streaming merge --- of sorted parts — and stays wherever dedup is needed. Version-ordered --- `ORDER BY .. LIMIT 1 BY` is not an alternative: it buffers a full sort --- of the read (measured ~2x the memory of FINAL at scale). The two reads --- that avoid FINAL do so because they need no dedup at all: the identity --- vote in pr_commit_emails aggregates by uniqExact, which duplicate row --- versions cannot inflate. file_changes pre-aggregates to commit x category --- grain before joining, so per-file rows and file_path strings never enter --- a join side or a measure aggregation. - -WITH -commits_source AS ( - SELECT - tenant_id, - source_id, - project_key, - repo_slug, - commit_hash, - -- Match the API's entity-id normalization exactly (trim + lower): - -- observations keyed on an untrimmed id would never be matched by a - -- request the frontend normalizes. - lower(trimBoth(author_email)) AS entity_id, - toDate(date) AS metric_date, - lines_added, - lines_removed, - -- coalesce nullable inputs to '': source_id (and project_key/repo_slug) - -- can be NULL (e.g. GitHub silver), which made these concat() results - -- NULL — rejected by the non-Nullable tuple `value` field on CH 25.7 (#1858). - if(coalesce(project_key, '') = '', '__unknown__', concat(coalesce(toString(source_id), ''), ':', project_key)) AS project_value, - if(coalesce(project_key, '') = '', 'Unknown', project_key) AS project_label, - concat(coalesce(toString(source_id), ''), ':', coalesce(project_key, ''), '/', coalesce(repo_slug, '')) AS repository_value, - if(coalesce(project_key, '') = '', coalesce(repo_slug, ''), concat(project_key, '/', repo_slug)) AS repository_label, - replaceOne(data_source, 'insight_', '') AS source_value, - {{ git_source_label('source_value') }} AS source_label, - CAST( - [ - tuple('repository', repository_value, repository_label), - tuple('project', project_value, project_label), - tuple('source', source_value, source_label) - ] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS source_dimensions - FROM {{ ref('class_git_commits') }} FINAL - WHERE trimBoth(author_email) != '' - AND date IS NOT NULL - AND is_merge_commit = 0 - -- Deterministic survivor per (tenant, source, hash): without an ORDER BY, - -- LIMIT 1 BY picks an arbitrary repo copy of a forked commit, which would - -- vary across runs and shift the project_key/repo_slug used by the - -- file-change join. - ORDER BY tenant_id, data_source, commit_hash, source_id, project_key, repo_slug - LIMIT 1 BY tenant_id, data_source, commit_hash -), -file_changes_source AS ( - SELECT - commits.tenant_id AS tenant_id, - commits.entity_id AS entity_id, - commits.metric_date AS metric_date, - file_changes.category AS category, - {{ git_file_category_label('file_changes.category') }} AS category_label, - file_changes.file_extension_value AS file_extension, - file_changes.file_extension_label AS file_extension_label, - file_changes.change_type_value AS change_type, - file_changes.change_type_label AS change_type_label, - file_changes.lines_added AS lines_added, - file_changes.lines_removed AS lines_removed, - commits.repository_value AS repository_value, - commits.repository_label AS repository_label, - commits.source_dimensions AS source_dimensions, - CAST( - [ - tuple('file_extension', file_extension, file_extension_label), - tuple('change_type', change_type, change_type_label), - tuple('repository', repository_value, repository_label), - tuple('project', commits.project_value, commits.project_label), - tuple('source', commits.source_value, commits.source_label) - ] AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS file_source_dimensions, - CAST( - [ - tuple('category', category, category_label), - tuple('file_extension', file_extension, file_extension_label), - tuple('change_type', change_type, change_type_label), - tuple('repository', repository_value, repository_label), - tuple('project', commits.project_value, commits.project_label), - tuple('source', commits.source_value, commits.source_label) - ] AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS category_source_dimensions - FROM ( - -- Aggregated to commit x category grain before the join, so per-file - -- rows and file_path strings never reach the join or the measure - -- aggregations. - SELECT - tenant_id, - source_id, - project_key, - repo_slug, - commit_hash, - {{ git_file_category('file_path') }} AS category, - if(raw_file_change.file_extension = '', '__unknown__', lower(raw_file_change.file_extension)) AS file_extension_value, - if(raw_file_change.file_extension = '', 'Unknown', lower(raw_file_change.file_extension)) AS file_extension_label, - if(raw_file_change.change_type = '', '__unknown__', lower(raw_file_change.change_type)) AS change_type_value, - multiIf( - raw_file_change.change_type = '', 'Unknown', - lower(raw_file_change.change_type) = 'added', 'Added', - lower(raw_file_change.change_type) = 'modified', 'Modified', - lower(raw_file_change.change_type) = 'renamed', 'Renamed', - lower(raw_file_change.change_type) = 'deleted', 'Deleted', - raw_file_change.change_type - ) AS change_type_label, - sum(lines_added) AS lines_added, - sum(lines_removed) AS lines_removed - FROM {{ ref('class_git_file_changes') }} AS raw_file_change FINAL - GROUP BY tenant_id, source_id, project_key, repo_slug, commit_hash, category, file_extension_value, file_extension_label, change_type_value, change_type_label - ) AS file_changes - INNER JOIN commits_source AS commits - ON commits.tenant_id = file_changes.tenant_id - AND commits.source_id = file_changes.source_id - AND commits.project_key = file_changes.project_key - AND commits.repo_slug = file_changes.repo_slug - AND commits.commit_hash = file_changes.commit_hash -), -pr_commit_emails AS ( - -- Dominant commit author email per pull request (tie -> NULL): the - -- strongest identity signal for PR authors whose source hides emails. - SELECT - tenant_id, - source_id, - project_key, - repo_slug, - pr_id, - if(uniqExact(email) = 1, any(email), CAST(NULL AS Nullable(String))) AS email - FROM ( - SELECT - links.tenant_id AS tenant_id, - links.source_id AS source_id, - links.project_key AS project_key, - links.repo_slug AS repo_slug, - links.pr_id AS pr_id, - lower(trimBoth(commits.author_email)) AS email, - -- Vote by distinct linked commits, not join rows: a hash present - -- in more than one repo of the source must not double-count. - uniqExact(commits.commit_hash) AS email_count, - max(uniqExact(commits.commit_hash)) OVER ( - PARTITION BY links.tenant_id, links.source_id, - links.project_key, links.repo_slug, links.pr_id - ) AS max_count - -- No dedup on either side: duplicate versions of a link or commit - -- row cannot inflate the uniqExact(commit_hash) vote below. - FROM {{ ref('class_git_pull_requests_commits') }} AS links - INNER JOIN {{ ref('class_git_commits') }} AS commits - ON commits.tenant_id = links.tenant_id - AND commits.source_id = links.source_id - AND commits.project_key = links.project_key - AND commits.repo_slug = links.repo_slug - AND commits.commit_hash = links.commit_hash - -- Non-merge authorship only, matching the commit observations; a merge - -- commit's author should not vote in the PR's identity election. - WHERE trimBoth(commits.author_email) != '' - AND commits.is_merge_commit = 0 - GROUP BY tenant_id, source_id, project_key, repo_slug, pr_id, email - ) - WHERE email_count = max_count - GROUP BY tenant_id, source_id, project_key, repo_slug, pr_id -), -pull_requests_source AS ( - SELECT - prs.tenant_id AS tenant_id, - multiIf( - trimBoth(prs.author_email) != '', lower(trimBoth(prs.author_email)), - pr_commit_emails.email IS NOT NULL AND pr_commit_emails.email != '', pr_commit_emails.email, - CAST(NULL AS Nullable(String)) - ) AS entity_id, - prs.state AS state, - prs.created_on AS created_on, - prs.closed_on AS closed_on, - prs.lines_added + prs.lines_removed AS change_size, - if( - prs.state = 'MERGED' - AND prs.closed_on IS NOT NULL - AND prs.created_on IS NOT NULL - AND prs.closed_on >= prs.created_on, - dateDiff('second', prs.created_on, prs.closed_on) / 3600.0, - CAST(NULL AS Nullable(Float64)) - ) AS cycle_hours, - if(prs.project_key = '', '__unknown__', concat(toString(prs.source_id), ':', prs.project_key)) AS project_value, - if(prs.project_key = '', 'Unknown', prs.project_key) AS project_label, - concat(toString(prs.source_id), ':', prs.project_key, '/', prs.repo_slug) AS repository_value, - if(prs.project_key = '', prs.repo_slug, concat(prs.project_key, '/', prs.repo_slug)) AS repository_label, - if(prs.destination_branch = '', '__unknown__', prs.destination_branch) AS destination_branch_value, - if(prs.destination_branch = '', 'Unknown', prs.destination_branch) AS destination_branch_label, - replaceOne(prs.data_source, 'insight_', '') AS source_value, - {{ git_source_label('source_value') }} AS source_label, - CAST( - [ - tuple('destination_branch', destination_branch_value, destination_branch_label), - tuple('repository', repository_value, repository_label), - tuple('project', project_value, project_label), - tuple('source', source_value, source_label) - ] - AS Array(Tuple(key String, value String, label Nullable(String))) - ) AS source_dimensions - FROM {{ ref('class_git_pull_requests') }} AS prs FINAL - LEFT JOIN pr_commit_emails - ON pr_commit_emails.tenant_id = prs.tenant_id - AND pr_commit_emails.source_id = prs.source_id - AND pr_commit_emails.project_key = prs.project_key - AND pr_commit_emails.repo_slug = prs.repo_slug - AND pr_commit_emails.pr_id = prs.pr_id - SETTINGS join_use_nulls = 1 -), -prs_created_source AS ( - SELECT - tenant_id, - assumeNotNull(entity_id) AS entity_id, - toDate(created_on) AS metric_date, - state, - change_size, - source_dimensions - FROM pull_requests_source - WHERE entity_id IS NOT NULL - AND entity_id != '' - AND created_on IS NOT NULL -), -prs_merged_source AS ( - SELECT - tenant_id, - assumeNotNull(entity_id) AS entity_id, - toDate(closed_on) AS metric_date, - cycle_hours, - source_dimensions - FROM pull_requests_source - WHERE entity_id IS NOT NULL - AND entity_id != '' - AND state = 'MERGED' - AND closed_on IS NOT NULL -), -measure_observations AS ( - {{ sum_measure('commit_count', 'commits_source', '1', 'source_dimensions') }} - - UNION ALL - - {{ presence_measure('commit_day', ['commits_source']) }} - - UNION ALL - - {{ event_measure('commit_change_size', 'commits_source', 'lines_added + lines_removed', 'source_dimensions') }} - - UNION ALL - - {{ sum_measure('code_lines_added', 'file_changes_source', 'lines_added', 'file_source_dimensions', where="category = 'code'") }} - - UNION ALL - - {{ sum_measure('lines_added', 'file_changes_source', 'lines_added', 'category_source_dimensions') }} - - UNION ALL - - {{ sum_measure('lines_removed', 'file_changes_source', 'lines_removed', 'category_source_dimensions') }} - - UNION ALL - - {{ sum_measure('pr_created', 'prs_created_source', '1', 'source_dimensions') }} - - UNION ALL - - -- Merge-rate numerator: PRs *created* in the period that have merged, - -- dated at creation so numerator and denominator share the created-cohort. - -- (pr_merged below is merge-dated throughput for the standalone metric.) - {{ sum_measure('pr_created_merged', 'prs_created_source', '1', 'source_dimensions', where="state = 'MERGED'") }} - - UNION ALL - - {{ sum_measure('pr_merged', 'prs_merged_source', '1', 'source_dimensions') }} - - UNION ALL - - {{ event_measure('pr_cycle_hours', 'prs_merged_source', 'cycle_hours', 'source_dimensions') }} +SELECT + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + toNullable(sum(contribution)) AS value, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions +FROM {{ ref('git_metric_evidence') }} +WHERE measure_key NOT IN ('commit_change_size', 'pr_cycle_hours', 'pr_change_size') +GROUP BY tenant_id, source_key, entity_type, entity_id, metric_date, measure_key, dimensions - UNION ALL +UNION ALL - {{ event_measure('pr_change_size', 'prs_created_source', 'change_size', 'source_dimensions', where='change_size > 0') }} -) SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'git' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, measure_key, - value, - CAST(NULL AS Nullable(String)) AS subject_key, + contribution AS value, + subject_key, dimensions -FROM measure_observations -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL +FROM {{ ref('git_metric_evidence') }} +WHERE measure_key IN ('commit_change_size', 'pr_cycle_hours', 'pr_change_size') diff --git a/src/ingestion/gold/schema.yml b/src/ingestion/gold/schema.yml index 0bee1f286..462c2cc22 100644 --- a/src/ingestion/gold/schema.yml +++ b/src/ingestion/gold/schema.yml @@ -1,6 +1,98 @@ version: 2 models: + - name: ai_metric_evidence + description: > + Materialized MergeTree evidence serving table for AI source measures. + AI silver retains daily usage summaries, so rows use source_summary + granularity except active-day population rows. + columns: &metric_evidence_columns + - name: tenant_id + description: "Tenant isolation field" + tests: + - not_null + - name: source_key + description: "Logical source family" + tests: + - not_null + - accepted_values: + arguments: + values: ['ai_usage', 'collab', 'git', 'task', 'wiki'] + - name: entity_type + description: "Measured entity type" + tests: + - not_null + - accepted_values: + arguments: + values: ['person'] + - name: entity_id + description: "Normalized measured entity identifier" + tests: + - not_null + - name: metric_date + description: "Date used by metric period filtering" + tests: + - not_null + - name: observed_at + description: "Optional event timestamp" + - name: measure_key + description: "Source measure key" + tests: + - not_null + - name: record_id + description: "Deterministic total-order key within the measure scope" + tests: + - not_null + - name: record_kind + description: "Source record classification" + tests: + - not_null + - name: granularity + description: "Evidence grain: event, source_summary, or derived_population" + tests: + - not_null + - accepted_values: + arguments: + values: ['event', 'source_summary', 'derived_population'] + - name: record_label + description: "Human-readable record label" + - name: contribution + description: "Nullable numeric contribution to the source measure" + - name: subject_key + description: "Nullable subject for distinct-count computations" + - name: dimensions + description: "Typed metric dimensions inherited by filtering and grouping" + - name: details + description: "Additional display fields keyed by stable field name" + + - name: git_metric_evidence + description: > + Materialized MergeTree evidence serving table for Git source measures. + Median inputs retain event grain; additive measures retain their + established daily and dimension summaries. + columns: *metric_evidence_columns + + - name: collab_metric_evidence + description: > + Materialized MergeTree evidence serving table for collaboration source + measures. Connector class contracts retain daily summaries; focus, + activity, and meeting-free populations are marked as derived. + columns: *metric_evidence_columns + + - name: task_metric_evidence + description: > + Materialized MergeTree evidence serving table for task source measures. + Issue duration inputs retain event grain; lifecycle ratios, snapshots, + and daily flows are marked as derived populations. + columns: *metric_evidence_columns + + - name: wiki_metric_evidence + description: > + Materialized MergeTree evidence serving table for wiki source measures. + Page creation rows retain event grain; edits and engagement expose the + established daily summaries retained by the silver contracts. + columns: *metric_evidence_columns + - name: ai_metric_observations description: > Source measure observations for the unified metrics runtime. One row per diff --git a/src/ingestion/gold/task_metric_evidence.sql b/src/ingestion/gold/task_metric_evidence.sql new file mode 100644 index 000000000..c04d5cf33 --- /dev/null +++ b/src/ingestion/gold/task_metric_evidence.sql @@ -0,0 +1,289 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + schema='insight', + alias='task_metric_evidence', + tags=['gold'], + query_settings={ + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +issue_state AS ( + SELECT * + FROM {{ ref('task_issue_state') }} +), +status_intervals AS ( + SELECT * + FROM {{ ref('task_status_spans') }} +), +issue_facts AS ( + SELECT + s.tenant_id AS tenant_id, + s.entity_id AS entity_id, + s.insight_source_id AS insight_source_id, + toDate(s.final_close_at) AS metric_date, + any(s.final_close_at) AS observed_at, + s.issue_id AS issue_id, + any(s.issue_type) AS issue_type, + any(s.status_category) = 'done' AS is_done, + toDate(s.final_close_at) AS close_date, + any(s.due_date) AS due_date, + any(s.time_estimate_seconds) AS time_estimate_seconds, + any(s.time_spent_seconds) AS time_spent_seconds, + sumIf(i.duration_seconds, i.interval_start < s.final_close_at) AS dev_seconds, + if(any(s.created_at) IS NULL, + CAST(NULL AS Nullable(Float64)), + toFloat64(greatest(toInt64(0), + dateDiff('second', any(s.created_at), any(s.final_close_at))))) AS lead_seconds, + if(any(s.created_at) IS NULL + OR minIf(i.interval_start, i.interval_start < s.final_close_at) IS NULL, + CAST(NULL AS Nullable(Float64)), + toFloat64(greatest(toInt64(0), + dateDiff('second', any(s.created_at), + minIf(i.interval_start, i.interval_start < s.final_close_at))))) AS pickup_seconds, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM issue_state AS s + LEFT JOIN status_intervals AS i + ON i.insight_source_id = s.insight_source_id + AND i.issue_id = s.issue_id + AND i.status_category = 'in_progress' + WHERE s.final_close_at IS NOT NULL + GROUP BY s.tenant_id, s.entity_id, s.insight_source_id, s.issue_id, toDate(s.final_close_at) +), +issue_item_evidence AS ( + SELECT + tenant_id, + entity_id, + insight_source_id, + toDate(final_close_at) AS metric_date, + final_close_at AS observed_at, + issue_id, + issue_type, + item_measure.1 AS measure_key, + toFloat64(item_measure.2) AS contribution, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM issue_state + ARRAY JOIN arrayConcat( + [tuple('tasks_closed', toFloat64(1))], + if(issue_type = 'Bug', [tuple('bugs_fixed', toFloat64(1))], []), + if( + due_date IS NOT NULL AND toDate(final_close_at) <= due_date, + [tuple('due_date_on_time', toFloat64(1))], + [] + ), + if(due_date IS NOT NULL, [tuple('due_date_with_due', toFloat64(1))], []), + if( + due_date IS NOT NULL AND toDate(final_close_at) > due_date, + [ + tuple( + 'slip_days_total', + toFloat64(dateDiff('day', due_date, toDate(final_close_at))) + ), + tuple('late_count', toFloat64(1)) + ], + [] + ) + ) AS item_measure + WHERE final_close_at IS NOT NULL + AND status_category = 'done' +), +estimation_day AS ( + SELECT + tenant_id, + entity_id, + metric_date, + 100 * avgIf(time_estimate_seconds, is_done AND ifNull(time_estimate_seconds, 0) > 0 AND time_spent_seconds IS NOT NULL) + / nullIf(avgIf(time_spent_seconds, is_done AND ifNull(time_estimate_seconds, 0) > 0 AND time_spent_seconds IS NOT NULL), 0) + AS estimation_pct, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM issue_facts + GROUP BY tenant_id, entity_id, metric_date +), +transitions AS ( + SELECT + insight_source_id, + issue_id, + interval_start AS event_at, + status_category, + lagInFrame(status_category) OVER ( + PARTITION BY insight_source_id, issue_id ORDER BY interval_start + ) AS prev_category + FROM status_intervals +), +closes AS ( + SELECT insight_source_id, issue_id, event_at AS close_at + FROM transitions + WHERE status_category = 'done' AND (prev_category IS NULL OR prev_category != 'done') +), +reopens AS ( + SELECT insight_source_id, issue_id, event_at AS reopen_at + FROM transitions + WHERE prev_category = 'done' AND (status_category != 'done' OR status_category IS NULL) +), +close_reopen AS ( + SELECT + s.tenant_id AS tenant_id, + s.entity_id AS entity_id, + toDate(c.close_at) AS metric_date, + toFloat64(1) AS close_event, + if(minIf(r.reopen_at, r.reopen_at > c.close_at) IS NOT NULL + AND minIf(r.reopen_at, r.reopen_at > c.close_at) <= c.close_at + INTERVAL 14 DAY, + toFloat64(1), CAST(NULL AS Nullable(Float64))) AS reopened_14d, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM closes AS c + INNER JOIN issue_state AS s + ON s.insight_source_id = c.insight_source_id AND s.issue_id = c.issue_id + LEFT JOIN reopens AS r + ON r.insight_source_id = c.insight_source_id AND r.issue_id = c.issue_id + GROUP BY s.tenant_id, s.entity_id, c.insight_source_id, c.issue_id, c.close_at +), +worklog_flow AS ( + SELECT + tenant_id, + entity_id, + metric_date, + in_progress_seconds, + worklog_seconds, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM {{ ref('task_worklog_flow') }} +), +stale AS ( + SELECT + s.tenant_id AS tenant_id, + s.entity_id AS entity_id, + toDate(s.last_status_event_at) AS metric_date, + toFloat64(count()) AS stale_count, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM issue_state AS s + WHERE (s.status_category IS NULL OR s.status_category != 'done') + AND s.last_status_event_at IS NOT NULL + AND dateDiff('day', s.last_status_event_at, now()) > 14 + GROUP BY s.tenant_id, s.entity_id, toDate(s.last_status_event_at) +), +value_measures AS ( + {{ sum_measure('estimation_error_pct', 'estimation_day', 'if(estimation_pct > 0 AND estimation_pct <= 200, abs(100 - estimation_pct), NULL)', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('estimation_samples', 'estimation_day', 'if(estimation_pct > 0 AND estimation_pct <= 200, 1, NULL)', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('flow_dev_seconds', 'issue_facts', 'if(ifNull(dev_seconds, 0) > 0 AND ifNull(lead_seconds, 0) > 0, dev_seconds, NULL)', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('flow_lead_seconds', 'issue_facts', 'if(ifNull(dev_seconds, 0) > 0 AND ifNull(lead_seconds, 0) > 0, lead_seconds, NULL)', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('close_events', 'close_reopen', 'close_event', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('reopened_within_14d', 'close_reopen', 'reopened_14d', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('worklog_seconds', 'worklog_flow', 'worklog_seconds', 'no_dimensions', where='in_progress_seconds > 0') }} + + UNION ALL + + {{ sum_measure('in_progress_seconds', 'worklog_flow', 'in_progress_seconds', 'no_dimensions', where='in_progress_seconds > 0') }} + + UNION ALL + + {{ sum_measure('stale_in_progress', 'stale', 'stale_count', 'no_dimensions') }} +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'task' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(arrayMap(d -> tuple(d.1, d.2), dimensions)))) + ) AS record_id, + measure_key AS record_kind, + 'derived_population' AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM value_measures +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'task' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + measure_key, + concat(toString(insight_source_id), ':', toString(issue_id), ':', measure_key) AS record_id, + 'issue' AS record_kind, + 'event' AS granularity, + toString(issue_id) AS record_label, + toNullable(contribution) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + no_dimensions AS dimensions, + map( + 'ref', toString(issue_id), + 'issue_type', ifNull(issue_type, '') + ) AS details +FROM issue_item_evidence +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND entity_id != '' + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'task' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + duration_measure.1 AS measure_key, + concat(toString(insight_source_id), ':', toString(issue_id), ':', duration_measure.1) AS record_id, + 'issue' AS record_kind, + 'event' AS granularity, + toString(issue_id) AS record_label, + toNullable(toFloat64(duration_measure.2)) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + no_dimensions AS dimensions, + map( + 'ref', toString(issue_id), + 'issue_type', ifNull(issue_type, '') + ) AS details +FROM issue_facts +ARRAY JOIN arrayConcat( + if(ifNull(dev_seconds, 0) > 0, [tuple('dev_time_hours', toFloat64(dev_seconds / 3600.0))], []), + if(ifNull(lead_seconds, 0) > 0, [tuple('resolution_days', toFloat64(lead_seconds / 86400.0))], []), + if(pickup_seconds IS NOT NULL, [tuple('pickup_days', toFloat64(pickup_seconds / 86400.0))], []) +) AS duration_measure +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND entity_id != '' + AND metric_date IS NOT NULL diff --git a/src/ingestion/gold/task_metric_observations.sql b/src/ingestion/gold/task_metric_observations.sql index 172aa99ee..d920ebce9 100644 --- a/src/ingestion/gold/task_metric_observations.sql +++ b/src/ingestion/gold/task_metric_observations.sql @@ -6,283 +6,40 @@ alias='task_metric_observations', tags=['gold'], query_settings={ - 'max_memory_usage': 3221225472, + 'max_memory_usage': 1610612736, 'max_threads': 4, 'max_bytes_before_external_group_by': 805306368, - 'max_bytes_before_external_sort': 805306368, - 'join_algorithm': 'grace_hash,hash' + 'max_bytes_before_external_sort': 805306368 } ) }} --- Task-delivery measure observations for the unified metrics runtime; every --- measure is emitted through macros/metric_observation_measures.sql. --- --- The per-issue reconstruction is materialized upstream (task_issue_state, --- task_status_spans): ClickHouse re-inlines every WITH reference, so the --- measure branches would otherwise re-run it once per branch. --- --- Grain: event rows per closed issue for the median metrics (dev_time_hours, --- resolution_days, pickup_days); day-grain sums for everything else; --- stale_in_progress is a build-date snapshot. No distinct-count measure, so --- subject_key is always NULL. - -WITH --- Worklog attribution (account id → lowercased email, tenant on the same --- row); issue attribution is already resolved on task_issue_state. -task_users AS ( - SELECT - tenant_id, - insight_source_id, - user_id, - lower(email) AS email - FROM {{ ref('class_task_users') }} FINAL - WHERE email LIKE '%@%' -), -issue_state AS ( - SELECT * - FROM {{ ref('task_issue_state') }} -), -status_intervals AS ( - SELECT * - FROM {{ ref('task_status_spans') }} -), --- Per closed issue (metric_date = close date): dev seconds, lead, pickup. --- Only spans started before the close count — live rework on a reopened --- issue belongs to its next close, never retroactively to one reported. -issue_facts AS ( - SELECT - s.tenant_id AS tenant_id, - s.entity_id AS entity_id, - toDate(s.final_close_at) AS metric_date, - s.issue_id AS issue_id, - any(s.issue_type) AS issue_type, - -- Gates count/due-date/estimation measures to CURRENT-status-done - -- issues; duration measures cover every ever-closed issue. - any(s.status_category) = 'done' AS is_done, - toDate(s.final_close_at) AS close_date, - any(s.due_date) AS due_date, - any(s.time_estimate_seconds) AS time_estimate_seconds, - any(s.time_spent_seconds) AS time_spent_seconds, - sumIf(i.duration_seconds, i.interval_start < s.final_close_at) AS dev_seconds, - if(any(s.created_at) IS NULL, - CAST(NULL AS Nullable(Float64)), - toFloat64(greatest(toInt64(0), - dateDiff('second', any(s.created_at), any(s.final_close_at))))) AS lead_seconds, - if(any(s.created_at) IS NULL - OR minIf(i.interval_start, i.interval_start < s.final_close_at) IS NULL, - CAST(NULL AS Nullable(Float64)), - toFloat64(greatest(toInt64(0), - dateDiff('second', any(s.created_at), - minIf(i.interval_start, i.interval_start < s.final_close_at))))) AS pickup_seconds, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM issue_state AS s - LEFT JOIN status_intervals AS i - ON i.insight_source_id = s.insight_source_id - AND i.issue_id = s.issue_id - AND i.status_category = 'in_progress' - WHERE s.final_close_at IS NOT NULL - GROUP BY s.tenant_id, s.entity_id, s.issue_id, toDate(s.final_close_at) -), --- Day-grain estimation pct: avg estimate / avg spent, both averages pinned to --- the SAME set (done, positive estimate, logged time). pct outside (0, 200] --- emits nothing — blown estimates read as unknowable, not as signal. -estimation_day AS ( - SELECT - tenant_id, - entity_id, - metric_date, - 100 * avgIf(time_estimate_seconds, is_done AND ifNull(time_estimate_seconds, 0) > 0 AND time_spent_seconds IS NOT NULL) - / nullIf(avgIf(time_spent_seconds, is_done AND ifNull(time_estimate_seconds, 0) > 0 AND time_spent_seconds IS NOT NULL), 0) - AS estimation_pct, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM issue_facts - GROUP BY tenant_id, entity_id, metric_date -), --- A close is a transition into a done category, a reopen out of one; the --- first reopen after a close decides reopened-within-14d. -transitions AS ( - SELECT - insight_source_id, - issue_id, - interval_start AS event_at, - status_category, - lagInFrame(status_category) OVER ( - PARTITION BY insight_source_id, issue_id ORDER BY interval_start - ) AS prev_category - FROM status_intervals -), -closes AS ( - SELECT insight_source_id, issue_id, event_at AS close_at - FROM transitions - WHERE status_category = 'done' AND (prev_category IS NULL OR prev_category != 'done') -), -reopens AS ( - SELECT insight_source_id, issue_id, event_at AS reopen_at - FROM transitions - WHERE prev_category = 'done' AND (status_category != 'done' OR status_category IS NULL) -), -close_reopen AS ( - SELECT - s.tenant_id AS tenant_id, - s.entity_id AS entity_id, - toDate(c.close_at) AS metric_date, - toFloat64(1) AS close_event, - if(minIf(r.reopen_at, r.reopen_at > c.close_at) IS NOT NULL - AND minIf(r.reopen_at, r.reopen_at > c.close_at) <= c.close_at + INTERVAL 14 DAY, - toFloat64(1), CAST(NULL AS Nullable(Float64))) AS reopened_14d, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM closes AS c - INNER JOIN issue_state AS s - ON s.insight_source_id = c.insight_source_id AND s.issue_id = c.issue_id - LEFT JOIN reopens AS r - ON r.insight_source_id = c.insight_source_id AND r.issue_id = c.issue_id - GROUP BY s.tenant_id, s.entity_id, c.issue_id, c.close_at -), --- In-progress seconds per (assignee, calendar day); worklog-accuracy --- denominator. -in_progress_per_day AS ( - SELECT - s.tenant_id AS tenant_id, - s.entity_id AS entity_id, - day AS metric_date, - sum(toFloat64(greatest(toInt64(0), - dateDiff('second', - greatest(i.interval_start, toDateTime(day)), - least(i.interval_end, toDateTime(day) + toIntervalDay(1)))))) AS in_progress_seconds - FROM status_intervals AS i - INNER JOIN issue_state AS s - ON s.insight_source_id = i.insight_source_id AND s.issue_id = i.issue_id - ARRAY JOIN - arrayMap(d -> toDate(i.interval_start) + toIntervalDay(d), - range(toUInt32(dateDiff('day', toDate(i.interval_start), toDate(i.interval_end)) + 1))) AS day - WHERE i.status_category = 'in_progress' - GROUP BY s.tenant_id, s.entity_id, day -), -worklog_per_day AS ( - SELECT - u.tenant_id AS tenant_id, - u.email AS entity_id, - toDate(w.work_date) AS metric_date, - sum(ifNull(w.duration_seconds, 0)) AS worklog_seconds - FROM {{ ref('class_task_worklogs') }} AS w FINAL - INNER JOIN task_users AS u - ON u.insight_source_id = w.insight_source_id AND u.user_id = w.author_id - WHERE w.work_date IS NOT NULL - GROUP BY u.tenant_id, u.email, toDate(w.work_date) -), --- Both sides of the accuracy ratio are gated to days with in-progress time — --- logging on a day with no tracked development is not comparable. -worklog_flow AS ( - SELECT - coalesce(ip.tenant_id, wl.tenant_id) AS tenant_id, - coalesce(ip.entity_id, wl.entity_id) AS entity_id, - coalesce(ip.metric_date, wl.metric_date) AS metric_date, - ifNull(ip.in_progress_seconds, 0) AS in_progress_seconds, - ifNull(wl.worklog_seconds, 0) AS worklog_seconds, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM in_progress_per_day AS ip - FULL OUTER JOIN worklog_per_day AS wl - ON wl.tenant_id = ip.tenant_id - AND wl.entity_id = ip.entity_id - AND wl.metric_date = ip.metric_date -), --- Snapshot at build date: open (non-done) issues idle more than 14 days. -stale AS ( - SELECT - s.tenant_id AS tenant_id, - s.entity_id AS entity_id, - today() AS metric_date, - toFloat64(count()) AS stale_count, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM issue_state AS s - WHERE (s.status_category IS NULL OR s.status_category != 'done') - AND s.last_status_event_at IS NOT NULL - AND dateDiff('day', s.last_status_event_at, now()) > 14 - GROUP BY s.tenant_id, s.entity_id -), -value_measures AS ( - {{ sum_measure('tasks_closed', 'issue_facts', 'if(is_done, 1, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('bugs_fixed', 'issue_facts', "if(is_done AND issue_type = 'Bug', 1, NULL)", 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('due_date_on_time', 'issue_facts', 'if(is_done AND due_date IS NOT NULL AND close_date <= due_date, 1, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('due_date_with_due', 'issue_facts', 'if(is_done AND due_date IS NOT NULL, 1, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('slip_days_total', 'issue_facts', 'if(is_done AND due_date IS NOT NULL AND close_date > due_date, toFloat64(dateDiff(\'day\', due_date, close_date)), NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('late_count', 'issue_facts', 'if(is_done AND due_date IS NOT NULL AND close_date > due_date, 1, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('estimation_error_pct', 'estimation_day', 'if(estimation_pct > 0 AND estimation_pct <= 200, abs(100 - estimation_pct), NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('estimation_samples', 'estimation_day', 'if(estimation_pct > 0 AND estimation_pct <= 200, 1, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('flow_dev_seconds', 'issue_facts', 'if(ifNull(dev_seconds, 0) > 0 AND ifNull(lead_seconds, 0) > 0, dev_seconds, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('flow_lead_seconds', 'issue_facts', 'if(ifNull(dev_seconds, 0) > 0 AND ifNull(lead_seconds, 0) > 0, lead_seconds, NULL)', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('close_events', 'close_reopen', 'close_event', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('reopened_within_14d', 'close_reopen', 'reopened_14d', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('worklog_seconds', 'worklog_flow', 'worklog_seconds', 'no_dimensions', where='in_progress_seconds > 0') }} - - UNION ALL - - {{ sum_measure('in_progress_seconds', 'worklog_flow', 'in_progress_seconds', 'no_dimensions', where='in_progress_seconds > 0') }} - - UNION ALL - - {{ sum_measure('stale_in_progress', 'stale', 'stale_count', 'no_dimensions') }} - - UNION ALL - - {{ event_measure('dev_time_hours', 'issue_facts', 'dev_seconds / 3600.0', 'no_dimensions', where='ifNull(dev_seconds, 0) > 0') }} - - UNION ALL - - {{ event_measure('resolution_days', 'issue_facts', 'lead_seconds / 86400.0', 'no_dimensions', where='ifNull(lead_seconds, 0) > 0') }} +SELECT + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + toNullable(sum(contribution)) AS value, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions +FROM {{ ref('task_metric_evidence') }} +WHERE measure_key NOT IN ('dev_time_hours', 'resolution_days', 'pickup_days') +GROUP BY tenant_id, source_key, entity_type, entity_id, metric_date, measure_key, dimensions - UNION ALL +UNION ALL - {{ event_measure('pickup_days', 'issue_facts', 'pickup_seconds / 86400.0', 'no_dimensions', where='pickup_seconds IS NOT NULL') }} -) SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'task' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, measure_key, - value, - CAST(NULL AS Nullable(String)) AS subject_key, + contribution AS value, + subject_key, dimensions -FROM value_measures -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL +FROM {{ ref('task_metric_evidence') }} +WHERE measure_key IN ('dev_time_hours', 'resolution_days', 'pickup_days') diff --git a/src/ingestion/gold/task_worklog_flow.sql b/src/ingestion/gold/task_worklog_flow.sql new file mode 100644 index 000000000..9c5f22ea5 --- /dev/null +++ b/src/ingestion/gold/task_worklog_flow.sql @@ -0,0 +1,71 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'entity_id', 'metric_date'], + schema='insight', + alias='task_worklog_flow', + tags=['gold'], + query_settings={ + 'join_use_nulls': 1, + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +task_users AS ( + SELECT + tenant_id, + insight_source_id, + user_id, + lower(email) AS email + FROM {{ ref('class_task_users') }} FINAL + WHERE email LIKE '%@%' +), +in_progress_per_day AS ( + SELECT + s.tenant_id AS tenant_id, + s.entity_id AS entity_id, + day AS metric_date, + sum(toFloat64(greatest(toInt64(0), + dateDiff('second', + greatest(i.interval_start, toDateTime(day)), + least(i.interval_end, toDateTime(day) + toIntervalDay(1)))))) AS in_progress_seconds + FROM {{ ref('task_status_spans') }} AS i + INNER JOIN {{ ref('task_issue_state') }} AS s + ON s.insight_source_id = i.insight_source_id AND s.issue_id = i.issue_id + ARRAY JOIN + arrayMap(d -> toDate(i.interval_start) + toIntervalDay(d), + range(toUInt32(greatest( + toInt64(0), + dateDiff('day', toDate(i.interval_start), toDate(i.interval_end)) + 1 + )))) AS day + WHERE i.status_category = 'in_progress' + GROUP BY s.tenant_id, s.entity_id, day +), +worklog_per_day AS ( + SELECT + u.tenant_id AS tenant_id, + u.email AS entity_id, + toDate(w.work_date) AS metric_date, + sum(ifNull(w.duration_seconds, 0)) AS worklog_seconds + FROM {{ ref('class_task_worklogs') }} AS w FINAL + INNER JOIN task_users AS u + ON u.insight_source_id = w.insight_source_id AND u.user_id = w.author_id + WHERE w.work_date IS NOT NULL + GROUP BY u.tenant_id, u.email, toDate(w.work_date) +) +SELECT + assumeNotNull(coalesce(ip.tenant_id, wl.tenant_id)) AS tenant_id, + assumeNotNull(coalesce(ip.entity_id, wl.entity_id)) AS entity_id, + assumeNotNull(coalesce(ip.metric_date, wl.metric_date)) AS metric_date, + ifNull(ip.in_progress_seconds, 0) AS in_progress_seconds, + ifNull(wl.worklog_seconds, 0) AS worklog_seconds +FROM in_progress_per_day AS ip +FULL OUTER JOIN worklog_per_day AS wl + ON wl.tenant_id = ip.tenant_id + AND wl.entity_id = ip.entity_id + AND wl.metric_date = ip.metric_date diff --git a/src/ingestion/gold/wiki_metric_evidence.sql b/src/ingestion/gold/wiki_metric_evidence.sql new file mode 100644 index 000000000..378cc8d01 --- /dev/null +++ b/src/ingestion/gold/wiki_metric_evidence.sql @@ -0,0 +1,136 @@ +{{ config( + materialized='table', + engine='MergeTree', + order_by=['tenant_id', 'source_key', 'measure_key', 'entity_id', 'metric_date', 'record_id'], + schema='insight', + alias='wiki_metric_evidence', + tags=['gold'], + query_settings={ + 'max_memory_usage': 1610612736, + 'max_threads': 4, + 'max_bytes_before_external_group_by': 805306368, + 'max_bytes_before_external_sort': 805306368 + } +) }} + + +WITH +pages AS ( + SELECT + tenant_id, + source_id, + page_id, + lower(author_email) AS entity_id + FROM {{ ref('class_wiki_pages') }} FINAL + WHERE author_email LIKE '%@%' +), +page_creations AS ( + SELECT + tenant_id, + source_id, + page_id, + title, + lower(author_email) AS entity_id, + toDate(created_at) AS metric_date, + created_at AS observed_at, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM {{ ref('class_wiki_pages') }} FINAL + WHERE author_email LIKE '%@%' + AND created_at IS NOT NULL + AND page_id IS NOT NULL +), +activity AS ( + SELECT + tenant_id, + lower(author_email) AS entity_id, + day AS metric_date, + total_edits, + pages_edited, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM {{ ref('class_wiki_activity') }} FINAL + WHERE author_email LIKE '%@%' + AND day IS NOT NULL +), +engagement AS ( + SELECT + e.tenant_id AS tenant_id, + p.entity_id AS entity_id, + e.day AS metric_date, + e.total_comments AS total_comments, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions + FROM ( + SELECT + tenant_id, + source_id, + page_id, + day, + total_comments + FROM {{ ref('class_wiki_engagement') }} FINAL + WHERE day IS NOT NULL + ) AS e + INNER JOIN pages AS p + ON e.tenant_id = p.tenant_id + AND e.source_id = p.source_id + AND e.page_id = p.page_id +), +value_measures AS ( + {{ sum_measure('edits', 'activity', 'total_edits', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('pages_edited', 'activity', 'pages_edited', 'no_dimensions') }} + + UNION ALL + + {{ sum_measure('comments', 'engagement', 'total_comments', 'no_dimensions') }} +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'wiki' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + concat( + toString(metric_date), + ':', + measure_key, + ':', + hex(sipHash128(toString(arrayMap(d -> tuple(d.1, d.2), dimensions)))) + ) AS record_id, + measure_key AS record_kind, + 'source_summary' AS granularity, + replaceAll(measure_key, '_', ' ') AS record_label, + value AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions, + CAST(map() AS Map(String, String)) AS details +FROM value_measures +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL + +UNION ALL + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'wiki' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + toNullable(toDateTime64(observed_at, 3)) AS observed_at, + 'pages_created' AS measure_key, + concat(coalesce(toString(source_id), ''), ':', assumeNotNull(page_id), ':pages_created') AS record_id, + 'page' AS record_kind, + 'event' AS granularity, + if(coalesce(title, '') = '', assumeNotNull(page_id), assumeNotNull(title)) AS record_label, + toNullable(toFloat64(1)) AS contribution, + CAST(NULL AS Nullable(String)) AS subject_key, + no_dimensions AS dimensions, + map( + 'ref', assumeNotNull(page_id), + 'title', coalesce(title, '') + ) AS details +FROM page_creations +WHERE tenant_id IS NOT NULL diff --git a/src/ingestion/gold/wiki_metric_observations.sql b/src/ingestion/gold/wiki_metric_observations.sql index c38208167..10f6ec633 100644 --- a/src/ingestion/gold/wiki_metric_observations.sql +++ b/src/ingestion/gold/wiki_metric_observations.sql @@ -14,136 +14,16 @@ } ) }} --- Source measure observations for the unified metrics runtime, wiki family. --- Reads the wiki class contracts only (class_wiki_pages, class_wiki_activity, --- class_wiki_engagement); every measure is emitted through the shape macros --- in macros/metric_observation_measures.sql. No dimensions: wiki sources --- (Confluence, Outline) feed one undifferentiated family. --- --- Materialized as a sorted table: the pipeline runs once per dbt build — --- the only time the silver inputs can have changed — and the ordering key --- mirrors the runtime's filter shape (source_key, measure_key, entity_id, --- metric_date), so single-measure queries read index-pruned ranges. --- --- Grain per measure: --- day-grain sums (creation date, page author): pages_created (1/page — --- the page object's own author/created_at are the --- canonical creation facts; a version-derived proxy --- undercounts imported pages) --- day-grain sums (edit date, version author): edits (logical edit --- sessions — autosave bursts collapsed in silver, see --- class_wiki_activity), pages_edited (distinct pages --- touched that day) --- day-grain sums (comment date, page author): comments (engagement --- RECEIVED on the person's pages — footer + inline + --- replies; the commenter is deliberately not the entity, --- see class_wiki_engagement's page-centric design note) --- --- Attribution: entity_id = lower(author_email); only email-shaped keys pass. --- Confluence resolves emails through the Jira directory join in staging and --- yields NULL on tenants without Jira — those rows are excluded as --- unmatchable rather than carried as dead entities (cohorts and API requests --- address people by email). Outline resolves from its own user stream. --- --- Memory shape: every class read keeps FINAL (ReplacingMergeTree dedup) over --- a pruned column set. class_wiki_activity is already (author, day) grain, --- so its sums are near-free. The single join in the model attributes --- page-day comment rollups to the page author: engagement ⋈ pages on --- (tenant_id, source_id, page_id) — source_id is part of the key so a --- page_id colliding across two wiki instances of one tenant cannot fan out. --- --- Peer measurability (who enters a metric's peer pool) is decided HERE, by --- row emission — the runtime never fabricates zeros. All four measures are --- engagement-gated: a row exists only where the source recorded authorship, --- editing, or received comments. Rostered-but-inactive people take no --- standing rather than dragging peer medians toward zero. - -WITH --- Page -> author attribution for the comment join. No created_at gate: --- comment attribution needs only the page's author, so a page snapshot --- without a creation timestamp still receives its comments. -pages AS ( - SELECT - tenant_id, - source_id, - page_id, - lower(author_email) AS entity_id - FROM {{ ref('class_wiki_pages') }} FINAL - WHERE author_email LIKE '%@%' -), --- Dated branch for pages_created; the creation date is the metric date, so --- only here does a missing created_at exclude the page. -page_creations AS ( - SELECT - tenant_id, - lower(author_email) AS entity_id, - toDate(created_at) AS metric_date, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM {{ ref('class_wiki_pages') }} FINAL - WHERE author_email LIKE '%@%' - AND created_at IS NOT NULL -), -activity AS ( - SELECT - tenant_id, - lower(author_email) AS entity_id, - day AS metric_date, - total_edits, - pages_edited, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM {{ ref('class_wiki_activity') }} FINAL - WHERE author_email LIKE '%@%' - AND day IS NOT NULL -), -engagement AS ( - SELECT - e.tenant_id AS tenant_id, - p.entity_id AS entity_id, - e.day AS metric_date, - e.total_comments AS total_comments, - CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions - FROM ( - SELECT - tenant_id, - source_id, - page_id, - day, - total_comments - FROM {{ ref('class_wiki_engagement') }} FINAL - WHERE day IS NOT NULL - ) AS e - INNER JOIN pages AS p - ON e.tenant_id = p.tenant_id - AND e.source_id = p.source_id - AND e.page_id = p.page_id -), -value_measures AS ( - {{ sum_measure('pages_created', 'page_creations', '1', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('edits', 'activity', 'total_edits', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('pages_edited', 'activity', 'pages_edited', 'no_dimensions') }} - - UNION ALL - - {{ sum_measure('comments', 'engagement', 'total_comments', 'no_dimensions') }} -) SELECT - assumeNotNull(tenant_id) AS tenant_id, - 'wiki' AS source_key, - 'person' AS entity_type, - assumeNotNull(entity_id) AS entity_id, - assumeNotNull(metric_date) AS metric_date, + tenant_id, + source_key, + entity_type, + entity_id, + metric_date, CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, measure_key, - value, + toNullable(sum(contribution)) AS value, CAST(NULL AS Nullable(String)) AS subject_key, dimensions -FROM value_measures -WHERE tenant_id IS NOT NULL - AND entity_id IS NOT NULL - AND metric_date IS NOT NULL +FROM {{ ref('wiki_metric_evidence') }} +GROUP BY tenant_id, source_key, entity_type, entity_id, metric_date, measure_key, dimensions diff --git a/src/ingestion/tests/e2e/api/test_metric_drilldown.py b/src/ingestion/tests/e2e/api/test_metric_drilldown.py new file mode 100644 index 000000000..48ba58a32 --- /dev/null +++ b/src/ingestion/tests/e2e/api/test_metric_drilldown.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import pytest + +from api.endpoint_helpers import text_body_request + +pytestmark = pytest.mark.api + + +def _request() -> dict: + return { + "metric_key": "tasks.closed", + "entity": {"type": "person", "id": ""}, + "period": {"from": "2026-01-01", "to": "2026-01-31"}, + "filters": [], + "display_dimensions": [], + "limit": 100, + } + + +def test_metric_drilldown_400_empty_entity(api) -> None: + response = api.post("/v1/metric-drilldown", json=_request()) + assert response.status_code == 400, f"status={response.status_code} body={response.text}" + + +def test_metric_drilldown_415_wrong_content_type(api) -> None: + response = text_body_request(api, "POST", "/v1/metric-drilldown") + assert response.status_code == 415, f"status={response.status_code} body={response.text}" + + +def test_metric_drilldown_export_400_empty_entity(api) -> None: + request = _request() + request["format"] = "csv" + request.pop("limit") + response = api.post("/v1/metric-drilldown/export", json=request) + assert response.status_code == 400, f"status={response.status_code} body={response.text}" + + +def test_metric_drilldown_export_415_wrong_content_type(api) -> None: + response = text_body_request(api, "POST", "/v1/metric-drilldown/export") + assert response.status_code == 415, f"status={response.status_code} body={response.text}"