diff --git a/AGENTS.md b/AGENTS.md index 6d64014e7..9e7a16f73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,3 +6,6 @@ cf-studio-path = ".cf-studio" ALWAYS resolve and enforce prerequisites of skills/workflows/commands BEFORE applying user intent. +## Project Rules + +ALWAYS open and follow `docs/domain/metrics/specs/DESIGN.md` WHEN adding or changing metrics, metric definitions, metric seeds, observation sources, or gold metric views diff --git a/charts/insight/templates/clickhouse-migrate-job.yaml b/charts/insight/templates/clickhouse-migrate-job.yaml index 39bf65613..c333bfee1 100644 --- a/charts/insight/templates/clickhouse-migrate-job.yaml +++ b/charts/insight/templates/clickhouse-migrate-job.yaml @@ -36,6 +36,13 @@ Idempotent: migrations are re-run on every upgrade and are CREATE OR REPLACE / IF NOT EXISTS; placeholders are guarded by `ch_table_exists`. No migration ledger — same contract as the legacy init.sh path. +The script's final step builds the dbt gold models (`dbt run --select +tag:gold`) so dbt-owned views exist right after deploy instead of after +the first connector sync — the analytics service reports metrics as +unavailable while an observation view is missing. The toolbox image +already bundles the dbt project; placeholders guarantee the view DDL +type-checks on a fresh cluster. + Gated on `clickhouse.runMigrations` (default true) AND the availability of `ingestion.toolboxImage` (the image that ships the migration SQL). */}} diff --git a/docs/components/backend/analytics/openapi.json b/docs/components/backend/analytics/openapi.json index 58a0ee136..447fcac82 100644 --- a/docs/components/backend/analytics/openapi.json +++ b/docs/components/backend/analytics/openapi.json @@ -1855,6 +1855,99 @@ "summary": "List queryable columns for a table" } }, + "/v1/metric-results": { + "post": { + "operationId": "analytics_api.metric_results.create", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Metric results" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Compute metric results" + } + }, "/v1/metrics": { "get": { "operationId": "analytics_api.metrics.list", diff --git a/docs/domain/README.md b/docs/domain/README.md index dde5fe40f..eb273bb0e 100644 --- a/docs/domain/README.md +++ b/docs/domain/README.md @@ -9,3 +9,4 @@ Domain-level specifications for the Insight platform. Each domain represents a b | [`ingestion/`](ingestion/) | Data pipeline from source APIs to Silver step 1 (Airbyte + Argo Workflows + dbt) | Accepted | | [`connector/`](connector/) | Connector development: Insight Connector packages, nocode and CDK patterns, packaging, debugging | Accepted | | [`identity-resolution/`](identity-resolution/) | Person identity matching and resolution across sources | Proposed | +| [`metrics/`](metrics/) | Unified metrics: typed registry + generic result runtime over source measure observations (`/v1/metric-results`) | Accepted | diff --git a/docs/domain/metrics/README.md b/docs/domain/metrics/README.md new file mode 100644 index 000000000..226d6b815 --- /dev/null +++ b/docs/domain/metrics/README.md @@ -0,0 +1,78 @@ +# Metrics Domain + +The unified metrics system: metrics are defined once in a typed registry, +computed by one generic runtime over normalized source measure observations, +and served self-describing through `POST /v1/metric-results`. All new metrics +are authored through this system. + +## Concepts + +Three layers, one handshake in the middle. + +An **observation** is a recorded fact: "alice, June 3rd, Claude Code, +`accepted_edit_actions` = 17". It says what happened, to whom, when, and how +much — and deliberately nothing about what it means. No name a user would +see, no formula, no chart. The data side (dbt gold models over silver +classes) produces millions of these in one fixed row shape, and its only job +is to record them honestly. + +A **definition** is a catalog card holding meaning: "there is a metric +`ai.tool_acceptance_rate`; compute it as `accepted_edit_actions` divided by +`tool_use_offered`, times 100; show as percent; higher is better; may be +split by tool; compare within org unit". No data lives here — only meaning +and instructions, stored in the registry and authored as one Rust struct per +metric. + +A **metric result** is the computed answer the user sees. It is not stored +anywhere: at request time the runtime applies a definition to the matching +observations — "alice, January: 312 ÷ 405 = 77%" — and returns it labeled and +ready to render. + +The split exists because one fact serves many meanings and one meaning serves +many questions. The same `accepted_edit_actions` observation is the whole +value of `ai.accepted_edit_actions` and the numerator of +`ai.tool_acceptance_rate` — recorded once, interpreted twice. The same +definition answers any period, person, team, dimension split, or peer +comparison without new code. Each side changes without touching the other: +renaming a metric edits a card; a vendor API change fixes fact recording +while every card keeps working. + +The handshake is the source measure observation contract (see +[`specs/DESIGN.md`](specs/DESIGN.md)): the data side promises to emit facts +in that shape, definitions reference facts only by measure key, and the +runtime can therefore connect any definition to any matching facts without +either side knowing the other exists. + +| | Observation | Definition | Metric result | +|---|---|---|---| +| What | a fact | the meaning of facts | the computed answer | +| Lives | ClickHouse views over silver (computed on read, nothing stored) | registry (MariaDB, seeded from Rust) | nowhere — made per request | +| Knows | what happened | what it is called, how to compute, how to show | both, combined | +| Authored by | connector + gold model | one struct per metric | nobody — the runtime derives it | + +## Documents + +| Document | Description | +|---|---| +| [`specs/DESIGN.md`](specs/DESIGN.md) | System contract: observation contract, registry model, computations, result API, validation, authoring guide ("Adding a Metric") | + +## Implementation + +| Layer | Location | +|---|---| +| Metric registry (builtin seeds) | [`src/backend/services/analytics/src/domain/metric_definitions/builtin.rs`](../../../src/backend/services/analytics/src/domain/metric_definitions/builtin.rs) | +| Definition loading, reconciler, schema validator | [`src/backend/services/analytics/src/domain/metric_definitions/`](../../../src/backend/services/analytics/src/domain/metric_definitions/) | +| Result runtime (validation, query compiler, response builder) | [`src/backend/services/analytics/src/domain/metric_results/`](../../../src/backend/services/analytics/src/domain/metric_results/) | +| Result endpoint | [`src/backend/services/analytics/src/api/metric_results.rs`](../../../src/backend/services/analytics/src/api/metric_results.rs) | +| Registry schema migration | [`src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs`](../../../src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs) | +| Managed observation sources (dbt gold models) | [`src/ingestion/gold/`](../../../src/ingestion/gold/) | +| Class-contract data-quality tests | [`src/ingestion/dbt/tests/ai/`](../../../src/ingestion/dbt/tests/ai/) | + +## Boundaries + +- The AI class contracts feeding the observation models are documented in + [`src/ingestion/silver/ai/schema.yml`](../../../src/ingestion/silver/ai/schema.yml) + (activity invariant, label and conversation-count semantics). +- The legacy metric path ([`metric-catalog/`](../metric-catalog/) + + ad-hoc `insight.*` gold views) is frozen for new metrics and remains only + until its consumers migrate. diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md new file mode 100644 index 000000000..62ad4323b --- /dev/null +++ b/docs/domain/metrics/specs/DESIGN.md @@ -0,0 +1,468 @@ +# Technical Design — Metrics + +Status: active implementation contract. + +The metrics system computes metric result views from typed metric definitions +and normalized source measure observations. Metrics are authored, requested, +and rendered through one structured path: a registry defines metric semantics, +dbt gold models emit source measure observations, and one generic runtime +compiles and serves every metric. + +New metrics MUST be added through this system. The legacy path — ad-hoc +`insight.*` gold views in `src/ingestion/scripts/migrations/` plus +`metric_catalog` seed migrations — is frozen for new metrics and remains only +until its existing consumers migrate. + +## Goals + +- Define metrics once and query them through one generic runtime. +- Model metrics by semantic computation, not by current UI cards. +- Support multiple entity types, with `person` as the first consumer. +- Keep backend responses self-describing enough for frontend rendering. +- Keep chart choice and layout out of backend metric contracts. +- Use typed Rust and TypeScript unions for states that cannot coexist. + +## Source Measure Observation Contract + +Managed observation sources expose rows shaped like: + +```sql +tenant_id String, +source_key String, +entity_type String, +entity_id String, +metric_date Date, +observed_at Nullable(DateTime64(3)), +measure_key String, +value Nullable(Float64), +subject_key Nullable(String), +dimensions Array(Tuple(key String, value String, label Nullable(String))) +``` + +Rules: + +- Observations belong to source measures, not final metrics. +- `source_key` identifies the logical source. +- `measure_key` identifies the source measure. +- `entity_type` and `entity_id` identify the measured entity. +- `observed_at` is reserved for future point-in-time semantics. +- `subject_key` is reserved for future distinct-count semantics. +- A row is emitted only when the source provides a value; `value` is never + NULL (the column stays nullable in the contract). +- Dimension values and labels come from class-contract columns declared by + staging models; gold does not synthesize fallbacks. +- Observations do not contain chart metadata. +- Observations do not contain cohort membership. Peer comparison reads the + cohort view directly. + +## Managed Source Ownership + +Managed observation sources and the cohort view are dbt gold models +(`src/ingestion/gold/`), materialized as views in the `insight` database: + +- `insight.ai_metric_observations` +- `insight.metric_entity_cohorts_current` + +dbt owns lineage to silver, build ordering, column documentation, and data +tests (including cohort uniqueness). The backend owns the registry, query +compilation, and runtime schema validation against these relations. Column +changes are coordinated changes: dbt model + `schema.yml` + backend +`OBSERVATION_COLUMNS`/`COHORT_COLUMNS` + this document. + +Gold models are built at deploy time by the ClickHouse migrate hook +(`dbt run --select tag:gold`, final step of +`src/ingestion/scripts/apply-ch-migrations.sh`), so the views exist before +any connector sync — bronze/silver placeholders guarantee the DDL +type-checks on a fresh cluster. Per-connector scoped dbt runs keep them +current afterwards. + +The cohort view is unique per `(tenant_id, entity_type, entity_id, +cohort_key)`. The peer query relies on this; a dbt build-integrity test +asserts it. + +## Computations + +The computation vocabulary is closed and fully executable: + +```text +sum +ratio +``` + +Semantics: + +- `sum`: sum one numeric measure. +- `ratio`: aggregate numerator and denominator measures first, then divide. + +Ratios use: + +```text +sum(numerator) / nullIf(sum(denominator), 0) * scale +``` + +They are not averages of row-level ratios. + +Ratio numerator and denominator inputs must resolve to measures of the same +source. Cross-source ratios are a configuration error. + +Extending the vocabulary (anticipated kinds: count-distinct over +`subject_key`, distribution statistics, point-in-time gauges over +`observed_at`, derived expressions over other metrics) is one coordinated +change: a `ComputationSpec` variant, a compiler arm, the `computation_type` +DB enum, a shape macro if the observation shape is new, and the response +`computation` tag. Nothing is stored before it executes. + +## Storage Model + +Metric definitions are stored separately from legacy metric/catalog concepts. + +Tables: + +```text +metric_sources +metric_source_measures +metric_source_dimensions +metric_definitions +metric_definition_inputs +metric_definition_dimensions +``` + +`metric_sources` stores typed source refs. + +`metric_source_measures` stores measures available from a source. + +`metric_source_dimensions` stores dimensions available from a source. + +`metric_definitions` stores metric metadata and computation type: + +```text +metric_key +label +description +explanation +unit +format +direction +entity_type +computation_type +scale +peer_cohort_key +origin +is_enabled +schema_status +schema_error_code +``` + +`metric_definition_inputs` maps input roles to source measures: + +```text +value +numerator +denominator +``` + +`metric_definition_dimensions` maps metrics to source dimensions. + +Rules: + +- Product definitions have `tenant_id = NULL`. +- Tenant definitions override product definitions for the same key. +- Disabled definitions, sources, or measures are unavailable. +- Schema-error definitions, sources, or measures are unavailable. +- A disabled or schema-error tenant definition falls back to the product + definition for the same key instead of shadowing it. +- Raw DB source refs are converted into typed backend enums before SQL generation. + +## Builtin Seed Reconciliation + +Builtin definitions are declared in one code registry +(`src/backend/services/analytics/src/domain/metric_definitions/builtin.rs`) +and converged into the DB by a startup reconciler, not by migrations. +Migrations own schema only. + +Rules: + +- The reconciler runs synchronously after migrations, before serving traffic, + and on the `migrate` CLI command. +- Upserts are idempotent and race-safe across replicas. +- Builtin sources, measures, and definitions absent from the registry are + disabled, never deleted. +- Source dimension rows have no enabled flag; rows removed from the registry + stay in place and are inert unless a definition links them. +- Tenant-owned rows are never touched by reconciliation. +- Warm environments converge to the registry state on every deploy. + +## Result API + +Endpoint: + +```http +POST /v1/metric-results +``` + +Request: + +```ts +type MetricResultsRequest = { + entity: { type: string; ids: string[] } + period: { from: string; to: string } + metrics: Array<{ + metric_key: string + views: Array< + | { view: "period" } + | { view: "peer"; cohort_key?: string } + | { view: "timeseries"; bucket?: "day" | "week" | "month"; dimensions?: string[] } + | { view: "breakdown"; dimensions: string[] } + > + }> +} +``` + +Response: + +```ts +type MetricResult = { + metric_key: string + label: string + description?: string + explanation?: string + unit: string | null + format: "integer" | "decimal" | "currency" | "percent" + direction: "higher_is_better" | "lower_is_better" | "neutral" + views: MetricResultView[] +} & ( + | { computation: "sum" } + | { computation: "ratio"; scale: number } +) +``` + +The computation tag and its fields are flattened into the result object; a +serde wire-shape test in `metric_results/builder.rs` pins this layout. + +View values use `entity_id`, not person-specific fields. + +## Runtime Flow + +1. Resolve tenant from request context. +2. Validate entity, period, metric keys, view specs, and dimensions. +3. Load visible metric definitions from DB. +4. Convert DB rows into Rust discriminated unions. +5. Compile one ClickHouse query per requested metric view. +6. Execute queries with bounded concurrency. +7. Shape rows into typed result views. +8. Enforce final response row cap. +9. Return metrics in request order. + +Execution rules: + +- `sum` no rows returns `0`. +- `ratio` missing or zero denominator returns `null`. +- Ungrouped timeseries are dense per requested entity and bucket. +- Dimensioned timeseries are dense per requested entity, observed dimension group, and bucket. +- Rows missing a requested dimension group under value `__unknown__` with + label `Unknown` (runtime guard; the schema validator's coverage probe makes + this rare). +- Breakdown returns observed dimension groups only. +- The cohort view scopes who counts as a peer; only members with observed + values contribute to the percentiles. The peer query never fabricates zero + observations: absence of rows is indistinguishable from "not covered by the + source" (no seat, no account), so inventing zeros would rank people the + data never measured. A source for which covered-but-inactive genuinely + means zero can emit explicit zero observations — the coverage knowledge + lives in the connector, not the runtime. +- Target entities missing cohort membership are omitted from peer values. +- Target entities without observed values get a null `target_value`. +- Null values are excluded from peer percentiles and `n`. +- Peer percentiles and min/max are suppressed (returned as null) when the + peer pool has fewer than 5 distinct observed members; `n` reports that + distinct count. Quartiles over a handful of people are noise, and tiny + pools disclose individual values. Enforced server-side so every consumer + inherits it, and counted with `uniqExact` so duplicate cohort membership + rows can neither inflate the pool nor defeat the floor. + +## Validation + +Request caps, checked before any per-request enumeration work: + +- at most 50 metrics per request. +- at most 1000 entity ids per request. +- at most 400 days per period. + +Entity id normalization is a property of the entity type: `person` ids are +emails and are trimmed and lowercased to match observation sources, which +emit lowercased emails; other entity types are trimmed only. + +Reject with a client error when: + +- entity type or ids are empty. +- a request cap is exceeded. +- period dates are invalid or reversed. +- metrics are empty. +- metric keys are empty, duplicated, unknown, disabled, or schema-error. +- a metric requests no views. +- a metric requests the same view twice. +- a requested dimension is empty, duplicated, or not declared for the metric. +- a breakdown has no dimensions. +- a peer view has no requested or default cohort key. +- projected or final result size exceeds the row cap. + +## Authorization + +v1 decision: any authenticated member of a tenant may query metric results +for any entity ids in that tenant. Peer views expose aggregates only (no peer +entity ids); period, timeseries, and breakdown views expose per-entity values. +Entity-level scoping (self, reports, role-based) is deferred to the real +authorization system; this endpoint must adopt it when it lands. + +Warehouse tenant isolation is not implemented platform-wide: compiled queries +do not filter on the warehouse `tenant_id` column, matching the rest of the +platform's single-tenant posture. The control-plane tenant id has no defined +mapping to the warehouse `tenant_id` strings stamped at ingestion; defining +that mapping and adding the predicate (one place: the compiler's shared WHERE +clause) is the multi-tenant unlock. The observation and cohort contracts keep +the column so that change needs no contract migration. + +Schema validation checks: + +- managed source refs map to backend source enums. +- source observation views expose required columns. +- generic cohort view exposes required columns. +- declared dimensions are present on every recent row of each observed input + measure; a covered-measure gap is a schema error. +- input measures without recent observations downgrade the definition to + `unchecked`, never `error`: filtered measures legitimately go quiet, and + absence of data is indistinguishable from an unemitted measure. +- probe failures never overwrite a previously established status. +- the validator sweeps periodically, not once at startup: managed relations + are dbt-created and may appear after the service boots (fresh deploys) or + regress later (a bad model change); both converge within one sweep with no + restart. +- warehouse diagnostics stay server-side. + +## Adding a Metric + +Built-in metrics are authored by Insight developers through the registry and +the managed observation models. There are exactly three cases; pick the first +one that applies. + +### Case 1: metric over an existing measure + +The measure already appears in a managed observation source (check the +`measures` list of the source in `builtin.rs` and the emitting gold model). + +1. Add one `MetricSeed` to `BUILTIN_METRICS` in + `src/backend/services/analytics/src/domain/metric_definitions/builtin.rs`: + metric key (`namespace.metric_name`, lowercase snake case), label, + description, unit, format, direction, entity type, computation type, + input role mapping to the measure, allowed dimensions, peer cohort key. +2. Run `cargo test -p analytics` — the registry invariant tests validate + key shapes, input/measure references, and computation field combinations. + +The reconciler seeds the definition on the next deploy. No SQL, no migration, +no dbt change. + +### Case 2: new measure from an existing source + +The source exists but does not emit the measure yet. + +1. Add the measure branch to the source's gold model in `src/ingestion/gold/`: + one `UNION ALL` entry calling a shape macro from + `src/ingestion/dbt/macros/metric_observation_measures.sql` — + `sum_measure(measure_key, relation, value_expr, dimensions_col, + where=none)` for aggregated numerics, `presence_measure(measure_key, + relations)` for row-existence markers. Every branch is a shape-macro call; + a new macro is added only when a new computation kind becomes executable. + Read only class-contract columns; never vendor-specific ones — if the fact + you need is not in the class contract, extend the class contract first + (staging models declare semantics, see the class `schema.yml`). +2. Add the `measure_key` to the gold model's `schema.yml` `accepted_values` + test. +3. Add the measure key to the source's `measures` list in `builtin.rs`. +4. Add the `MetricSeed` as in case 1. +5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation + commands). + +### Case 3: new observation source + +The metric family reads data no managed source covers. + +1. Create a dbt gold model in `src/ingestion/gold/` emitting the source + measure observation contract, `schema=insight`, `ref()`-ing silver models + (medallion layering rules: `docs/domain/ingestion-data-flow/specs/DESIGN.md`). + Document columns and measure keys in `src/ingestion/gold/schema.yml`. +2. Add an `ObservationSource` enum variant and `from_ref`/`table_ref` mapping + in `src/backend/services/analytics/src/domain/metric_definitions/definition.rs` + (the `db_strings_round_trip` test covers the new pair). +3. Add a `BuiltinSource` (source + measures + dimensions) to `builtin.rs`. +4. Add `MetricSeed`s as in case 1. +5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation + commands). The runtime schema validator probes the new relation at + startup. + +### Rules that hold for every case + +- No metric-key-specific branches in runtime code. +- No vendor names, vendor columns, or label mappings in gold models — labels + and taxonomy come from class-contract columns declared by staging. +- Measure filter predicates (`where=` on shape macros) may reference only + class-contract dimension columns and their normalized values — never vendor + columns, tool names, or label text. +- Adding a class-contract column to existing staging models needs a history + backfill plan: columns derived from source data require re-materialization + — major-bump every affected connector (ADR-0015 dispatches a scoped + one-shot full refresh; CDK connectors need an explicit invocation until + the toolkit closes its semver-storage gap). Declared-constant columns + (labels) are repaired in place by the migrate hook; any rebuild + independently converges to the same values. +- No new `metric_catalog` seed migrations and no new ad-hoc `insight.*` views + for metrics. +- Do not add runtime formula JSON until generation exists. + +### Validation commands + +```sh +# from src/backend — registry invariants, enum round-trips, compiler tests +cargo test -p analytics + +# from src/ingestion/dbt — manifest validation, no warehouse connection. +# CI runs the same gate (build-images.yml, toolbox job). +dbt parse --profiles-dir +``` + +The dummy profile is a `profiles.yml` with profile name `ingestion` and any +unreachable `type: clickhouse` output; `dbt parse` loads the adapter but never +connects. + +Future developer-side generation may use source models and formulas to produce +the managed observation SQL and seed rows, but runtime execution still +consumes typed definitions and source measure observations. + +## Custom Metric Gate + +Runtime-authored metrics require one of: + +- generated managed observation SQL plus generated definition/source seed rows. +- validated custom observation SQL that emits the source measure observation contract. + +Until one exists, custom definitions can be stored but cannot produce new source observations. The runtime only executes metrics whose inputs resolve to available, validated source measures. + +## Frontend Contract + +Frontend collection rendering: + +- requests metric keys and views. +- treats configured required views as required. +- normalizes response arrays only for local lookup. +- renders using returned label, description, explanation, unit, format, + direction, and computation. +- owns chart choice and layout. + +Backend responses do not include chart metadata. + +## Non-Goals + +- No custom metric authoring UI in this pass. +- No custom SQL execution in this pass. +- No public source labels in metric results. +- No metric-key-specific branches in result compilation. +- No partial responses for oversized results. diff --git a/src/backend/services/analytics/src/api/metric_results.rs b/src/backend/services/analytics/src/api/metric_results.rs new file mode 100644 index 000000000..0bb48a7fb --- /dev/null +++ b/src/backend/services/analytics/src/api/metric_results.rs @@ -0,0 +1,226 @@ +use std::sync::Arc; +use std::time::Duration; + +use axum::Json; +use axum::extract::Extension; +use futures::stream::{self, StreamExt}; +use serde::de::DeserializeOwned; +use toolkit_canonical_errors::CanonicalError; + +use super::AppState; +use super::error::MetricError; +use crate::domain::metric_definitions::MetricDefinition; +use crate::domain::metric_results::{ + BreakdownQueryRow, CompiledQuery, MetricResultViewDto, MetricResultsRequest, + MetricResultsResponse, PeerQueryRow, PeriodQueryRow, TimeseriesQueryRow, + ValidatedMetricResultsRequest, ValidatedMetricView, build_breakdown_view, build_metric_result, + build_peer_view, build_period_view, build_timeseries_view, compile_view_query, + enforce_row_limit, validate_request, +}; +use toolkit_security::SecurityContext; + +const QUERY_CONCURRENCY: usize = 4; +// Client-side bound on one view query, network stalls included. The +// insight-clickhouse client already caps server-side execution at 30s +// (`max_execution_time`); this covers the transport path that setting +// cannot reach (dead peer, half-open connection). +const QUERY_FETCH_TIMEOUT: Duration = Duration::from_mins(1); + +pub async fn query_metric_results( + Extension(state): Extension>, + Extension(ctx): Extension, + Json(req): Json, +) -> Result, CanonicalError> { + let req = validate_request(&state.db, ctx.subject_tenant_id(), req).await?; + let tasks = compile_tasks(&req); + + let mut views_by_metric: Vec>> = req + .metrics + .iter() + .map(|metric| (0..metric.views.len()).map(|_| None).collect()) + .collect(); + + // Consuming results as they complete bails on the first error; dropping + // the stream cancels the in-flight and queued view queries. + let mut results = stream::iter(tasks) + .map(|task| execute_task(&state, &req, task)) + .buffer_unordered(QUERY_CONCURRENCY); + while let Some(result) = results.next().await { + let result = result?; + views_by_metric[result.metric_index][result.view_index] = Some(result.view); + } + + let mut metrics = Vec::with_capacity(req.metrics.len()); + for (idx, metric) in req.metrics.iter().enumerate() { + let mut views = Vec::with_capacity(metric.views.len()); + for view in views_by_metric[idx].drain(..) { + let Some(view) = view else { + return Err(CanonicalError::internal("missing metric view result").create()); + }; + views.push(view); + } + metrics.push(build_metric_result(&metric.def, views)); + } + + let response = MetricResultsResponse { metrics }; + enforce_row_limit(&response)?; + Ok(Json(response)) +} + +struct MetricViewTask { + metric_index: usize, + view_index: usize, + def: MetricDefinition, + view: ValidatedMetricView, + query: CompiledQuery, +} + +struct MetricViewTaskResult { + metric_index: usize, + view_index: usize, + view: MetricResultViewDto, +} + +fn compile_tasks(req: &ValidatedMetricResultsRequest) -> Vec { + req.metrics + .iter() + .enumerate() + .flat_map(|(metric_index, metric)| { + metric + .views + .iter() + .enumerate() + .map(move |(view_index, view)| MetricViewTask { + metric_index, + view_index, + def: metric.def.clone(), + view: view.clone(), + query: compile_view_query(&metric.def, req, view), + }) + }) + .collect() +} + +async fn execute_task( + state: &Arc, + req: &ValidatedMetricResultsRequest, + task: MetricViewTask, +) -> Result { + let MetricViewTask { + metric_index, + view_index, + def, + view, + query, + } = task; + + let view = match view { + ValidatedMetricView::Period => { + let rows = fetch_rows::(state, query).await?; + build_period_view(&def, req, rows) + } + ValidatedMetricView::Peer { .. } => { + let rows = fetch_rows::(state, query).await?; + build_peer_view(rows) + } + ValidatedMetricView::Timeseries { bucket, dimensions } => { + let rows = fetch_rows::(state, query).await?; + build_timeseries_view(&def, req, bucket, &dimensions, rows)? + } + ValidatedMetricView::Breakdown { dimensions } => { + let rows = fetch_rows::(state, query).await?; + build_breakdown_view(&dimensions, rows)? + } + }; + + Ok(MetricViewTaskResult { + metric_index, + view_index, + view, + }) +} + +async fn fetch_rows( + state: &Arc, + query: CompiledQuery, +) -> Result, CanonicalError> +where + T: DeserializeOwned, +{ + let mut ch_query = state.ch.query(&query.sql); + for param in &query.params { + ch_query = ch_query.bind(param.as_str()); + } + + let mut cursor = ch_query.fetch_bytes("JSONEachRow").map_err(|e| { + tracing::error!(error = %e, sql = %query.sql, "ClickHouse metric-results query failed"); + map_query_error(&e.to_string()) + })?; + + let raw_bytes = tokio::time::timeout(QUERY_FETCH_TIMEOUT, cursor.collect()) + .await + .map_err(|_| { + tracing::error!(sql = %query.sql, "ClickHouse metric-results fetch timed out"); + CanonicalError::internal("query execution failed").create() + })? + .map_err(|e| { + tracing::error!(error = %e, sql = %query.sql, "ClickHouse metric-results fetch failed"); + map_query_error(&e.to_string()) + })?; + + if raw_bytes.is_empty() { + return Ok(Vec::new()); + } + + raw_bytes + .split(|&b| b == b'\n') + .filter(|line| !line.is_empty()) + .map(serde_json::from_slice) + .collect::, _>>() + .map_err(|e| { + tracing::error!(error = %e, "failed to parse metric-results rows"); + CanonicalError::internal("failed to parse query results").create() + }) +} + +// A missing observation/cohort relation is a known transient state (dbt has +// not built the view yet, or a model regressed) that the validator sweep +// converges on — surface it as a typed precondition failure instead of a +// 500. UNKNOWN_TABLE is ClickHouse error code 60. +fn map_query_error(message: &str) -> CanonicalError { + if message.contains("UNKNOWN_TABLE") || message.contains("Code: 60") { + return MetricError::failed_precondition() + .with_precondition_violation( + "metric source relation", + "The observation or cohort view backing this metric has not been built yet; it converges on the next validation sweep.", + "SOURCE_RELATION_MISSING", + ) + .create(); + } + CanonicalError::internal("query execution failed").create() +} + +#[cfg(test)] +mod tests { + use axum::http::StatusCode; + use axum::response::IntoResponse; + + use super::map_query_error; + + #[test] + fn missing_relation_maps_to_precondition_failure_not_500() { + let err = map_query_error( + "bad response: Code: 60. DB::Exception: Table insight.ai_metric_observations does not exist. (UNKNOWN_TABLE)", + ); + let status = err.into_response().status(); + assert_ne!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!(status.is_client_error()); + } + + #[test] + fn other_query_errors_stay_internal() { + let err = map_query_error("Code: 241. DB::Exception: Memory limit exceeded"); + let status = err.into_response().status(); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + } +} diff --git a/src/backend/services/analytics/src/api/mod.rs b/src/backend/services/analytics/src/api/mod.rs index f4af00ad6..ea5492610 100644 --- a/src/backend/services/analytics/src/api/mod.rs +++ b/src/backend/services/analytics/src/api/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod canonical_json; mod catalog; pub(crate) mod error; mod handlers; +mod metric_results; #[cfg(test)] mod tenant_resolution_tests; @@ -210,6 +211,16 @@ fn build_operations(router: Router, openapi: &dyn OpenApiRegistry) -> Router { .handler(handlers::query_metrics_batch) .register(router, openapi); + router = OperationBuilder::post("/v1/metric-results") + .operation_id("analytics_api.metric_results.create") + .summary("Compute metric results") + .authenticated() + .no_license_required() + .json_response(StatusCode::OK, "Metric results") + .standard_errors(openapi) + .handler(metric_results::query_metric_results) + .register(router, openapi); + // Thresholds (legacy) router = OperationBuilder::get("/v1/metrics/{id}/thresholds") .operation_id("analytics_api.thresholds.list") diff --git a/src/backend/services/analytics/src/domain/metric_definitions/README.md b/src/backend/services/analytics/src/domain/metric_definitions/README.md new file mode 100644 index 000000000..61a7234b4 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/README.md @@ -0,0 +1,7 @@ +# Metric Definitions + +Implementation of the metrics domain. The system contract and the authoring +guide live in [`docs/domain/metrics/specs/DESIGN.md`](../../../../../../../docs/domain/metrics/specs/DESIGN.md). + +Read that document before adding or changing metrics, metric seeds, +observation sources, or gold metric views. diff --git a/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs b/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs new file mode 100644 index 000000000..db5a98e71 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/builtin.rs @@ -0,0 +1,433 @@ +use crate::domain::metric_definitions::definition::{ + MetricComputation, MetricDirection, MetricFormat, MetricInputRole, ObservationSource, + SourceKind, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntityType { + Person, +} + +impl EntityType { + pub fn as_db(self) -> &'static str { + match self { + Self::Person => "person", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CohortKey { + OrgUnit, +} + +impl CohortKey { + pub fn as_db(self) -> &'static str { + match self { + Self::OrgUnit => "org_unit", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SeedComputation { + Sum, + Ratio { scale: f64 }, +} + +impl SeedComputation { + pub fn computation(self) -> MetricComputation { + match self { + Self::Sum => MetricComputation::Sum, + Self::Ratio { .. } => MetricComputation::Ratio, + } + } + + pub fn scale(self) -> Option { + match self { + Self::Sum => None, + Self::Ratio { scale } => Some(scale), + } + } +} + +pub struct SourceSeed { + pub key: &'static str, + pub kind: SourceKind, + pub source_ref: ObservationSource, +} + +pub struct BuiltinSource { + pub source: SourceSeed, + pub measures: &'static [&'static str], + pub dimensions: &'static [&'static str], +} + +pub struct MetricSeed { + pub metric_key: &'static str, + pub source_key: &'static str, + pub label: &'static str, + pub description: Option<&'static str>, + pub explanation: Option<&'static str>, + pub unit: Option<&'static str>, + pub format: MetricFormat, + pub direction: MetricDirection, + pub entity_type: EntityType, + pub computation: SeedComputation, + pub peer_cohort_key: Option, + pub inputs: &'static [InputSeed], + pub dimensions: &'static [&'static str], +} + +pub struct InputSeed { + pub input_role: MetricInputRole, + pub measure_key: &'static str, +} + +pub const BUILTIN_SOURCES: &[BuiltinSource] = &[BuiltinSource { + source: SourceSeed { + key: "ai_usage", + kind: SourceKind::ManagedObservation, + source_ref: ObservationSource::AiMetricObservations, + }, + measures: &[ + "accepted_lines", + "removed_lines", + "active_day", + "cost_usd", + "accepted_edit_actions", + "tool_use_offered", + "assistant_messages", + "assistant_actions", + "dev_conversations", + "chat_assistant_conversations", + ], + dimensions: &["tool", "surface"], +}]; + +pub const BUILTIN_METRICS: &[MetricSeed] = &[ + MetricSeed { + metric_key: "ai.accepted_lines", + source_key: "ai_usage", + label: "AI-added lines", + description: Some("Accepted added coding output"), + explanation: Some("Accepted AI-generated added lines across coding AI tools."), + unit: Some("lines"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "accepted_lines", + }], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.removed_lines", + source_key: "ai_usage", + label: "AI-removed lines", + description: Some("Accepted deleted coding output"), + explanation: Some("Accepted AI-generated removed lines across coding AI tools."), + unit: Some("lines"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "removed_lines", + }], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.active_days", + source_key: "ai_usage", + label: "AI active days", + description: Some("Days with any AI activity across dev and assistant tools"), + explanation: Some( + "Distinct days with person-attributed AI activity across dev and assistant tools.", + ), + unit: Some("days"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "active_day", + }], + dimensions: &[], + }, + MetricSeed { + metric_key: "ai.cost", + source_key: "ai_usage", + label: "AI cost", + description: Some("Reported AI spend across dev and assistant tools"), + explanation: Some( + "Person-attributed AI spend across dev and assistant tools, where the connector reports cost.", + ), + unit: None, + format: MetricFormat::Currency, + direction: MetricDirection::LowerIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "cost_usd", + }], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.accepted_edit_actions", + source_key: "ai_usage", + label: "Accepted AI edits", + description: Some("Accepted tool or edit suggestions"), + explanation: Some("Accepted AI edit or tool suggestions across supported coding AI tools."), + unit: Some("actions"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "accepted_edit_actions", + }], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.tool_acceptance_rate", + source_key: "ai_usage", + label: "AI tool acceptance", + description: Some("Accepted divided by offered AI edits"), + explanation: Some("Accepted AI edit or tool suggestions divided by offered suggestions."), + unit: Some("percent"), + format: MetricFormat::Percent, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Ratio { scale: 100.0 }, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[ + InputSeed { + input_role: MetricInputRole::Numerator, + measure_key: "accepted_edit_actions", + }, + InputSeed { + input_role: MetricInputRole::Denominator, + measure_key: "tool_use_offered", + }, + ], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.assistant_messages", + source_key: "ai_usage", + label: "AI assistant messages", + description: Some("Assistant messages"), + explanation: Some( + "Person-attributed assistant messages from supported AI assistant tools.", + ), + unit: Some("messages"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "assistant_messages", + }], + dimensions: &["tool", "surface"], + }, + MetricSeed { + metric_key: "ai.assistant_actions", + source_key: "ai_usage", + label: "AI assistant actions", + description: Some("Assistant actions"), + explanation: Some("Person-attributed assistant actions from supported AI assistant tools."), + unit: Some("actions"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "assistant_actions", + }], + dimensions: &["tool", "surface"], + }, + MetricSeed { + metric_key: "ai.dev_conversations", + source_key: "ai_usage", + label: "AI dev conversations", + description: Some("Coding tool conversations where the source reports them"), + explanation: Some( + "Person-attributed coding conversations from dev tools that report them.", + ), + unit: Some("conversations"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "dev_conversations", + }], + dimensions: &["tool"], + }, + MetricSeed { + metric_key: "ai.chat_assistant_conversations", + source_key: "ai_usage", + label: "AI chat conversations", + description: Some("Chat assistant conversations"), + explanation: Some( + "Person-attributed chat assistant conversations from supported AI chat tools.", + ), + unit: Some("conversations"), + format: MetricFormat::Integer, + direction: MetricDirection::HigherIsBetter, + entity_type: EntityType::Person, + computation: SeedComputation::Sum, + peer_cohort_key: Some(CohortKey::OrgUnit), + inputs: &[InputSeed { + input_role: MetricInputRole::Value, + measure_key: "chat_assistant_conversations", + }], + dimensions: &["tool", "surface"], + }, +]; + +#[cfg(test)] +mod tests { + use std::collections::{BTreeSet, HashMap}; + + use super::*; + + fn is_snake_case(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + && value.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + } + + fn is_metric_key(value: &str) -> bool { + let parts = value.split('.').collect::>(); + parts.len() == 2 && parts.iter().all(|part| is_snake_case(part)) + } + + #[test] + fn source_keys_are_unique_and_shaped() { + let mut seen = BTreeSet::new(); + for builtin_source in BUILTIN_SOURCES { + assert!(is_snake_case(builtin_source.source.key)); + assert!(seen.insert(builtin_source.source.key)); + } + } + + #[test] + fn measure_and_dimension_keys_are_unique_per_source() { + for builtin_source in BUILTIN_SOURCES { + let mut measures = BTreeSet::new(); + for measure_key in builtin_source.measures { + assert!(is_snake_case(measure_key)); + assert!(measures.insert(*measure_key)); + } + let mut dimensions = BTreeSet::new(); + for dimension_key in builtin_source.dimensions { + assert!(is_snake_case(dimension_key)); + assert!(dimensions.insert(*dimension_key)); + } + } + } + + #[test] + fn metric_keys_are_unique_and_shaped() { + let mut seen = BTreeSet::new(); + for metric in BUILTIN_METRICS { + assert!(is_metric_key(metric.metric_key), "{}", metric.metric_key); + assert!(seen.insert(metric.metric_key)); + } + } + + #[test] + fn metric_inputs_reference_declared_measures() { + let measures_by_source: HashMap<&str, BTreeSet<&str>> = BUILTIN_SOURCES + .iter() + .map(|builtin_source| { + ( + builtin_source.source.key, + builtin_source.measures.iter().copied().collect(), + ) + }) + .collect(); + + for metric in BUILTIN_METRICS { + let measures = measures_by_source + .get(metric.source_key) + .unwrap_or_else(|| panic!("unknown source for {}", metric.metric_key)); + assert!(!metric.inputs.is_empty(), "{}", metric.metric_key); + for input in metric.inputs { + assert!( + measures.contains(input.measure_key), + "{} references undeclared measure {}", + metric.metric_key, + input.measure_key + ); + } + } + } + + #[test] + fn metric_dimensions_reference_declared_source_dimensions() { + let dimensions_by_source: HashMap<&str, BTreeSet<&str>> = BUILTIN_SOURCES + .iter() + .map(|builtin_source| { + ( + builtin_source.source.key, + builtin_source.dimensions.iter().copied().collect(), + ) + }) + .collect(); + + for metric in BUILTIN_METRICS { + let Some(dimensions) = dimensions_by_source.get(metric.source_key) else { + panic!("unknown source for {}", metric.metric_key); + }; + for dimension in metric.dimensions { + assert!( + dimensions.contains(dimension), + "{} references undeclared dimension {dimension}", + metric.metric_key + ); + } + } + } + + #[test] + fn ratio_metrics_have_numerator_and_denominator_roles() { + for metric in BUILTIN_METRICS { + let SeedComputation::Ratio { .. } = metric.computation else { + continue; + }; + let has_role = |role| metric.inputs.iter().any(|input| input.input_role == role); + assert!( + has_role(MetricInputRole::Numerator), + "{}", + metric.metric_key + ); + assert!( + has_role(MetricInputRole::Denominator), + "{}", + metric.metric_key + ); + } + } +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/definition.rs b/src/backend/services/analytics/src/domain/metric_definitions/definition.rs new file mode 100644 index 000000000..ac9443930 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/definition.rs @@ -0,0 +1,284 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricDirection { + HigherIsBetter, + LowerIsBetter, + Neutral, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricFormat { + Integer, + Decimal, + Currency, + Percent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricComputation { + Sum, + Ratio, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricInputRole { + Value, + Numerator, + Denominator, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceKind { + ManagedObservation, + CustomObservationSql, +} + +impl SourceKind { + pub fn as_db(self) -> &'static str { + match self { + Self::ManagedObservation => "managed_observation", + Self::CustomObservationSql => "custom_observation_sql", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "managed_observation" => Some(Self::ManagedObservation), + "custom_observation_sql" => Some(Self::CustomObservationSql), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObservationSource { + AiMetricObservations, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CohortSource { + MetricEntityCohortsCurrent, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MetricDefinition { + pub base: MetricBase, + pub spec: ComputationSpec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MetricBase { + pub key: String, + pub label: String, + pub description: Option, + pub explanation: Option, + pub entity_type: String, + pub format: MetricFormat, + pub unit: Option, + pub direction: MetricDirection, + pub peer_cohort_key: Option, + pub allowed_dimensions: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ComputationSpec { + Sum { + value: MetricInput, + }, + Ratio { + numerator: MetricInput, + denominator: MetricInput, + scale: f64, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MetricInput { + pub role: MetricInputRole, + pub observation_source: ObservationSource, + pub source_key: String, + pub measure_key: String, +} + +impl MetricDefinition { + pub fn key(&self) -> &str { + self.base.key.as_str() + } + + pub fn allowed_dimension(&self, dimension: &str) -> Option<&str> { + self.base + .allowed_dimensions + .iter() + .map(String::as_str) + .find(|d| *d == dimension) + } + + pub fn is_zero_filled(&self) -> bool { + matches!(self.spec, ComputationSpec::Sum { .. }) + } + + pub fn observation_source(&self) -> ObservationSource { + match &self.spec { + ComputationSpec::Sum { value } => value.observation_source, + ComputationSpec::Ratio { numerator, .. } => numerator.observation_source, + } + } +} + +impl ObservationSource { + pub fn source_ref(self) -> &'static str { + match self { + Self::AiMetricObservations => "ai_metric_observations", + } + } + + pub fn from_ref(value: &str) -> Option { + match value { + "ai_metric_observations" => Some(Self::AiMetricObservations), + _ => None, + } + } + + pub fn table_ref(self) -> (&'static str, &'static str) { + match self { + Self::AiMetricObservations => ("insight", "ai_metric_observations"), + } + } +} + +impl CohortSource { + pub fn table_ref(self) -> (&'static str, &'static str) { + match self { + Self::MetricEntityCohortsCurrent => ("insight", "metric_entity_cohorts_current"), + } + } +} + +impl MetricFormat { + pub fn as_db(self) -> &'static str { + match self { + Self::Integer => "integer", + Self::Decimal => "decimal", + Self::Currency => "currency", + Self::Percent => "percent", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "integer" => Some(Self::Integer), + "decimal" => Some(Self::Decimal), + "currency" => Some(Self::Currency), + "percent" => Some(Self::Percent), + _ => None, + } + } +} + +impl MetricDirection { + pub fn as_db(self) -> &'static str { + match self { + Self::HigherIsBetter => "higher_is_better", + Self::LowerIsBetter => "lower_is_better", + Self::Neutral => "neutral", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "higher_is_better" => Some(Self::HigherIsBetter), + "lower_is_better" => Some(Self::LowerIsBetter), + "neutral" => Some(Self::Neutral), + _ => None, + } + } +} + +impl MetricComputation { + pub fn as_db(self) -> &'static str { + match self { + Self::Sum => "sum", + Self::Ratio => "ratio", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "sum" => Some(Self::Sum), + "ratio" => Some(Self::Ratio), + _ => None, + } + } +} + +impl MetricInputRole { + pub fn as_db(self) -> &'static str { + match self { + Self::Value => "value", + Self::Numerator => "numerator", + Self::Denominator => "denominator", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "value" => Some(Self::Value), + "numerator" => Some(Self::Numerator), + "denominator" => Some(Self::Denominator), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn db_strings_round_trip() { + for format in [ + MetricFormat::Integer, + MetricFormat::Decimal, + MetricFormat::Currency, + MetricFormat::Percent, + ] { + assert_eq!(MetricFormat::from_db(format.as_db()), Some(format)); + } + for direction in [ + MetricDirection::HigherIsBetter, + MetricDirection::LowerIsBetter, + MetricDirection::Neutral, + ] { + assert_eq!(MetricDirection::from_db(direction.as_db()), Some(direction)); + } + for computation in [MetricComputation::Sum, MetricComputation::Ratio] { + assert_eq!( + MetricComputation::from_db(computation.as_db()), + Some(computation) + ); + } + for role in [ + MetricInputRole::Value, + MetricInputRole::Numerator, + MetricInputRole::Denominator, + ] { + assert_eq!(MetricInputRole::from_db(role.as_db()), Some(role)); + } + let source = ObservationSource::AiMetricObservations; + assert_eq!( + ObservationSource::from_ref(source.source_ref()), + Some(source) + ); + for kind in [ + SourceKind::ManagedObservation, + SourceKind::CustomObservationSql, + ] { + assert_eq!(SourceKind::from_db(kind.as_db()), Some(kind)); + } + } +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/error_code.rs b/src/backend/services/analytics/src/domain/metric_definitions/error_code.rs new file mode 100644 index 000000000..2726e32ce --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/error_code.rs @@ -0,0 +1,89 @@ +use std::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetricSchemaErrorCode { + TableNotFound, + ColumnNotFound, + DimensionNotCovered, + Unknown, +} + +#[cfg(test)] +pub const ALL_METRIC_SCHEMA_ERROR_CODES: &[MetricSchemaErrorCode] = &[ + MetricSchemaErrorCode::TableNotFound, + MetricSchemaErrorCode::ColumnNotFound, + MetricSchemaErrorCode::DimensionNotCovered, + MetricSchemaErrorCode::Unknown, +]; + +impl MetricSchemaErrorCode { + #[must_use] + pub fn as_db(self) -> &'static str { + match self { + Self::TableNotFound => "table_not_found", + Self::ColumnNotFound => "column_not_found", + Self::DimensionNotCovered => "dimension_not_covered", + Self::Unknown => "unknown", + } + } +} + +impl fmt::Display for MetricSchemaErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_db()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SchemaStatus { + Ok, + Error, + Unchecked, +} + +impl SchemaStatus { + pub fn as_db(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Error => "error", + Self::Unchecked => "unchecked", + } + } + + pub fn from_db(value: &str) -> Option { + match value { + "ok" => Some(Self::Ok), + "error" => Some(Self::Error), + "unchecked" => Some(Self::Unchecked), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_status_round_trips() { + for status in [ + SchemaStatus::Ok, + SchemaStatus::Error, + SchemaStatus::Unchecked, + ] { + assert_eq!(SchemaStatus::from_db(status.as_db()), Some(status)); + } + assert_eq!(SchemaStatus::from_db("nope"), None); + } + + #[test] + fn all_codes_listed_once() { + let mut strings = ALL_METRIC_SCHEMA_ERROR_CODES + .iter() + .map(|code| code.as_db()) + .collect::>(); + strings.sort_unstable(); + strings.dedup(); + assert_eq!(strings.len(), ALL_METRIC_SCHEMA_ERROR_CODES.len()); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/mod.rs b/src/backend/services/analytics/src/domain/metric_definitions/mod.rs new file mode 100644 index 000000000..14bdc8f92 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/mod.rs @@ -0,0 +1,14 @@ +pub mod builtin; +pub mod definition; +pub mod error_code; +mod repository; +mod seeds; +pub mod validator; + +pub use definition::{ + CohortSource, ComputationSpec, MetricDefinition, MetricDirection, MetricFormat, + ObservationSource, +}; +pub use repository::load_definitions; +pub use seeds::reconcile_builtin_definitions; +pub use validator::MetricDefinitionValidator; diff --git a/src/backend/services/analytics/src/domain/metric_definitions/repository.rs b/src/backend/services/analytics/src/domain/metric_definitions/repository.rs new file mode 100644 index 000000000..1765488d9 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/repository.rs @@ -0,0 +1,909 @@ +use std::collections::{BTreeMap, HashMap}; + +use sea_orm::{ConnectionTrait, DatabaseConnection, FromQueryResult, Statement, Value}; +use toolkit_canonical_errors::CanonicalError; +use uuid::Uuid; + +use crate::api::error::MetricError; +use crate::domain::metric_definitions::error_code::{MetricSchemaErrorCode, SchemaStatus}; + +use crate::domain::metric_definitions::definition::{ + ComputationSpec, MetricBase, MetricComputation, MetricDefinition, MetricDirection, + MetricFormat, MetricInput, MetricInputRole, ObservationSource, SourceKind, +}; + +#[derive(Debug, FromQueryResult)] +struct DefinitionRow { + definition_id: Uuid, + tenant_id: Option, + metric_key: String, + label: String, + description: Option, + explanation: Option, + unit: Option, + format: String, + direction: String, + entity_type: String, + computation_type: String, + scale: Option, + peer_cohort_key: Option, + definition_enabled: bool, + definition_schema_status: String, +} + +#[derive(Debug, FromQueryResult)] +struct InputRow { + metric_definition_id: Uuid, + input_role: String, + measure_key: String, + measure_enabled: bool, + measure_schema_status: String, + source_key: String, + source_kind: String, + source_ref: String, + source_enabled: bool, + source_schema_status: String, +} + +#[derive(Debug, FromQueryResult)] +struct DimensionRow { + metric_definition_id: Uuid, + dimension_key: String, +} + +#[derive(Debug)] +enum ClassifiedInputs { + Available(Vec), + Unavailable, + Corrupt, +} + +#[derive(Debug, Clone)] +pub struct MetricDefinitionValidationSpec { + pub definition_id: Uuid, + pub metric_key: String, + pub entity_type: String, + pub inputs: Vec, + pub dimensions: Vec, +} + +#[derive(Debug, FromQueryResult)] +struct ValidationDefinitionRow { + definition_id: Uuid, + metric_key: String, + entity_type: String, +} + +pub async fn load_definitions( + db: &DatabaseConnection, + tenant_id: Uuid, + metric_keys: &[String], +) -> Result, CanonicalError> { + if metric_keys.is_empty() { + return Ok(HashMap::new()); + } + + let rows = fetch_definition_rows(db, tenant_id, metric_keys) + .await + .map_err(|error| db_error(&error))?; + let all_ids = rows.iter().map(|row| row.definition_id).collect::>(); + let input_rows = fetch_input_rows(db, &all_ids) + .await + .map_err(|error| db_error(&error))?; + let inputs = classify_inputs(input_rows); + let dimensions = fetch_dimensions(db, &all_ids) + .await + .map_err(|error| db_error(&error))?; + + let mut definitions = HashMap::new(); + for (metric_key, candidates) in group_by_key(rows) { + let Some(row) = select_available_row(&metric_key, candidates, &inputs)? else { + continue; + }; + let definition_id = row.definition_id; + let row_inputs: &[MetricInput] = match inputs.get(&definition_id) { + Some(ClassifiedInputs::Available(row_inputs)) => row_inputs, + Some(ClassifiedInputs::Unavailable | ClassifiedInputs::Corrupt) | None => &[], + }; + let definition = build_definition( + &row, + row_inputs, + dimensions.get(&definition_id).cloned().unwrap_or_default(), + )?; + definitions.insert(metric_key, definition); + } + + for key in metric_keys { + if !definitions.contains_key(key) { + return Err(unavailable(key)); + } + } + + Ok(definitions) +} + +pub async fn managed_definition_validation_specs( + db: &DatabaseConnection, + source_id: Uuid, +) -> Result, sea_orm::DbErr> { + let rows = fetch_validation_definition_rows(db, source_id).await?; + let definition_ids = rows.iter().map(|row| row.definition_id).collect::>(); + let inputs = classify_inputs(fetch_input_rows(db, &definition_ids).await?); + let dimensions = fetch_dimensions(db, &definition_ids).await?; + + Ok(rows + .into_iter() + .filter_map(|row| { + let definition_id = row.definition_id; + match inputs.get(&definition_id) { + Some(ClassifiedInputs::Available(row_inputs)) => { + Some(MetricDefinitionValidationSpec { + definition_id, + metric_key: row.metric_key, + entity_type: row.entity_type, + inputs: row_inputs.clone(), + dimensions: dimensions.get(&definition_id).cloned().unwrap_or_default(), + }) + } + Some(ClassifiedInputs::Unavailable | ClassifiedInputs::Corrupt) | None => None, + } + }) + .collect()) +} + +async fn fetch_validation_definition_rows( + db: &DatabaseConnection, + source_id: Uuid, +) -> Result, sea_orm::DbErr> { + ValidationDefinitionRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + "SELECT DISTINCT \ + d.id AS definition_id, \ + d.metric_key AS metric_key, \ + d.entity_type AS entity_type \ + FROM metric_definitions d \ + INNER JOIN metric_definition_inputs i ON i.metric_definition_id = d.id \ + INNER JOIN metric_source_measures m ON m.id = i.source_measure_id \ + WHERE d.is_enabled = TRUE \ + AND m.is_enabled = TRUE \ + AND m.source_id = ? \ + ORDER BY d.metric_key", + [uuid_value(source_id)], + )) + .all(db) + .await +} + +async fn fetch_definition_rows( + db: &DatabaseConnection, + tenant_id: Uuid, + metric_keys: &[String], +) -> Result, sea_orm::DbErr> { + let placeholders = vec!["?"; metric_keys.len()].join(", "); + let sql = format!( + "SELECT \ + d.id AS definition_id, \ + d.tenant_id AS tenant_id, \ + d.metric_key AS metric_key, \ + d.label AS label, \ + d.description AS description, \ + d.explanation AS explanation, \ + d.unit AS unit, \ + d.format AS format, \ + d.direction AS direction, \ + d.entity_type AS entity_type, \ + d.computation_type AS computation_type, \ + CAST(d.scale AS DOUBLE) AS scale, \ + d.peer_cohort_key AS peer_cohort_key, \ + d.is_enabled AS definition_enabled, \ + d.schema_status AS definition_schema_status \ + FROM metric_definitions d \ + WHERE d.metric_key IN ({placeholders}) \ + AND (d.tenant_id IS NULL OR d.tenant_id = ?)" + ); + + let mut values = metric_keys.iter().map(Value::from).collect::>(); + values.push(Value::Bytes(Some(Box::new(tenant_id.as_bytes().to_vec())))); + + DefinitionRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .all(db) + .await +} + +async fn fetch_input_rows( + db: &DatabaseConnection, + definition_ids: &[Uuid], +) -> Result, sea_orm::DbErr> { + if definition_ids.is_empty() { + return Ok(Vec::new()); + } + + let placeholders = vec!["?"; definition_ids.len()].join(", "); + let sql = format!( + "SELECT \ + i.metric_definition_id AS metric_definition_id, \ + i.input_role AS input_role, \ + m.measure_key AS measure_key, \ + m.is_enabled AS measure_enabled, \ + m.schema_status AS measure_schema_status, \ + s.source_key AS source_key, \ + s.source_kind AS source_kind, \ + s.source_ref AS source_ref, \ + s.is_enabled AS source_enabled, \ + s.schema_status AS source_schema_status \ + FROM metric_definition_inputs i \ + INNER JOIN metric_source_measures m ON m.id = i.source_measure_id \ + INNER JOIN metric_sources s ON s.id = m.source_id \ + WHERE i.metric_definition_id IN ({placeholders}) \ + ORDER BY i.metric_definition_id, i.input_role, m.measure_key" + ); + let values = definition_ids + .iter() + .map(|id| Value::Bytes(Some(Box::new(id.as_bytes().to_vec())))) + .collect::>(); + + InputRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .all(db) + .await +} + +fn classify_inputs(rows: Vec) -> HashMap { + let mut out: HashMap = HashMap::new(); + for row in rows { + let entry = out + .entry(row.metric_definition_id) + .or_insert_with(|| ClassifiedInputs::Available(Vec::new())); + + // Corrupt config must stay loud regardless of row order: it wins + // over Unavailable, which wins over Available. + let role = MetricInputRole::from_db(&row.input_role); + let kind = SourceKind::from_db(&row.source_kind); + let observation_source = ObservationSource::from_ref(&row.source_ref); + let parsed = match (role, kind, observation_source) { + (Some(role), Some(SourceKind::ManagedObservation), Some(observation_source)) => { + Some((role, observation_source)) + } + (Some(_), Some(SourceKind::CustomObservationSql), _) => { + if !matches!(entry, ClassifiedInputs::Corrupt) { + *entry = ClassifiedInputs::Unavailable; + } + continue; + } + _ => None, + }; + let Some((role, observation_source)) = parsed else { + tracing::error!( + input_role = %row.input_role, + source_ref = %row.source_ref, + source_kind = %row.source_kind, + "corrupt metric definition input" + ); + *entry = ClassifiedInputs::Corrupt; + continue; + }; + if matches!(entry, ClassifiedInputs::Corrupt) { + continue; + } + + if !row.measure_enabled + || !row.source_enabled + || schema_status_blocks(&row.measure_schema_status) + || schema_status_blocks(&row.source_schema_status) + { + *entry = ClassifiedInputs::Unavailable; + continue; + } + + if let ClassifiedInputs::Available(inputs) = entry { + inputs.push(MetricInput { + role, + observation_source, + source_key: row.source_key, + measure_key: row.measure_key, + }); + } + } + out +} + +fn schema_status_blocks(status: &str) -> bool { + !matches!( + SchemaStatus::from_db(status), + Some(SchemaStatus::Ok | SchemaStatus::Unchecked) + ) +} + +fn group_by_key(rows: Vec) -> BTreeMap> { + let mut grouped: BTreeMap> = BTreeMap::new(); + for row in rows { + grouped.entry(row.metric_key.clone()).or_default().push(row); + } + grouped +} + +fn select_available_row( + metric_key: &str, + rows: Vec, + inputs: &HashMap, +) -> Result, CanonicalError> { + let tenant_rows = rows.iter().filter(|row| row.tenant_id.is_some()).count(); + if tenant_rows > 1 { + return Err(config_error(&format!( + "multiple tenant metric definitions for {metric_key}" + ))); + } + let product_rows = rows.iter().filter(|row| row.tenant_id.is_none()).count(); + if product_rows > 1 { + return Err(config_error(&format!( + "multiple product metric definitions for {metric_key}" + ))); + } + + let mut candidates = rows; + candidates.sort_by_key(|row| row.tenant_id.is_none()); + + for row in candidates { + if matches!( + inputs.get(&row.definition_id), + Some(ClassifiedInputs::Corrupt) + ) { + return Err(config_error(&format!( + "corrupt inputs for metric definition {metric_key}" + ))); + } + let inputs_available = !matches!( + inputs.get(&row.definition_id), + Some(ClassifiedInputs::Unavailable) + ); + if row.definition_enabled + && !schema_status_blocks(&row.definition_schema_status) + && inputs_available + { + return Ok(Some(row)); + } + } + Ok(None) +} + +async fn fetch_dimensions( + db: &DatabaseConnection, + definition_ids: &[Uuid], +) -> Result>, sea_orm::DbErr> { + if definition_ids.is_empty() { + return Ok(HashMap::new()); + } + + let placeholders = vec!["?"; definition_ids.len()].join(", "); + let sql = format!( + "SELECT \ + d.metric_definition_id AS metric_definition_id, \ + s.dimension_key AS dimension_key \ + FROM metric_definition_dimensions d \ + INNER JOIN metric_source_dimensions s ON s.id = d.source_dimension_id \ + WHERE d.metric_definition_id IN ({placeholders}) \ + ORDER BY d.metric_definition_id, d.display_order, s.dimension_key" + ); + let values = definition_ids + .iter() + .map(|id| Value::Bytes(Some(Box::new(id.as_bytes().to_vec())))) + .collect::>(); + + DimensionRow::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .all(db) + .await + .map(|rows| { + let mut out: HashMap> = HashMap::new(); + for row in rows { + out.entry(row.metric_definition_id) + .or_default() + .push(row.dimension_key); + } + out + }) +} + +fn build_definition( + row: &DefinitionRow, + inputs: &[MetricInput], + allowed_dimensions: Vec, +) -> Result { + let computation = MetricComputation::from_db(&row.computation_type).ok_or_else(|| { + config_error(&format!( + "unknown metric computation for {}", + row.metric_key + )) + })?; + let base = build_base(row, allowed_dimensions)?; + + let spec = match computation { + MetricComputation::Sum => ComputationSpec::Sum { + value: one_input(&row.metric_key, inputs, MetricInputRole::Value)?, + }, + MetricComputation::Ratio => { + let numerator = one_input(&row.metric_key, inputs, MetricInputRole::Numerator)?; + let denominator = one_input(&row.metric_key, inputs, MetricInputRole::Denominator)?; + if numerator.observation_source != denominator.observation_source + || numerator.source_key != denominator.source_key + { + return Err(config_error(&format!( + "ratio inputs must share one source for {}", + row.metric_key + ))); + } + let scale = row.scale.ok_or_else(|| { + config_error(&format!("missing ratio scale for {}", row.metric_key)) + })?; + ComputationSpec::Ratio { + numerator, + denominator, + scale, + } + } + }; + + Ok(MetricDefinition { base, spec }) +} + +fn build_base( + row: &DefinitionRow, + allowed_dimensions: Vec, +) -> Result { + let format = MetricFormat::from_db(&row.format) + .ok_or_else(|| config_error(&format!("unknown metric format for {}", row.metric_key)))?; + let direction = MetricDirection::from_db(&row.direction) + .ok_or_else(|| config_error(&format!("unknown metric direction for {}", row.metric_key)))?; + + Ok(MetricBase { + key: row.metric_key.clone(), + label: row.label.clone(), + description: row.description.clone(), + explanation: row.explanation.clone(), + entity_type: row.entity_type.clone(), + format, + unit: row.unit.clone(), + direction, + peer_cohort_key: row.peer_cohort_key.clone(), + allowed_dimensions, + }) +} + +fn one_input( + metric_key: &str, + inputs: &[MetricInput], + role: MetricInputRole, +) -> Result { + let matches = inputs + .iter() + .filter(|input| input.role == role) + .cloned() + .collect::>(); + match matches.as_slice() { + [input] => Ok(input.clone()), + [] => Err(config_error(&format!( + "missing {role:?} input for {metric_key}" + ))), + _ => Err(config_error(&format!( + "duplicate {role:?} inputs for {metric_key}" + ))), + } +} + +pub async fn all_managed_sources( + db: &DatabaseConnection, +) -> Result, sea_orm::DbErr> { + #[derive(FromQueryResult)] + struct Row { + id: Uuid, + source_kind: String, + source_ref: String, + } + + Row::find_by_statement(Statement::from_string( + db.get_database_backend(), + "SELECT id, source_kind, source_ref \ + FROM metric_sources \ + WHERE is_enabled = TRUE", + )) + .all(db) + .await + .map(|rows| { + rows.into_iter() + .map(|row| (row.id, row.source_kind, row.source_ref)) + .collect() + }) +} + +// `updated_at = updated_at` in the status writers below pins the column so +// ON UPDATE CURRENT_TIMESTAMP(3) does not fire: updated_at tracks config +// edits, not validator sweeps. +pub async fn update_source_status( + db: &DatabaseConnection, + source_id: Uuid, + status: SchemaStatus, + error_code: Option, +) -> Result<(), sea_orm::DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "UPDATE metric_sources \ + SET schema_status = ?, \ + schema_checked_at = CURRENT_TIMESTAMP(3), \ + schema_error_code = ?, \ + updated_at = updated_at \ + WHERE id = ?", + [ + Value::from(status.as_db()), + match error_code { + Some(code) => Value::from(code.as_db()), + None => Value::String(None), + }, + Value::Bytes(Some(Box::new(source_id.as_bytes().to_vec()))), + ], + )) + .await?; + Ok(()) +} + +pub async fn update_definitions_for_source_status( + db: &DatabaseConnection, + source_id: Uuid, + status: SchemaStatus, + error_code: Option, +) -> Result<(), sea_orm::DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "UPDATE metric_definitions \ + SET schema_status = ?, \ + schema_checked_at = CURRENT_TIMESTAMP(3), \ + schema_error_code = ?, \ + updated_at = updated_at \ + WHERE id IN ( \ + SELECT metric_definition_id \ + FROM metric_definition_inputs i \ + INNER JOIN metric_source_measures m ON m.id = i.source_measure_id \ + WHERE m.source_id = ? \ + )", + [ + Value::from(status.as_db()), + match error_code { + Some(code) => Value::from(code.as_db()), + None => Value::String(None), + }, + Value::Bytes(Some(Box::new(source_id.as_bytes().to_vec()))), + ], + )) + .await?; + Ok(()) +} + +pub async fn update_definition_status( + db: &DatabaseConnection, + definition_id: Uuid, + status: SchemaStatus, + error_code: Option, +) -> Result<(), sea_orm::DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "UPDATE metric_definitions \ + SET schema_status = ?, \ + schema_checked_at = CURRENT_TIMESTAMP(3), \ + schema_error_code = ?, \ + updated_at = updated_at \ + WHERE id = ?", + [ + Value::from(status.as_db()), + match error_code { + Some(code) => Value::from(code.as_db()), + None => Value::String(None), + }, + uuid_value(definition_id), + ], + )) + .await?; + Ok(()) +} + +fn uuid_value(value: Uuid) -> Value { + Value::Bytes(Some(Box::new(value.as_bytes().to_vec()))) +} + +fn unavailable(metric_key: &str) -> CanonicalError { + MetricError::invalid_argument() + .with_field_violation( + "metrics.metric_key", + format!("unknown or unavailable metric key: {metric_key}"), + "UNAVAILABLE", + ) + .create() +} + +fn config_error(message: &str) -> CanonicalError { + tracing::error!(message = %message, "metric definition configuration error"); + CanonicalError::internal("metric definition configuration error").create() +} + +fn db_error(error: &sea_orm::DbErr) -> CanonicalError { + tracing::error!(error = %error, "metric definition database query failed"); + CanonicalError::internal("metric definition lookup failed").create() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn definition_row( + metric_key: &str, + tenant: Option, + enabled: bool, + schema_status: &str, + ) -> DefinitionRow { + DefinitionRow { + definition_id: Uuid::now_v7(), + tenant_id: tenant, + metric_key: metric_key.to_owned(), + label: "Label".to_owned(), + description: None, + explanation: None, + unit: None, + format: "integer".to_owned(), + direction: "neutral".to_owned(), + entity_type: "person".to_owned(), + computation_type: "sum".to_owned(), + scale: None, + peer_cohort_key: Some("org_unit".to_owned()), + definition_enabled: enabled, + definition_schema_status: schema_status.to_owned(), + } + } + + fn input_row(definition_id: Uuid, role: &str, enabled: bool, status: &str) -> InputRow { + InputRow { + metric_definition_id: definition_id, + input_role: role.to_owned(), + measure_key: "accepted_lines".to_owned(), + measure_enabled: enabled, + measure_schema_status: status.to_owned(), + source_key: "ai_usage".to_owned(), + source_kind: "managed_observation".to_owned(), + source_ref: "ai_metric_observations".to_owned(), + source_enabled: true, + source_schema_status: "ok".to_owned(), + } + } + + fn available_inputs(id: Uuid) -> HashMap { + classify_inputs(vec![input_row(id, "value", true, "ok")]) + } + + #[test] + fn selects_tenant_row_over_product() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), true, "ok"); + let product_row = definition_row("ai.x", None, true, "ok"); + let tenant_id = tenant_row.definition_id; + let mut inputs = available_inputs(tenant_id); + inputs.extend(available_inputs(product_row.definition_id)); + + let Ok(Some(selected)) = + select_available_row("ai.x", vec![product_row, tenant_row], &inputs) + else { + panic!("expected selected row"); + }; + assert_eq!(selected.definition_id, tenant_id); + } + + #[test] + fn disabled_tenant_row_falls_back_to_product() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), false, "ok"); + let product_row = definition_row("ai.x", None, true, "ok"); + let product_id = product_row.definition_id; + let mut inputs = available_inputs(tenant_row.definition_id); + inputs.extend(available_inputs(product_id)); + + let Ok(Some(selected)) = + select_available_row("ai.x", vec![tenant_row, product_row], &inputs) + else { + panic!("expected selected row"); + }; + assert_eq!(selected.definition_id, product_id); + } + + #[test] + fn schema_error_tenant_row_falls_back_to_product() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), true, "error"); + let product_row = definition_row("ai.x", None, true, "ok"); + let product_id = product_row.definition_id; + let mut inputs = available_inputs(tenant_row.definition_id); + inputs.extend(available_inputs(product_id)); + + let Ok(Some(selected)) = + select_available_row("ai.x", vec![tenant_row, product_row], &inputs) + else { + panic!("expected selected row"); + }; + assert_eq!(selected.definition_id, product_id); + } + + #[test] + fn unavailable_inputs_fall_back_to_product() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), true, "ok"); + let product_row = definition_row("ai.x", None, true, "ok"); + let product_id = product_row.definition_id; + let mut inputs = classify_inputs(vec![input_row( + tenant_row.definition_id, + "value", + false, + "ok", + )]); + inputs.extend(available_inputs(product_id)); + + let Ok(Some(selected)) = + select_available_row("ai.x", vec![tenant_row, product_row], &inputs) + else { + panic!("expected selected row"); + }; + assert_eq!(selected.definition_id, product_id); + } + + #[test] + fn no_available_row_yields_none() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), false, "ok"); + let product_row = definition_row("ai.x", None, true, "error"); + let mut inputs = available_inputs(tenant_row.definition_id); + inputs.extend(available_inputs(product_row.definition_id)); + + let Ok(selected) = select_available_row("ai.x", vec![tenant_row, product_row], &inputs) + else { + panic!("expected ok selection"); + }; + assert!(selected.is_none()); + } + + #[test] + fn product_only_selection_works() { + let product_row = definition_row("ai.x", None, true, "ok"); + let product_id = product_row.definition_id; + let inputs = available_inputs(product_id); + + let Ok(Some(selected)) = select_available_row("ai.x", vec![product_row], &inputs) else { + panic!("expected selected row"); + }; + assert_eq!(selected.definition_id, product_id); + } + + #[test] + fn duplicate_tenant_rows_are_config_errors() { + let tenant = Uuid::now_v7(); + let rows = vec![ + definition_row("ai.x", Some(tenant), true, "ok"), + definition_row("ai.x", Some(tenant), true, "ok"), + ]; + assert!(select_available_row("ai.x", rows, &HashMap::new()).is_err()); + } + + #[test] + fn corrupt_inputs_are_config_errors_not_fallback() { + let tenant = Uuid::now_v7(); + let tenant_row = definition_row("ai.x", Some(tenant), true, "ok"); + let product_row = definition_row("ai.x", None, true, "ok"); + let mut inputs = classify_inputs(vec![InputRow { + input_role: "nonsense".to_owned(), + ..input_row(tenant_row.definition_id, "value", true, "ok") + }]); + inputs.extend(available_inputs(product_row.definition_id)); + + assert!(select_available_row("ai.x", vec![tenant_row, product_row], &inputs).is_err()); + } + + #[test] + fn classify_corrupt_wins_over_earlier_unavailable_row() { + let id = Uuid::now_v7(); + let disabled = input_row(id, "value", false, "ok"); + let corrupt = InputRow { + input_role: "nonsense".to_owned(), + ..input_row(id, "value", true, "ok") + }; + let classified = classify_inputs(vec![disabled, corrupt]); + assert!(matches!( + classified.get(&id), + Some(ClassifiedInputs::Corrupt) + )); + } + + #[test] + fn classify_corrupt_is_not_downgraded_by_later_rows() { + let id = Uuid::now_v7(); + let corrupt = InputRow { + input_role: "nonsense".to_owned(), + ..input_row(id, "value", true, "ok") + }; + let disabled = input_row(id, "value", false, "ok"); + let available = input_row(id, "value", true, "ok"); + let classified = classify_inputs(vec![corrupt, disabled, available]); + assert!(matches!( + classified.get(&id), + Some(ClassifiedInputs::Corrupt) + )); + } + + #[test] + fn classify_corrupt_is_not_downgraded_by_later_custom_sql_row() { + let id = Uuid::now_v7(); + let corrupt = InputRow { + input_role: "nonsense".to_owned(), + ..input_row(id, "value", true, "ok") + }; + let mut custom = input_row(id, "value", true, "ok"); + custom.source_kind = "custom_observation_sql".to_owned(); + let classified = classify_inputs(vec![corrupt, custom]); + assert!(matches!( + classified.get(&id), + Some(ClassifiedInputs::Corrupt) + )); + } + + #[test] + fn classify_marks_custom_sql_source_unavailable_not_corrupt() { + let id = Uuid::now_v7(); + let mut row = input_row(id, "value", true, "ok"); + row.source_kind = "custom_observation_sql".to_owned(); + let classified = classify_inputs(vec![row]); + assert!(matches!( + classified.get(&id), + Some(ClassifiedInputs::Unavailable) + )); + } + + #[test] + fn classify_marks_disabled_source_unavailable() { + let id = Uuid::now_v7(); + let mut row = input_row(id, "value", true, "ok"); + row.source_enabled = false; + let classified = classify_inputs(vec![row]); + assert!(matches!( + classified.get(&id), + Some(ClassifiedInputs::Unavailable) + )); + } + + #[test] + fn classify_keeps_available_inputs() { + let id = Uuid::now_v7(); + let classified = classify_inputs(vec![ + input_row(id, "numerator", true, "ok"), + input_row(id, "denominator", true, "ok"), + ]); + match classified.get(&id) { + Some(ClassifiedInputs::Available(inputs)) => assert_eq!(inputs.len(), 2), + other => panic!("expected available inputs, got {other:?}"), + } + } + + #[test] + fn one_input_rejects_missing_and_duplicate_roles() { + let input = MetricInput { + role: MetricInputRole::Value, + observation_source: ObservationSource::AiMetricObservations, + source_key: "ai_usage".to_owned(), + measure_key: "accepted_lines".to_owned(), + }; + assert!(one_input("ai.x", &[], MetricInputRole::Value).is_err()); + assert!(one_input("ai.x", std::slice::from_ref(&input), MetricInputRole::Value).is_ok()); + assert!(one_input("ai.x", &[input.clone(), input], MetricInputRole::Value).is_err()); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs b/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs new file mode 100644 index 000000000..55e3e6505 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/seeds.rs @@ -0,0 +1,350 @@ +use sea_orm::{ConnectionTrait, DatabaseConnection, DbErr, Statement, Value}; +use uuid::Uuid; + +use crate::domain::metric_definitions::builtin::{ + BUILTIN_METRICS, BUILTIN_SOURCES, BuiltinSource, CohortKey, InputSeed, MetricSeed, +}; + +pub async fn reconcile_builtin_definitions(db: &DatabaseConnection) -> Result<(), DbErr> { + for builtin_source in BUILTIN_SOURCES { + reconcile_source(db, builtin_source).await?; + } + + for metric in BUILTIN_METRICS { + let source_id = fetch_source_id(db, metric.source_key).await?; + upsert_metric(db, metric).await?; + let metric_id = fetch_metric_id(db, metric.metric_key).await?; + replace_inputs(db, source_id, metric_id, metric.inputs).await?; + replace_dimensions(db, source_id, metric_id, metric.dimensions).await?; + } + + disable_missing_builtin_rows(db).await?; + Ok(()) +} + +async fn reconcile_source( + db: &DatabaseConnection, + builtin_source: &BuiltinSource, +) -> Result<(), DbErr> { + upsert_source(db, builtin_source).await?; + let source_id = fetch_source_id(db, builtin_source.source.key).await?; + + for measure_key in builtin_source.measures { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_source_measures \ + (id, source_id, measure_key, is_enabled) \ + VALUES (?, ?, ?, TRUE) \ + ON DUPLICATE KEY UPDATE \ + is_enabled = VALUES(is_enabled)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(source_id), + Value::from(*measure_key), + ], + )) + .await?; + } + + for (idx, dimension_key) in builtin_source.dimensions.iter().enumerate() { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_source_dimensions \ + (id, source_id, dimension_key, display_order) \ + VALUES (?, ?, ?, ?) \ + ON DUPLICATE KEY UPDATE \ + display_order = VALUES(display_order)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(source_id), + Value::from(*dimension_key), + Value::from(order_value(idx)), + ], + )) + .await?; + } + + Ok(()) +} + +async fn upsert_source( + db: &DatabaseConnection, + builtin_source: &BuiltinSource, +) -> Result<(), DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_sources \ + (id, tenant_id, source_key, source_kind, source_ref, origin, is_enabled) \ + VALUES (?, NULL, ?, ?, ?, 'builtin', TRUE) \ + ON DUPLICATE KEY UPDATE \ + source_kind = VALUES(source_kind), \ + source_ref = VALUES(source_ref), \ + origin = VALUES(origin), \ + is_enabled = VALUES(is_enabled)", + [ + uuid_value(Uuid::now_v7()), + Value::from(builtin_source.source.key), + Value::from(builtin_source.source.kind.as_db()), + Value::from(builtin_source.source.source_ref.source_ref()), + ], + )) + .await?; + Ok(()) +} + +async fn upsert_metric(db: &DatabaseConnection, metric: &MetricSeed) -> Result<(), DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definitions \ + (id, tenant_id, metric_key, label, description, explanation, unit, format, direction, entity_type, \ + computation_type, scale, peer_cohort_key, origin, is_enabled) \ + VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'builtin', TRUE) \ + ON DUPLICATE KEY UPDATE \ + label = VALUES(label), \ + description = VALUES(description), \ + explanation = VALUES(explanation), \ + unit = VALUES(unit), \ + format = VALUES(format), \ + direction = VALUES(direction), \ + entity_type = VALUES(entity_type), \ + computation_type = VALUES(computation_type), \ + scale = VALUES(scale), \ + peer_cohort_key = VALUES(peer_cohort_key), \ + origin = VALUES(origin), \ + is_enabled = VALUES(is_enabled)", + [ + uuid_value(Uuid::now_v7()), + Value::from(metric.metric_key), + Value::from(metric.label), + nullable_str(metric.description), + nullable_str(metric.explanation), + nullable_str(metric.unit), + Value::from(metric.format.as_db()), + Value::from(metric.direction.as_db()), + Value::from(metric.entity_type.as_db()), + Value::from(metric.computation.computation().as_db()), + match metric.computation.scale() { + Some(scale) => Value::from(scale), + None => Value::Double(None), + }, + nullable_str(metric.peer_cohort_key.map(CohortKey::as_db)), + ], + )) + .await?; + Ok(()) +} + +async fn replace_inputs( + db: &DatabaseConnection, + source_id: Uuid, + metric_id: Uuid, + inputs: &[InputSeed], +) -> Result<(), DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "DELETE FROM metric_definition_inputs WHERE metric_definition_id = ?", + [uuid_value(metric_id)], + )) + .await?; + + for input in inputs { + let measure_id = fetch_measure_id(db, source_id, input.measure_key).await?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definition_inputs \ + (id, metric_definition_id, input_role, source_measure_id) \ + VALUES (?, ?, ?, ?)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(metric_id), + Value::from(input.input_role.as_db()), + uuid_value(measure_id), + ], + )) + .await?; + } + Ok(()) +} + +async fn replace_dimensions( + db: &DatabaseConnection, + source_id: Uuid, + metric_id: Uuid, + dimensions: &[&str], +) -> Result<(), DbErr> { + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "DELETE FROM metric_definition_dimensions WHERE metric_definition_id = ?", + [uuid_value(metric_id)], + )) + .await?; + + for (idx, dimension) in dimensions.iter().enumerate() { + let dimension_id = fetch_source_dimension_id(db, source_id, dimension).await?; + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO metric_definition_dimensions \ + (id, metric_definition_id, source_dimension_id, display_order) \ + VALUES (?, ?, ?, ?)", + [ + uuid_value(Uuid::now_v7()), + uuid_value(metric_id), + uuid_value(dimension_id), + Value::from(order_value(idx)), + ], + )) + .await?; + } + Ok(()) +} + +async fn disable_missing_builtin_rows(db: &DatabaseConnection) -> Result<(), DbErr> { + let metric_keys = BUILTIN_METRICS + .iter() + .map(|metric| metric.metric_key) + .collect::>(); + disable_missing( + db, + "UPDATE metric_definitions SET is_enabled = FALSE \ + WHERE tenant_id IS NULL AND origin = 'builtin' AND is_enabled = TRUE", + "metric_key", + &metric_keys, + ) + .await?; + + let source_keys = BUILTIN_SOURCES + .iter() + .map(|builtin_source| builtin_source.source.key) + .collect::>(); + disable_missing( + db, + "UPDATE metric_sources SET is_enabled = FALSE \ + WHERE tenant_id IS NULL AND origin = 'builtin' AND is_enabled = TRUE", + "source_key", + &source_keys, + ) + .await?; + + for builtin_source in BUILTIN_SOURCES { + let source_id = fetch_source_id(db, builtin_source.source.key).await?; + let measure_keys = builtin_source.measures.to_vec(); + let placeholders = vec!["?"; measure_keys.len()].join(", "); + let sql = format!( + "UPDATE metric_source_measures SET is_enabled = FALSE \ + WHERE source_id = ? AND is_enabled = TRUE \ + AND measure_key NOT IN ({placeholders})" + ); + let mut values = vec![uuid_value(source_id)]; + values.extend(measure_keys.iter().map(|key| Value::from(*key))); + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .await?; + } + + Ok(()) +} + +async fn disable_missing( + db: &DatabaseConnection, + base_sql: &str, + key_column: &str, + keys: &[&str], +) -> Result<(), DbErr> { + let sql = if keys.is_empty() { + base_sql.to_owned() + } else { + let placeholders = vec!["?"; keys.len()].join(", "); + format!("{base_sql} AND {key_column} NOT IN ({placeholders})") + }; + let values = keys.iter().map(|key| Value::from(*key)).collect::>(); + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .await?; + Ok(()) +} + +async fn fetch_source_id(db: &DatabaseConnection, source_key: &str) -> Result { + fetch_uuid( + db, + "SELECT id FROM metric_sources WHERE tenant_id IS NULL AND source_key = ?", + &[Value::from(source_key)], + source_key, + ) + .await +} + +async fn fetch_measure_id( + db: &DatabaseConnection, + source_id: Uuid, + measure_key: &str, +) -> Result { + fetch_uuid( + db, + "SELECT id FROM metric_source_measures WHERE source_id = ? AND measure_key = ?", + &[uuid_value(source_id), Value::from(measure_key)], + measure_key, + ) + .await +} + +async fn fetch_source_dimension_id( + db: &DatabaseConnection, + source_id: Uuid, + dimension_key: &str, +) -> Result { + fetch_uuid( + db, + "SELECT id FROM metric_source_dimensions WHERE source_id = ? AND dimension_key = ?", + &[uuid_value(source_id), Value::from(dimension_key)], + dimension_key, + ) + .await +} + +async fn fetch_metric_id(db: &DatabaseConnection, metric_key: &str) -> Result { + fetch_uuid( + db, + "SELECT id FROM metric_definitions WHERE tenant_id IS NULL AND metric_key = ?", + &[Value::from(metric_key)], + metric_key, + ) + .await +} + +async fn fetch_uuid( + db: &DatabaseConnection, + sql: &str, + values: &[Value], + key: &str, +) -> Result { + let row = db + .query_one(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values.to_vec(), + )) + .await? + .ok_or_else(|| DbErr::Custom(format!("missing seeded row for {key}")))?; + row.try_get("", "id") +} + +fn order_value(idx: usize) -> i32 { + i32::try_from(idx).unwrap_or(i32::MAX) +} + +fn uuid_value(id: Uuid) -> Value { + Value::Bytes(Some(Box::new(id.as_bytes().to_vec()))) +} + +fn nullable_str(value: Option<&str>) -> Value { + match value { + Some(value) => Value::from(value), + None => Value::String(None), + } +} diff --git a/src/backend/services/analytics/src/domain/metric_definitions/validator.rs b/src/backend/services/analytics/src/domain/metric_definitions/validator.rs new file mode 100644 index 000000000..4077e878a --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_definitions/validator.rs @@ -0,0 +1,531 @@ +use std::collections::{BTreeSet, HashMap}; + +use clickhouse::Row; +use sea_orm::DatabaseConnection; +use serde::Deserialize; + +use crate::domain::metric_definitions::definition::{CohortSource, ObservationSource, SourceKind}; +use crate::domain::metric_definitions::error_code::{MetricSchemaErrorCode, SchemaStatus}; +use crate::domain::metric_definitions::repository::{ + MetricDefinitionValidationSpec, all_managed_sources, managed_definition_validation_specs, + update_definition_status, update_definitions_for_source_status, update_source_status, +}; + +const PROBE_WINDOW_DAYS: u32 = 35; +// Managed observation relations are dbt-created and can appear (or regress) +// while the service is running — a one-shot startup scan would pin +// `table_not_found` until the next pod restart. Sweeps are idempotent: +// transient probe failures never overwrite an established status, and +// status writes pin `updated_at`. +const SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); + +#[derive(Clone)] +pub struct MetricDefinitionValidator { + db: DatabaseConnection, + ch: insight_clickhouse::Client, +} + +impl MetricDefinitionValidator { + pub fn new(db: DatabaseConnection, ch: insight_clickhouse::Client) -> Self { + Self { db, ch } + } + + /// Periodic sweep: validates immediately, then every [`SWEEP_INTERVAL`]. + /// Never returns; run it on a spawned task. + pub async fn run(self) { + let mut ticks = tokio::time::interval(SWEEP_INTERVAL); + ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + ticks.tick().await; + self.validate_all().await; + } + } + + pub async fn validate_all(&self) { + let sources = match all_managed_sources(&self.db).await { + Ok(sources) => sources, + Err(error) => { + tracing::warn!(error = %error, "metric definition validation source load failed"); + return; + } + }; + + for (source_id, source_kind, source_ref) in sources { + let outcome = self + .validate_source(source_kind.as_str(), source_ref.as_str()) + .await; + + match outcome { + ProbeOutcome::Definitive(state) => { + let (status, error_code) = state.as_db(); + if let Err(error) = + update_source_status(&self.db, source_id, status, error_code).await + { + tracing::warn!(error = %error, "metric definition source status update failed"); + continue; + } + if state.is_ok() { + self.validate_definitions_for_source(source_id, source_ref.as_str()) + .await; + } else if let Err(error) = update_definitions_for_source_status( + &self.db, source_id, status, error_code, + ) + .await + { + tracing::warn!(error = %error, "metric definition status update failed"); + } + } + ProbeOutcome::Inconclusive => { + tracing::warn!( + source_ref = %source_ref, + "metric source validation inconclusive; keeping previous status" + ); + } + } + } + } + + async fn validate_source(&self, source_kind: &str, source_ref: &str) -> ProbeOutcome { + match SourceKind::from_db(source_kind) { + Some(SourceKind::ManagedObservation) => {} + Some(SourceKind::CustomObservationSql) => return ProbeOutcome::Inconclusive, + None => { + return ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::Unknown, + )); + } + } + + let Some(source) = ObservationSource::from_ref(source_ref) else { + return ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::Unknown, + )); + }; + let cohort = CohortSource::MetricEntityCohortsCurrent; + + match self + .has_columns(source.table_ref(), OBSERVATION_COLUMNS) + .await + { + Ok(ColumnCheck::Present) => {} + Ok(missing) => { + return ProbeOutcome::Definitive(ValidationState::Error(missing.error_code())); + } + Err(error) => { + tracing::warn!(error = %error, "metric observation source validation failed"); + return ProbeOutcome::Inconclusive; + } + } + + match self.has_columns(cohort.table_ref(), COHORT_COLUMNS).await { + Ok(ColumnCheck::Present) => ProbeOutcome::Definitive(ValidationState::Ok), + Ok(missing) => ProbeOutcome::Definitive(ValidationState::Error(missing.error_code())), + Err(error) => { + tracing::warn!(error = %error, "metric cohort source validation failed"); + ProbeOutcome::Inconclusive + } + } + } + + async fn validate_definitions_for_source(&self, source_id: uuid::Uuid, source_ref: &str) { + let Some(source) = ObservationSource::from_ref(source_ref) else { + return; + }; + let specs = match managed_definition_validation_specs(&self.db, source_id).await { + Ok(specs) => specs, + Err(error) => { + tracing::warn!(error = %error, "metric definition validation spec load failed"); + return; + } + }; + + for spec in specs { + match self.validate_definition(source, &spec).await { + ProbeOutcome::Definitive(state) => { + let (status, error_code) = state.as_db(); + if let Err(error) = + update_definition_status(&self.db, spec.definition_id, status, error_code) + .await + { + tracing::warn!( + error = %error, + metric_key = %spec.metric_key, + "metric definition status update failed" + ); + } + } + ProbeOutcome::Inconclusive => { + tracing::warn!( + metric_key = %spec.metric_key, + "metric definition validation inconclusive; keeping previous status" + ); + } + } + } + } + + async fn validate_definition( + &self, + source: ObservationSource, + spec: &MetricDefinitionValidationSpec, + ) -> ProbeOutcome { + let inputs = spec + .inputs + .iter() + .filter(|input| input.observation_source == source) + .collect::>(); + if inputs.is_empty() { + return ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::Unknown, + )); + } + + let source_keys = inputs + .iter() + .map(|input| input.source_key.as_str()) + .collect::>() + .into_iter() + .collect::>(); + let Some(source_key) = source_keys.first().copied() else { + return ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::Unknown, + )); + }; + if source_keys.len() != 1 { + return ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::Unknown, + )); + } + + let measure_keys = inputs + .iter() + .map(|input| input.measure_key.as_str()) + .collect::>(); + + match self + .has_source_rows(source, source_key, spec.entity_type.as_str()) + .await + { + Ok(true) => {} + Ok(false) => return ProbeOutcome::Definitive(ValidationState::Unchecked), + Err(error) => { + tracing::warn!(error = %error, "metric observation row probe failed"); + return ProbeOutcome::Inconclusive; + } + } + + // Absence of recent rows for a declared measure is a data condition, + // not a schema error: filtered measures (e.g. tool-scoped + // conversations) legitimately go quiet. Only measures that ARE + // observed can be checked definitively; unobserved ones downgrade + // the definition to unchecked, which stays runtime-available. + let observed = match self + .observed_measure_keys( + source, + source_key, + spec.entity_type.as_str(), + &measure_keys.iter().copied().collect::>(), + ) + .await + { + Ok(observed) => observed, + Err(error) => { + tracing::warn!(error = %error, "metric measure probe failed"); + return ProbeOutcome::Inconclusive; + } + }; + let observed_keys = measure_keys + .iter() + .copied() + .filter(|key| observed.contains(*key)) + .collect::>(); + + if let Some(outcome) = self + .check_dimension_coverage(source, source_key, spec, &observed_keys) + .await + { + return outcome; + } + + if observed_keys.len() < measure_keys.len() { + let unobserved = measure_keys + .iter() + .copied() + .filter(|key| !observed.contains(*key)) + .collect::>(); + tracing::warn!( + metric_key = %spec.metric_key, + unobserved = ?unobserved, + "declared measures without recent observations; definition stays unchecked" + ); + return ProbeOutcome::Definitive(ValidationState::Unchecked); + } + + ProbeOutcome::Definitive(ValidationState::Ok) + } + + async fn check_dimension_coverage( + &self, + source: ObservationSource, + source_key: &str, + spec: &MetricDefinitionValidationSpec, + observed_keys: &[&str], + ) -> Option { + if observed_keys.is_empty() { + return None; + } + for dimension in &spec.dimensions { + match self + .dimension_present_on_all_rows( + source, + source_key, + spec.entity_type.as_str(), + observed_keys.iter().copied(), + dimension, + ) + .await + { + Ok(true) => {} + Ok(false) => { + return Some(ProbeOutcome::Definitive(ValidationState::Error( + MetricSchemaErrorCode::DimensionNotCovered, + ))); + } + Err(error) => { + tracing::warn!(error = %error, "metric dimension probe failed"); + return Some(ProbeOutcome::Inconclusive); + } + } + } + None + } + + async fn has_columns( + &self, + table: (&str, &str), + columns: &[&str], + ) -> Result { + let (database, table) = table; + let column_list = columns + .iter() + .map(|column| format!("'{column}'")) + .collect::>() + .join(", "); + let sql = format!( + "SELECT \ + count() AS total_columns, \ + countIf(name IN ({column_list})) AS matching_columns \ + FROM system.columns \ + WHERE database = ? AND table = ?" + ); + let mut query = self.ch.query(&sql); + query = query.bind(database).bind(table); + let row: ColumnProbeRow = query.fetch_one().await?; + if row.total_columns == 0 { + return Ok(ColumnCheck::TableMissing); + } + if row.matching_columns < columns.len() as u64 { + return Ok(ColumnCheck::ColumnsMissing); + } + Ok(ColumnCheck::Present) + } + + async fn has_source_rows( + &self, + source: ObservationSource, + source_key: &str, + entity_type: &str, + ) -> Result { + let (database, table) = source.table_ref(); + let sql = format!( + "SELECT count() AS rows \ + FROM ( \ + SELECT 1 \ + FROM {database}.{table} \ + WHERE source_key = ? \ + AND entity_type = ? \ + AND metric_date >= today() - {PROBE_WINDOW_DAYS} \ + LIMIT 1 \ + )" + ); + let row: CountProbeRow = self + .ch + .query(&sql) + .bind(source_key) + .bind(entity_type) + .fetch_one() + .await?; + Ok(row.rows > 0) + } + + async fn observed_measure_keys( + &self, + source: ObservationSource, + source_key: &str, + entity_type: &str, + measure_keys: &[&str], + ) -> Result, clickhouse::error::Error> { + let (database, table) = source.table_ref(); + let placeholders = vec!["?"; measure_keys.len()].join(", "); + let sql = format!( + "SELECT measure_key \ + FROM {database}.{table} \ + WHERE source_key = ? \ + AND entity_type = ? \ + AND metric_date >= today() - {PROBE_WINDOW_DAYS} \ + AND measure_key IN ({placeholders}) \ + GROUP BY measure_key" + ); + let mut query = self.ch.query(&sql).bind(source_key).bind(entity_type); + for measure_key in measure_keys { + query = query.bind(*measure_key); + } + let rows = query.fetch_all::().await?; + Ok(rows.into_iter().map(|row| row.measure_key).collect()) + } + + async fn dimension_present_on_all_rows<'a>( + &self, + source: ObservationSource, + source_key: &str, + entity_type: &str, + measure_keys: impl Iterator, + dimension: &str, + ) -> Result { + let measure_keys = measure_keys.collect::>(); + let rows = self + .dimension_coverage(source, source_key, entity_type, &measure_keys, dimension) + .await?; + let by_measure = rows + .into_iter() + .map(|row| (row.measure_key.clone(), row)) + .collect::>(); + + Ok(measure_keys.iter().all(|measure_key| { + by_measure + .get(*measure_key) + .is_some_and(|row| row.total_rows > 0 && row.total_rows == row.matching_rows) + })) + } + + async fn dimension_coverage( + &self, + source: ObservationSource, + source_key: &str, + entity_type: &str, + measure_keys: &[&str], + dimension: &str, + ) -> Result, clickhouse::error::Error> { + let (database, table) = source.table_ref(); + let placeholders = vec!["?"; measure_keys.len()].join(", "); + let sql = format!( + "SELECT \ + measure_key, \ + count() AS total_rows, \ + countIf(has(arrayMap(d -> d.key, dimensions), ?)) AS matching_rows \ + FROM {database}.{table} \ + WHERE source_key = ? \ + AND entity_type = ? \ + AND metric_date >= today() - {PROBE_WINDOW_DAYS} \ + AND measure_key IN ({placeholders}) \ + GROUP BY measure_key" + ); + let mut query = self + .ch + .query(&sql) + .bind(dimension) + .bind(source_key) + .bind(entity_type); + for measure_key in measure_keys { + query = query.bind(*measure_key); + } + query.fetch_all().await + } +} + +const OBSERVATION_COLUMNS: &[&str] = &[ + "tenant_id", + "source_key", + "entity_type", + "entity_id", + "metric_date", + "observed_at", + "measure_key", + "value", + "subject_key", + "dimensions", +]; + +const COHORT_COLUMNS: &[&str] = &[ + "tenant_id", + "entity_type", + "entity_id", + "cohort_key", + "cohort_id", +]; + +#[derive(Row, Deserialize)] +struct ColumnProbeRow { + total_columns: u64, + matching_columns: u64, +} + +#[derive(Row, Deserialize)] +struct CountProbeRow { + rows: u64, +} + +#[derive(Row, Deserialize)] +struct MeasureKeyProbeRow { + measure_key: String, +} + +#[derive(Row, Deserialize)] +struct DimensionCoverageProbeRow { + measure_key: String, + total_rows: u64, + matching_rows: u64, +} + +#[derive(Debug, Clone, Copy)] +enum ColumnCheck { + Present, + ColumnsMissing, + TableMissing, +} + +impl ColumnCheck { + fn error_code(self) -> MetricSchemaErrorCode { + match self { + Self::TableMissing => MetricSchemaErrorCode::TableNotFound, + Self::ColumnsMissing | Self::Present => MetricSchemaErrorCode::ColumnNotFound, + } + } +} + +#[derive(Debug, Clone, Copy)] +enum ProbeOutcome { + Definitive(ValidationState), + Inconclusive, +} + +#[derive(Debug, Clone, Copy)] +enum ValidationState { + Ok, + Error(MetricSchemaErrorCode), + Unchecked, +} + +impl ValidationState { + fn is_ok(self) -> bool { + matches!(self, Self::Ok) + } + + fn as_db(self) -> (SchemaStatus, Option) { + match self { + Self::Ok => (SchemaStatus::Ok, None), + Self::Error(code) => (SchemaStatus::Error, Some(code)), + Self::Unchecked => (SchemaStatus::Unchecked, None), + } + } +} diff --git a/src/backend/services/analytics/src/domain/metric_results/builder.rs b/src/backend/services/analytics/src/domain/metric_results/builder.rs new file mode 100644 index 000000000..a59eb9626 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/builder.rs @@ -0,0 +1,457 @@ +use std::collections::{BTreeMap, HashMap}; + +use toolkit_canonical_errors::CanonicalError; + +use crate::domain::metric_definitions::{ComputationSpec, MetricDefinition}; + +use super::compiler::{ + BreakdownQueryRow, PeerQueryRow, PeriodQueryRow, TimeseriesQueryRow, UNKNOWN_DIMENSION_LABEL, + UNKNOWN_DIMENSION_VALUE, dimension_aliases, +}; +use super::dto::{ + BreakdownValueDto, ComputationDto, MetricDimensionDto, MetricResultDto, MetricResultViewDto, + MetricResultsResponse, PeerValueDto, PeriodValueDto, TimeseriesDto, TimeseriesPointDto, +}; +use super::validation::{ + ValidatedMetricResultsRequest, enumerate_buckets, metric_result_too_large, row_limit, +}; +use super::view::Bucket; + +type DimensionKey = Vec<(String, String, Option)>; +type SeriesKey = (String, DimensionKey); +type PointsByBucket = HashMap>; + +pub fn build_period_view( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + rows: Vec, +) -> MetricResultViewDto { + let values_by_entity: HashMap> = rows + .into_iter() + .map(|row| (row.entity_id, row.value)) + .collect(); + let values = req + .entity_ids + .iter() + .map(|entity_id| PeriodValueDto { + entity_id: entity_id.clone(), + value: values_by_entity + .get(entity_id) + .copied() + .flatten() + .or_else(|| { + if def.is_zero_filled() { + Some(0.0) + } else { + None + } + }), + }) + .collect(); + MetricResultViewDto::Period { values } +} + +pub fn build_timeseries_view( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + bucket: Bucket, + dimensions: &[String], + rows: Vec, +) -> Result { + let buckets = enumerate_buckets(req.from, req.to, bucket); + let mut by_series: BTreeMap = BTreeMap::new(); + + if dimensions.is_empty() { + for entity_id in &req.entity_ids { + by_series + .entry((entity_id.clone(), Vec::new())) + .or_default(); + } + } + + for row in rows { + let dims = row_dimensions(&row.extra, dimensions)?; + by_series + .entry((row.entity_id, dims.clone())) + .or_default() + .insert(row.bucket_start, row.value); + } + + let series = by_series + .into_iter() + .map(|((entity_id, dims), points_by_bucket)| { + let points = buckets + .iter() + .map(|bucket| TimeseriesPointDto { + bucket_start: bucket.clone(), + value: points_by_bucket.get(bucket).copied().flatten().or_else(|| { + if def.is_zero_filled() { + Some(0.0) + } else { + None + } + }), + }) + .collect(); + TimeseriesDto { + entity_id, + dimensions: dims + .into_iter() + .map(|(key, value, label)| MetricDimensionDto { key, value, label }) + .collect(), + points, + } + }) + .collect(); + + Ok(MetricResultViewDto::Timeseries { bucket, series }) +} + +pub fn build_peer_view(rows: Vec) -> MetricResultViewDto { + MetricResultViewDto::Peer { + values: rows + .into_iter() + .map(|row| PeerValueDto { + entity_id: row.entity_id, + target_value: row.target_value, + p25: row.p25, + median: row.median, + p75: row.p75, + min: row.min, + max: row.max, + n: row.n.unwrap_or(0), + }) + .collect(), + } +} + +pub fn build_breakdown_view( + dimensions: &[String], + rows: Vec, +) -> Result { + let values = rows + .into_iter() + .map(|row| { + Ok(BreakdownValueDto { + entity_id: row.entity_id, + dimensions: row_dimensions(&row.extra, dimensions)? + .into_iter() + .map(|(key, value, label)| MetricDimensionDto { key, value, label }) + .collect(), + value: row.value, + }) + }) + .collect::, CanonicalError>>()?; + Ok(MetricResultViewDto::Breakdown { + dimensions: dimensions.iter().map(|d| (*d).clone()).collect(), + values, + }) +} + +pub fn build_metric_result( + def: &MetricDefinition, + views: Vec, +) -> MetricResultDto { + let computation = match &def.spec { + ComputationSpec::Sum { .. } => ComputationDto::Sum, + ComputationSpec::Ratio { scale, .. } => ComputationDto::Ratio { scale: *scale }, + }; + MetricResultDto { + metric_key: def.base.key.clone(), + label: def.base.label.clone(), + description: def.base.description.clone(), + explanation: def.base.explanation.clone(), + unit: def.base.unit.clone(), + format: def.base.format, + direction: def.base.direction, + computation, + views, + } +} + +pub fn enforce_row_limit(response: &MetricResultsResponse) -> Result<(), CanonicalError> { + if response_size(response) > row_limit() { + return Err(metric_result_too_large()); + } + Ok(()) +} + +fn response_size(response: &MetricResultsResponse) -> usize { + response + .metrics + .iter() + .flat_map(|metric| &metric.views) + .map(|view| match view { + MetricResultViewDto::Period { values } => values.len(), + MetricResultViewDto::Timeseries { series, .. } => { + series.iter().map(|s| s.points.len()).sum() + } + MetricResultViewDto::Peer { values } => values.len(), + MetricResultViewDto::Breakdown { values, .. } => values.len(), + }) + .sum() +} + +fn row_dimensions( + extra: &HashMap, + dimensions: &[String], +) -> Result)>, CanonicalError> { + dimensions + .iter() + .enumerate() + .map(|(idx, key)| { + let (value_alias, label_alias) = dimension_aliases(idx); + let value_field = extra.get(&value_alias).ok_or_else(|| { + tracing::error!(alias = %value_alias, "metric result row missing dimension alias"); + CanonicalError::internal("metric result shape mismatch").create() + })?; + let label_field = extra.get(&label_alias).ok_or_else(|| { + tracing::error!(alias = %label_alias, "metric result row missing dimension alias"); + CanonicalError::internal("metric result shape mismatch").create() + })?; + let value = json_string(Some(value_field)) + .unwrap_or_else(|| UNKNOWN_DIMENSION_VALUE.to_owned()); + let label = + json_string(Some(label_field)).or_else(|| Some(UNKNOWN_DIMENSION_LABEL.to_owned())); + Ok((key.clone(), value, label)) + }) + .collect() +} + +fn json_string(value: Option<&serde_json::Value>) -> Option { + match value { + Some(serde_json::Value::String(s)) => Some(s.clone()), + Some(serde_json::Value::Null) | None => None, + Some(v) => Some(v.to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::NaiveDate; + use serde_json::json; + + use crate::domain::metric_definitions::definition::{ + MetricBase, MetricDirection, MetricFormat, MetricInput, MetricInputRole, ObservationSource, + }; + use crate::domain::metric_results::view::Bucket; + + fn base() -> MetricBase { + MetricBase { + key: "ai.accepted_lines".to_owned(), + label: "AI-added lines".to_owned(), + description: None, + explanation: None, + entity_type: "person".to_owned(), + format: MetricFormat::Integer, + unit: None, + direction: MetricDirection::HigherIsBetter, + peer_cohort_key: None, + allowed_dimensions: vec!["tool".to_owned()], + } + } + + fn input(role: MetricInputRole, measure_key: &str) -> MetricInput { + MetricInput { + role, + observation_source: ObservationSource::AiMetricObservations, + source_key: "ai_usage".to_owned(), + measure_key: measure_key.to_owned(), + } + } + + fn sum_metric() -> MetricDefinition { + MetricDefinition { + base: base(), + spec: ComputationSpec::Sum { + value: input(MetricInputRole::Value, "accepted_lines"), + }, + } + } + + fn ratio_metric() -> MetricDefinition { + MetricDefinition { + base: base(), + spec: ComputationSpec::Ratio { + numerator: input(MetricInputRole::Numerator, "accepted_edit_actions"), + denominator: input(MetricInputRole::Denominator, "tool_use_offered"), + scale: 100.0, + }, + } + } + + fn request(entity_ids: Vec<&str>, from: &str, to: &str) -> ValidatedMetricResultsRequest { + ValidatedMetricResultsRequest { + entity_type: "person".to_owned(), + entity_ids: entity_ids.into_iter().map(str::to_owned).collect(), + from: match NaiveDate::parse_from_str(from, "%Y-%m-%d") { + Ok(date) => date, + Err(error) => panic!("bad test date {from}: {error}"), + }, + to: match NaiveDate::parse_from_str(to, "%Y-%m-%d") { + Ok(date) => date, + Err(error) => panic!("bad test date {to}: {error}"), + }, + metrics: Vec::new(), + } + } + + #[test] + fn period_view_zero_fills_sum_and_keeps_request_order() { + let req = request(vec!["b@x.io", "a@x.io"], "2026-01-01", "2026-01-31"); + let rows = vec![PeriodQueryRow { + entity_id: "a@x.io".to_owned(), + value: Some(5.0), + }]; + let MetricResultViewDto::Period { values } = build_period_view(&sum_metric(), &req, rows) + else { + panic!("expected period view"); + }; + assert_eq!(values[0].entity_id, "b@x.io"); + assert_eq!(values[0].value, Some(0.0)); + assert_eq!(values[1].entity_id, "a@x.io"); + assert_eq!(values[1].value, Some(5.0)); + } + + #[test] + fn period_view_keeps_ratio_nulls() { + let req = request(vec!["a@x.io"], "2026-01-01", "2026-01-31"); + let MetricResultViewDto::Period { values } = + build_period_view(&ratio_metric(), &req, Vec::new()) + else { + panic!("expected period view"); + }; + assert_eq!(values[0].value, None); + } + + #[test] + fn timeseries_densifies_all_buckets_per_entity() { + let req = request(vec!["a@x.io"], "2026-01-01", "2026-01-03"); + let rows = vec![TimeseriesQueryRow { + entity_id: "a@x.io".to_owned(), + bucket_start: "2026-01-02".to_owned(), + value: Some(3.0), + extra: HashMap::new(), + }]; + let Ok(MetricResultViewDto::Timeseries { series, .. }) = + build_timeseries_view(&sum_metric(), &req, Bucket::Day, &[], rows) + else { + panic!("expected timeseries view"); + }; + assert_eq!(series.len(), 1); + let points = &series[0].points; + assert_eq!(points.len(), 3); + assert_eq!(points[0].value, Some(0.0)); + assert_eq!(points[1].value, Some(3.0)); + assert_eq!(points[2].value, Some(0.0)); + } + + #[test] + fn ungrouped_timeseries_emits_series_for_entities_without_rows() { + let req = request(vec!["a@x.io", "b@x.io"], "2026-01-01", "2026-01-02"); + let Ok(MetricResultViewDto::Timeseries { series, .. }) = + build_timeseries_view(&ratio_metric(), &req, Bucket::Day, &[], Vec::new()) + else { + panic!("expected timeseries view"); + }; + assert_eq!(series.len(), 2); + assert!(series.iter().all(|s| s.points.len() == 2)); + assert!( + series + .iter() + .all(|s| s.points.iter().all(|p| p.value.is_none())) + ); + } + + #[test] + fn dimensioned_timeseries_groups_by_observed_dimensions() { + let req = request(vec!["a@x.io"], "2026-01-01", "2026-01-02"); + let mut extra = HashMap::new(); + extra.insert("dim_0_value".to_owned(), json!("cursor")); + extra.insert("dim_0_label".to_owned(), json!("Cursor")); + let rows = vec![TimeseriesQueryRow { + entity_id: "a@x.io".to_owned(), + bucket_start: "2026-01-01".to_owned(), + value: Some(2.0), + extra, + }]; + let dimensions = vec!["tool".to_owned()]; + let Ok(MetricResultViewDto::Timeseries { series, .. }) = + build_timeseries_view(&sum_metric(), &req, Bucket::Day, &dimensions, rows) + else { + panic!("expected timeseries view"); + }; + assert_eq!(series.len(), 1); + assert_eq!(series[0].dimensions[0].key, "tool"); + assert_eq!(series[0].dimensions[0].value, "cursor"); + assert_eq!(series[0].dimensions[0].label.as_deref(), Some("Cursor")); + assert_eq!(series[0].points.len(), 2); + } + + #[test] + fn missing_dimension_alias_is_internal_error() { + let req = request(vec!["a@x.io"], "2026-01-01", "2026-01-02"); + let rows = vec![TimeseriesQueryRow { + entity_id: "a@x.io".to_owned(), + bucket_start: "2026-01-01".to_owned(), + value: Some(2.0), + extra: HashMap::new(), + }]; + let dimensions = vec!["tool".to_owned()]; + assert!( + build_timeseries_view(&sum_metric(), &req, Bucket::Day, &dimensions, rows).is_err() + ); + } + + #[test] + fn breakdown_null_dimension_value_maps_to_unknown() { + let mut extra = HashMap::new(); + extra.insert("dim_0_value".to_owned(), serde_json::Value::Null); + extra.insert("dim_0_label".to_owned(), serde_json::Value::Null); + let rows = vec![BreakdownQueryRow { + entity_id: "a@x.io".to_owned(), + value: Some(1.0), + extra, + }]; + let dimensions = vec!["tool".to_owned()]; + let Ok(MetricResultViewDto::Breakdown { values, .. }) = + build_breakdown_view(&dimensions, rows) + else { + panic!("expected breakdown view"); + }; + assert_eq!(values[0].dimensions[0].value, UNKNOWN_DIMENSION_VALUE); + assert_eq!( + values[0].dimensions[0].label.as_deref(), + Some(UNKNOWN_DIMENSION_LABEL) + ); + } + + #[test] + fn metric_result_wire_shape_is_flat_with_computation_tag() { + let sum = build_metric_result(&sum_metric(), Vec::new()); + let sum_json = serde_json::to_value(&sum).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(sum_json["computation"], "sum"); + assert_eq!(sum_json["metric_key"], "ai.accepted_lines"); + assert_eq!(sum_json["format"], "integer"); + assert!(sum_json.get("scale").is_none()); + + let ratio = build_metric_result(&ratio_metric(), Vec::new()); + let ratio_json = serde_json::to_value(&ratio).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(ratio_json["computation"], "ratio"); + assert_eq!(ratio_json["scale"], 100.0); + } + + #[test] + fn response_size_counts_densified_points() { + let req = request(vec!["a@x.io"], "2026-01-01", "2026-01-10"); + let Ok(view) = build_timeseries_view(&sum_metric(), &req, Bucket::Day, &[], Vec::new()) + else { + panic!("expected timeseries view"); + }; + let response = MetricResultsResponse { + metrics: vec![build_metric_result(&sum_metric(), vec![view])], + }; + assert_eq!(response_size(&response), 10); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_results/compiler.rs b/src/backend/services/analytics/src/domain/metric_results/compiler.rs new file mode 100644 index 000000000..571172f6d --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/compiler.rs @@ -0,0 +1,702 @@ +use std::collections::HashMap; +use std::fmt::Write; + +use serde::Deserialize; + +use super::validation::{ValidatedMetricResultsRequest, ValidatedMetricView, query_row_limit}; +use super::view::Bucket; +use crate::domain::metric_definitions::{ + CohortSource, ComputationSpec, MetricDefinition, ObservationSource, +}; + +pub(crate) const UNKNOWN_DIMENSION_VALUE: &str = "__unknown__"; +pub(crate) const UNKNOWN_DIMENSION_LABEL: &str = "Unknown"; + +/// Minimum peer-pool size for percentile disclosure. Below this, quartiles +/// over a handful of people are noise presented as signal (someone is always +/// "bottom 25%" of three), and with n=2 the "median" discloses the one +/// colleague's value. Enforced here, server-side, so every consumer inherits +/// it: the peer view still reports `n`, but p25/median/p75/min/max come back +/// NULL and clients render "no peer data". +pub(crate) const MIN_PEER_N: u32 = 5; + +#[derive(Debug)] +pub struct CompiledQuery { + pub sql: String, + pub params: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct PeriodQueryRow { + pub entity_id: String, + pub value: Option, +} + +#[derive(Debug, Deserialize)] +pub struct TimeseriesQueryRow { + pub entity_id: String, + pub bucket_start: String, + pub value: Option, + #[serde(flatten)] + pub extra: HashMap, +} + +#[derive(Debug, Deserialize)] +pub struct PeerQueryRow { + pub entity_id: String, + pub target_value: Option, + pub p25: Option, + pub median: Option, + pub p75: Option, + pub min: Option, + pub max: Option, + #[serde(default, deserialize_with = "optional_u64")] + pub n: Option, +} + +#[derive(Debug, Deserialize)] +pub struct BreakdownQueryRow { + pub entity_id: String, + pub value: Option, + #[serde(flatten)] + pub extra: HashMap, +} + +pub fn compile_view_query( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + view: &ValidatedMetricView, +) -> CompiledQuery { + match view { + ValidatedMetricView::Period => compile_period_query(def, req), + ValidatedMetricView::Peer { cohort_key } => compile_peer_query(def, req, cohort_key), + ValidatedMetricView::Timeseries { bucket, dimensions } => { + compile_timeseries_query(def, req, *bucket, dimensions) + } + ValidatedMetricView::Breakdown { dimensions } => { + compile_breakdown_query(def, req, dimensions) + } + } +} + +fn compile_period_query( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, +) -> CompiledQuery { + let mut params = metric_params(def, req); + params.extend(req.entity_ids.iter().cloned()); + let entities = placeholders(req.entity_ids.len()); + let observation_table = observation_table(def.observation_source()); + let limit = query_row_limit(); + let sql = match &def.spec { + ComputationSpec::Sum { .. } => format!( + r" + SELECT + entity_id, + sumIf(value, value IS NOT NULL) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY entity_id + LIMIT {limit} + ", + metric_where = metric_where(def), + ), + ComputationSpec::Ratio { scale, .. } => format!( + r" + SELECT + entity_id, + {scale} * sumIf(value, measure_key = ? AND value IS NOT NULL) + / nullIf(sumIf(value, measure_key = ? AND value IS NOT NULL), 0) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY entity_id + LIMIT {limit} + ", + scale = scale, + metric_where = metric_where(def), + ), + }; + CompiledQuery { sql, params } +} + +fn compile_timeseries_query( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + bucket: Bucket, + dimensions: &[String], +) -> CompiledQuery { + let mut params = metric_params(def, req); + params.extend(req.entity_ids.iter().cloned()); + let entities = placeholders(req.entity_ids.len()); + let bucket = bucket_expr(bucket); + let (dim_select, dim_group) = dimension_select_group(dimensions); + let group = if dim_group.is_empty() { + "entity_id, bucket_start".to_owned() + } else { + format!("entity_id, bucket_start, {dim_group}") + }; + let observation_table = observation_table(def.observation_source()); + let limit = query_row_limit(); + let sql = match &def.spec { + ComputationSpec::Sum { .. } => format!( + r" + SELECT + entity_id, + toString({bucket}) AS bucket_start{dim_select}, + sumIf(value, value IS NOT NULL) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY {group} + ORDER BY entity_id, bucket_start + LIMIT {limit} + ", + metric_where = metric_where(def), + ), + ComputationSpec::Ratio { scale, .. } => format!( + r" + SELECT + entity_id, + toString({bucket}) AS bucket_start{dim_select}, + {scale} * sumIf(value, measure_key = ? AND value IS NOT NULL) + / nullIf(sumIf(value, measure_key = ? AND value IS NOT NULL), 0) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY {group} + ORDER BY entity_id, bucket_start + LIMIT {limit} + ", + metric_where = metric_where(def), + scale = scale, + ), + }; + CompiledQuery { sql, params } +} + +fn compile_breakdown_query( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + dimensions: &[String], +) -> CompiledQuery { + let mut params = metric_params(def, req); + params.extend(req.entity_ids.iter().cloned()); + let entities = placeholders(req.entity_ids.len()); + let (dim_select, dim_group) = dimension_select_group(dimensions); + let group = if dim_group.is_empty() { + "entity_id".to_owned() + } else { + format!("entity_id, {dim_group}") + }; + let observation_table = observation_table(def.observation_source()); + let limit = query_row_limit(); + let sql = match &def.spec { + ComputationSpec::Sum { .. } => format!( + r" + SELECT + entity_id{dim_select}, + sumIf(value, value IS NOT NULL) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY {group} + ORDER BY entity_id + LIMIT {limit} + ", + metric_where = metric_where(def), + ), + ComputationSpec::Ratio { scale, .. } => format!( + r" + SELECT + entity_id{dim_select}, + {scale} * sumIf(value, measure_key = ? AND value IS NOT NULL) + / nullIf(sumIf(value, measure_key = ? AND value IS NOT NULL), 0) AS value + FROM {observation_table} + WHERE {metric_where} + AND entity_id IN ({entities}) + GROUP BY {group} + ORDER BY entity_id + LIMIT {limit} + ", + metric_where = metric_where(def), + scale = scale, + ), + }; + CompiledQuery { sql, params } +} + +fn compile_peer_query( + def: &MetricDefinition, + req: &ValidatedMetricResultsRequest, + cohort_key: &str, +) -> CompiledQuery { + let mut params = Vec::new(); + params.push(req.entity_type.clone()); + params.push(cohort_key.to_owned()); + params.extend(req.entity_ids.iter().cloned()); + params.push(req.entity_type.clone()); + params.push(cohort_key.to_owned()); + params.extend(metric_params(def, req)); + + let entities = placeholders(req.entity_ids.len()); + let observation_table = observation_table(def.observation_source()); + let cohort_table = cohort_table(CohortSource::MetricEntityCohortsCurrent); + let metric_value = match &def.spec { + ComputationSpec::Sum { .. } => "sumIf(value, value IS NOT NULL)".to_owned(), + ComputationSpec::Ratio { scale, .. } => format!( + "{scale} * sumIf(value, measure_key = ? AND value IS NOT NULL) / nullIf(sumIf(value, measure_key = ? AND value IS NOT NULL), 0)" + ), + }; + let limit = query_row_limit(); + let sql = format!( + r" + WITH + targets AS ( + SELECT DISTINCT + entity_id, + cohort_id + FROM {cohort_table} + WHERE entity_type = ? + AND cohort_key = ? + AND entity_id IN ({entities}) + AND cohort_id IS NOT NULL + ), + cohort AS ( + SELECT DISTINCT + entity_id, + cohort_id + FROM {cohort_table} + WHERE entity_type = ? + AND cohort_key = ? + AND cohort_id IN (SELECT cohort_id FROM targets) + ), + metric_values AS ( + SELECT + entity_id, + {metric_value} AS value + FROM {observation_table} + WHERE {metric_where} + GROUP BY entity_id + ), + entity_values AS ( + SELECT + cohort.entity_id AS entity_id, + cohort.cohort_id AS cohort_id, + metric_values.value AS value + FROM cohort + LEFT JOIN metric_values + ON metric_values.entity_id = cohort.entity_id + ), + peers AS ( + SELECT + cohort_id, + entity_id, + value + FROM entity_values + WHERE value IS NOT NULL + ) + SELECT + targets.entity_id AS entity_id, + target_values.value AS target_value, + if(uniqExact(peers.entity_id) >= {min_peer_n}, toNullable(quantileExact(0.25)(peers.value)), NULL) AS p25, + if(uniqExact(peers.entity_id) >= {min_peer_n}, toNullable(quantileExact(0.5)(peers.value)), NULL) AS median, + if(uniqExact(peers.entity_id) >= {min_peer_n}, toNullable(quantileExact(0.75)(peers.value)), NULL) AS p75, + if(uniqExact(peers.entity_id) >= {min_peer_n}, toNullable(min(peers.value)), NULL) AS min, + if(uniqExact(peers.entity_id) >= {min_peer_n}, toNullable(max(peers.value)), NULL) AS max, + toUInt64(uniqExact(peers.entity_id)) AS n + FROM targets + LEFT JOIN entity_values AS target_values + ON target_values.entity_id = targets.entity_id + LEFT JOIN peers + ON peers.cohort_id = targets.cohort_id + GROUP BY targets.entity_id, target_values.value + LIMIT {limit} + SETTINGS join_use_nulls = 1 + ", + metric_where = metric_where(def), + min_peer_n = MIN_PEER_N, + ); + CompiledQuery { sql, params } +} + +// No tenant_id predicate: warehouse tenant isolation is not implemented +// platform-wide (the legacy query engine also queries without it), and the +// control-plane tenant UUID has no defined mapping to the warehouse +// tenant_id strings stamped at ingestion. The observation and cohort +// contracts keep the tenant_id column so isolation can be added here in one +// place once the platform defines that mapping. +fn metric_where(def: &MetricDefinition) -> &'static str { + match &def.spec { + ComputationSpec::Sum { .. } => { + "source_key = ? AND entity_type = ? AND metric_date >= toDate(?) AND metric_date <= toDate(?) AND measure_key = ?" + } + ComputationSpec::Ratio { .. } => { + "source_key = ? AND entity_type = ? AND metric_date >= toDate(?) AND metric_date <= toDate(?) AND measure_key IN (?, ?)" + } + } +} + +fn metric_params(def: &MetricDefinition, req: &ValidatedMetricResultsRequest) -> Vec { + match &def.spec { + ComputationSpec::Sum { value } => vec![ + value.source_key.clone(), + req.entity_type.clone(), + req.from.to_string(), + req.to.to_string(), + value.measure_key.clone(), + ], + ComputationSpec::Ratio { + numerator, + denominator, + .. + } => { + let mut params = vec![ + numerator.measure_key.clone(), + denominator.measure_key.clone(), + ]; + params.extend([ + numerator.source_key.clone(), + req.entity_type.clone(), + req.from.to_string(), + req.to.to_string(), + numerator.measure_key.clone(), + denominator.measure_key.clone(), + ]); + params + } + } +} + +fn placeholders(count: usize) -> String { + vec!["?"; count].join(", ") +} + +fn bucket_expr(bucket: Bucket) -> &'static str { + match bucket { + Bucket::Day => "metric_date", + Bucket::Week => "toStartOfWeek(metric_date, 1)", + Bucket::Month => "toStartOfMonth(metric_date)", + } +} + +fn observation_table(source: ObservationSource) -> &'static str { + match source { + ObservationSource::AiMetricObservations => "insight.ai_metric_observations", + } +} + +fn cohort_table(source: CohortSource) -> &'static str { + match source { + CohortSource::MetricEntityCohortsCurrent => "insight.metric_entity_cohorts_current", + } +} + +pub(crate) fn dimension_aliases(idx: usize) -> (String, String) { + (format!("dim_{idx}_value"), format!("dim_{idx}_label")) +} + +fn dimension_select_group(dimensions: &[String]) -> (String, String) { + let mut select = String::new(); + let mut groups = Vec::with_capacity(dimensions.len() * 2); + for (idx, dimension) in dimensions.iter().enumerate() { + let (value_alias, label_alias) = dimension_aliases(idx); + let _ = write!( + select, + ", {value} AS {value_alias}, {label} AS {label_alias}", + value = dimension_value_expr(dimension), + label = dimension_label_expr(dimension) + ); + groups.push(value_alias); + groups.push(label_alias); + } + (select, groups.join(", ")) +} + +fn dimension_value_expr(dimension: &str) -> String { + format!( + r" + if( + length(arrayFilter(d -> tupleElement(d, 1) = '{dimension}', dimensions)) = 0, + '{UNKNOWN_DIMENSION_VALUE}', + coalesce( + tupleElement(arrayFilter(d -> tupleElement(d, 1) = '{dimension}', dimensions)[1], 2), + '{UNKNOWN_DIMENSION_VALUE}' + ) + ) + " + ) +} + +fn dimension_label_expr(dimension: &str) -> String { + format!( + r" + if( + length(arrayFilter(d -> tupleElement(d, 1) = '{dimension}', dimensions)) = 0, + '{UNKNOWN_DIMENSION_LABEL}', + coalesce( + tupleElement(arrayFilter(d -> tupleElement(d, 1) = '{dimension}', dimensions)[1], 3), + '{UNKNOWN_DIMENSION_LABEL}' + ) + ) + " + ) +} + +fn optional_u64<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + match value { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::Number(number)) => number + .as_u64() + .ok_or_else(|| serde::de::Error::custom("expected unsigned integer")) + .map(Some), + Some(serde_json::Value::String(value)) => value + .parse::() + .map(Some) + .map_err(serde::de::Error::custom), + Some(_) => Err(serde::de::Error::custom("expected unsigned integer")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::NaiveDate; + + use crate::domain::metric_definitions::definition::{ + MetricBase, MetricDirection, MetricFormat, MetricInput, MetricInputRole, + }; + + fn base(dimensions: Vec<&str>) -> MetricBase { + MetricBase { + key: "ai.accepted_lines".to_owned(), + label: "AI-added lines".to_owned(), + description: None, + explanation: None, + entity_type: "person".to_owned(), + format: MetricFormat::Integer, + unit: None, + direction: MetricDirection::HigherIsBetter, + peer_cohort_key: Some("org_unit".to_owned()), + allowed_dimensions: dimensions.into_iter().map(str::to_owned).collect(), + } + } + + fn input(role: MetricInputRole, measure_key: &str) -> MetricInput { + MetricInput { + role, + observation_source: ObservationSource::AiMetricObservations, + source_key: "ai_usage".to_owned(), + measure_key: measure_key.to_owned(), + } + } + + fn sum_metric() -> MetricDefinition { + MetricDefinition { + base: base(vec!["tool"]), + spec: ComputationSpec::Sum { + value: input(MetricInputRole::Value, "accepted_lines"), + }, + } + } + + fn ratio_metric() -> MetricDefinition { + MetricDefinition { + base: base(vec!["tool"]), + spec: ComputationSpec::Ratio { + numerator: input(MetricInputRole::Numerator, "accepted_edit_actions"), + denominator: input(MetricInputRole::Denominator, "tool_use_offered"), + scale: 100.0, + }, + } + } + + fn request() -> ValidatedMetricResultsRequest { + ValidatedMetricResultsRequest { + entity_type: "person".to_owned(), + entity_ids: vec!["a@x.io".to_owned(), "b@x.io".to_owned()], + from: NaiveDate::from_ymd_opt(2026, 1, 1).unwrap_or_default(), + to: NaiveDate::from_ymd_opt(2026, 1, 31).unwrap_or_default(), + metrics: Vec::new(), + } + } + + #[test] + fn sum_period_query_binds_scope_then_entities() { + let query = compile_view_query(&sum_metric(), &request(), &ValidatedMetricView::Period); + assert!(query.sql.contains("FROM insight.ai_metric_observations")); + assert!(!query.sql.contains("tenant_id")); + assert!(query.sql.contains("measure_key = ?")); + assert!(query.sql.contains("GROUP BY entity_id")); + assert_eq!( + query.params, + vec![ + "ai_usage", + "person", + "2026-01-01", + "2026-01-31", + "accepted_lines", + "a@x.io", + "b@x.io", + ] + ); + } + + #[test] + fn ratio_period_query_binds_select_measures_first() { + let query = compile_view_query(&ratio_metric(), &request(), &ValidatedMetricView::Period); + assert!(query.sql.contains("nullIf")); + assert!(query.sql.contains("100 *")); + assert!(query.sql.contains("measure_key IN (?, ?)")); + assert_eq!( + query.params, + vec![ + "accepted_edit_actions", + "tool_use_offered", + "ai_usage", + "person", + "2026-01-01", + "2026-01-31", + "accepted_edit_actions", + "tool_use_offered", + "a@x.io", + "b@x.io", + ] + ); + } + + #[test] + fn timeseries_query_uses_bucket_expression() { + for (bucket, expr) in [ + (Bucket::Day, "metric_date"), + (Bucket::Week, "toStartOfWeek(metric_date, 1)"), + (Bucket::Month, "toStartOfMonth(metric_date)"), + ] { + let query = compile_view_query( + &sum_metric(), + &request(), + &ValidatedMetricView::Timeseries { + bucket, + dimensions: vec![], + }, + ); + assert!( + query + .sql + .contains(&format!("toString({expr}) AS bucket_start")) + ); + assert!(query.sql.contains("GROUP BY entity_id, bucket_start")); + } + } + + #[test] + fn dimensioned_query_emits_value_and_label_aliases() { + let query = compile_view_query( + &sum_metric(), + &request(), + &ValidatedMetricView::Breakdown { + dimensions: vec!["tool".to_owned()], + }, + ); + assert!(query.sql.contains("AS dim_0_value")); + assert!(query.sql.contains("AS dim_0_label")); + assert!(query.sql.contains("tupleElement(d, 1) = 'tool'")); + assert!( + query + .sql + .contains("GROUP BY entity_id, dim_0_value, dim_0_label") + ); + } + + #[test] + fn peer_query_binds_cohort_scopes_then_metric_scope() { + let query = compile_view_query( + &sum_metric(), + &request(), + &ValidatedMetricView::Peer { + cohort_key: "org_unit".to_owned(), + }, + ); + assert!( + query + .sql + .contains("FROM insight.metric_entity_cohorts_current") + ); + assert!(query.sql.contains("WHERE value IS NOT NULL")); + assert!(!query.sql.contains("AND peer.value IS NOT NULL")); + assert_eq!( + query.params, + vec![ + "person", + "org_unit", + "a@x.io", + "b@x.io", + "person", + "org_unit", + "ai_usage", + "person", + "2026-01-01", + "2026-01-31", + "accepted_lines", + ] + ); + } + + #[test] + fn peer_queries_never_fabricate_zero_observations() { + // Honest-null through the runtime: cohort members without observed + // values stay NULL and drop out of the peer pool — absence of rows + // cannot be distinguished from "not covered by the source", so the + // peer query must not invent zeros for them. + for def in [sum_metric(), ratio_metric()] { + let query = compile_view_query( + &def, + &request(), + &ValidatedMetricView::Peer { + cohort_key: "org_unit".to_owned(), + }, + ); + assert!(query.sql.contains("metric_values.value AS value")); + assert!(!query.sql.contains("coalesce(metric_values.value, 0)")); + } + } + + #[test] + fn peer_queries_suppress_percentiles_below_min_pool_size() { + for def in [sum_metric(), ratio_metric()] { + let query = compile_view_query( + &def, + &request(), + &ValidatedMetricView::Peer { + cohort_key: "org_unit".to_owned(), + }, + ); + let guard = format!("uniqExact(peers.entity_id) >= {MIN_PEER_N}"); + assert_eq!( + query.sql.matches(&guard).count(), + 5, + "every percentile/min/max must carry the disclosure guard" + ); + assert!( + query + .sql + .contains("toUInt64(uniqExact(peers.entity_id)) AS n") + ); + // Duplicate cohort membership must not fan out the pool. + assert_eq!(query.sql.matches("SELECT DISTINCT").count(), 2); + // Honest-null must not depend on server config or column typing. + assert!(query.sql.contains("SETTINGS join_use_nulls = 1")); + } + } + + #[test] + fn queries_carry_row_limit() { + let query = compile_view_query(&sum_metric(), &request(), &ValidatedMetricView::Period); + assert!(query.sql.contains(&format!("LIMIT {}", query_row_limit()))); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_results/dto.rs b/src/backend/services/analytics/src/domain/metric_results/dto.rs new file mode 100644 index 000000000..f7905341a --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/dto.rs @@ -0,0 +1,150 @@ +use serde::{Deserialize, Serialize}; + +use super::view::{Bucket, MetricResultViewKind}; +use crate::domain::metric_definitions::{MetricDirection, MetricFormat}; + +#[derive(Debug, Deserialize)] +pub struct MetricResultsRequest { + pub entity: MetricResultsEntity, + pub period: MetricResultsPeriod, + pub metrics: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct MetricResultsEntity { + pub r#type: String, + pub ids: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct MetricResultsPeriod { + pub from: String, + pub to: String, +} + +#[derive(Debug, Deserialize)] +pub struct MetricRequest { + pub metric_key: String, + pub views: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "view", rename_all = "snake_case")] +pub enum MetricViewRequest { + Period, + Peer { + cohort_key: Option, + }, + Timeseries { + bucket: Option, + #[serde(default)] + dimensions: Vec, + }, + Breakdown { + dimensions: Vec, + }, +} + +impl MetricViewRequest { + pub fn kind(&self) -> MetricResultViewKind { + match self { + Self::Period => MetricResultViewKind::Period, + Self::Peer { .. } => MetricResultViewKind::Peer, + Self::Timeseries { .. } => MetricResultViewKind::Timeseries, + Self::Breakdown { .. } => MetricResultViewKind::Breakdown, + } + } +} + +#[derive(Debug, Serialize)] +pub struct MetricResultsResponse { + pub metrics: Vec, +} + +#[derive(Debug, Serialize)] +pub struct MetricResultDto { + pub metric_key: String, + pub label: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub explanation: Option, + pub unit: Option, + pub format: MetricFormat, + pub direction: MetricDirection, + #[serde(flatten)] + pub computation: ComputationDto, + pub views: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "computation", rename_all = "snake_case")] +pub enum ComputationDto { + Sum, + Ratio { scale: f64 }, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "view", rename_all = "snake_case")] +pub enum MetricResultViewDto { + Period { + values: Vec, + }, + Timeseries { + bucket: Bucket, + series: Vec, + }, + Peer { + values: Vec, + }, + Breakdown { + dimensions: Vec, + values: Vec, + }, +} + +#[derive(Debug, Clone, Serialize)] +pub struct MetricDimensionDto { + pub key: String, + pub value: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, +} + +#[derive(Debug, Serialize)] +pub struct PeriodValueDto { + pub entity_id: String, + pub value: Option, +} + +#[derive(Debug, Serialize)] +pub struct TimeseriesDto { + pub entity_id: String, + pub dimensions: Vec, + pub points: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TimeseriesPointDto { + pub bucket_start: String, + pub value: Option, +} + +#[derive(Debug, Serialize)] +pub struct PeerValueDto { + pub entity_id: String, + pub target_value: Option, + pub p25: Option, + pub median: Option, + pub p75: Option, + pub min: Option, + pub max: Option, + pub n: u64, +} + +#[derive(Debug, Serialize)] +pub struct BreakdownValueDto { + pub entity_id: String, + pub dimensions: Vec, + pub value: Option, +} diff --git a/src/backend/services/analytics/src/domain/metric_results/mod.rs b/src/backend/services/analytics/src/domain/metric_results/mod.rs new file mode 100644 index 000000000..6bcaedc20 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/mod.rs @@ -0,0 +1,16 @@ +mod builder; +mod compiler; +mod dto; +mod validation; +mod view; + +pub use builder::{ + build_breakdown_view, build_metric_result, build_peer_view, build_period_view, + build_timeseries_view, enforce_row_limit, +}; +pub use compiler::{ + BreakdownQueryRow, CompiledQuery, PeerQueryRow, PeriodQueryRow, TimeseriesQueryRow, + compile_view_query, +}; +pub use dto::{MetricResultViewDto, MetricResultsRequest, MetricResultsResponse}; +pub use validation::{ValidatedMetricResultsRequest, ValidatedMetricView, validate_request}; diff --git a/src/backend/services/analytics/src/domain/metric_results/validation.rs b/src/backend/services/analytics/src/domain/metric_results/validation.rs new file mode 100644 index 000000000..573dec8cc --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/validation.rs @@ -0,0 +1,685 @@ +use std::collections::BTreeSet; + +use chrono::{Datelike, Duration, NaiveDate}; +use sea_orm::DatabaseConnection; +use toolkit_canonical_errors::CanonicalError; +use uuid::Uuid; + +use crate::api::error::MetricError; +use crate::domain::metric_definitions::{MetricDefinition, load_definitions}; + +use super::dto::{MetricResultsRequest, MetricViewRequest}; +use super::view::Bucket; + +const ROW_LIMIT: usize = 5000; +const MAX_METRICS: usize = 50; +const MAX_ENTITY_IDS: usize = 1000; +const MAX_PERIOD_DAYS: i64 = 400; + +#[derive(Debug)] +pub struct ValidatedMetricResultsRequest { + pub entity_type: String, + pub entity_ids: Vec, + pub from: NaiveDate, + pub to: NaiveDate, + pub metrics: Vec, +} + +#[derive(Debug)] +pub struct ValidatedMetricRequest { + pub def: MetricDefinition, + pub views: Vec, +} + +#[derive(Debug, Clone)] +pub enum ValidatedMetricView { + Period, + Peer { + cohort_key: String, + }, + Timeseries { + bucket: Bucket, + dimensions: Vec, + }, + Breakdown { + dimensions: Vec, + }, +} + +struct RequestShape { + entity_type: String, + entity_ids: Vec, + from: NaiveDate, + to: NaiveDate, + metric_keys: Vec, +} + +pub async fn validate_request( + db: &DatabaseConnection, + tenant_id: Uuid, + req: MetricResultsRequest, +) -> Result { + let shape = validate_request_shape(&req)?; + let RequestShape { + entity_type, + entity_ids, + from, + to, + metric_keys, + } = shape; + + let mut definitions = load_definitions(db, tenant_id, &metric_keys).await?; + let mut metrics = Vec::with_capacity(req.metrics.len()); + + for metric in req.metrics { + let metric_key = metric.metric_key.trim(); + let def = definitions.remove(metric_key).ok_or_else(|| { + tracing::error!(metric_key = %metric_key, "definition missing after successful load"); + CanonicalError::internal("metric definition lookup failed").create() + })?; + if def.base.entity_type != entity_type { + return invalid( + "entity.type", + format!( + "metric {} is defined for entity type {}", + def.key(), + def.base.entity_type + ), + ); + } + if metric.views.is_empty() { + return invalid( + "metrics.views", + format!("metric {} must request at least one view", def.key()), + ); + } + + let mut view_kinds = BTreeSet::new(); + let mut views = Vec::with_capacity(metric.views.len()); + for view in metric.views { + let kind = view.kind(); + if !view_kinds.insert(kind) { + return invalid( + "metrics.views", + format!("metric {} has duplicate {kind:?} view", def.key()), + ); + } + views.push(validate_view(&def, view)?); + } + + metrics.push(ValidatedMetricRequest { def, views }); + } + + let validated = ValidatedMetricResultsRequest { + entity_type, + entity_ids, + from, + to, + metrics, + }; + validate_projected_row_limit(&validated)?; + Ok(validated) +} + +fn validate_request_shape(req: &MetricResultsRequest) -> Result { + if req.metrics.is_empty() { + return invalid("metrics", "metrics must not be empty"); + } + if req.metrics.len() > MAX_METRICS { + return invalid( + "metrics", + format!("at most {MAX_METRICS} metrics per request"), + ); + } + + let entity_type = normalize_entity_type(&req.entity.r#type)?; + let entity_ids = normalize_entity_ids(&entity_type, &req.entity.ids)?; + if entity_ids.len() > MAX_ENTITY_IDS { + return invalid( + "entity.ids", + format!("at most {MAX_ENTITY_IDS} entity ids per request"), + ); + } + let from = parse_date("period.from", &req.period.from)?; + let to = parse_date("period.to", &req.period.to)?; + if from > to { + return invalid("period", "period.from must be before or equal to period.to"); + } + if (to - from).num_days() >= MAX_PERIOD_DAYS { + return invalid( + "period", + format!("period must not exceed {MAX_PERIOD_DAYS} days"), + ); + } + + let mut seen_metric_keys = BTreeSet::new(); + let mut metric_keys = Vec::with_capacity(req.metrics.len()); + for metric in &req.metrics { + let metric_key = metric.metric_key.trim(); + if metric_key.is_empty() { + return invalid("metrics.metric_key", "metric_key must not be empty"); + } + if !seen_metric_keys.insert(metric_key.to_owned()) { + return invalid( + "metrics.metric_key", + format!("duplicate metric key: {metric_key}"), + ); + } + metric_keys.push(metric_key.to_owned()); + } + + Ok(RequestShape { + entity_type, + entity_ids, + from, + to, + metric_keys, + }) +} + +pub const fn row_limit() -> usize { + ROW_LIMIT +} + +// One past the response cap: a query returning ROW_LIMIT + 1 rows proves +// truncation, which the final enforce_row_limit pass then converts into a +// whole-request failure instead of a silently clipped result. +pub const fn query_row_limit() -> usize { + ROW_LIMIT + 1 +} + +pub fn metric_result_too_large() -> CanonicalError { + MetricError::invalid_argument() + .with_field_violation( + "metric_results", + "Requested metric result exceeds the row limit. Reduce the date range, entities, metrics, or dimensions.", + "metric_result_too_large", + ) + .create() +} + +fn validate_view( + def: &MetricDefinition, + view: MetricViewRequest, +) -> Result { + match view { + MetricViewRequest::Period => Ok(ValidatedMetricView::Period), + MetricViewRequest::Peer { cohort_key } => { + let cohort_key = match cohort_key { + Some(key) => { + let key = normalize_key("metrics.views.cohort_key", &key)?; + if def.base.peer_cohort_key.as_deref() != Some(key.as_str()) { + return Err(MetricError::invalid_argument() + .with_field_violation( + "metrics.views.cohort_key", + format!("cohort {key} is not declared for metric {}", def.key()), + "INVALID", + ) + .create()); + } + key + } + None => def.base.peer_cohort_key.clone().ok_or_else(|| { + MetricError::invalid_argument() + .with_field_violation( + "metrics.views.cohort_key", + format!("metric {} has no default peer cohort", def.key()), + "INVALID", + ) + .create() + })?, + }; + Ok(ValidatedMetricView::Peer { cohort_key }) + } + MetricViewRequest::Timeseries { bucket, dimensions } => { + Ok(ValidatedMetricView::Timeseries { + bucket: bucket.unwrap_or(Bucket::Day), + dimensions: validate_dimensions(def, "metrics.views.dimensions", dimensions)?, + }) + } + MetricViewRequest::Breakdown { dimensions } => { + if dimensions.is_empty() { + return invalid( + "metrics.views.dimensions", + format!( + "metric {} breakdown dimensions must not be empty", + def.key() + ), + ); + } + Ok(ValidatedMetricView::Breakdown { + dimensions: validate_dimensions(def, "metrics.views.dimensions", dimensions)?, + }) + } + } +} + +fn validate_dimensions( + def: &MetricDefinition, + field: &'static str, + dimensions: Vec, +) -> Result, CanonicalError> { + let mut seen = BTreeSet::new(); + let mut out = Vec::with_capacity(dimensions.len()); + for dimension in dimensions { + let dimension = normalize_key(field, &dimension)?; + if !seen.insert(dimension.clone()) { + return invalid(field, format!("duplicate dimension: {dimension}")); + } + let Some(valid_dimension) = def.allowed_dimension(&dimension) else { + return invalid( + field, + format!( + "metric {} does not support dimension {dimension}", + def.key() + ), + ); + }; + out.push(valid_dimension.to_owned()); + } + Ok(out) +} + +fn normalize_entity_type(entity_type: &str) -> Result { + normalize_key("entity.type", entity_type) +} + +fn normalize_entity_ids( + entity_type: &str, + entity_ids: &[String], +) -> Result, CanonicalError> { + let mut seen = BTreeSet::new(); + let mut out = Vec::with_capacity(entity_ids.len()); + for entity_id in entity_ids { + let entity_id = normalize_entity_id(entity_type, entity_id); + if entity_id.is_empty() { + continue; + } + if seen.insert(entity_id.clone()) { + out.push(entity_id); + } + } + if out.is_empty() { + return invalid("entity.ids", "entity.ids must not be empty"); + } + Ok(out) +} + +// Id normalization is a property of the entity type: person ids are emails +// and the observation sources emit them lowercased, so equality requires +// lowercasing here too. Other entity types keep their casing. +fn normalize_entity_id(entity_type: &str, entity_id: &str) -> String { + let trimmed = entity_id.trim(); + match entity_type { + "person" => trimmed.to_ascii_lowercase(), + _ => trimmed.to_owned(), + } +} + +fn normalize_key(field: &'static str, value: &str) -> Result { + let value = value.trim().to_ascii_lowercase(); + if value.is_empty() { + return invalid(field, "value must not be empty"); + } + if !value + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + || value + .as_bytes() + .first() + .is_some_and(|b| b.is_ascii_digit() || *b == b'_') + { + return invalid(field, "expected lowercase snake case"); + } + Ok(value) +} + +fn validate_projected_row_limit(req: &ValidatedMetricResultsRequest) -> Result<(), CanonicalError> { + let mut projected = 0usize; + + for metric in &req.metrics { + for view in &metric.views { + match view { + ValidatedMetricView::Period | ValidatedMetricView::Peer { .. } => { + projected = projected.saturating_add(req.entity_ids.len()); + } + ValidatedMetricView::Timeseries { bucket, .. } => { + projected = projected.saturating_add( + req.entity_ids + .len() + .saturating_mul(enumerate_buckets(req.from, req.to, *bucket).len()), + ); + } + ValidatedMetricView::Breakdown { .. } => {} + } + } + } + + if projected > ROW_LIMIT { + return Err(metric_result_too_large()); + } + Ok(()) +} + +pub fn enumerate_buckets(from: NaiveDate, to: NaiveDate, bucket: Bucket) -> Vec { + let mut out = Vec::new(); + let mut seen = BTreeSet::new(); + let mut day = from; + while day <= to { + let bucket_start = match bucket { + Bucket::Day => day, + Bucket::Week => day - Duration::days(i64::from(day.weekday().num_days_from_monday())), + Bucket::Month => NaiveDate::from_ymd_opt(day.year(), day.month(), 1).unwrap_or(day), + }; + if seen.insert(bucket_start) { + out.push(bucket_start.to_string()); + } + day += Duration::days(1); + } + out +} + +fn parse_date(field: &'static str, value: &str) -> Result { + NaiveDate::parse_from_str(value, "%Y-%m-%d").map_err(|_| { + MetricError::invalid_argument() + .with_field_violation(field, "expected YYYY-MM-DD", "INVALID") + .create() + }) +} + +fn invalid(field: &'static str, message: impl Into) -> Result { + Err(MetricError::invalid_argument() + .with_field_violation(field, message.into(), "INVALID") + .create()) +} + +#[cfg(test)] +mod tests { + use super::super::dto::MetricResultsEntity; + use super::*; + use crate::domain::metric_definitions::definition::{ + ComputationSpec, MetricBase, MetricDefinition, MetricDirection, MetricFormat, MetricInput, + MetricInputRole, ObservationSource, + }; + + fn shape_request( + entity_ids: Vec<&str>, + from: &str, + to: &str, + metric_keys: Vec<&str>, + ) -> MetricResultsRequest { + MetricResultsRequest { + entity: MetricResultsEntity { + r#type: "person".to_owned(), + ids: entity_ids.into_iter().map(str::to_owned).collect(), + }, + period: super::super::dto::MetricResultsPeriod { + from: from.to_owned(), + to: to.to_owned(), + }, + metrics: metric_keys + .into_iter() + .map(|key| super::super::dto::MetricRequest { + metric_key: key.to_owned(), + views: vec![MetricViewRequest::Period], + }) + .collect(), + } + } + + fn sum_definition(dimensions: Vec<&str>) -> MetricDefinition { + MetricDefinition { + base: MetricBase { + key: "ai.accepted_lines".to_owned(), + label: "AI-added lines".to_owned(), + description: None, + explanation: None, + entity_type: "person".to_owned(), + format: MetricFormat::Integer, + unit: None, + direction: MetricDirection::HigherIsBetter, + peer_cohort_key: Some("org_unit".to_owned()), + allowed_dimensions: dimensions.into_iter().map(str::to_owned).collect(), + }, + spec: ComputationSpec::Sum { + value: MetricInput { + role: MetricInputRole::Value, + observation_source: ObservationSource::AiMetricObservations, + source_key: "ai_usage".to_owned(), + measure_key: "accepted_lines".to_owned(), + }, + }, + } + } + + fn day(value: &str) -> NaiveDate { + match NaiveDate::parse_from_str(value, "%Y-%m-%d") { + Ok(date) => date, + Err(error) => panic!("bad test date {value}: {error}"), + } + } + + #[test] + fn shape_accepts_valid_request() { + let Ok(shape) = validate_request_shape(&shape_request( + vec!["A@x.io "], + "2026-01-01", + "2026-01-31", + vec!["ai.x"], + )) else { + panic!("expected valid shape"); + }; + assert_eq!(shape.entity_type, "person"); + assert_eq!(shape.entity_ids, vec!["a@x.io".to_owned()]); + assert_eq!(shape.metric_keys, vec!["ai.x".to_owned()]); + } + + #[test] + fn shape_rejects_too_many_metrics() { + let keys: Vec = (0..=MAX_METRICS).map(|i| format!("ai.m{i}")).collect(); + let req = shape_request( + vec!["a@x.io"], + "2026-01-01", + "2026-01-31", + keys.iter().map(String::as_str).collect(), + ); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn shape_rejects_too_many_entity_ids() { + let ids: Vec = (0..=MAX_ENTITY_IDS).map(|i| format!("p{i}@x.io")).collect(); + let req = shape_request( + ids.iter().map(String::as_str).collect(), + "2026-01-01", + "2026-01-31", + vec!["ai.x"], + ); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn shape_rejects_oversized_period_before_enumeration() { + let req = shape_request(vec!["a@x.io"], "0001-01-01", "9999-12-31", vec!["ai.x"]); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn shape_rejects_reversed_period() { + let req = shape_request(vec!["a@x.io"], "2026-02-01", "2026-01-01", vec!["ai.x"]); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn shape_rejects_duplicate_metric_keys() { + let req = shape_request( + vec!["a@x.io"], + "2026-01-01", + "2026-01-31", + vec!["ai.x", "ai.x"], + ); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn shape_rejects_all_blank_entity_ids() { + let req = shape_request(vec![" ", ""], "2026-01-01", "2026-01-31", vec!["ai.x"]); + assert!(validate_request_shape(&req).is_err()); + } + + #[test] + fn person_entity_ids_are_lowercased() { + let Ok(ids) = normalize_entity_ids("person", &[" A@X.io ".to_owned()]) else { + panic!("expected normalized ids"); + }; + assert_eq!(ids, vec!["a@x.io".to_owned()]); + } + + #[test] + fn non_person_entity_ids_keep_case() { + let Ok(ids) = normalize_entity_ids("repo", &[" Org/Repo-Name ".to_owned()]) else { + panic!("expected normalized ids"); + }; + assert_eq!(ids, vec!["Org/Repo-Name".to_owned()]); + } + + #[test] + fn normalize_key_enforces_snake_case() { + assert_eq!(normalize_key("f", " Tool ").ok().as_deref(), Some("tool")); + assert!(normalize_key("f", "").is_err()); + assert!(normalize_key("f", "1tool").is_err()); + assert!(normalize_key("f", "_tool").is_err()); + assert!(normalize_key("f", "tool-x").is_err()); + assert!(normalize_key("f", "tool x").is_err()); + assert_eq!( + normalize_key("f", "org_unit2").ok().as_deref(), + Some("org_unit2") + ); + } + + #[test] + fn validate_view_rejects_undeclared_dimension() { + let def = sum_definition(vec!["tool"]); + let view = MetricViewRequest::Breakdown { + dimensions: vec!["surface".to_owned()], + }; + assert!(validate_view(&def, view).is_err()); + } + + #[test] + fn validate_view_defaults_timeseries_bucket_to_day() { + let def = sum_definition(vec!["tool"]); + let view = MetricViewRequest::Timeseries { + bucket: None, + dimensions: vec![], + }; + match validate_view(&def, view) { + Ok(ValidatedMetricView::Timeseries { bucket, .. }) => assert_eq!(bucket, Bucket::Day), + other => panic!("expected timeseries, got {other:?}"), + } + } + + #[test] + fn validate_view_peer_uses_definition_default_cohort() { + let def = sum_definition(vec![]); + match validate_view(&def, MetricViewRequest::Peer { cohort_key: None }) { + Ok(ValidatedMetricView::Peer { cohort_key }) => assert_eq!(cohort_key, "org_unit"), + other => panic!("expected peer, got {other:?}"), + } + } + + #[test] + fn validate_view_peer_accepts_explicit_declared_cohort() { + let def = sum_definition(vec![]); + let view = MetricViewRequest::Peer { + cohort_key: Some("org_unit".to_owned()), + }; + match validate_view(&def, view) { + Ok(ValidatedMetricView::Peer { cohort_key }) => assert_eq!(cohort_key, "org_unit"), + other => panic!("expected peer, got {other:?}"), + } + } + + #[test] + fn validate_view_peer_rejects_undeclared_cohort() { + let def = sum_definition(vec![]); + let view = MetricViewRequest::Peer { + cohort_key: Some("team".to_owned()), + }; + assert!(validate_view(&def, view).is_err()); + } + + #[test] + fn validate_view_rejects_empty_breakdown_dimensions() { + let def = sum_definition(vec!["tool"]); + let view = MetricViewRequest::Breakdown { dimensions: vec![] }; + assert!(validate_view(&def, view).is_err()); + } + + #[test] + fn enumerate_day_buckets_counts_days() { + let buckets = enumerate_buckets(day("2026-01-30"), day("2026-02-02"), Bucket::Day); + assert_eq!( + buckets, + vec!["2026-01-30", "2026-01-31", "2026-02-01", "2026-02-02"] + ); + } + + #[test] + fn enumerate_week_buckets_start_monday() { + let buckets = enumerate_buckets(day("2026-07-01"), day("2026-07-14"), Bucket::Week); + assert_eq!(buckets, vec!["2026-06-29", "2026-07-06", "2026-07-13"]); + } + + #[test] + fn enumerate_month_buckets_cross_year() { + let buckets = enumerate_buckets(day("2025-12-15"), day("2026-02-01"), Bucket::Month); + assert_eq!(buckets, vec!["2025-12-01", "2026-01-01", "2026-02-01"]); + } + + #[test] + fn enumerate_single_day_range() { + let buckets = enumerate_buckets(day("2026-07-02"), day("2026-07-02"), Bucket::Week); + assert_eq!(buckets, vec!["2026-06-29"]); + } + + #[test] + fn projected_row_limit_counts_timeseries_buckets() { + let def = sum_definition(vec![]); + let validated = ValidatedMetricResultsRequest { + entity_type: "person".to_owned(), + entity_ids: (0..100).map(|i| format!("p{i}@x.io")).collect(), + from: day("2026-01-01"), + to: day("2026-03-31"), + metrics: vec![ValidatedMetricRequest { + def, + views: vec![ValidatedMetricView::Timeseries { + bucket: Bucket::Day, + dimensions: vec![], + }], + }], + }; + assert!(validate_projected_row_limit(&validated).is_err()); + } + + #[test] + fn projected_row_limit_allows_small_requests() { + let def = sum_definition(vec![]); + let validated = ValidatedMetricResultsRequest { + entity_type: "person".to_owned(), + entity_ids: vec!["a@x.io".to_owned()], + from: day("2026-01-01"), + to: day("2026-01-31"), + metrics: vec![ValidatedMetricRequest { + def, + views: vec![ + ValidatedMetricView::Period, + ValidatedMetricView::Peer { + cohort_key: "org_unit".to_owned(), + }, + ], + }], + }; + assert!(validate_projected_row_limit(&validated).is_ok()); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_results/view.rs b/src/backend/services/analytics/src/domain/metric_results/view.rs new file mode 100644 index 000000000..dca2543be --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_results/view.rs @@ -0,0 +1,18 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Bucket { + Day, + Week, + Month, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricResultViewKind { + Period, + Timeseries, + Peer, + Breakdown, +} diff --git a/src/backend/services/analytics/src/domain/mod.rs b/src/backend/services/analytics/src/domain/mod.rs index b435a9b9b..8095c105a 100644 --- a/src/backend/services/analytics/src/domain/mod.rs +++ b/src/backend/services/analytics/src/domain/mod.rs @@ -2,6 +2,8 @@ pub mod admin_threshold; pub mod auth; pub mod catalog; pub mod metric; +pub mod metric_definitions; +pub mod metric_results; pub mod query; pub mod schema_validator; pub mod threshold; diff --git a/src/backend/services/analytics/src/gear.rs b/src/backend/services/analytics/src/gear.rs index b00bdaf91..904d84963 100644 --- a/src/backend/services/analytics/src/gear.rs +++ b/src/backend/services/analytics/src/gear.rs @@ -59,6 +59,12 @@ impl Gear for AnalyticsApiGear { // Run pending migrations. infra::db::run_migrations(&db).await?; + // Converge builtin metric definitions to the code registry before + // serving traffic. MySQL-only, so it does not violate the + // post-readiness ClickHouse rule; failure aborts startup because the + // registry state must be consistent before the first request. + crate::domain::metric_definitions::reconcile_builtin_definitions(&db).await?; + // Refuse to start if any required CHECK constraint is missing. See // `infra/db/check_probe` and DESIGN §2.2 // `cpt-metric-cat-constraint-mariadb-check`. @@ -125,7 +131,11 @@ impl Gear for AnalyticsApiGear { // Schema-validator (Refs #521). Held in AppState (admin-crud per-write // hook) and cloned into the post-init startup pass below. let validator = SchemaValidator::new(db.clone(), ch.clone()); - + let metric_definition_validator = + crate::domain::metric_definitions::MetricDefinitionValidator::new( + db.clone(), + ch.clone(), + ); // Catalog auth-trait (Refs #522 / #525). v1 stub — see `domain::auth`. let tenant_auth: Arc = Arc::new( ConfigTenantAuthorization::new(cfg.metric_catalog.tenant_default_id), @@ -160,6 +170,10 @@ impl Gear for AnalyticsApiGear { tokio::spawn(async move { validator.validate_all().await; }); + // Periodic, not one-shot: the managed observation views are + // dbt-created after boot on a fresh deploy, and the registry has no + // write path that would re-trigger probing. + tokio::spawn(metric_definition_validator.run()); Ok(()) } @@ -199,6 +213,10 @@ pub async fn run_migrate(app: &toolkit::bootstrap::AppConfig) -> anyhow::Result< let db = infra::db::connect(&cfg.database_url).await?; infra::db::run_migrations(&db).await?; + // Same convergence as `init`: `migrate` run as a standalone step must + // leave builtin metric definitions matching the code registry. + crate::domain::metric_definitions::reconcile_builtin_definitions(&db).await?; + // Same probes as `init`. An operator running `migrate` standalone wants // the integrity signals too (DESIGN §2.2 / §3.6). infra::db::check_probe::assert_required_checks(&db).await?; diff --git a/src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs b/src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs new file mode 100644 index 000000000..f1753bbff --- /dev/null +++ b/src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs @@ -0,0 +1,194 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +pub const REQUIRED_SOURCE_CHECKS: &[&str] = &[ + "chk_metric_sources_source_key_shape", + "chk_metric_sources_schema_error_biconditional", + "chk_metric_sources_schema_error_enum", +]; + +pub const REQUIRED_SOURCE_MEASURE_CHECKS: &[&str] = &[ + "chk_metric_source_measures_measure_key_shape", + "chk_metric_source_measures_schema_error_biconditional", + "chk_metric_source_measures_schema_error_enum", +]; + +pub const REQUIRED_SOURCE_DIMENSION_CHECKS: &[&str] = &[ + "chk_metric_source_dimensions_dimension_key_shape", + "chk_metric_source_dimensions_display_order_nonnegative", +]; + +pub const REQUIRED_DEFINITION_CHECKS: &[&str] = &[ + "chk_metric_definitions_metric_key_shape", + "chk_metric_definitions_entity_type_shape", + "chk_metric_definitions_peer_cohort_key_shape", + "chk_metric_definitions_computation_fields", + "chk_metric_definitions_schema_error_biconditional", + "chk_metric_definitions_schema_error_enum", +]; + +pub const REQUIRED_DIMENSION_CHECKS: &[&str] = + &["chk_metric_definition_dimensions_display_order_nonnegative"]; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let conn = manager.get_connection(); + for statement in SCHEMA_STATEMENTS { + conn.execute_unprepared(statement).await?; + } + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Err(DbErr::Custom("we have only forward migrations".to_owned())) + } +} + +const SCHEMA_STATEMENTS: &[&str] = &[ + "CREATE TABLE IF NOT EXISTS metric_sources ( + id BINARY(16) NOT NULL PRIMARY KEY, + tenant_id BINARY(16) NULL, + tenant_id_sentinel BINARY(16) GENERATED ALWAYS AS (COALESCE(tenant_id, 0x00000000000000000000000000000000)) STORED, + source_key VARCHAR(128) NOT NULL, + source_kind ENUM('managed_observation','custom_observation_sql') NOT NULL, + source_ref VARCHAR(256) NOT NULL, + origin ENUM('builtin','custom') NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + schema_status ENUM('ok','error','unchecked') NOT NULL DEFAULT 'unchecked', + schema_checked_at DATETIME(3) NULL, + schema_error_code VARCHAR(64) NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_sources_tenant_key (tenant_id_sentinel, source_key), + CONSTRAINT chk_metric_sources_source_key_shape CHECK (source_key REGEXP BINARY '^[a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_sources_schema_error_biconditional CHECK ((schema_status = 'error') = (schema_error_code IS NOT NULL)), + CONSTRAINT chk_metric_sources_schema_error_enum CHECK (schema_error_code IS NULL OR schema_error_code IN ('table_not_found','column_not_found','dimension_not_covered','unknown')) + )", + "CREATE TABLE IF NOT EXISTS metric_source_measures ( + id BINARY(16) NOT NULL PRIMARY KEY, + source_id BINARY(16) NOT NULL, + measure_key VARCHAR(128) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + schema_status ENUM('ok','error','unchecked') NOT NULL DEFAULT 'unchecked', + schema_checked_at DATETIME(3) NULL, + schema_error_code VARCHAR(64) NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_source_measures_key (source_id, measure_key), + CONSTRAINT fk_metric_source_measures_source FOREIGN KEY (source_id) REFERENCES metric_sources(id) ON DELETE CASCADE, + CONSTRAINT chk_metric_source_measures_measure_key_shape CHECK (measure_key REGEXP BINARY '^[a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_source_measures_schema_error_biconditional CHECK ((schema_status = 'error') = (schema_error_code IS NOT NULL)), + CONSTRAINT chk_metric_source_measures_schema_error_enum CHECK (schema_error_code IS NULL OR schema_error_code IN ('table_not_found','column_not_found','dimension_not_covered','unknown')) + )", + "CREATE TABLE IF NOT EXISTS metric_source_dimensions ( + id BINARY(16) NOT NULL PRIMARY KEY, + source_id BINARY(16) NOT NULL, + dimension_key VARCHAR(64) NOT NULL, + display_order INT NOT NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_source_dimensions_key (source_id, dimension_key), + CONSTRAINT fk_metric_source_dimensions_source FOREIGN KEY (source_id) REFERENCES metric_sources(id) ON DELETE CASCADE, + CONSTRAINT chk_metric_source_dimensions_dimension_key_shape CHECK (dimension_key REGEXP BINARY '^[a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_source_dimensions_display_order_nonnegative CHECK (display_order >= 0) + )", + "CREATE TABLE IF NOT EXISTS metric_definitions ( + id BINARY(16) NOT NULL PRIMARY KEY, + tenant_id BINARY(16) NULL, + tenant_id_sentinel BINARY(16) GENERATED ALWAYS AS (COALESCE(tenant_id, 0x00000000000000000000000000000000)) STORED, + metric_key VARCHAR(128) NOT NULL, + label VARCHAR(128) NOT NULL, + description VARCHAR(2048) NULL, + explanation VARCHAR(4096) NULL, + unit VARCHAR(32) NULL, + format ENUM('integer','decimal','currency','percent') NOT NULL, + direction ENUM('higher_is_better','lower_is_better','neutral') NOT NULL, + entity_type VARCHAR(64) NOT NULL, + computation_type ENUM('sum','ratio') NOT NULL, + scale DOUBLE NULL, + peer_cohort_key VARCHAR(64) NULL, + origin ENUM('builtin','custom') NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + schema_status ENUM('ok','error','unchecked') NOT NULL DEFAULT 'unchecked', + schema_checked_at DATETIME(3) NULL, + schema_error_code VARCHAR(64) NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_definitions_tenant_key (tenant_id_sentinel, metric_key), + KEY idx_metric_definitions_metric_key (metric_key), + CONSTRAINT chk_metric_definitions_metric_key_shape CHECK (metric_key REGEXP BINARY '^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_definitions_entity_type_shape CHECK (entity_type REGEXP BINARY '^[a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_definitions_peer_cohort_key_shape CHECK (peer_cohort_key IS NULL OR peer_cohort_key REGEXP BINARY '^[a-z][a-z0-9_]*$'), + CONSTRAINT chk_metric_definitions_computation_fields CHECK ( + (computation_type = 'sum' AND scale IS NULL) + OR (computation_type = 'ratio' AND scale IS NOT NULL) + ), + CONSTRAINT chk_metric_definitions_schema_error_biconditional CHECK ((schema_status = 'error') = (schema_error_code IS NOT NULL)), + CONSTRAINT chk_metric_definitions_schema_error_enum CHECK (schema_error_code IS NULL OR schema_error_code IN ('table_not_found','column_not_found','dimension_not_covered','unknown')) + )", + "CREATE TABLE IF NOT EXISTS metric_definition_inputs ( + id BINARY(16) NOT NULL PRIMARY KEY, + metric_definition_id BINARY(16) NOT NULL, + input_role ENUM('value','numerator','denominator') NOT NULL, + source_measure_id BINARY(16) NOT NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_definition_inputs_role_measure (metric_definition_id, input_role, source_measure_id), + CONSTRAINT fk_metric_definition_inputs_definition FOREIGN KEY (metric_definition_id) REFERENCES metric_definitions(id) ON DELETE CASCADE, + CONSTRAINT fk_metric_definition_inputs_measure FOREIGN KEY (source_measure_id) REFERENCES metric_source_measures(id) ON DELETE RESTRICT + )", + "CREATE TABLE IF NOT EXISTS metric_definition_dimensions ( + id BINARY(16) NOT NULL PRIMARY KEY, + metric_definition_id BINARY(16) NOT NULL, + source_dimension_id BINARY(16) NOT NULL, + display_order INT NOT NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + UNIQUE KEY uq_metric_definition_dimensions_dimension (metric_definition_id, source_dimension_id), + CONSTRAINT fk_metric_definition_dimensions_definition FOREIGN KEY (metric_definition_id) REFERENCES metric_definitions(id) ON DELETE CASCADE, + CONSTRAINT fk_metric_definition_dimensions_source_dimension FOREIGN KEY (source_dimension_id) REFERENCES metric_source_dimensions(id) ON DELETE RESTRICT, + CONSTRAINT chk_metric_definition_dimensions_display_order_nonnegative CHECK (display_order >= 0) + )", +]; + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::metric_definitions::error_code::ALL_METRIC_SCHEMA_ERROR_CODES; + + #[test] + fn schema_error_check_lists_match_error_code_enum() { + let expected = ALL_METRIC_SCHEMA_ERROR_CODES + .iter() + .map(|code| format!("'{}'", code.as_db())) + .collect::>() + .join(","); + let expected_clause = format!("schema_error_code IN ({expected})"); + + let mut check_count = 0; + for statement in SCHEMA_STATEMENTS { + if statement.contains("schema_error_code IN (") { + assert!( + statement.contains(&expected_clause), + "CHECK list out of sync with MetricSchemaErrorCode in: {statement}" + ); + check_count += 1; + } + } + assert_eq!(check_count, 3); + } + + #[test] + fn every_required_check_appears_in_schema() { + let all_sql = SCHEMA_STATEMENTS.join("\n"); + for check in REQUIRED_SOURCE_CHECKS + .iter() + .chain(REQUIRED_SOURCE_MEASURE_CHECKS) + .chain(REQUIRED_SOURCE_DIMENSION_CHECKS) + .chain(REQUIRED_DEFINITION_CHECKS) + .chain(REQUIRED_DIMENSION_CHECKS) + { + assert!(all_sql.contains(check), "missing CHECK {check}"); + } + } +} diff --git a/src/backend/services/analytics/src/migration/mod.rs b/src/backend/services/analytics/src/migration/mod.rs index dacff1485..768352c75 100644 --- a/src/backend/services/analytics/src/migration/mod.rs +++ b/src/backend/services/analytics/src/migration/mod.rs @@ -45,6 +45,7 @@ mod m20260623_000002_seed_ai_personal_metric_catalog; mod m20260624_000001_collab_zulip_chat; mod m20260624_000002_seed_zulip_collab_catalog; mod m20260624_000003_ic_chart_loc_git_breakdown; +mod m20260625_000001_metric_definitions; mod m20260701_000001_devendor_email_labels; mod m20260702_000001_collab_messaging_queries; mod m20260702_000002_seed_collab_messaging_catalog; @@ -102,6 +103,7 @@ impl MigratorTrait for Migrator { Box::new(m20260624_000001_collab_zulip_chat::Migration), Box::new(m20260624_000002_seed_zulip_collab_catalog::Migration), Box::new(m20260624_000003_ic_chart_loc_git_breakdown::Migration), + Box::new(m20260625_000001_metric_definitions::Migration), Box::new(m20260701_000001_devendor_email_labels::Migration), Box::new(m20260702_000001_collab_messaging_queries::Migration), Box::new(m20260702_000002_seed_collab_messaging_catalog::Migration), @@ -127,6 +129,26 @@ pub const REQUIRED_CHECKS_BY_TABLE: &[(&str, &[&str])] = &[ "threshold_lock_audit", m20260522_000003_threshold_lock_audit::REQUIRED_CHECKS, ), + ( + "metric_sources", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_CHECKS, + ), + ( + "metric_source_measures", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_MEASURE_CHECKS, + ), + ( + "metric_source_dimensions", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_DIMENSION_CHECKS, + ), + ( + "metric_definitions", + m20260625_000001_metric_definitions::REQUIRED_DEFINITION_CHECKS, + ), + ( + "metric_definition_dimensions", + m20260625_000001_metric_definitions::REQUIRED_DIMENSION_CHECKS, + ), ]; #[cfg(test)] @@ -163,6 +185,26 @@ mod tests { "threshold_lock_audit", m20260522_000003_threshold_lock_audit::REQUIRED_CHECKS, ), + ( + "metric_sources", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_CHECKS, + ), + ( + "metric_source_measures", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_MEASURE_CHECKS, + ), + ( + "metric_source_dimensions", + m20260625_000001_metric_definitions::REQUIRED_SOURCE_DIMENSION_CHECKS, + ), + ( + "metric_definitions", + m20260625_000001_metric_definitions::REQUIRED_DEFINITION_CHECKS, + ), + ( + "metric_definition_dimensions", + m20260625_000001_metric_definitions::REQUIRED_DIMENSION_CHECKS, + ), ]; for &(table, checks) in expected { diff --git a/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_assistant_usage.sql b/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_assistant_usage.sql index 070436e43..b6565d6ba 100644 --- a/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_assistant_usage.sql +++ b/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_assistant_usage.sql @@ -38,7 +38,9 @@ SELECT lower(trim(email)) AS email, toDate(date) AS day, 'chatgpt' AS tool, + 'ChatGPT' AS tool_label, 'chat' AS surface, + 'Chat' AS surface_label, -- chat is not session-bounded in the API → session_count NULL CAST(NULL AS Nullable(UInt32)) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, diff --git a/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_dev_usage.sql b/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_dev_usage.sql index 8ea0e3411..2d2c3de0a 100644 --- a/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/chatgpt-team/dbt/chatgpt_team__ai_dev_usage.sql @@ -42,8 +42,10 @@ SELECT CAST(NULL AS Nullable(String)) AS api_key_id, toDate(date) AS day, 'codex' AS tool, + 'Codex' AS tool_label, -- Codex threads ≈ coding sessions. Non-nullable UInt32 per contract. toUInt32(coalesce(toUInt32OrNull(toString(n_threads)), 0)) AS session_count, + toUInt32OrNull(toString(n_threads)) AS conversation_count, toUInt32(coalesce(toUInt32OrNull(toString(lines_added)), 0)) AS lines_added, -- Codex does not surface AI-removed lines / total keystrokes. CAST(NULL AS Nullable(UInt32)) AS lines_removed, diff --git a/src/ingestion/connectors/ai/chatgpt-team/descriptor.yaml b/src/ingestion/connectors/ai/chatgpt-team/descriptor.yaml index 351bd6e22..4a82eec5a 100644 --- a/src/ingestion/connectors/ai/chatgpt-team/descriptor.yaml +++ b/src/ingestion/connectors/ai/chatgpt-team/descriptor.yaml @@ -12,7 +12,11 @@ name: chatgpt-team # leaderboard (daily range only), remove the hardcoded min_datetime floor, # add lookback_window P2D, dedup on lower(trim(email)) and add a Codex # activity filter. Behaviour + dbt output change → reconcile must re-run. -version: "1.1.1" +# 2.0.0: dbt staging maps Codex threads (n_threads) into class-contract +# conversation_count and declares tool_label / surface_label — MAJOR per +# ADR-0015: pre-existing rows must re-materialize from Bronze to populate +# the new columns (one-shot scoped full refresh). +version: "2.0.0" # Daily at 04:00 UTC — off-peak. The customer's proxy runs a single browser # instance, so we avoid scheduling chatgpt-team next to anything else that # may also hit it (same rationale as claude-team). diff --git a/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql index 7b4e82e06..02cda0f0a 100644 --- a/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql @@ -99,7 +99,9 @@ SELECT CASE WHEN u.actor_type = 'api_actor' THEN k.api_key_id END AS api_key_id, u.day, 'claude_code' AS tool, + 'Claude Code' AS tool_label, toUInt32(u.sessions_sum) AS session_count, + toUInt32OrNull(toString(u.sessions_sum)) AS conversation_count, toUInt32(u.lines_added_sum) AS lines_added, toUInt32(u.lines_removed_sum) AS lines_removed, -- total_lines_added/removed: Admin code_usage doesn't expose total keystrokes. NULL. @@ -115,6 +117,10 @@ SELECT -- CE-specific columns — NULL for Admin (Admin code_usage does not expose git attribution). CAST(NULL AS Nullable(UInt32)) AS commits_count, CAST(NULL AS Nullable(UInt32)) AS pull_requests_count, + -- prs_with_cc_count / prs_total_count: Claude Team-only (Anthropic GitHub-app attribution). + -- Structural NULL per Silver NULL-policy (presence of column required for UNION ALL parity). + CAST(NULL AS Nullable(UInt32)) AS prs_with_cc_count, + CAST(NULL AS Nullable(UInt32)) AS prs_total_count, CAST(NULL AS Nullable(String)) AS tool_action_breakdown_json, 'claude_admin' AS source, 'insight_claude_admin' AS data_source, diff --git a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_assistant_usage.sql b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_assistant_usage.sql index ad661cc5a..179aab778 100644 --- a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_assistant_usage.sql +++ b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_assistant_usage.sql @@ -90,7 +90,9 @@ chat AS ( email, day, 'claude' AS tool, + 'Claude' AS tool_label, 'chat' AS surface, + 'Chat' AS surface_label, -- chat is not session-bounded in the API → session_count NULL CAST(NULL AS Nullable(UInt32)) AS session_count, toUInt32OrNull(toString(chat_conversation_count)) AS conversation_count, @@ -138,7 +140,9 @@ excel AS ( email, day, 'claude' AS tool, + 'Claude' AS tool_label, 'excel' AS surface, + 'Excel' AS surface_label, toUInt32OrNull(toString(excel_session_count)) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32OrNull(toString(excel_message_count)) AS message_count, @@ -173,7 +177,9 @@ powerpoint AS ( email, day, 'claude' AS tool, + 'Claude' AS tool_label, 'powerpoint' AS surface, + 'PowerPoint' AS surface_label, toUInt32OrNull(toString(powerpoint_session_count)) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32OrNull(toString(powerpoint_message_count)) AS message_count, @@ -208,7 +214,9 @@ cowork AS ( email, day, 'claude' AS tool, + 'Claude' AS tool_label, 'cowork' AS surface, + 'Cowork' AS surface_label, toUInt32OrNull(toString(cowork_session_count)) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32OrNull(toString(cowork_message_count)) AS message_count, @@ -251,7 +259,9 @@ cross_surface AS ( email, day, 'claude' AS tool, + 'Claude' AS tool_label, 'cross' AS surface, + 'Cross' AS surface_label, CAST(NULL AS Nullable(UInt32)) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, CAST(NULL AS Nullable(UInt32)) AS message_count, diff --git a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql index 95169756e..e77c77ceb 100644 --- a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql @@ -53,7 +53,9 @@ SELECT CAST(NULL AS Nullable(String)) AS api_key_id, toDate(parseDateTimeBestEffortOrNull(date)) AS day, 'claude_code' AS tool, + 'Claude Code' AS tool_label, toUInt32(coalesce(code_session_count, 0)) AS session_count, + toUInt32OrNull(toString(code_session_count)) AS conversation_count, toUInt32(coalesce(code_lines_added, 0)) AS lines_added, toUInt32(coalesce(code_lines_removed, 0)) AS lines_removed, -- Enterprise reports AI-accepted lines only — no view of total user keystrokes. diff --git a/src/ingestion/connectors/ai/claude-enterprise/descriptor.yaml b/src/ingestion/connectors/ai/claude-enterprise/descriptor.yaml index 8ac027906..5d31f1668 100644 --- a/src/ingestion/connectors/ai/claude-enterprise/descriptor.yaml +++ b/src/ingestion/connectors/ai/claude-enterprise/descriptor.yaml @@ -1,5 +1,10 @@ name: claude-enterprise -version: "2.0.0" +# 3.0.0: dbt staging maps code_session_count / chat_conversation_count into +# class-contract conversation_count and declares tool_label / +# surface_label — MAJOR per ADR-0015: pre-existing rows must +# re-materialize from Bronze to populate the new columns (one-shot scoped +# full refresh). +version: "3.0.0" schedule: "0 11 * * *" # daily at 11:00 UTC — accommodates Anthropic's 10:00 UTC aggregation workflow: sync diff --git a/src/ingestion/connectors/ai/claude-team/dbt/claude_team__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-team/dbt/claude_team__ai_dev_usage.sql index f27f9c352..15ae92f83 100644 --- a/src/ingestion/connectors/ai/claude-team/dbt/claude_team__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-team/dbt/claude_team__ai_dev_usage.sql @@ -54,7 +54,9 @@ SELECT CAST(NULL AS Nullable(String)) AS api_key_id, toDate(metric_date) AS day, 'claude_code' AS tool, + 'Claude Code' AS tool_label, toUInt32(coalesce(total_sessions, 0)) AS session_count, + toUInt32OrNull(toString(total_sessions)) AS conversation_count, toUInt32(coalesce(total_lines_accepted, 0)) AS lines_added, -- NULL per NULL-policy (PR #553): Claude Team does not expose AI-removed -- lines — structural absence, not zero. diff --git a/src/ingestion/connectors/ai/claude-team/descriptor.yaml b/src/ingestion/connectors/ai/claude-team/descriptor.yaml index ff1dcf85f..4013a873a 100644 --- a/src/ingestion/connectors/ai/claude-team/descriptor.yaml +++ b/src/ingestion/connectors/ai/claude-team/descriptor.yaml @@ -11,7 +11,11 @@ name: claude-team # limit_type) and new dbt models (claude_team__ai_overage → class_ai_overage). # Populates only once the proxy sessionKey holds billing:view / Owner; # otherwise the stream stays empty and the Silver rows are zero (no failure). -version: "1.3.0" +# 2.0.0: dbt staging maps total_sessions into class-contract +# conversation_count and declares tool_label — MAJOR per ADR-0015: +# pre-existing rows must re-materialize from Bronze to populate the new +# columns (one-shot scoped full refresh). +version: "2.0.0" # Daily at 04:00 UTC — off-peak vs other connectors. The customer's # proxy runs a single browser instance, so we avoid scheduling # claude-team next to anything else that may also hit it. diff --git a/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql b/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql index c977b7843..9e58ac57e 100644 --- a/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql @@ -39,7 +39,9 @@ SELECT CAST(NULL AS Nullable(String)) AS api_key_id, toDate(fromUnixTimestamp64Milli(CAST(date AS Int64))) AS day, 'cursor' AS tool, + 'Cursor' AS tool_label, toUInt32(1) AS session_count, + CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32(coalesce(acceptedLinesAdded, 0)) AS lines_added, toUInt32(coalesce(acceptedLinesDeleted, 0)) AS lines_removed, -- total_lines_added/removed = ALL lines the user wrote/deleted that day diff --git a/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql b/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql index 8fb5c408e..e3345132d 100644 --- a/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql @@ -104,10 +104,12 @@ SELECT CAST(NULL AS Nullable(String)) AS api_key_id, toDate(parseDateTimeBestEffortOrNull(m.day)) AS day, 'copilot' AS tool, + 'GitHub Copilot' AS tool_label, -- session_count: Copilot doesn't expose a per-day session counter; -- presence of an activity row implies at least one active session. -- Match Cursor's convention: 1 per active day. toUInt32(1) AS session_count, + CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32(coalesce(m.loc_added_sum, 0)) AS lines_added, toUInt32(coalesce(m.loc_deleted_sum, 0)) AS lines_removed, -- See header comment — Copilot reports AI-accepted lines only. diff --git a/src/ingestion/dbt/dbt_project.yml b/src/ingestion/dbt/dbt_project.yml index 61cf264a5..0dafc2eec 100644 --- a/src/ingestion/dbt/dbt_project.yml +++ b/src/ingestion/dbt/dbt_project.yml @@ -7,6 +7,7 @@ profile: ingestion model-paths: - identity - ../silver + - ../gold - ../connectors macro-paths: diff --git a/src/ingestion/dbt/macros/metric_observation_measures.sql b/src/ingestion/dbt/macros/metric_observation_measures.sql new file mode 100644 index 000000000..f831b6466 --- /dev/null +++ b/src/ingestion/dbt/macros/metric_observation_measures.sql @@ -0,0 +1,35 @@ +{% macro sum_measure(measure_key, relation, value_expr, dimensions_col, where=none) %} + SELECT + tenant_id, + entity_id, + metric_date, + '{{ measure_key }}' AS measure_key, + toNullable(sumIf(toFloat64({{ value_expr }}), ({{ value_expr }}) IS NOT NULL)) AS value, + {{ dimensions_col }} AS dimensions + FROM {{ relation }} + {% if where %}WHERE {{ where }} + {% endif %}GROUP BY tenant_id, entity_id, metric_date, {{ dimensions_col }} + HAVING countIf(({{ value_expr }}) IS NOT NULL) > 0 +{% endmacro %} + +{% macro presence_measure(measure_key, relations) %} + SELECT + tenant_id, + entity_id, + metric_date, + '{{ measure_key }}' AS measure_key, + toNullable(toFloat64(1)) AS value, + CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS dimensions + FROM ( + {%- for relation in relations %} + SELECT DISTINCT + tenant_id, + entity_id, + metric_date + FROM {{ relation }} + {%- if not loop.last %} + UNION DISTINCT + {%- endif %} + {%- endfor %} + ) +{% endmacro %} diff --git a/src/ingestion/dbt/tests/ai/assert_ai_assistant_usage_rows_active.sql b/src/ingestion/dbt/tests/ai/assert_ai_assistant_usage_rows_active.sql new file mode 100644 index 000000000..77b7e5c13 --- /dev/null +++ b/src/ingestion/dbt/tests/ai/assert_ai_assistant_usage_rows_active.sql @@ -0,0 +1,34 @@ +{{ config( + tags=['data_quality'], + severity='warn', + store_failures=true, + meta={ + 'title': 'class_ai_assistant_usage rows carry real activity', + 'domain': 'ai', + 'category': 'grain', + 'tier': 'error', + 'remediation': 'The class contract guarantees that a (person, day, tool, surface) row exists only when the person actually used the surface that day — insight.ai_metric_observations derives active_day from row existence. A row here means a staging model in the silver:class_ai_assistant_usage tag emitted a zero-activity row: fix that model''s emission filter, do not patch consumers.' + } +) }} +SELECT + insight_tenant_id, + email, + day, + tool, + surface, + source +FROM {{ ref('class_ai_assistant_usage') }} +WHERE coalesce(session_count, 0) = 0 + AND coalesce(conversation_count, 0) = 0 + AND coalesce(message_count, 0) = 0 + AND coalesce(action_count, 0) = 0 + AND coalesce(files_uploaded_count, 0) = 0 + AND coalesce(artifacts_created_count, 0) = 0 + AND coalesce(projects_created_count, 0) = 0 + AND coalesce(projects_used_count, 0) = 0 + AND coalesce(skills_used_count, 0) = 0 + AND coalesce(connectors_used_count, 0) = 0 + AND coalesce(thinking_message_count, 0) = 0 + AND coalesce(dispatch_turn_count, 0) = 0 + AND coalesce(search_count, 0) = 0 + AND coalesce(cost_cents, 0) = 0 diff --git a/src/ingestion/dbt/tests/ai/assert_ai_class_labels_nonempty.sql b/src/ingestion/dbt/tests/ai/assert_ai_class_labels_nonempty.sql new file mode 100644 index 000000000..6f605f777 --- /dev/null +++ b/src/ingestion/dbt/tests/ai/assert_ai_class_labels_nonempty.sql @@ -0,0 +1,30 @@ +{{ config( + tags=['data_quality'], + severity='warn', + store_failures=true, + meta={ + 'title': 'AI class label columns are non-empty', + 'domain': 'ai', + 'category': 'contract', + 'tier': 'error', + 'remediation': 'The class contract guarantees connector-declared non-empty tool_label / surface_label — insight.ai_metric_observations builds dimension tuples from them verbatim, with no downstream fallback. An empty label means either a staging model stopped declaring it (fix the staging model) or historical rows predate the column and the label backfill migration (20260707000000_ai_class_label_backfill.sql) has not been applied.' + } +) }} + +SELECT + 'class_ai_dev_usage' AS relation, + insight_tenant_id, + tool AS discriminator, + day +FROM {{ ref('class_ai_dev_usage') }} +WHERE tool_label = '' + +UNION ALL + +SELECT + 'class_ai_assistant_usage' AS relation, + insight_tenant_id, + concat(tool, '/', surface) AS discriminator, + day +FROM {{ ref('class_ai_assistant_usage') }} +WHERE tool_label = '' OR surface_label = '' diff --git a/src/ingestion/dbt/tests/ai/assert_ai_dev_usage_rows_active.sql b/src/ingestion/dbt/tests/ai/assert_ai_dev_usage_rows_active.sql new file mode 100644 index 000000000..8fb08420f --- /dev/null +++ b/src/ingestion/dbt/tests/ai/assert_ai_dev_usage_rows_active.sql @@ -0,0 +1,34 @@ +{{ config( + tags=['data_quality'], + severity='warn', + store_failures=true, + meta={ + 'title': 'class_ai_dev_usage rows carry real activity', + 'domain': 'ai', + 'category': 'grain', + 'tier': 'error', + 'remediation': 'The class contract guarantees that a (person, day, tool) row exists only when the person actually used the tool that day — insight.ai_metric_observations derives active_day from row existence. A row here means a staging model in the silver:class_ai_dev_usage tag emitted a zero-activity row (seat/roster entry): fix that model''s emission filter, do not patch consumers.' + } +) }} +SELECT + insight_tenant_id, + email, + day, + tool, + source +FROM {{ ref('class_ai_dev_usage') }} +WHERE coalesce(session_count, 0) = 0 + AND coalesce(conversation_count, 0) = 0 + AND coalesce(lines_added, 0) = 0 + AND coalesce(lines_removed, 0) = 0 + AND coalesce(total_lines_added, 0) = 0 + AND coalesce(total_lines_removed, 0) = 0 + AND coalesce(tool_use_offered, 0) = 0 + AND coalesce(tool_use_accepted, 0) = 0 + AND coalesce(agent_sessions, 0) = 0 + AND coalesce(chat_requests, 0) = 0 + AND coalesce(cost_cents, 0) = 0 + AND coalesce(commits_count, 0) = 0 + AND coalesce(pull_requests_count, 0) = 0 + AND coalesce(prs_with_cc_count, 0) = 0 + AND coalesce(prs_total_count, 0) = 0 diff --git a/src/ingestion/dbt/tests/gold/assert_metric_entity_cohorts_unique.sql b/src/ingestion/dbt/tests/gold/assert_metric_entity_cohorts_unique.sql new file mode 100644 index 000000000..9f2bf5a27 --- /dev/null +++ b/src/ingestion/dbt/tests/gold/assert_metric_entity_cohorts_unique.sql @@ -0,0 +1,14 @@ +-- Build-integrity check (untagged → error severity under `dbt build`). +-- The analytics service peer view joins insight.metric_entity_cohorts_current on +-- (tenant_id, entity_type, entity_id, cohort_key) and assumes exactly one row +-- per key — duplicate rows fan out the join and corrupt peer percentiles. +-- Any returned row is a violation of that contract. +SELECT + tenant_id, + entity_type, + entity_id, + cohort_key, + count() AS row_count +FROM {{ ref('metric_entity_cohorts_current') }} +GROUP BY tenant_id, entity_type, entity_id, cohort_key +HAVING count() > 1 diff --git a/src/ingestion/gold/ai_metric_observations.sql b/src/ingestion/gold/ai_metric_observations.sql new file mode 100644 index 000000000..b97dd1e79 --- /dev/null +++ b/src/ingestion/gold/ai_metric_observations.sql @@ -0,0 +1,121 @@ +{{ config( + materialized='view', + schema='insight', + alias='ai_metric_observations', + tags=['gold'] +) }} + +-- Source measure observations for the unified metrics runtime. Reads only +-- class-contract fields: activity is row existence (the class contract +-- guarantees rows exist only for real activity — see silver/ai/schema.yml), +-- display labels come from tool_label / surface_label, and conversation +-- semantics come from data presence (conversation_count is NULL for sources +-- without a conversation concept). Discriminator and label columns are +-- non-null non-empty by the class contract (enforced by silver schema tests); +-- this model consumes them as-is. No vendor-specific columns, tool names, or +-- label mappings may appear in this model. Every measure is emitted through +-- the shape macros in macros/metric_observation_measures.sql; filter +-- predicates may reference only class-contract dimension values. + +WITH +ai_dev_usage_source AS ( + SELECT + insight_tenant_id AS tenant_id, + lower(email) AS entity_id, + day AS metric_date, + CAST( + [tuple('tool', tool, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + conversation_count, + lines_added, + lines_removed, + tool_use_offered, + tool_use_accepted, + cost_cents + FROM {{ ref('class_ai_dev_usage') }} + WHERE email IS NOT NULL + AND email != '' +), +ai_assistant_usage_source AS ( + SELECT + insight_tenant_id AS tenant_id, + lower(email) AS entity_id, + day AS metric_date, + surface, + CAST( + [tuple('tool', tool, tool_label)] + AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_dimensions, + CAST( + [ + tuple('tool', tool, tool_label), + tuple('surface', surface, surface_label) + ] AS Array(Tuple(key String, value String, label Nullable(String))) + ) AS tool_surface_dimensions, + conversation_count, + message_count, + action_count, + cost_cents + FROM {{ ref('class_ai_assistant_usage') }} + WHERE email IS NOT NULL + AND email != '' +), +measure_observations AS ( + {{ sum_measure('accepted_lines', 'ai_dev_usage_source', 'lines_added', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('removed_lines', 'ai_dev_usage_source', 'lines_removed', 'tool_dimensions') }} + + UNION ALL + + {{ presence_measure('active_day', ['ai_dev_usage_source', 'ai_assistant_usage_source']) }} + + UNION ALL + + {{ sum_measure('cost_usd', 'ai_dev_usage_source', 'cost_cents / 100', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('cost_usd', 'ai_assistant_usage_source', 'cost_cents / 100', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('accepted_edit_actions', 'ai_dev_usage_source', 'tool_use_accepted', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('tool_use_offered', 'ai_dev_usage_source', 'tool_use_offered', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('dev_conversations', 'ai_dev_usage_source', 'conversation_count', 'tool_dimensions') }} + + UNION ALL + + {{ sum_measure('assistant_messages', 'ai_assistant_usage_source', 'message_count', 'tool_surface_dimensions') }} + + UNION ALL + + {{ sum_measure('assistant_actions', 'ai_assistant_usage_source', 'action_count', 'tool_surface_dimensions') }} + + UNION ALL + + {{ sum_measure('chat_assistant_conversations', 'ai_assistant_usage_source', 'conversation_count', 'tool_surface_dimensions', where="surface = 'chat'") }} +) +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'ai_usage' AS source_key, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + assumeNotNull(metric_date) AS metric_date, + CAST(NULL AS Nullable(DateTime64(3))) AS observed_at, + measure_key, + value, + CAST(NULL AS Nullable(String)) AS subject_key, + dimensions +FROM measure_observations +WHERE tenant_id IS NOT NULL + AND entity_id IS NOT NULL + AND metric_date IS NOT NULL diff --git a/src/ingestion/gold/metric_entity_cohorts_current.sql b/src/ingestion/gold/metric_entity_cohorts_current.sql new file mode 100644 index 000000000..90b98f861 --- /dev/null +++ b/src/ingestion/gold/metric_entity_cohorts_current.sql @@ -0,0 +1,37 @@ +{{ config( + materialized='view', + schema='insight', + alias='metric_entity_cohorts_current', + tags=['gold'] +) }} + +SELECT + assumeNotNull(tenant_id) AS tenant_id, + 'person' AS entity_type, + assumeNotNull(entity_id) AS entity_id, + 'org_unit' AS cohort_key, + cohort_id +FROM ( + SELECT + workspace_id AS tenant_id, + lower(assumeNotNull(email)) AS entity_id, + coalesce( + nullIf(toString(org_unit_id), ''), + nullIf(department_name, '') + ) AS cohort_id + FROM {{ ref('class_people') }} + WHERE email IS NOT NULL + AND email != '' + AND workspace_id IS NOT NULL + AND workspace_id != '' + ORDER BY + tenant_id, + entity_id, + coalesce(parseDateTimeBestEffortOrNull(toString(valid_from)), toDateTime('1970-01-01')) DESC, + unique_key DESC + LIMIT 1 BY tenant_id, entity_id +) +WHERE tenant_id IS NOT NULL + AND tenant_id != '' + AND entity_id IS NOT NULL + AND entity_id != '' diff --git a/src/ingestion/gold/schema.yml b/src/ingestion/gold/schema.yml new file mode 100644 index 000000000..79dbd2987 --- /dev/null +++ b/src/ingestion/gold/schema.yml @@ -0,0 +1,112 @@ +version: 2 + +models: + - name: ai_metric_observations + description: > + Source measure observations for the unified metrics runtime. One row per + (tenant, entity, day, measure, dimension tuple). Emits measures, not + metrics: the analytics service metric registry binds input roles to + measure keys and compiles result queries against this relation. The + column set is a published contract — the analytics service schema validator + probes it at runtime (`OBSERVATION_COLUMNS`); changing it requires a + coordinated backend change. + columns: + - name: tenant_id + description: "Tenant isolation field" + tests: + - not_null + - name: source_key + description: "Logical source key registered in the metric registry" + tests: + - not_null + - accepted_values: + arguments: + values: ['ai_usage'] + - name: entity_type + description: "Measured entity type" + tests: + - not_null + - accepted_values: + arguments: + values: ['person'] + - name: entity_id + description: "Normalized entity identifier (lowercased email for person)" + tests: + - not_null + - name: metric_date + description: "Observation date" + tests: + - not_null + - name: observed_at + description: > + Intra-day observation timestamp for future point-in-time semantics. + NULL for day-grain aggregate measures. + - name: measure_key + description: "Source measure key registered in the metric registry" + tests: + - not_null + - accepted_values: + arguments: + values: + - accepted_lines + - removed_lines + - active_day + - cost_usd + - accepted_edit_actions + - tool_use_offered + - dev_conversations + - assistant_messages + - assistant_actions + - chat_assistant_conversations + - name: value + description: > + Measure value. Rows are emitted only when the source provides a + value, so value is never NULL; the column stays Nullable(Float64) + in the published contract. + tests: + - not_null + - name: subject_key + description: > + Distinct-count subject for future distinct-count semantics. NULL for + measures without distinct-count support. + - name: dimensions + description: > + Array(Tuple(key, value, label)) of all dimensions the source emits + for the measure. Values and labels come straight from class-contract + columns, which the silver contract guarantees non-null. + + - name: metric_entity_cohorts_current + description: > + Current cohort membership per entity for peer comparison. One row per + (tenant, entity_type, entity_id, cohort_key) — uniqueness is asserted by + tests/gold/assert_metric_entity_cohorts_unique.sql and relied on by the + analytics service peer view (join fan-out would corrupt percentiles). The + column set is a published contract probed by the analytics service schema + validator (`COHORT_COLUMNS`). + columns: + - name: tenant_id + description: "Tenant isolation field" + tests: + - not_null + - name: entity_type + description: "Entity type the cohort row applies to" + tests: + - not_null + - accepted_values: + arguments: + values: ['person'] + - name: entity_id + description: "Normalized entity identifier (lowercased email for person)" + tests: + - not_null + - name: cohort_key + description: "Cohort basis identifier" + tests: + - not_null + - accepted_values: + arguments: + values: ['org_unit'] + - name: cohort_id + description: > + Cohort membership value: org unit id when present, department name + fallback otherwise. NULL when the person has neither. diff --git a/src/ingestion/scripts/apply-ch-migrations.sh b/src/ingestion/scripts/apply-ch-migrations.sh index 1fe60a678..727a4d52c 100755 --- a/src/ingestion/scripts/apply-ch-migrations.sh +++ b/src/ingestion/scripts/apply-ch-migrations.sh @@ -14,6 +14,8 @@ # stubs so gold-view CREATE VIEW type-checks on a fresh cluster # (CH validates referenced tables at parse time). See ADR-0007. # 3. Apply migrations/*.sql in lexicographic order. +# 4. Build the dbt gold models (tag:gold) so dbt-owned views exist at +# deploy time instead of after the first connector sync. # # Bookkeeping: none — every migration is re-run on every invocation and # MUST stay idempotent/re-runnable (CREATE OR REPLACE / IF NOT EXISTS). @@ -49,4 +51,98 @@ for migration in "$SCRIPT_DIR/migrations"/*.sql; do run_ch < "$migration" done +echo "=== Repairing class-contract labels on AI staging history ===" +# Staging rows ingested before the label columns existed read them as '' +# (String DEFAULT materialized by append_new_columns), and incremental +# models never re-read old rows. Labels are DECLARED CONSTANTS in the +# staging models, so writing the same constants here is byte-identical to +# what a full re-materialization would produce — any later full refresh +# independently converges to the same values and these updates become +# permanent no-ops. Data-bearing contract columns are NOT repairable this +# way; those go through the ADR-0015 major-bump full refresh instead. +# Guarded per table: staging tables do not exist before the connector's +# first dbt run. Idempotent: re-runs match zero rows. +repair_staging_tool_label() { + local table="$1" label="$2" + ch_table_exists staging "${table}" || return 0 + echo " staging.${table}" + run_ch < + Display label for the tool, declared by the emitting staging model + (the connector owns its display name). Consumed by Gold dimension + tuples; no label mapping may live downstream. + tests: + - not_null + - name: conversation_count + description: > + Dev-tool conversations/threads on this day, populated ONLY by + connectors whose source actually reports them (Claude Code sessions, + Codex threads). Structural NULL where the source has no conversation + concept (Cursor, Copilot) — never a synthetic marker. Gold + dev_conversations sums this column with no tool filter; a connector + opts in by mapping the column. - name: session_count description: "Sessions recorded on this day. For Cursor: 1 per active day (Cursor only exposes an is-active flag, not session counts)." tests: @@ -64,8 +82,8 @@ models: - name: lines_added description: > Lines of code added and accepted from AI. For Cursor: acceptedLinesAdded. - For Claude Code: lines_added from claude_admin_code_usage. Used in - ai_loc_share_pct / team_ai_loc Gold metrics (joined with class_git_commits). + For Claude Code: lines_added from claude_admin_code_usage. Summed by + insight.ai_metric_observations as the accepted_lines measure. tests: - not_null - name: lines_removed @@ -78,7 +96,6 @@ models: Total lines added by the user that day (AI-accepted + manual keystrokes). Populated for Cursor only (totalLinesAdded). NULL for Claude Code — Enterprise only sees AI-accepted lines, not total keystrokes. - Used as the denominator for ai_loc_share_pct (cursor-only metric). - name: total_lines_removed description: > Total lines deleted by the user that day (all edits, not just AI-assisted). @@ -91,8 +108,8 @@ models: - name: tool_use_accepted description: > Tool invocations accepted. For Cursor: totalTabsAccepted. - For Claude Code: tool_use_accepted (Edit/Write/MultiEdit/NotebookEdit). Used in - cursor_acceptance / cc_tool_acceptance ratios (accepted / offered). + For Claude Code: tool_use_accepted (Edit/Write/MultiEdit/NotebookEdit). + Numerator for accepted/offered acceptance ratios downstream. - name: agent_sessions description: > Cursor agent sessions (agentRequests). NULL for other tools. @@ -125,8 +142,8 @@ models: - name: prs_total_count description: > Total PRs in the measurement window. Populated for Claude Team only - (total_prs from the web API). Denominator for prs_with_cc_percentage - Gold metric. ⚠️ May be a period-aggregate (cumulative across the + (total_prs from the web API). Denominator for a PRs-with-assistant + share downstream. ⚠️ May be a period-aggregate (cumulative across the billing window), not a daily count — verify against a tenant with the Anthropic GitHub-app connected. NULL for all other sources. - name: tool_action_breakdown_json @@ -250,10 +267,14 @@ models: Unified per-person per-day AI assistant surface usage. Covers Claude Enterprise chat, office (Excel/PowerPoint), cowork, and cross (surface-agnostic counters, currently web_search) surfaces. - One row per (tenant, email, day, surface). Feeds Gold metrics for - assistant adoption: chat_active / excel_active / powerpoint_active / - cowork_active / cross_active / ai_adoption_pct (email IS NOT NULL - AND conversation_count > 0). + One row per (tenant, email, day, surface). + CONTRACT — activity invariant: a row exists ONLY when the person had real + activity on the surface that day. Every staging model in the + silver:class_ai_assistant_usage tag must gate emission on its surface + counters (claude-enterprise: per-surface counter filters; chatgpt-team: + chat counter filter). Downstream consumers + (insight.ai_metric_observations active_day) rely on row existence and + must NOT re-derive activity from counters. columns: - name: insight_tenant_id description: "Tenant isolation field" @@ -286,6 +307,13 @@ models: - accepted_values: arguments: values: ['claude', 'chatgpt', 'gemini'] + - name: tool_label + description: > + Display label for the assistant vendor, declared by the emitting + staging model. Consumed by Gold dimension tuples; no label mapping + may live downstream. + tests: + - not_null - name: surface description: > Per-vendor surface discriminator. For tool='claude': 'chat' = Claude.ai @@ -298,6 +326,13 @@ models: - accepted_values: arguments: values: ['chat', 'excel', 'powerpoint', 'cowork', 'cross'] + - name: surface_label + description: > + Display label for the surface, declared by the emitting staging + model. Consumed by Gold dimension tuples; no label mapping may live + downstream. + tests: + - not_null - name: session_count description: > Number of distinct sessions on the surface.