From 3fa69b5555b7157a727bd22aeb84266c3ceda6b2 Mon Sep 17 00:00:00 2001 From: Aleksandr Barkhatov Date: Thu, 30 Jul 2026 13:25:24 +0200 Subject: [PATCH] feat(metrics): add metric drilldown CSV/XLSX export Signed-off-by: Aleksandr Barkhatov --- .../components/backend/analytics/openapi.json | 161 ++++ docs/domain/metrics/README.md | 27 +- docs/domain/metrics/specs/DESIGN.md | 115 +-- src/backend/Cargo.lock | 85 +++ src/backend/Cargo.toml | 3 +- src/backend/services/analytics/Cargo.toml | 2 + .../analytics/src/api/http_live_tests.rs | 13 + .../analytics/src/api/metric_drilldown.rs | 157 +++- src/backend/services/analytics/src/api/mod.rs | 98 ++- .../src/domain/metric_drilldown/dto.rs | 30 + .../src/domain/metric_drilldown/error.rs | 10 + .../src/domain/metric_drilldown/export.rs | 694 ++++++++++++++++++ .../src/domain/metric_drilldown/mod.rs | 21 +- .../src/domain/metric_drilldown/validation.rs | 32 +- .../tests/e2e/api/test_metric_drilldown.py | 13 + 15 files changed, 1395 insertions(+), 66 deletions(-) create mode 100644 src/backend/services/analytics/src/domain/metric_drilldown/export.rs diff --git a/docs/components/backend/analytics/openapi.json b/docs/components/backend/analytics/openapi.json index fd3fb6aa3..74fdad8d3 100644 --- a/docs/components/backend/analytics/openapi.json +++ b/docs/components/backend/analytics/openapi.json @@ -911,6 +911,48 @@ ], "type": "object" }, + "MetricDrilldownExportFormat": { + "enum": [ + "csv", + "xlsx" + ], + "type": "string" + }, + "MetricDrilldownExportRequest": { + "properties": { + "display_dimensions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "entity": { + "$ref": "#/components/schemas/MetricDrilldownEntity" + }, + "filters": { + "items": { + "$ref": "#/components/schemas/MetricDrilldownFilter" + }, + "type": "array" + }, + "format": { + "$ref": "#/components/schemas/MetricDrilldownExportFormat" + }, + "metric_key": { + "type": "string" + }, + "period": { + "$ref": "#/components/schemas/MetricDrilldownPeriod" + } + }, + "required": [ + "metric_key", + "entity", + "period", + "format" + ], + "type": "object" + }, "MetricDrilldownFilter": { "properties": { "dimension": { @@ -3380,6 +3422,125 @@ "summary": "List metric evidence" } }, + "/v1/metric-drilldown/export": { + "post": { + "operationId": "analytics_api.metric_drilldown.export", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetricDrilldownExportRequest" + } + } + }, + "description": "Metric evidence export selection", + "required": true + }, + "responses": { + "200": { + "content": { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "schema": { + "format": "binary", + "type": "string" + } + }, + "text/csv": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Complete metric evidence export", + "headers": { + "Content-Disposition": { + "description": "Attachment filename", + "schema": { + "type": "string" + } + } + } + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Export metric evidence" + } + }, "/v1/metric-results": { "post": { "operationId": "analytics_api.metric_results.create", diff --git a/docs/domain/metrics/README.md b/docs/domain/metrics/README.md index 226d6b815..9644116d3 100644 --- a/docs/domain/metrics/README.md +++ b/docs/domain/metrics/README.md @@ -28,6 +28,14 @@ anywhere: at request time the runtime applies a definition to the matching observations — "alice, January: 312 ÷ 405 = 77%" — and returns it labeled and ready to render. +Metric **evidence** is the source-level population behind that answer. Each +managed source exposes a normalized serving table. +Definitions inherit drilldown support when all compatible inputs are backed by +the same validated evidence relation, so adding a metric over an existing +measure requires no drilldown-specific configuration. +The backend applies the same entity, period, dimension, input-role, and +computation semantics used by metric results. + The split exists because one fact serves many meanings and one meaning serves many questions. The same `accepted_edit_actions` observation is the whole value of `ai.accepted_edit_actions` and the numerator of @@ -43,12 +51,12 @@ in that shape, definitions reference facts only by measure key, and the runtime can therefore connect any definition to any matching facts without either side knowing the other exists. -| | Observation | Definition | Metric result | -|---|---|---|---| -| What | a fact | the meaning of facts | the computed answer | -| Lives | ClickHouse views over silver (computed on read, nothing stored) | registry (MariaDB, seeded from Rust) | nowhere — made per request | -| Knows | what happened | what it is called, how to compute, how to show | both, combined | -| Authored by | connector + gold model | one struct per metric | nobody — the runtime derives it | +| | Observation | Evidence | Definition | Metric result | +|---|---|---|---|---| +| What | an aggregate-ready fact | the participating source population | the meaning of facts | the computed answer | +| Lives | ClickHouse views or tables over silver | ClickHouse serving tables | registry | nowhere | +| Knows | what happened numerically | which records participated | how to compute and display | all three, combined | +| Authored by | connector + gold model | connector + gold model | one struct per metric | nobody | ## Documents @@ -64,12 +72,19 @@ either side knowing the other exists. | Definition loading, reconciler, schema validator | [`src/backend/services/analytics/src/domain/metric_definitions/`](../../../src/backend/services/analytics/src/domain/metric_definitions/) | | Result runtime (validation, query compiler, response builder) | [`src/backend/services/analytics/src/domain/metric_results/`](../../../src/backend/services/analytics/src/domain/metric_results/) | | Result endpoint | [`src/backend/services/analytics/src/api/metric_results.rs`](../../../src/backend/services/analytics/src/api/metric_results.rs) | +| Drilldown runtime | [`src/backend/services/analytics/src/domain/metric_drilldown/`](../../../src/backend/services/analytics/src/domain/metric_drilldown/) | +| Drilldown endpoints | [`src/backend/services/analytics/src/api/metric_drilldown.rs`](../../../src/backend/services/analytics/src/api/metric_drilldown.rs) | | Registry schema migration | [`src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs`](../../../src/backend/services/analytics/src/migration/m20260625_000001_metric_definitions.rs) | | Managed observation sources (dbt gold models) | [`src/ingestion/gold/`](../../../src/ingestion/gold/) | | Class-contract data-quality tests | [`src/ingestion/dbt/tests/ai/`](../../../src/ingestion/dbt/tests/ai/) | ## Boundaries +- Current deployments isolate one tenant per instance. Drilldown entity IDs + remain source-derived identifiers, commonly normalized email addresses. + Multi-tenant warehouse predicates, canonical person IDs, cross-source alias + resolution, and subordinate authorization belong to the identity-resolution + epic and are required before a multi-tenant instance enables drilldown. - The AI class contracts feeding the observation models are documented in [`src/ingestion/silver/ai/schema.yml`](../../../src/ingestion/silver/ai/schema.yml) (activity invariant, label and conversation-count semantics). diff --git a/docs/domain/metrics/specs/DESIGN.md b/docs/domain/metrics/specs/DESIGN.md index e06f034c4..120547e69 100644 --- a/docs/domain/metrics/specs/DESIGN.md +++ b/docs/domain/metrics/specs/DESIGN.md @@ -115,10 +115,10 @@ details Map(String, String) ``` Evidence relations are MergeTree serving tables built from silver. Their -ordering follows the evidence key access pattern, avoiding repeated silver -reconstruction on read. Observation models derive their values from these -evidence tables. Each source exposes one evidence relation and each measure -has one granularity: +ordering follows the drilldown predicate and cursor access pattern, avoiding +repeated silver reconstruction for every page and export. Observation models +derive their values from these evidence tables. The registry stores one +evidence relation per source and one granularity per measure: - `event`: one source event, such as a commit. - `source_summary`: the finest summary preserved by silver. @@ -162,6 +162,11 @@ cursor is versioned and bound to the normalized selection and request tenant. It is not an authorization token and modifying its ordering key cannot widen the server-owned relation or selection. +`POST /v1/metric-drilldown/export` produces the complete selected population +with the same projected columns as CSV or XLSX. Export is server-side and +rejects results exceeding its row, byte, cell, execution-time, or concurrency +limits. It never silently truncates. + The evidence contract has these limitations: - Summary-grain silver cannot produce event-grain evidence. AI and @@ -172,16 +177,17 @@ The evidence contract has these limitations: - Metric results and evidence are not transactionally snapshot-isolated from each other during a dbt rebuild. They reconcile after the complete gold build because observations derive from evidence. -- Pagination is bound to the evidence table UUID. A rebuild during the - operation fails with `EVIDENCE_SNAPSHOT_EXPIRED`; the client must restart - the selection rather than mix rows from two builds. Previous table +- Pagination and each export are bound to the evidence table UUID. A rebuild + during the operation fails with `EVIDENCE_SNAPSHOT_EXPIRED`; the client must + restart the selection rather than mix rows from two builds. Previous table snapshots are not retained. -- Drilldown preserves the existing metric entity and tenant behavior. This - does not add identity-tree authorization or warehouse tenant enforcement. - Source links are omitted. Hosted services commonly use custom domains, and the current silver contract does not preserve a canonical web base URL. A future source registry can add a non-secret `web_base_url` keyed by source instance and combine it with provider-specific record identifiers. +- Drilldown preserves the existing metric entity and tenant behavior. This + change does not add identity-tree authorization or warehouse tenant + enforcement. ## Computations @@ -527,8 +533,9 @@ one that applies. ### Case 1: metric over an existing measure -The measure already appears in a managed observation source (check the -`measures` list of the source in `builtin.rs` and the emitting gold model). +The measure already appears in a managed source. Check the source's `measures` +list in `builtin.rs`, the emitting evidence model, and the observation model +derived from it. 1. Add one `MetricSeed` to `BUILTIN_METRICS` in `src/backend/services/analytics/src/domain/metric_definitions/builtin.rs`: @@ -538,54 +545,72 @@ The measure already appears in a managed observation source (check the 2. Run `cargo test -p analytics` — the registry invariant tests validate key shapes, input/measure references, and computation field combinations. -The reconciler seeds the definition on the next deploy. No SQL, no migration, -no dbt change. +The reconciler seeds the definition on the next deploy. If every input measure +has healthy evidence metadata, the metric automatically receives drilldown, +table, and CSV/XLSX export support. No drilldown-specific SQL, frontend +configuration, migration, or dbt change is required. ### Case 2: new measure from an existing source The source exists but does not emit the measure yet. -1. Add the measure branch to the source's gold model in `src/ingestion/gold/`: - one `UNION ALL` entry calling a shape macro from - `src/ingestion/dbt/macros/metric_observation_measures.sql` — - `sum_measure(measure_key, relation, value_expr, dimensions_col, - where=none)` for aggregated numerics, `presence_measure(measure_key, - relations)` for row-existence markers, `event_measure(measure_key, - relation, value_expr, dimensions_col, where=none)` for per-event values - feeding median metrics. Every branch is a shape-macro call; a new macro - is added only when a new computation kind becomes executable. - Read only class-contract columns; never vendor-specific ones — if the fact - you need is not in the class contract, extend the class contract first - (staging models declare semantics, see the class `schema.yml`). -2. Add the `measure_key` to the gold model's `schema.yml` `accepted_values` - test. -3. Add the measure key to the source's `measures` list in `builtin.rs`. -4. Add the `MetricSeed` as in case 1. -5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation +1. Add the measure to the source's `_metric_evidence` model in + `src/ingestion/gold/`. Summary measures use the shared shape macros from + `src/ingestion/dbt/macros/metric_observation_measures.sql`; + event measures select stable source records into the evidence contract. + Read only class-contract columns; never vendor-specific ones. If the fact + is absent from the class contract, extend that contract first (staging + models declare semantics in their `schema.yml`). +2. Choose the evidence granularity deliberately: + - emit one stable row per source record for `event`. + - emit the finest source-retained grouping for `source_summary`. + - emit the reconstructed participating population for + `derived_population`. + Event rows need deterministic `record_id` values and should place reusable + grouping fields in `dimensions` and human-facing fields in `details`. +3. Derive the matching observation measure from the evidence relation. The + observation remains the aggregate-ready runtime input; the evidence row is + the population that explains it. +4. Add the `measure_key` to the observation model's `schema.yml` + `accepted_values` test. +5. Add the measure key to the source's `measures` list in `builtin.rs` and + classify its evidence granularity. +6. Add or reuse a source-measure presentation rule when the default + date-plus-value table is insufficient. Declare detail keys there and add + explicit column metadata only for fields that are not humanized strings. +7. Add the `MetricSeed` as in case 1. +8. Validate: `dbt parse` + `cargo test -p analytics` (see Validation commands). ### Case 3: new observation source The metric family reads data no managed source covers. -1. Create a dbt gold model in `src/ingestion/gold/` named - `_metric_observations`, emitting the source measure observation - contract, `schema=insight`, `ref()`-ing silver models (medallion layering - rules: `docs/domain/ingestion-data-flow/specs/DESIGN.md`). Document columns - and measure keys in `src/ingestion/gold/schema.yml`. +1. Create `_metric_evidence` and + `_metric_observations` dbt gold models in + `src/ingestion/gold/`, `schema=insight`, `ref()`-ing silver models + (medallion layering rules: + `docs/domain/ingestion-data-flow/specs/DESIGN.md`). The evidence model emits + the evidence contract; the observation model derives the aggregate-ready + observation contract from it. Document both in + `src/ingestion/gold/schema.yml`. 2. Add a `BuiltinSource` (source + measures + dimensions) to `builtin.rs`, - with `source_ref` set to the relation name. No backend enum or table-name - code changes: the relation name is data, validated on load against the - `_metric_observations` shape (`ObservationRelation`) and probed at - runtime by the schema validator. -3. Add `MetricSeed`s as in case 1. -4. Validate: `dbt parse` + `cargo test -p analytics` (see Validation + with `source_ref` and `evidence_ref` set to their relation names and every + measure assigned an evidence granularity. No backend enum or table-name + code changes are required: relation names are validated data and both + contracts are probed by the runtime schema validator. +3. Add source-measure presentation rules for event shapes that need + human-facing detail columns. +4. Add `MetricSeed`s as in case 1. +5. Validate: `dbt parse` + `cargo test -p analytics` (see Validation commands). The runtime schema validator probes the new relation at startup. ### Rules that hold for every case - No metric-key-specific branches in runtime code. +- Evidence presentation branches may depend on source and measure, never on + final metric key. - No vendor names, vendor columns, or label mappings in gold models — labels and taxonomy come from class-contract columns declared by staging. - Measure filter predicates (`where=` on shape macros) may reference only @@ -643,6 +668,14 @@ Until one exists, custom definitions can be stored but cannot produce new source Frontend collection rendering: - requests metric keys and views. +- treats the optional `drilldown` capability as the only evidence-action + switch; there is no frontend metric allowlist. +- forwards the canonical metric selection returned by `/v1/metric-results`, + narrowing period and dimension filters for chart-point interactions. +- renders server-owned typed evidence columns and rows without interpreting + the internal evidence contract. +- uses the same canonical selection for table pagination and server-side + CSV/XLSX export. - treats configured required views as required. - normalizes response arrays only for local lookup. - renders using returned label, description, explanation, unit, format, diff --git a/src/backend/Cargo.lock b/src/backend/Cargo.lock index 9904abb55..d8f838b37 100644 --- a/src/backend/Cargo.lock +++ b/src/backend/Cargo.lock @@ -93,11 +93,13 @@ dependencies = [ "chrono", "clap", "clickhouse", + "csv", "futures", "futures-util", "insight-clickhouse", "redis", "reqwest 0.12.28", + "rust_xlsxwriter", "sea-orm", "sea-orm-migration", "serde", @@ -198,6 +200,15 @@ dependencies = [ "object", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.1" @@ -1715,6 +1726,27 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1872,6 +1904,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2274,6 +2317,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -5108,6 +5152,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "rust_xlsxwriter" +version = "0.90.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2be778223b36bb449b2ef2df4856ced2d311680818a7310db5c5dc370170f935" +dependencies = [ + "zip", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -7685,8 +7738,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/src/backend/Cargo.toml b/src/backend/Cargo.toml index 9da7058a4..a077844ee 100644 --- a/src/backend/Cargo.toml +++ b/src/backend/Cargo.toml @@ -67,6 +67,8 @@ uuid = { version = "1.19", features = ["serde", "v7"] } chrono = { version = "0.4", features = ["serde"] } base64 = "0.22" sha2 = "0.10" +csv = "1.3" +rust_xlsxwriter = "0.90" time = { version = "0.3", features = ["serde", "formatting", "parsing"] } # Logging @@ -128,4 +130,3 @@ authenticator-sdk = { path = "libs/authenticator-sdk" } # CI rebuild marker — bumped to retrigger the analytics # image build on the cf → gears crate migration (2026-06-12). - diff --git a/src/backend/services/analytics/Cargo.toml b/src/backend/services/analytics/Cargo.toml index 84ad7180f..f21738aae 100644 --- a/src/backend/services/analytics/Cargo.toml +++ b/src/backend/services/analytics/Cargo.toml @@ -67,6 +67,8 @@ uuid = { workspace = true } chrono = { workspace = true } base64 = { workspace = true } sha2 = { workspace = true } +csv = { workspace = true } +rust_xlsxwriter = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } clap = { workspace = true } diff --git a/src/backend/services/analytics/src/api/http_live_tests.rs b/src/backend/services/analytics/src/api/http_live_tests.rs index ce6e1201f..879d149ae 100644 --- a/src/backend/services/analytics/src/api/http_live_tests.rs +++ b/src/backend/services/analytics/src/api/http_live_tests.rs @@ -712,9 +712,22 @@ async fn metric_drilldown_validates_selection_before_clickhouse_error() -> TestR "limit": 100 }); let resp = app + .clone() .oneshot(json_req("POST", "/v1/metric-drilldown", &body)?) .await?; anyhow::ensure!(resp.status() == StatusCode::BAD_REQUEST); + let export = json!({ + "metric_key": "git.commits", + "entity": {"type": "person", "id": "person@example.com"}, + "period": {"from": "2026-07-01", "to": "2026-07-28"}, + "filters": [], + "display_dimensions": [], + "format": "csv" + }); + let resp = app + .oneshot(json_req("POST", "/v1/metric-drilldown/export", &export)?) + .await?; + anyhow::ensure!(resp.status() == StatusCode::BAD_REQUEST); Ok(()) } .await; diff --git a/src/backend/services/analytics/src/api/metric_drilldown.rs b/src/backend/services/analytics/src/api/metric_drilldown.rs index 20990f9a6..562c8027c 100644 --- a/src/backend/services/analytics/src/api/metric_drilldown.rs +++ b/src/backend/services/analytics/src/api/metric_drilldown.rs @@ -2,7 +2,10 @@ use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; use axum::Json; +use axum::body::Body; use axum::extract::Extension; +use axum::http::header::{CONTENT_DISPOSITION, CONTENT_TYPE}; +use axum::http::{HeaderValue, Response}; use tokio::sync::Semaphore; use toolkit_canonical_errors::CanonicalError; use toolkit_security::SecurityContext; @@ -11,14 +14,22 @@ use super::AppState; use crate::api::error::MetricError; use crate::domain::metric_drilldown::{ EVIDENCE_QUERY_MEMORY_BYTES, EVIDENCE_QUERY_READ_BYTES, EVIDENCE_QUERY_RESULT_BYTES, - EVIDENCE_QUERY_TIMEOUT_SECS, EvidenceQueryRow, MetricDrilldownRequest, MetricDrilldownResponse, - build_response, compile_query, decode_evidence_rows, evidence_unavailable, validate_request, - verify_evidence_snapshot, + EVIDENCE_QUERY_TIMEOUT_SECS, EvidenceQueryRow, MAX_EXPORT_BYTES, MAX_EXPORT_ROWS, + MetricDrilldownColumn, MetricDrilldownExportFormat, MetricDrilldownExportRequest, + MetricDrilldownRequest, MetricDrilldownResponse, MetricDrilldownRow, ValidatedMetricDrilldown, + build_export, build_response, compile_query, decode_evidence_rows, evidence_unavailable, + export_filename, export_internal, export_limit, presentation, validate_export_request, + validate_request, verify_evidence_snapshot, }; const QUERY_TIMEOUT: Duration = Duration::from_secs(EVIDENCE_QUERY_TIMEOUT_SECS); +const EXPORT_TIMEOUT: Duration = Duration::from_mins(1); +const EXPORT_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(2); const QUERY_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(2); +const MAX_CONCURRENT_EXPORTS: usize = 2; const MAX_CONCURRENT_QUERIES: usize = 8; +static EXPORT_SEMAPHORE: LazyLock = + LazyLock::new(|| Semaphore::new(MAX_CONCURRENT_EXPORTS)); static QUERY_SEMAPHORE: LazyLock = LazyLock::new(|| Semaphore::new(MAX_CONCURRENT_QUERIES)); @@ -45,6 +56,139 @@ pub async fn query_metric_drilldown( Ok(Json(response)) } +pub async fn export_metric_drilldown( + Extension(state): Extension>, + Extension(ctx): Extension, + Json(req): Json, +) -> Result, CanonicalError> { + let started = Instant::now(); + let permit = acquire_export_permit().await?; + let deadline = tokio::time::Instant::now() + EXPORT_TIMEOUT; + + let validated = validate_export_request( + &state.db, + &state.ch, + ctx.subject_tenant_id(), + &req, + MAX_EXPORT_ROWS + 1, + ) + .await?; + let evidence = collect_export_rows(&state, &validated, deadline).await?; + let exported_rows = evidence.len(); + + let (columns, rows) = presentation( + &evidence, + &validated.plan, + &validated.selection.filters, + &validated.selection.display_dimensions, + )?; + drop(evidence); + + let format = req.format; + let (body, content_type, extension) = + serialize_export(permit, format, columns, rows, deadline).await?; + + tracing::info!( + duration_ms = started.elapsed().as_millis(), + rows = exported_rows, + bytes = body.len(), + format = format.as_str(), + row_limit = MAX_EXPORT_ROWS, + byte_limit = MAX_EXPORT_BYTES, + capacity = MAX_CONCURRENT_EXPORTS, + "metric drilldown export completed" + ); + + attachment_response(body, content_type, &export_name(&validated, extension)) +} + +async fn acquire_export_permit() -> Result, CanonicalError> { + // INVARIANT: the caller holds this permit across validation, fetch, and the + // blocking serialization — the hold is the MAX_CONCURRENT_EXPORTS cap. + tokio::time::timeout(EXPORT_ACQUIRE_TIMEOUT, EXPORT_SEMAPHORE.acquire()) + .await + .map_err(|_| { + tracing::warn!( + capacity = MAX_CONCURRENT_EXPORTS, + available = EXPORT_SEMAPHORE.available_permits(), + "metric drilldown export capacity exhausted" + ); + export_busy() + })? + .map_err(|_| export_busy()) +} + +async fn collect_export_rows( + state: &Arc, + validated: &ValidatedMetricDrilldown, + deadline: tokio::time::Instant, +) -> Result, CanonicalError> { + let log_comment = format!( + "metric-drilldown:export:{}", + validated.plan.definition.key() + ); + let rows = tokio::time::timeout_at(deadline, fetch_rows(state, validated, &log_comment)) + .await + .map_err(|_| export_limit("Export exceeded the execution time limit."))??; + verify_evidence_snapshot(&state.ch, &validated.plan.relation, &validated.snapshot_id).await?; + + if rows.len() > MAX_EXPORT_ROWS { + return Err(export_limit(format!( + "Export exceeds the {MAX_EXPORT_ROWS} row limit." + ))); + } + Ok(rows) +} + +async fn serialize_export( + permit: tokio::sync::SemaphorePermit<'static>, + format: MetricDrilldownExportFormat, + columns: Vec, + rows: Vec, + deadline: tokio::time::Instant, +) -> Result<(Vec, &'static str, &'static str), CanonicalError> { + let blocking_deadline = deadline.into_std(); + let export = tokio::task::spawn_blocking(move || { + let _permit = permit; + build_export(format, &columns, &rows, blocking_deadline) + }); + tokio::time::timeout_at(deadline, export) + .await + .map_err(|_| export_limit("Export exceeded the execution time limit."))? + .map_err(|_| export_internal())? +} + +fn export_name(validated: &ValidatedMetricDrilldown, extension: &str) -> String { + export_filename( + &validated.plan.definition.base.label, + &validated.selection.metric_key, + &validated.selection.period.from, + &validated.selection.period.to, + validated + .selection + .filters + .iter() + .any(|filter| !filter.values.is_empty()), + extension, + ) +} + +fn attachment_response( + body: Vec, + content_type: &'static str, + filename: &str, +) -> Result, CanonicalError> { + Response::builder() + .header(CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header( + CONTENT_DISPOSITION, + HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")) + .map_err(|_| export_internal())?, + ) + .body(Body::from(body)) + .map_err(|_| export_internal()) +} + async fn fetch_rows( state: &Arc, req: &crate::domain::metric_drilldown::ValidatedMetricDrilldown, @@ -123,6 +267,13 @@ fn is_clickhouse_resource_limit(message: &str) -> bool { .any(|marker| message.contains(marker)) } +fn export_busy() -> CanonicalError { + MetricError::resource_exhausted("Metric evidence export capacity is busy.") + .with_quota_violation("metric evidence exports", "concurrency limit reached") + .with_quota_violation_retry_after_seconds(EXPORT_ACQUIRE_TIMEOUT.as_secs()) + .create() +} + fn query_busy() -> CanonicalError { MetricError::resource_exhausted("Metric evidence query capacity is busy.") .with_quota_violation("metric evidence queries", "concurrency limit reached") diff --git a/src/backend/services/analytics/src/api/mod.rs b/src/backend/services/analytics/src/api/mod.rs index 0d07491ca..6f637484d 100644 --- a/src/backend/services/analytics/src/api/mod.rs +++ b/src/backend/services/analytics/src/api/mod.rs @@ -20,7 +20,15 @@ use axum::http::StatusCode; use axum::{Extension, Router}; use sea_orm::DatabaseConnection; use std::sync::Arc; -use toolkit::api::{OpenApiInfo, OpenApiRegistry, OpenApiRegistryImpl, OperationBuilder}; +use toolkit::api::{ + OpenApiInfo, OpenApiRegistry, OpenApiRegistryImpl, OperationBuilder, ResponseSpec, +}; +use utoipa::openapi::RefOr; +use utoipa::openapi::content::ContentBuilder; +use utoipa::openapi::header::HeaderBuilder; +use utoipa::openapi::schema::{ + KnownFormat, ObjectBuilder, Schema, SchemaFormat, SchemaType, Type as OpenApiType, +}; use crate::config::GearConfig; use crate::domain::admin_threshold::AdminThresholdService; @@ -338,6 +346,25 @@ fn build_operations(router: Router, openapi: &dyn OpenApiRegistry) -> Router { .handler(metric_drilldown::query_metric_drilldown) .register(router, openapi); + router = OperationBuilder::post("/v1/metric-drilldown/export") + .operation_id("analytics_api.metric_drilldown.export") + .summary("Export metric evidence") + .authenticated() + .no_license_required() + .json_request::( + openapi, + "Metric evidence export selection", + ) + .response(ResponseSpec { + status: StatusCode::OK.as_u16(), + content_type: "text/csv", + description: "Complete metric evidence export".to_owned(), + schema_name: None, + }) + .standard_errors(openapi) + .handler(metric_drilldown::export_metric_drilldown) + .register(router, openapi); + // Thresholds (legacy) router = OperationBuilder::get("/v1/metrics/{id}/thresholds") .operation_id("analytics_api.thresholds.list") @@ -560,9 +587,44 @@ fn build_operations(router: Router, openapi: &dyn OpenApiRegistry) -> Router { pub fn openapi_document() -> anyhow::Result { let openapi = OpenApiRegistryImpl::new(); let _ = build_operations(Router::new(), &openapi); - openapi + let mut document = openapi .build_openapi(&openapi_info()) - .map_err(|e| anyhow::anyhow!("failed to build analytics OpenAPI document: {e}")) + .map_err(|e| anyhow::anyhow!("failed to build analytics OpenAPI document: {e}"))?; + let response = document + .paths + .paths + .get_mut("/v1/metric-drilldown/export") + .and_then(|path| path.post.as_mut()) + .and_then(|operation| operation.responses.responses.get_mut("200")) + .ok_or_else(|| anyhow::anyhow!("metric drilldown export response is missing"))?; + let RefOr::T(response) = response else { + return Err(anyhow::anyhow!( + "metric drilldown export response must be inline" + )); + }; + let schema = Schema::Object( + ObjectBuilder::new() + .schema_type(SchemaType::Type(OpenApiType::String)) + .format(Some(SchemaFormat::KnownFormat(KnownFormat::Binary))) + .build(), + ); + for media_type in [ + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ] { + response.content.insert( + media_type.to_owned(), + ContentBuilder::new().schema(Some(schema.clone())).build(), + ); + } + response.headers.insert( + "Content-Disposition".to_owned(), + HeaderBuilder::new() + .schema(ObjectBuilder::new().schema_type(OpenApiType::String)) + .description(Some("Attachment filename")) + .build(), + ); + Ok(document) } #[cfg(test)] @@ -579,4 +641,34 @@ mod tests { let openapi = OpenApiRegistryImpl::new(); let _router: Router = build_operations(Router::new(), &openapi); } + + #[test] + fn export_response_advertises_both_file_media_types_and_the_filename_header() { + let document = + openapi_document().unwrap_or_else(|error| panic!("document must build: {error}")); + let response = document + .paths + .paths + .get("/v1/metric-drilldown/export") + .and_then(|path| path.post.as_ref()) + .and_then(|operation| operation.responses.responses.get("200")) + .unwrap_or_else(|| panic!("export 200 response must be registered")); + let RefOr::T(response) = response else { + panic!("export response must be inline, not a $ref"); + }; + + for media_type in [ + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ] { + assert!( + response.content.contains_key(media_type), + "export must advertise {media_type}" + ); + } + assert!( + response.headers.contains_key("Content-Disposition"), + "export must advertise the attachment filename header" + ); + } } diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/dto.rs b/src/backend/services/analytics/src/domain/metric_drilldown/dto.rs index 623113187..a363eeb60 100644 --- a/src/backend/services/analytics/src/domain/metric_drilldown/dto.rs +++ b/src/backend/services/analytics/src/domain/metric_drilldown/dto.rs @@ -15,6 +15,7 @@ pub(super) const MAX_FILTERS: usize = 10; pub(super) const MAX_DISPLAY_DIMENSIONS: usize = 10; pub(super) const MAX_FILTER_VALUES: usize = 100; pub(super) const MAX_FILTER_VALUE_BYTES: usize = 512; +pub const MAX_EXPORT_ROWS: usize = 50_000; pub const EVIDENCE_QUERY_TIMEOUT_SECS: u64 = 45; pub const EVIDENCE_QUERY_MEMORY_BYTES: usize = 256 * 1024 * 1024; pub const EVIDENCE_QUERY_READ_BYTES: usize = 512 * 1024 * 1024; @@ -51,6 +52,25 @@ pub struct MetricDrilldownRequest { pub cursor: Option, } +#[derive(Debug, Clone, Copy, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum MetricDrilldownExportFormat { + Csv, + Xlsx, +} + +#[derive(Debug, Clone, Deserialize, utoipa::ToSchema)] +pub struct MetricDrilldownExportRequest { + pub metric_key: String, + pub entity: MetricDrilldownEntity, + pub period: MetricDrilldownPeriod, + #[serde(default)] + pub filters: Vec, + #[serde(default)] + pub display_dimensions: Vec, + pub format: MetricDrilldownExportFormat, +} + #[derive(Debug, Clone, Serialize, utoipa::ToSchema)] pub struct MetricDrilldownSelection { pub metric_key: String, @@ -94,6 +114,7 @@ pub struct MetricDrilldownResponse { } impl toolkit::api::api_dto::RequestApiDto for MetricDrilldownRequest {} +impl toolkit::api::api_dto::RequestApiDto for MetricDrilldownExportRequest {} impl toolkit::api::api_dto::ResponseApiDto for MetricDrilldownResponse {} #[derive(Debug)] @@ -145,3 +166,12 @@ pub struct EvidenceQueryRow { pub dimensions_json: String, pub details: serde_json::Value, } + +impl MetricDrilldownExportFormat { + pub fn as_str(self) -> &'static str { + match self { + Self::Csv => "csv", + Self::Xlsx => "xlsx", + } + } +} diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/error.rs b/src/backend/services/analytics/src/domain/metric_drilldown/error.rs index dd6de2836..5840de8ef 100644 --- a/src/backend/services/analytics/src/domain/metric_drilldown/error.rs +++ b/src/backend/services/analytics/src/domain/metric_drilldown/error.rs @@ -36,3 +36,13 @@ pub(super) fn invalid_error(field: &str, description: impl Into) -> Cano .with_field_violation(field, description.into(), "INVALID") .create() } + +pub(crate) fn export_limit(description: impl Into) -> CanonicalError { + MetricError::resource_exhausted("Metric evidence export exceeded resource limits.") + .with_quota_violation("metric evidence export", description.into()) + .create() +} + +pub(crate) fn export_internal() -> CanonicalError { + CanonicalError::internal("failed to build metric evidence export").create() +} diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/export.rs b/src/backend/services/analytics/src/domain/metric_drilldown/export.rs new file mode 100644 index 000000000..e8a292430 --- /dev/null +++ b/src/backend/services/analytics/src/domain/metric_drilldown/export.rs @@ -0,0 +1,694 @@ +use std::io::{Cursor, Seek, SeekFrom, Write}; + +use rust_xlsxwriter::{ExcelDateTime, Format, Table, TableStyle, Workbook}; +use toolkit_canonical_errors::CanonicalError; + +use super::dto::{ + MetricDrilldownColumn, MetricDrilldownColumnType, MetricDrilldownExportFormat, + MetricDrilldownRow, +}; +use super::error::{export_internal, export_limit}; + +pub const BYTE_LIMIT_MARKER: &str = "export byte limit exceeded"; +const DEADLINE_CHECK_EVERY_ROWS: usize = 512; +const MAX_FILENAME_SLUG_BYTES: usize = 80; +pub(crate) const MAX_EXPORT_BYTES: usize = 25 * 1024 * 1024; +const MAX_CELL_BYTES: usize = 32 * 1024; + +pub fn build_export( + format: MetricDrilldownExportFormat, + columns: &[MetricDrilldownColumn], + rows: &[MetricDrilldownRow], + deadline: std::time::Instant, +) -> Result<(Vec, &'static str, &'static str), CanonicalError> { + match format { + MetricDrilldownExportFormat::Csv => { + let formatted_rows = rows + .iter() + .map(|row| export_values(columns, row)) + .collect::, _>>()?; + + let mut budget = ExportInputBudget::new(columns)?; + for row in &formatted_rows { + budget.add_row(row)?; + } + Ok(( + build_csv(columns, formatted_rows, deadline)?, + "text/csv; charset=utf-8", + "csv", + )) + } + MetricDrilldownExportFormat::Xlsx => { + let mut budget = ExportInputBudget::new(columns)?; + for row in rows { + budget.add_row(&export_values(columns, row)?)?; + } + Ok(( + build_xlsx(columns, rows, deadline)?, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlsx", + )) + } + } +} + +fn build_csv( + columns: &[MetricDrilldownColumn], + rows: Vec>, + deadline: std::time::Instant, +) -> Result, CanonicalError> { + let mut writer = csv::Writer::from_writer(LimitedBuffer::new(MAX_EXPORT_BYTES)); + let headers = columns + .iter() + .map(|column| column.label.as_str()) + .collect::>(); + writer + .write_record(&headers) + .map_err(|_| export_limit("CSV export exceeds the byte limit."))?; + for (row_index, row) in rows.into_iter().enumerate() { + check_export_deadline(row_index, deadline)?; + let values = row.into_iter().map(csv_safe_cell).collect::>(); + writer + .write_record(values) + .map_err(|_| export_limit("CSV export exceeds the byte limit."))?; + } + writer + .into_inner() + .map(LimitedBuffer::into_inner) + .map_err(|_| export_limit("CSV export exceeds the byte limit.")) +} + +fn build_xlsx( + columns: &[MetricDrilldownColumn], + rows: &[MetricDrilldownRow], + deadline: std::time::Instant, +) -> Result, CanonicalError> { + let mut workbook = Workbook::new(); + let worksheet = workbook.add_worksheet(); + let date_format = Format::new().set_num_format("yyyy-mm-dd"); + let blank_format = Format::new(); + + for (column, header) in columns.iter().enumerate() { + let column = u16::try_from(column).map_err(|_| xlsx_error("header column index"))?; + worksheet + .write_string(0, column, &header.label) + .map_err(|_| xlsx_error("header cell"))?; + } + + for (row_index, row) in rows.iter().enumerate() { + check_export_deadline(row_index, deadline)?; + let row_index = u32::try_from(row_index + 1).map_err(|_| xlsx_error("row index"))?; + for (column_index, column) in columns.iter().enumerate() { + let column_index = + u16::try_from(column_index).map_err(|_| xlsx_error("column index"))?; + let value = row + .values + .get(&column.key) + .unwrap_or(&serde_json::Value::Null); + match column.r#type { + MetricDrilldownColumnType::Number => match value { + serde_json::Value::Null => worksheet + .write_blank(row_index, column_index, &blank_format) + .map_err(|_| xlsx_error("blank cell"))?, + value => match value.as_f64() { + Some(number) => worksheet + .write_number(row_index, column_index, number) + .map_err(|_| xlsx_error("number cell"))?, + None => worksheet + .write_string(row_index, column_index, cell_text(value)?) + .map_err(|_| xlsx_error("number cell as text"))?, + }, + }, + MetricDrilldownColumnType::Date => match value { + serde_json::Value::Null => worksheet + .write_blank(row_index, column_index, &blank_format) + .map_err(|_| xlsx_error("blank cell"))?, + serde_json::Value::String(text) => match ExcelDateTime::parse_from_str(text) { + Ok(date) => worksheet + .write_datetime_with_format( + row_index, + column_index, + &date, + &date_format, + ) + .map_err(|_| xlsx_error("date cell"))?, + Err(_) => worksheet + .write_string(row_index, column_index, text) + .map_err(|_| xlsx_error("date cell as text"))?, + }, + value => worksheet + .write_string(row_index, column_index, cell_text(value)?) + .map_err(|_| xlsx_error("date cell as text"))?, + }, + MetricDrilldownColumnType::String => match value { + serde_json::Value::Null => worksheet + .write_blank(row_index, column_index, &blank_format) + .map_err(|_| xlsx_error("blank cell"))?, + serde_json::Value::Bool(flag) => worksheet + .write_boolean(row_index, column_index, *flag) + .map_err(|_| xlsx_error("boolean cell"))?, + value => worksheet + .write_string(row_index, column_index, cell_text(value)?) + .map_err(|_| xlsx_error("string cell"))?, + }, + }; + } + } + + if !rows.is_empty() && !columns.is_empty() { + let last_row = u32::try_from(rows.len()).map_err(|_| xlsx_error("table last row"))?; + let last_column = + u16::try_from(columns.len() - 1).map_err(|_| xlsx_error("table last column"))?; + let table = Table::new() + .set_style(TableStyle::None) + .set_autofilter(false) + .set_banded_rows(false); + worksheet + .add_table(0, 0, last_row, last_column, &table) + .map_err(|_| xlsx_error("table"))?; + } + + let mut output = LimitedBuffer::new(MAX_EXPORT_BYTES); + workbook.save_to_writer(&mut output).map_err(|error| { + let message = error.to_string(); + if message.contains(BYTE_LIMIT_MARKER) { + tracing::warn!(error = %error, "metric drilldown XLSX exceeded the byte limit"); + return export_limit("XLSX export exceeds the byte limit."); + } + tracing::error!(error = %error, "metric drilldown XLSX serialization failed"); + export_internal() + })?; + Ok(output.into_inner()) +} + +// INVARIANT: serialization stops at the deadline so the concurrency permit is +// released at the client-visible limit, not when the blocking task finishes. +fn check_export_deadline( + row_index: usize, + deadline: std::time::Instant, +) -> Result<(), CanonicalError> { + if row_index.is_multiple_of(DEADLINE_CHECK_EVERY_ROWS) && std::time::Instant::now() >= deadline + { + return Err(export_limit("Export exceeded the execution time limit.")); + } + Ok(()) +} + +fn cell_text(value: &serde_json::Value) -> Result { + match value { + serde_json::Value::String(text) => Ok(text.clone()), + value => serde_json::to_string(value).map_err(|_| xlsx_error("cell encode")), + } +} + +fn xlsx_error(operation: &'static str) -> CanonicalError { + tracing::error!(operation, "metric drilldown XLSX serialization failed"); + export_internal() +} + +fn export_values( + columns: &[MetricDrilldownColumn], + row: &MetricDrilldownRow, +) -> Result, CanonicalError> { + let values = columns + .iter() + .map(|column| { + let value = row + .values + .get(&column.key) + .unwrap_or(&serde_json::Value::Null); + match value { + serde_json::Value::Null => Ok(String::new()), + serde_json::Value::String(value) => Ok(value.clone()), + serde_json::Value::Bool(value) => Ok(value.to_string()), + serde_json::Value::Number(value) => Ok(value.to_string()), + value => serde_json::to_string(value).map_err(|_| export_internal()), + } + }) + .collect::, _>>()?; + if values.iter().any(|value| value.len() > MAX_CELL_BYTES) { + return Err(export_limit(format!( + "Export contains a value exceeding the {MAX_CELL_BYTES} byte limit." + ))); + } + Ok(values) +} + +struct ExportInputBudget { + bytes: usize, +} + +impl ExportInputBudget { + fn new(columns: &[MetricDrilldownColumn]) -> Result { + let bytes = columns + .iter() + .try_fold(0usize, |total, column| { + total.checked_add(column.label.len() + 1) + }) + .ok_or_else(input_too_large)?; + Ok(Self { bytes }) + } + + fn add_row(&mut self, values: &[String]) -> Result<(), CanonicalError> { + for value in values { + self.bytes = self + .bytes + .checked_add(value.len() + 1) + .ok_or_else(input_too_large)?; + if self.bytes > MAX_EXPORT_BYTES { + return Err(input_too_large()); + } + } + Ok(()) + } +} + +fn input_too_large() -> CanonicalError { + export_limit("Export input exceeds the byte limit.") +} + +fn csv_safe_cell(value: String) -> String { + if value.as_bytes().first().is_some_and(|first| { + matches!( + first, + b'=' | b'+' | b'-' | b'@' | b'\t' | b'\r' | b'\n' | b' ' + ) + }) { + format!("'{value}") + } else { + value + } +} + +#[derive(Debug)] +struct LimitedBuffer { + inner: Cursor>, + limit: usize, +} + +impl LimitedBuffer { + fn new(limit: usize) -> Self { + Self { + inner: Cursor::new(Vec::new()), + limit, + } + } + + fn into_inner(self) -> Vec { + self.inner.into_inner() + } +} + +impl Write for LimitedBuffer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let end = self + .inner + .position() + .checked_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)) + .ok_or_else(|| std::io::Error::other(BYTE_LIMIT_MARKER))?; + if end > self.limit as u64 { + return Err(std::io::Error::other(BYTE_LIMIT_MARKER)); + } + self.inner.write(bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +impl Seek for LimitedBuffer { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + let offset = self.inner.seek(position)?; + if offset > self.limit as u64 { + return Err(std::io::Error::other(BYTE_LIMIT_MARKER)); + } + Ok(offset) + } +} + +pub fn export_filename( + metric_label: &str, + metric_key: &str, + from: &str, + to: &str, + filtered: bool, + extension: &str, +) -> String { + let metric = filename_slug(metric_label); + let metric = if metric.is_empty() { + filename_slug(metric_key) + } else { + metric + }; + let suffix = if filtered { "_filtered" } else { "" }; + format!("{metric}_{from}_{to}{suffix}.{extension}") +} + +fn filename_slug(value: &str) -> String { + let mut slug = String::with_capacity(value.len().min(MAX_FILENAME_SLUG_BYTES)); + let mut separated = true; + for character in value.chars() { + if character.is_ascii_alphanumeric() { + if slug.len() == MAX_FILENAME_SLUG_BYTES { + break; + } + slug.push(character.to_ascii_lowercase()); + separated = false; + } else if !separated && slug.len() < MAX_FILENAME_SLUG_BYTES { + slug.push('-'); + separated = true; + } + } + while slug.ends_with('-') { + slug.pop(); + } + slug +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::collections::BTreeMap; + + fn far_deadline() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_mins(1) + } + + fn columns() -> Vec { + vec![ + MetricDrilldownColumn { + key: "ref".to_owned(), + label: "Ref".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "date".to_owned(), + label: "Date".to_owned(), + r#type: MetricDrilldownColumnType::Date, + }, + MetricDrilldownColumn { + key: "value".to_owned(), + label: "Value".to_owned(), + r#type: MetricDrilldownColumnType::Number, + }, + MetricDrilldownColumn { + key: "active".to_owned(), + label: "Active".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + ] + } + + fn row() -> MetricDrilldownRow { + MetricDrilldownRow { + values: BTreeMap::from([ + ("ref".to_owned(), json!("=formula")), + ("date".to_owned(), json!("2026-07-28")), + ("value".to_owned(), json!(12.5)), + ("active".to_owned(), json!(true)), + ]), + } + } + + #[test] + fn csv_export_is_bounded_and_formula_safe() { + let (bytes, content_type, extension) = build_export( + MetricDrilldownExportFormat::Csv, + &columns(), + &[row()], + far_deadline(), + ) + .unwrap_or_else(|error| panic!("CSV export must succeed: {error}")); + let csv = String::from_utf8(bytes) + .unwrap_or_else(|error| panic!("CSV export must be UTF-8: {error}")); + assert_eq!(content_type, "text/csv; charset=utf-8"); + assert_eq!(extension, "csv"); + assert!(csv.contains("'=formula")); + assert!(csv.contains("2026-07-28")); + assert!(csv.contains("12.5")); + } + + #[test] + fn every_spreadsheet_formula_prefix_is_neutralized() { + for prefix in ['=', '+', '-', '@', '\t', '\r', '\n', ' '] { + let cell = format!("{prefix}cmd"); + assert_eq!( + csv_safe_cell(cell.clone()), + format!("'{cell}"), + "dangerous prefix {prefix:?} must be quoted" + ); + } + for safe in ["plain", "12.5", "", "a=b"] { + assert_eq!( + csv_safe_cell(safe.to_owned()), + safe, + "safe value {safe:?} must pass through" + ); + } + } + + #[test] + fn export_input_over_the_byte_limit_is_rejected() { + let columns = vec![MetricDrilldownColumn { + key: "value".to_owned(), + label: "Value".to_owned(), + r#type: MetricDrilldownColumnType::String, + }]; + let row = vec!["x".repeat(MAX_CELL_BYTES)]; + + let mut budget = ExportInputBudget::new(&columns) + .unwrap_or_else(|error| panic!("header budget must fit: {error}")); + let rejected = + (0..=MAX_EXPORT_BYTES / MAX_CELL_BYTES).any(|_| budget.add_row(&row).is_err()); + assert!(rejected, "input past the byte limit must be rejected"); + + let mut small = ExportInputBudget::new(&columns) + .unwrap_or_else(|error| panic!("header budget must fit: {error}")); + assert!(small.add_row(&["small".to_owned()]).is_ok()); + } + + #[test] + fn xlsx_export_contains_typed_cells() { + let (bytes, content_type, extension) = build_export( + MetricDrilldownExportFormat::Xlsx, + &columns(), + &[row()], + far_deadline(), + ) + .unwrap_or_else(|error| panic!("XLSX export must succeed: {error}")); + assert_eq!( + content_type, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ); + assert_eq!(extension, "xlsx"); + assert!(bytes.starts_with(b"PK")); + assert!(bytes.len() > 1_000); + } + + #[test] + fn export_values_serialize_supported_json_values() { + let columns = vec![ + MetricDrilldownColumn { + key: "missing".to_owned(), + label: "Missing".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "object".to_owned(), + label: "Object".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + ]; + let row = MetricDrilldownRow { + values: BTreeMap::from([("object".to_owned(), json!({"key": "value"}))]), + }; + assert_eq!( + export_values(&columns, &row) + .unwrap_or_else(|error| panic!("export values must serialize: {error}")), + ["", r#"{"key":"value"}"#] + ); + } + + #[test] + fn oversized_export_cells_are_rejected() { + let columns = vec![MetricDrilldownColumn { + key: "value".to_owned(), + label: "Value".to_owned(), + r#type: MetricDrilldownColumnType::String, + }]; + let row = MetricDrilldownRow { + values: BTreeMap::from([("value".to_owned(), json!("x".repeat(MAX_CELL_BYTES + 1)))]), + }; + assert!(export_values(&columns, &row).is_err()); + } + + #[test] + fn limited_buffer_enforces_write_and_seek_bounds() { + let mut buffer = LimitedBuffer::new(4); + assert_eq!( + buffer + .write(b"1234") + .unwrap_or_else(|error| panic!("bounded write must succeed: {error}")), + 4 + ); + assert!(buffer.write(b"5").is_err()); + assert!(buffer.seek(SeekFrom::Start(5)).is_err()); + assert_eq!(buffer.into_inner(), b"1234"); + } + + #[test] + fn filenames_are_human_readable_and_bounded() { + assert_eq!( + export_filename( + "Tasks closed", + "tasks.closed", + "2025-07-28", + "2026-07-27", + true, + "xlsx" + ), + "tasks-closed_2025-07-28_2026-07-27_filtered.xlsx" + ); + assert_eq!(filename_slug("***"), ""); + assert!(filename_slug(&"a".repeat(100)).len() <= 80); + } + + #[test] + fn xlsx_writes_every_cell_shape_the_contract_allows() { + let columns = vec![ + MetricDrilldownColumn { + key: "number".to_owned(), + label: "Number".to_owned(), + r#type: MetricDrilldownColumnType::Number, + }, + MetricDrilldownColumn { + key: "unnumeric".to_owned(), + label: "Not a number".to_owned(), + r#type: MetricDrilldownColumnType::Number, + }, + MetricDrilldownColumn { + key: "date".to_owned(), + label: "Date".to_owned(), + r#type: MetricDrilldownColumnType::Date, + }, + MetricDrilldownColumn { + key: "text".to_owned(), + label: "Text".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "flag".to_owned(), + label: "Flag".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "nested".to_owned(), + label: "Nested".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "absent".to_owned(), + label: "Absent".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + MetricDrilldownColumn { + key: "numeric_date".to_owned(), + label: "Numeric date".to_owned(), + r#type: MetricDrilldownColumnType::Date, + }, + MetricDrilldownColumn { + key: "numeric_text".to_owned(), + label: "Numeric text".to_owned(), + r#type: MetricDrilldownColumnType::String, + }, + ]; + let row = MetricDrilldownRow { + values: BTreeMap::from([ + ("number".to_owned(), json!(12.5)), + ("unnumeric".to_owned(), json!("not numeric")), + ("date".to_owned(), json!("2026-07-28")), + ("text".to_owned(), json!("plain")), + ("flag".to_owned(), json!(true)), + ("nested".to_owned(), json!({"key": "value"})), + ("numeric_date".to_owned(), json!(20_260_728)), + ("numeric_text".to_owned(), json!(7)), + ]), + }; + + let bytes = build_xlsx(&columns, &[row], far_deadline()) + .unwrap_or_else(|error| panic!("every cell shape must serialize: {error}")); + assert!(bytes.starts_with(b"PK"), "XLSX is a zip container"); + } + + #[test] + fn an_unparseable_date_is_written_as_text_not_an_error() { + let columns = vec![MetricDrilldownColumn { + key: "date".to_owned(), + label: "Date".to_owned(), + r#type: MetricDrilldownColumnType::Date, + }]; + let row = MetricDrilldownRow { + values: BTreeMap::from([("date".to_owned(), json!("not-a-date"))]), + }; + let bytes = build_xlsx(&columns, &[row], far_deadline()).unwrap_or_else(|error| { + panic!("unparseable warehouse dates must not fail the export: {error}") + }); + assert!(bytes.starts_with(b"PK")); + } + + #[test] + fn number_column_text_is_written_unquoted_like_the_csv_path() { + let columns = vec![MetricDrilldownColumn { + key: "count".to_owned(), + label: "Count".to_owned(), + r#type: MetricDrilldownColumnType::Number, + }]; + let value = json!("not numeric"); + assert_eq!( + cell_text(&value).unwrap_or_else(|error| panic!("text must encode: {error}")), + "not numeric", + "a string in a Number column keeps its bare text, no JSON quotes" + ); + assert_eq!( + cell_text(&json!({"key": "value"})) + .unwrap_or_else(|error| panic!("json must encode: {error}")), + r#"{"key":"value"}"#, + "non-string values stay JSON-encoded" + ); + + let row = MetricDrilldownRow { + values: BTreeMap::from([("count".to_owned(), value)]), + }; + assert!(build_xlsx(&columns, &[row], far_deadline()).is_ok()); + } + + #[test] + fn an_elapsed_deadline_aborts_serialization() { + let Some(elapsed) = + std::time::Instant::now().checked_sub(std::time::Duration::from_secs(1)) + else { + panic!("clock must support a past instant"); + }; + for format in [ + MetricDrilldownExportFormat::Csv, + MetricDrilldownExportFormat::Xlsx, + ] { + assert!( + build_export(format, &columns(), &[row()], elapsed).is_err(), + "should abort past the deadline: {format:?}" + ); + } + assert!( + check_export_deadline(1, elapsed).is_ok(), + "rows between checkpoints skip the clock read" + ); + } + + #[test] + fn export_format_strings_are_stable() { + assert_eq!(MetricDrilldownExportFormat::Csv.as_str(), "csv"); + assert_eq!(MetricDrilldownExportFormat::Xlsx.as_str(), "xlsx"); + } +} diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs b/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs index 8f64fd35d..570493b68 100644 --- a/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs +++ b/src/backend/services/analytics/src/domain/metric_drilldown/mod.rs @@ -3,20 +3,23 @@ mod compiler; mod cursor; mod dto; mod error; +mod export; mod presentation; mod validation; -pub use capability::load_capabilities; -pub use compiler::{compile_query, decode_evidence_rows}; -pub use cursor::verify_evidence_snapshot; -pub use dto::{ +pub(crate) use capability::load_capabilities; +pub(crate) use compiler::{compile_query, decode_evidence_rows}; +pub(crate) use cursor::verify_evidence_snapshot; +pub(crate) use dto::{ EVIDENCE_QUERY_MEMORY_BYTES, EVIDENCE_QUERY_READ_BYTES, EVIDENCE_QUERY_RESULT_BYTES, - EVIDENCE_QUERY_TIMEOUT_SECS, EvidenceQueryRow, MetricDrilldownCapability, - MetricDrilldownRequest, MetricDrilldownResponse, ValidatedMetricDrilldown, + EVIDENCE_QUERY_TIMEOUT_SECS, EvidenceQueryRow, MAX_EXPORT_ROWS, MetricDrilldownCapability, + MetricDrilldownColumn, MetricDrilldownExportFormat, MetricDrilldownExportRequest, + MetricDrilldownRequest, MetricDrilldownResponse, MetricDrilldownRow, ValidatedMetricDrilldown, }; -pub use error::evidence_unavailable; -pub use presentation::build_response; -pub use validation::validate_request; +pub(crate) use error::{evidence_unavailable, export_internal, export_limit}; +pub(crate) use export::{MAX_EXPORT_BYTES, build_export, export_filename}; +pub(crate) use presentation::{build_response, presentation}; +pub(crate) use validation::{validate_export_request, validate_request}; #[cfg(test)] mod test_support; diff --git a/src/backend/services/analytics/src/domain/metric_drilldown/validation.rs b/src/backend/services/analytics/src/domain/metric_drilldown/validation.rs index 4888a84dc..c2594edbc 100644 --- a/src/backend/services/analytics/src/domain/metric_drilldown/validation.rs +++ b/src/backend/services/analytics/src/domain/metric_drilldown/validation.rs @@ -17,10 +17,11 @@ use super::cursor::{ decode_cursor, evidence_snapshot_id, selection_fingerprint, verify_evidence_snapshot, }; use super::dto::{ - DEFAULT_PAGE_LIMIT, EvidenceInput, EvidencePlan, MAX_DISPLAY_DIMENSIONS, + DEFAULT_PAGE_LIMIT, EvidenceInput, EvidencePlan, MAX_DISPLAY_DIMENSIONS, MAX_EXPORT_ROWS, MAX_FILTER_VALUE_BYTES, MAX_FILTER_VALUES, MAX_FILTERS, MAX_PAGE_LIMIT, MAX_PERIOD_DAYS, - MetricDrilldownEntity, MetricDrilldownFilter, MetricDrilldownPeriod, MetricDrilldownRequest, - MetricDrilldownSelection, ValidatedMetricDrilldown, + MetricDrilldownEntity, MetricDrilldownExportRequest, MetricDrilldownFilter, + MetricDrilldownPeriod, MetricDrilldownRequest, MetricDrilldownSelection, + ValidatedMetricDrilldown, }; struct CommonRequest { @@ -61,6 +62,31 @@ pub async fn validate_request( .await } +pub async fn validate_export_request( + db: &DatabaseConnection, + ch: &insight_clickhouse::Client, + tenant_id: Uuid, + req: &MetricDrilldownExportRequest, + limit: usize, +) -> Result { + validate_common( + db, + ch, + tenant_id, + CommonRequest { + metric_key: req.metric_key.clone(), + entity: req.entity.clone(), + period: req.period.clone(), + filters: req.filters.clone(), + display_dimensions: req.display_dimensions.clone(), + limit, + max_limit: MAX_EXPORT_ROWS + 1, + cursor: None, + }, + ) + .await +} + async fn validate_common( db: &DatabaseConnection, ch: &insight_clickhouse::Client, diff --git a/src/ingestion/tests/e2e/api/test_metric_drilldown.py b/src/ingestion/tests/e2e/api/test_metric_drilldown.py index 807ad9f45..48ba58a32 100644 --- a/src/ingestion/tests/e2e/api/test_metric_drilldown.py +++ b/src/ingestion/tests/e2e/api/test_metric_drilldown.py @@ -26,3 +26,16 @@ def test_metric_drilldown_400_empty_entity(api) -> None: def test_metric_drilldown_415_wrong_content_type(api) -> None: response = text_body_request(api, "POST", "/v1/metric-drilldown") assert response.status_code == 415, f"status={response.status_code} body={response.text}" + + +def test_metric_drilldown_export_400_empty_entity(api) -> None: + request = _request() + request["format"] = "csv" + request.pop("limit") + response = api.post("/v1/metric-drilldown/export", json=request) + assert response.status_code == 400, f"status={response.status_code} body={response.text}" + + +def test_metric_drilldown_export_415_wrong_content_type(api) -> None: + response = text_body_request(api, "POST", "/v1/metric-drilldown/export") + assert response.status_code == 415, f"status={response.status_code} body={response.text}"