From 657d7173eaf99e4a93e0205af4c82f5fa2747051 Mon Sep 17 00:00:00 2001 From: Anton Zelenov Date: Wed, 5 Aug 2026 12:47:08 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(semantic):=20field=20catalog=20?= =?UTF-8?q?=E2=80=94=20types=20from=20ClickHouse=20+=20authored=20roles=20?= =?UTF-8?q?(#2208)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 slice 2 of the semantic-layer definition core: the field catalog, the typed, role-annotated validation universe measures are checked against. Capability becomes a projection of definitions, not of data. The dbt schema.yml files carry no meta: type/role annotations, so the catalog source is hybrid (per the chosen approach): - roles.yaml — authored semantic roles (tenant/entity/event_time/ dimension/value/subject) per exposed field. - types.snapshot.json — column types generated from ClickHouse system.columns, committed so build-time validation needs no live warehouse, and kept honest by an ignored drift test against live CH. domain/field_catalog/ (Rust) joins the two into a typed FieldCatalog, rejecting any role that names an absent column or an unmodeled ClickHouse type — so the embedded catalog is a consistent validation universe at build time. Offline consistency tests gate it in normal CI; the CH drift test runs where a warehouse is available. `analytics field-catalog` prints it. Seed datasets: the two git relations that carry a direct entity (class_git_commits, class_git_pull_requests). Relations without a self-contained entity are derived-dataset cases catalogued during extraction (slice 5). Validated: offline tests green; drift test passes against live ClickHouse 25.7. Extends the semantic-layer DESIGN field-catalog component with slice-2 status, recording the hybrid-source deviation. Part of #2208 Part of #1803 Co-Authored-By: Claude Opus 4.8 Signed-off-by: Anton Zelenov --- docs/domain/semantic-layer/specs/DESIGN.md | 2 + .../src/domain/field_catalog/live_tests.rs | 91 +++++ .../src/domain/field_catalog/loader.rs | 337 ++++++++++++++++++ .../analytics/src/domain/field_catalog/mod.rs | 37 ++ .../src/domain/field_catalog/model.rs | 120 +++++++ .../src/domain/field_catalog/roles.yaml | 53 +++ .../domain/field_catalog/types.snapshot.json | 50 +++ .../services/analytics/src/domain/mod.rs | 1 + src/backend/services/analytics/src/main.rs | 16 + 9 files changed, 707 insertions(+) create mode 100644 src/backend/services/analytics/src/domain/field_catalog/live_tests.rs create mode 100644 src/backend/services/analytics/src/domain/field_catalog/loader.rs create mode 100644 src/backend/services/analytics/src/domain/field_catalog/mod.rs create mode 100644 src/backend/services/analytics/src/domain/field_catalog/model.rs create mode 100644 src/backend/services/analytics/src/domain/field_catalog/roles.yaml create mode 100644 src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json diff --git a/docs/domain/semantic-layer/specs/DESIGN.md b/docs/domain/semantic-layer/specs/DESIGN.md index b0a114964..763555c8e 100644 --- a/docs/domain/semantic-layer/specs/DESIGN.md +++ b/docs/domain/semantic-layer/specs/DESIGN.md @@ -282,6 +282,8 @@ Definitions live in application-owned storage so they can be authored, versioned Every dataset publishes a typed, role-annotated catalog of its fields — the editor's palette, the compiler's validation universe, and the discovery API's vocabulary at once. It is what makes capability a projection of definitions rather than data. +**Implementation status (#2208, Phase 1 slice 2):** the build-time catalog is shipped in `domain/field_catalog/` (Rust). Because the dbt `schema.yml` files carry no `meta:` type/role annotations, the source is **hybrid**: column **types** come from a committed ClickHouse type snapshot (`types.snapshot.json`, generated from `system.columns` and kept honest by an ignored drift test against the live warehouse), and **roles** are authored in `roles.yaml`. The generator joins the two and rejects any role that names an absent column or an unmodeled type, so the embedded catalog is a consistent validation universe at build time (offline tests) with no live warehouse needed in CI. Seed datasets are the two git relations that carry a direct entity (`class_git_commits`, `class_git_pull_requests`); full dataset coverage lands during extraction (slice 5). Inspect via `analytics field-catalog`. + ##### Responsibility scope - Generated from dataset schemas (a Rust build-time generator sharing the backend's definition parsers, sourced from dbt `schema.yml` `meta:` blocks), never hand-maintained. Roles: entity, dimension, measurable, event time. diff --git a/src/backend/services/analytics/src/domain/field_catalog/live_tests.rs b/src/backend/services/analytics/src/domain/field_catalog/live_tests.rs new file mode 100644 index 000000000..8068098c8 --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/live_tests.rs @@ -0,0 +1,91 @@ +//! Drift test: the committed type snapshot (via the built catalog) must still +//! agree with the live ClickHouse schema. For every catalogued field, the +//! warehouse's column type — normalized — must equal the field's type in the +//! catalog. A mismatch means `types.snapshot.json` is stale and must be +//! regenerated from ClickHouse. +//! +//! Ignored by default; requires a live ClickHouse. Enable with +//! `INTEGRATION_TESTS_CLICKHOUSE_URL= cargo test -p analytics -- --ignored +//! field_catalog`. + +use std::collections::BTreeMap; + +use clickhouse::Row; +use serde::Deserialize; + +use crate::domain::field_catalog::field_catalog; +use crate::domain::field_catalog::model::FieldType; + +const CH_ENV: &str = "INTEGRATION_TESTS_CLICKHOUSE_URL"; +const CH_USER_ENV: &str = "INTEGRATION_TESTS_CLICKHOUSE_USER"; +const CH_PASSWORD_ENV: &str = "INTEGRATION_TESTS_CLICKHOUSE_PASSWORD"; + +#[derive(Debug, Row, Deserialize)] +struct ColumnRow { + name: String, + #[serde(rename = "type")] + ch_type: String, +} + +#[tokio::test] +#[ignore = "requires live ClickHouse; set INTEGRATION_TESTS_CLICKHOUSE_URL to enable"] +async fn types_snapshot_matches_clickhouse() { + let Ok(url) = std::env::var(CH_ENV) else { + eprintln!("skipping: {CH_ENV} not set"); + return; + }; + let mut config = insight_clickhouse::Config::new(&url, "default"); + if let (Ok(user), Ok(password)) = (std::env::var(CH_USER_ENV), std::env::var(CH_PASSWORD_ENV)) { + config = config.with_auth(user, password); + } + let client = insight_clickhouse::Client::new(config); + + let mut drift: Vec = Vec::new(); + + for dataset in &field_catalog().datasets { + let rows = client + .query("SELECT name, type FROM system.columns WHERE database = ? AND table = ?") + .bind(dataset.database.as_str()) + .bind(dataset.table.as_str()) + .fetch_all::() + .await + .unwrap_or_else(|e| panic!("querying columns of {}: {e}", dataset.relation())); + + let live: BTreeMap = + rows.into_iter().map(|r| (r.name, r.ch_type)).collect(); + + if live.is_empty() { + drift.push(format!( + "{}: relation absent from ClickHouse", + dataset.relation() + )); + continue; + } + + for field in &dataset.fields { + match live.get(&field.name) { + None => drift.push(format!( + "{}.{}: column absent from ClickHouse", + dataset.relation(), + field.name + )), + Some(ch_type) => match FieldType::normalize(ch_type) { + Some((ty, nullable)) if ty == field.ty && nullable == field.nullable => {} + other => drift.push(format!( + "{}.{}: catalog has {:?} (nullable={}), ClickHouse has {ch_type} -> {other:?}", + dataset.relation(), + field.name, + field.ty, + field.nullable + )), + }, + } + } + } + + assert!( + drift.is_empty(), + "field-catalog type snapshot is stale — regenerate types.snapshot.json from ClickHouse:\n{}", + drift.join("\n") + ); +} diff --git a/src/backend/services/analytics/src/domain/field_catalog/loader.rs b/src/backend/services/analytics/src/domain/field_catalog/loader.rs new file mode 100644 index 000000000..82b889f0d --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/loader.rs @@ -0,0 +1,337 @@ +//! Build the field catalog by joining the authored roles (`roles.yaml`) with +//! the ClickHouse type snapshot (`types.snapshot.json`). The join is where the +//! two halves are reconciled: a role that names a column the warehouse does not +//! have, or a column whose type the layer cannot model, is a hard error — so the +//! embedded catalog is the validation universe, guaranteed internally consistent +//! at build time (the offline tests below), and kept honest against the live +//! warehouse by the drift test (`live_tests`). + +use std::collections::BTreeMap; +use std::sync::OnceLock; + +use serde::Deserialize; +use thiserror::Error; + +use crate::domain::field_catalog::model::{ + Dataset, Field, FieldCatalog, FieldRole, FieldType, ReadDiscipline, +}; + +const ROLES_YAML: &str = include_str!("roles.yaml"); +const TYPES_SNAPSHOT_JSON: &str = include_str!("types.snapshot.json"); + +static CATALOG: OnceLock = OnceLock::new(); + +/// The embedded, validated field catalog. Panics on a malformed or inconsistent +/// embedded artifact — a build defect the offline tests already catch, never a +/// runtime condition. +pub fn field_catalog() -> &'static FieldCatalog { + CATALOG.get_or_init(|| { + #[expect( + clippy::expect_used, + reason = "roles.yaml + types.snapshot.json are embedded at compile time and pinned by the offline consistency tests; a failure here is a build defect, not a runtime condition" + )] + build(ROLES_YAML, TYPES_SNAPSHOT_JSON).expect("embedded field catalog must be consistent") + }) +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CatalogError { + #[error("invalid roles.yaml: {0}")] + Roles(String), + #[error("invalid types.snapshot.json: {0}")] + Snapshot(String), + #[error("dataset {dataset}: relation {relation} is absent from the type snapshot")] + UnknownRelation { dataset: String, relation: String }, + #[error("dataset {dataset}: field {field} is not a column of {relation}")] + UnknownColumn { + dataset: String, + relation: String, + field: String, + }, + #[error( + "dataset {dataset}: field {field} has ClickHouse type {ch_type}, which the semantic layer does not model" + )] + UnmodeledType { + dataset: String, + field: String, + ch_type: String, + }, + #[error("dataset {dataset}: expected exactly one {role} field, found {count}")] + RoleCardinality { + dataset: String, + role: &'static str, + count: usize, + }, + #[error("dataset {dataset}: no event_time field")] + NoEventTime { dataset: String }, + #[error("duplicate dataset key {0}")] + DuplicateDataset(String), +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RolesDoc { + datasets: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct DatasetRoles { + key: String, + database: String, + table: String, + read_discipline: ReadDiscipline, + fields: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FieldRoleEntry { + name: String, + role: FieldRole, +} + +/// Relation (`database.table`) -> column -> ClickHouse type. +type Snapshot = BTreeMap>; + +fn build(roles_yaml: &str, snapshot_json: &str) -> Result { + let roles: RolesDoc = + serde_yaml::from_str(roles_yaml).map_err(|e| CatalogError::Roles(e.to_string()))?; + let snapshot: Snapshot = + serde_json::from_str(snapshot_json).map_err(|e| CatalogError::Snapshot(e.to_string()))?; + + let mut datasets = Vec::with_capacity(roles.datasets.len()); + let mut seen = std::collections::BTreeSet::new(); + + for dataset in roles.datasets { + if !seen.insert(dataset.key.clone()) { + return Err(CatalogError::DuplicateDataset(dataset.key)); + } + datasets.push(build_dataset(&dataset, &snapshot)?); + } + + Ok(FieldCatalog { datasets }) +} + +fn build_dataset(roles: &DatasetRoles, snapshot: &Snapshot) -> Result { + let relation = format!("{}.{}", roles.database, roles.table); + let columns = snapshot + .get(&relation) + .ok_or_else(|| CatalogError::UnknownRelation { + dataset: roles.key.clone(), + relation: relation.clone(), + })?; + + let mut fields = Vec::with_capacity(roles.fields.len()); + for entry in &roles.fields { + let ch_type = columns + .get(&entry.name) + .ok_or_else(|| CatalogError::UnknownColumn { + dataset: roles.key.clone(), + relation: relation.clone(), + field: entry.name.clone(), + })?; + let (ty, nullable) = + FieldType::normalize(ch_type).ok_or_else(|| CatalogError::UnmodeledType { + dataset: roles.key.clone(), + field: entry.name.clone(), + ch_type: ch_type.clone(), + })?; + fields.push(Field { + name: entry.name.clone(), + role: entry.role, + ty, + nullable, + }); + } + + let dataset = Dataset { + key: roles.key.clone(), + database: roles.database.clone(), + table: roles.table.clone(), + read_discipline: roles.read_discipline, + fields, + }; + + assert_role_invariants(&dataset)?; + Ok(dataset) +} + +/// A dataset must expose exactly one tenant field, exactly one entity field, and +/// at least one event-time field — the minimum for the compiler to scope, +/// attribute, and bucket every measure over it. +fn assert_role_invariants(dataset: &Dataset) -> Result<(), CatalogError> { + let count = |role| dataset.fields_with_role(role).count(); + + for (role, name) in [(FieldRole::Tenant, "tenant"), (FieldRole::Entity, "entity")] { + let n = count(role); + if n != 1 { + return Err(CatalogError::RoleCardinality { + dataset: dataset.key.clone(), + role: name, + count: n, + }); + } + } + + if count(FieldRole::EventTime) == 0 { + return Err(CatalogError::NoEventTime { + dataset: dataset.key.clone(), + }); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dataset<'a>(catalog: &'a FieldCatalog, key: &str) -> &'a Dataset { + catalog + .datasets + .iter() + .find(|d| d.key == key) + .unwrap_or_else(|| panic!("dataset {key} missing")) + } + + fn field<'a>(dataset: &'a Dataset, name: &str) -> &'a Field { + dataset + .fields + .iter() + .find(|f| f.name == name) + .unwrap_or_else(|| panic!("field {name} missing")) + } + + #[test] + fn embedded_catalog_builds_and_is_nonempty() { + let catalog = field_catalog(); + assert!(!catalog.datasets.is_empty(), "catalog is empty"); + dataset(catalog, "git_commits"); + dataset(catalog, "git_pull_requests"); + } + + #[test] + fn every_authored_field_resolved_to_a_type() { + // build() only returns Ok if every field joined to a modeled type; this + // spells out the resulting shape for one dataset. + let catalog = field_catalog(); + let commits = dataset(catalog, "git_commits"); + assert_eq!(commits.read_discipline, ReadDiscipline::Final); + + let author = field(commits, "author_email"); + assert_eq!(author.role, FieldRole::Entity); + assert_eq!(author.ty, FieldType::String); + + let date = field(commits, "date"); + assert_eq!(date.role, FieldRole::EventTime); + assert_eq!(date.ty, FieldType::DateTime); + assert!(date.nullable, "date is Nullable(DateTime) in the snapshot"); + + let lines = field(commits, "lines_added"); + assert_eq!(lines.role, FieldRole::Value); + assert_eq!(lines.ty, FieldType::Int); + } + + #[test] + fn every_dataset_has_one_tenant_one_entity_and_a_time() { + for dataset in &field_catalog().datasets { + assert_eq!(dataset.fields_with_role(FieldRole::Tenant).count(), 1); + assert_eq!(dataset.fields_with_role(FieldRole::Entity).count(), 1); + assert!(dataset.fields_with_role(FieldRole::EventTime).count() >= 1); + } + } + + #[test] + fn type_normalization_peels_wrappers() { + let cases = [ + ("String", Some((FieldType::String, false))), + ("Nullable(String)", Some((FieldType::String, true))), + ("Int64", Some((FieldType::Int, false))), + ("Nullable(Int64)", Some((FieldType::Int, true))), + ("UInt8", Some((FieldType::UInt, false))), + ("DateTime", Some((FieldType::DateTime, false))), + ("DateTime64(3)", Some((FieldType::DateTime, false))), + ("Nullable(DateTime)", Some((FieldType::DateTime, true))), + ("LowCardinality(String)", Some((FieldType::String, false))), + ( + "LowCardinality(Nullable(String))", + Some((FieldType::String, true)), + ), + ("Float64", Some((FieldType::Float, false))), + ("Array(String)", None), + ("Map(String, String)", None), + ]; + for (input, expected) in cases { + assert_eq!(FieldType::normalize(input), expected, "normalize({input})"); + } + } + + const MIN_SNAPSHOT: &str = + r#"{"silver.t":{"tenant_id":"Nullable(String)","e":"String","ts":"DateTime"}}"#; + + fn roles(fields: &str) -> String { + format!( + "datasets:\n - key: d\n database: silver\n table: t\n \ + read_discipline: final\n fields:\n{fields}" + ) + } + + #[test] + fn rejects_field_absent_from_snapshot() { + let doc = roles( + " - {{ name: tenant_id, role: tenant }}\n - {{ name: e, role: entity }}\n \ + - {{ name: ts, role: event_time }}\n - {{ name: ghost, role: value }}\n", + ) + .replace("{{", "{") + .replace("}}", "}"); + let Err(err) = build(&doc, MIN_SNAPSHOT) else { + panic!("ghost column must be rejected"); + }; + assert!(matches!(err, CatalogError::UnknownColumn { field, .. } if field == "ghost")); + } + + #[test] + fn rejects_unmodeled_type() { + let snapshot = r#"{"silver.t":{"tenant_id":"Nullable(String)","e":"String","ts":"DateTime","weird":"Array(UInt8)"}}"#; + let doc = roles( + " - { name: tenant_id, role: tenant }\n - { name: e, role: entity }\n \ + - { name: ts, role: event_time }\n - { name: weird, role: value }\n", + ); + let Err(err) = build(&doc, snapshot) else { + panic!("Array type must be rejected"); + }; + assert!(matches!(err, CatalogError::UnmodeledType { field, .. } if field == "weird")); + } + + #[test] + fn rejects_dataset_without_entity() { + let doc = roles( + " - { name: tenant_id, role: tenant }\n - { name: ts, role: event_time }\n", + ); + let Err(err) = build(&doc, MIN_SNAPSHOT) else { + panic!("missing entity must be rejected"); + }; + assert!(matches!( + err, + CatalogError::RoleCardinality { + role: "entity", + count: 0, + .. + } + )); + } + + #[test] + fn rejects_relation_absent_from_snapshot() { + let doc = roles( + " - { name: tenant_id, role: tenant }\n - { name: e, role: entity }\n \ + - { name: ts, role: event_time }\n", + ) + .replace("table: t", "table: missing"); + let Err(err) = build(&doc, MIN_SNAPSHOT) else { + panic!("missing relation must be rejected"); + }; + assert!(matches!(err, CatalogError::UnknownRelation { .. })); + } +} diff --git a/src/backend/services/analytics/src/domain/field_catalog/mod.rs b/src/backend/services/analytics/src/domain/field_catalog/mod.rs new file mode 100644 index 000000000..b4cbcd5bd --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/mod.rs @@ -0,0 +1,37 @@ +//! The field catalog: the semantic layer's typed, role-annotated view of every +//! product dataset — the validation universe measures and expressions are +//! checked against. Built once from two committed halves: authored roles +//! (`roles.yaml`) joined with a ClickHouse type snapshot (`types.snapshot.json`). + +#[cfg(test)] +mod live_tests; +mod loader; +pub mod model; + +pub use loader::field_catalog; + +/// Render the catalog as a compact, human-readable listing for the +/// `field-catalog` subcommand. +pub fn render() -> String { + use std::fmt::Write as _; + + let mut out = String::from("# Field catalog\n"); + for dataset in &field_catalog().datasets { + let _ = write!( + out, + "\n## {} ({}, read: {:?})\n", + dataset.key, + dataset.relation(), + dataset.read_discipline + ); + for field in &dataset.fields { + let null = if field.nullable { " nullable" } else { "" }; + let _ = writeln!( + out, + " - {:<20} {:<10?} {:?}{}", + field.name, field.role, field.ty, null + ); + } + } + out +} diff --git a/src/backend/services/analytics/src/domain/field_catalog/model.rs b/src/backend/services/analytics/src/domain/field_catalog/model.rs new file mode 100644 index 000000000..a23b53200 --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/model.rs @@ -0,0 +1,120 @@ +use serde::Deserialize; + +/// Semantic role a dataset field plays. Authored in `roles.yaml`; the compiler +/// and the expression validator (slice 3) read capability from these. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FieldRole { + /// Tenant isolation field — the injected tenancy predicate binds here. + Tenant, + /// The measured entity key (e.g. a person email resolved downstream). + Entity, + /// A timestamp usable for period bucketing. A dataset may expose several. + EventTime, + /// A breakdown field. + Dimension, + /// A numeric field a measure can sum/avg/min/max. + Value, + /// A field a measure can count distinct over. + Subject, +} + +/// Normalized field type — the closed set the semantic layer reasons about, +/// derived from the ClickHouse column type in the snapshot. Warehouse-specific +/// spellings (`Nullable(...)`, `LowCardinality(...)`, `DateTime64(3)`) collapse +/// to one of these; nullability is tracked separately on [`Field`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FieldType { + String, + Int, + UInt, + Float, + Date, + DateTime, +} + +impl FieldType { + /// Normalize a ClickHouse column type into `(field_type, nullable)`. + /// Returns `None` for a base type the semantic layer does not model, so the + /// consistency test rejects an authored field the layer cannot type. + pub fn normalize(ch_type: &str) -> Option<(Self, bool)> { + let (inner, nullable) = match strip_wrapper(ch_type, "Nullable") { + Some(inner) => (inner, true), + None => (ch_type, false), + }; + // LowCardinality wraps the storage, not the logical type; peel it (and a + // Nullable it may itself wrap) without changing the result. + if let Some(unwrapped) = strip_wrapper(inner, "LowCardinality") { + return Self::normalize(unwrapped).map(|(ty, inner_null)| (ty, nullable || inner_null)); + } + + let base = inner.split('(').next().unwrap_or(inner).trim(); + let ty = match base { + "String" | "FixedString" | "UUID" => Self::String, + "Int8" | "Int16" | "Int32" | "Int64" | "Int128" | "Int256" => Self::Int, + "UInt8" | "UInt16" | "UInt32" | "UInt64" | "UInt128" | "UInt256" => Self::UInt, + "Float32" | "Float64" => Self::Float, + "Date" | "Date32" => Self::Date, + "DateTime" | "DateTime64" => Self::DateTime, + _ => return None, + }; + Some((ty, nullable)) + } +} + +/// Peel a single `Wrapper(...)` layer, returning the inner type spelling. +fn strip_wrapper<'a>(ch_type: &'a str, wrapper: &str) -> Option<&'a str> { + let rest = ch_type.strip_prefix(wrapper)?.strip_prefix('(')?; + rest.strip_suffix(')') +} + +/// Dedup strategy the compiler inherits when it reads the dataset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReadDiscipline { + /// `ReplacingMergeTree` and friends — read with `FINAL`/dedup. + Final, + /// Already unique — read directly. + None, +} + +/// One exposed field of a dataset: its authored role plus the type joined in +/// from the ClickHouse snapshot. +#[derive(Debug, Clone, PartialEq)] +pub struct Field { + pub name: String, + pub role: FieldRole, + pub ty: FieldType, + pub nullable: bool, +} + +/// A queryable relation with its exposed, typed, role-annotated fields — the +/// unit a measure aggregates and the validation universe an expression is +/// checked against. +#[derive(Debug, Clone, PartialEq)] +pub struct Dataset { + pub key: String, + pub database: String, + pub table: String, + pub read_discipline: ReadDiscipline, + pub fields: Vec, +} + +impl Dataset { + /// Fully-qualified relation name, the key used to look the dataset up in the + /// ClickHouse type snapshot. + pub fn relation(&self) -> String { + format!("{}.{}", self.database, self.table) + } + + pub fn fields_with_role(&self, role: FieldRole) -> impl Iterator { + self.fields.iter().filter(move |f| f.role == role) + } +} + +/// The whole catalog — every product dataset the semantic layer can reason +/// about. Built once from `roles.yaml` joined with `types.snapshot.json`. +#[derive(Debug, Clone, PartialEq)] +pub struct FieldCatalog { + pub datasets: Vec, +} diff --git a/src/backend/services/analytics/src/domain/field_catalog/roles.yaml b/src/backend/services/analytics/src/domain/field_catalog/roles.yaml new file mode 100644 index 000000000..6f185ea40 --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/roles.yaml @@ -0,0 +1,53 @@ +# Field roles for the semantic-layer field catalog (#2208, Phase 1 slice 2). +# +# This is the AUTHORED half of the hybrid catalog: the semantic role of each +# exposed dataset field. Column TYPES are NOT authored here — they are joined in +# from `types.snapshot.json`, which is generated from ClickHouse `system.columns` +# (see the drift test). A field named here must exist as a column in the +# snapshot, and its ClickHouse type must normalize to a known field type; the +# offline consistency test enforces both, so a role can never reference a column +# the warehouse does not have. +# +# Seed set: the two git datasets that carry a direct entity. Relations without a +# self-contained entity (e.g. class_git_file_changes, which needs a join to +# commits) belong to derived datasets and are catalogued during extraction +# (slice 5). +# +# Roles: tenant | entity | event_time | dimension | value | subject +datasets: + - key: git_commits + database: silver + table: class_git_commits + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: author_email, role: entity} + - {name: date, role: event_time} + - {name: commit_hash, role: subject} + - {name: repo_slug, role: dimension} + - {name: project_key, role: dimension} + - {name: branch, role: dimension} + - {name: data_source, role: dimension} + - {name: is_merge_commit, role: dimension} + - {name: lines_added, role: value} + - {name: lines_removed, role: value} + - {name: files_changed, role: value} + - key: git_pull_requests + database: silver + table: class_git_pull_requests + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: author_email, role: entity} + - {name: created_on, role: event_time} + - {name: closed_on, role: event_time} + - {name: pr_id, role: subject} + - {name: repo_slug, role: dimension} + - {name: project_key, role: dimension} + - {name: source_branch, role: dimension} + - {name: destination_branch, role: dimension} + - {name: state, role: dimension} + - {name: data_source, role: dimension} + - {name: files_changed, role: value} + - {name: lines_added, role: value} + - {name: lines_removed, role: value} diff --git a/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json b/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json new file mode 100644 index 000000000..ed6bc1d52 --- /dev/null +++ b/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json @@ -0,0 +1,50 @@ +{ + "silver.class_git_commits": { + "_airbyte_extracted_at": "DateTime64(3)", + "_version": "Int64", + "author_email": "String", + "author_name": "String", + "branch": "String", + "commit_hash": "String", + "committer_email": "String", + "committer_name": "String", + "data_source": "String", + "date": "Nullable(DateTime)", + "files_changed": "Nullable(Int64)", + "is_merge_commit": "UInt8", + "lines_added": "Nullable(Int64)", + "lines_removed": "Nullable(Int64)", + "message": "String", + "project_key": "String", + "repo_slug": "String", + "source_id": "Nullable(String)", + "tenant_id": "Nullable(String)", + "unique_key": "Nullable(String)" + }, + "silver.class_git_pull_requests": { + "_airbyte_extracted_at": "DateTime64(3)", + "_version": "Int64", + "author_email": "String", + "author_name": "String", + "closed_on": "Nullable(DateTime)", + "created_on": "Nullable(DateTime)", + "data_source": "String", + "description": "String", + "destination_branch": "String", + "files_changed": "Nullable(Int64)", + "lines_added": "Nullable(Int64)", + "lines_removed": "Nullable(Int64)", + "merge_commit_hash": "String", + "pr_id": "Int64", + "pr_number": "Int64", + "project_key": "String", + "repo_slug": "String", + "source_branch": "String", + "source_id": "Nullable(String)", + "state": "String", + "tenant_id": "Nullable(String)", + "title": "String", + "unique_key": "Nullable(String)", + "updated_on": "Nullable(DateTime)" + } +} diff --git a/src/backend/services/analytics/src/domain/mod.rs b/src/backend/services/analytics/src/domain/mod.rs index 24a4c9899..2ddfcdfef 100644 --- a/src/backend/services/analytics/src/domain/mod.rs +++ b/src/backend/services/analytics/src/domain/mod.rs @@ -2,6 +2,7 @@ pub mod admin_threshold; pub mod auth; pub mod catalog; pub mod contract_version; +pub mod field_catalog; pub mod metric; pub mod metric_definitions; pub mod metric_drilldown; diff --git a/src/backend/services/analytics/src/main.rs b/src/backend/services/analytics/src/main.rs index f4d0f5a79..cfe06d9b7 100644 --- a/src/backend/services/analytics/src/main.rs +++ b/src/backend/services/analytics/src/main.rs @@ -90,6 +90,10 @@ enum Commands { /// regenerate docs/components/backend/analytics/openapi.json and to /// drift-check it in CI. Openapi, + /// Print the semantic-layer field catalog to stdout and exit. Built offline + /// from the embedded roles + ClickHouse type snapshot — no database, no + /// config. The validation universe measures are checked against. + FieldCatalog, } #[tokio::main] @@ -114,6 +118,11 @@ async fn main() -> Result<()> { Commands::Check => gear::check_config(&config), // Emit the OpenAPI document offline (no backends) — see `print_openapi`. Commands::Openapi => print_openapi(), + // Emit the field catalog offline (no backends) — see `print_field_catalog`. + Commands::FieldCatalog => { + print!("{}", domain::field_catalog::render()); + Ok(()) + } } } @@ -134,4 +143,11 @@ mod tests { fn print_openapi_writes_the_document() -> anyhow::Result<()> { super::print_openapi() } + + /// The `field-catalog` subcommand's happy path: render the catalog offline + /// (building it from the embedded roles + type snapshot) and write it out. + #[test] + fn print_field_catalog_writes_the_document() { + assert!(super::domain::field_catalog::render().contains("git_commits")); + } } From 2fa16e57b68ccc13c0ef1828af9686aa450ed99f Mon Sep 17 00:00:00 2001 From: Anton Zelenov Date: Wed, 5 Aug 2026 13:09:09 +0800 Subject: [PATCH 2/2] feat(semantic): extend field catalog to ai/collab/wiki (#2208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the "why only git" gap: the catalog was under-scoped to two git relations. Extend coverage to every source family with a clean, self-contained entity + tenant at silver — git, ai, collab, wiki (12 datasets) — and record what genuinely can't be catalogued without the extraction analysis. - Add FieldType::Decimal: collab/ai amount columns are Decimal(p,s), kept distinct from Float for exact currency arithmetic. - Author roles for ai (4), collab (4), wiki (2) relations; regenerate the ClickHouse type snapshot for all 12 relations (245 columns). - Tenant column name is not uniform across families (tenant_id vs insight_tenant_id) — named per dataset in roles.yaml. Deferred to extraction (slice 5), where roles are verified against real measures: task (no tenant column, entity is author_id not email — needs identity resolution), reference/join relations with no direct entity, and derived populations (active-day, focus). Validated: offline tests green (incl. Decimal normalization); drift test passes against live ClickHouse 25.7 for all 12 relations. DESIGN field-catalog note updated with coverage + deferrals. Part of #2208 Part of #1803 Co-Authored-By: Claude Opus 4.8 Signed-off-by: Anton Zelenov --- docs/domain/semantic-layer/specs/DESIGN.md | 4 +- .../src/domain/field_catalog/loader.rs | 2 + .../src/domain/field_catalog/model.rs | 4 + .../src/domain/field_catalog/roles.yaml | 180 +++++++++++++- .../domain/field_catalog/types.snapshot.json | 221 ++++++++++++++++++ 5 files changed, 401 insertions(+), 10 deletions(-) diff --git a/docs/domain/semantic-layer/specs/DESIGN.md b/docs/domain/semantic-layer/specs/DESIGN.md index 763555c8e..3a79875d2 100644 --- a/docs/domain/semantic-layer/specs/DESIGN.md +++ b/docs/domain/semantic-layer/specs/DESIGN.md @@ -282,7 +282,9 @@ Definitions live in application-owned storage so they can be authored, versioned Every dataset publishes a typed, role-annotated catalog of its fields — the editor's palette, the compiler's validation universe, and the discovery API's vocabulary at once. It is what makes capability a projection of definitions rather than data. -**Implementation status (#2208, Phase 1 slice 2):** the build-time catalog is shipped in `domain/field_catalog/` (Rust). Because the dbt `schema.yml` files carry no `meta:` type/role annotations, the source is **hybrid**: column **types** come from a committed ClickHouse type snapshot (`types.snapshot.json`, generated from `system.columns` and kept honest by an ignored drift test against the live warehouse), and **roles** are authored in `roles.yaml`. The generator joins the two and rejects any role that names an absent column or an unmodeled type, so the embedded catalog is a consistent validation universe at build time (offline tests) with no live warehouse needed in CI. Seed datasets are the two git relations that carry a direct entity (`class_git_commits`, `class_git_pull_requests`); full dataset coverage lands during extraction (slice 5). Inspect via `analytics field-catalog`. +**Implementation status (#2208, Phase 1 slice 2):** the build-time catalog is shipped in `domain/field_catalog/` (Rust). Because the dbt `schema.yml` files carry no `meta:` type/role annotations, the source is **hybrid**: column **types** come from a committed ClickHouse type snapshot (`types.snapshot.json`, generated from `system.columns` and kept honest by an ignored drift test against the live warehouse), and **roles** are authored in `roles.yaml`. The generator joins the two and rejects any role that names an absent column or an unmodeled type, so the embedded catalog is a consistent validation universe at build time (offline tests) with no live warehouse needed in CI. Inspect via `analytics field-catalog`. + +Coverage is the source families with a clean, self-contained entity and tenant at silver — **git, ai, collab, wiki** (12 datasets). Deferred to the extraction slice (slice 5), where roles are verified against real measures: **task** (`class_task_worklogs`/`comments` carry no tenant column and key on `author_id`, not an email — needs a tenant join and identity resolution), the reference/join relations with no direct entity (repository/branch, git PR comment/commit/reviewer, task statuses/projects/sprints/field-metadata, wiki engagement), and derived populations (active-day, focus). Two cross-family facts the catalog already absorbs: the tenant column name is **not** uniform (`tenant_id` for git/collab/wiki, `insight_tenant_id` for ai), named per dataset in `roles.yaml`; and `Decimal(p,s)` amount columns are a first-class field type distinct from `Float`. ##### Responsibility scope diff --git a/src/backend/services/analytics/src/domain/field_catalog/loader.rs b/src/backend/services/analytics/src/domain/field_catalog/loader.rs index 82b889f0d..96300a4dd 100644 --- a/src/backend/services/analytics/src/domain/field_catalog/loader.rs +++ b/src/backend/services/analytics/src/domain/field_catalog/loader.rs @@ -259,6 +259,8 @@ mod tests { Some((FieldType::String, true)), ), ("Float64", Some((FieldType::Float, false))), + ("Decimal(18, 4)", Some((FieldType::Decimal, false))), + ("Nullable(Decimal(38, 9))", Some((FieldType::Decimal, true))), ("Array(String)", None), ("Map(String, String)", None), ]; diff --git a/src/backend/services/analytics/src/domain/field_catalog/model.rs b/src/backend/services/analytics/src/domain/field_catalog/model.rs index a23b53200..cca9af07e 100644 --- a/src/backend/services/analytics/src/domain/field_catalog/model.rs +++ b/src/backend/services/analytics/src/domain/field_catalog/model.rs @@ -29,6 +29,9 @@ pub enum FieldType { Int, UInt, Float, + /// Fixed-point (`Decimal(p, s)`) — a distinct measurable type from `Float` + /// so the compiler can preserve exact currency/amount arithmetic. + Decimal, Date, DateTime, } @@ -54,6 +57,7 @@ impl FieldType { "Int8" | "Int16" | "Int32" | "Int64" | "Int128" | "Int256" => Self::Int, "UInt8" | "UInt16" | "UInt32" | "UInt64" | "UInt128" | "UInt256" => Self::UInt, "Float32" | "Float64" => Self::Float, + "Decimal" | "Decimal32" | "Decimal64" | "Decimal128" | "Decimal256" => Self::Decimal, "Date" | "Date32" => Self::Date, "DateTime" | "DateTime64" => Self::DateTime, _ => return None, diff --git a/src/backend/services/analytics/src/domain/field_catalog/roles.yaml b/src/backend/services/analytics/src/domain/field_catalog/roles.yaml index 6f185ea40..8f76dbd68 100644 --- a/src/backend/services/analytics/src/domain/field_catalog/roles.yaml +++ b/src/backend/services/analytics/src/domain/field_catalog/roles.yaml @@ -2,19 +2,28 @@ # # This is the AUTHORED half of the hybrid catalog: the semantic role of each # exposed dataset field. Column TYPES are NOT authored here — they are joined in -# from `types.snapshot.json`, which is generated from ClickHouse `system.columns` -# (see the drift test). A field named here must exist as a column in the -# snapshot, and its ClickHouse type must normalize to a known field type; the -# offline consistency test enforces both, so a role can never reference a column -# the warehouse does not have. +# from `types.snapshot.json`, generated from ClickHouse `system.columns` (see the +# drift test). A field named here must exist as a column in the snapshot and its +# ClickHouse type must normalize to a known field type; the offline consistency +# test enforces both. # -# Seed set: the two git datasets that carry a direct entity. Relations without a -# self-contained entity (e.g. class_git_file_changes, which needs a join to -# commits) belong to derived datasets and are catalogued during extraction -# (slice 5). +# Coverage: the source families with a clean, self-contained entity and tenant at +# the silver layer — git, ai, collab, wiki. NOT yet catalogued (deferred to the +# extraction slice, #2208 slice 5, where roles are verified against real +# measures): +# - task (class_task_worklogs/comments carry no tenant column and key on +# author_id, not an email — needs identity resolution and a tenant join); +# - reference/join relations with no direct entity (class_git_repositories, +# *_branches, git PR comment/commit/reviewer tables, class_task_statuses/ +# projects/sprints/field_metadata, class_wiki_engagement); +# - derived populations (active-day, focus) that are not a single raw relation. +# +# Note the tenant column name is NOT uniform across families (git/collab/wiki: +# tenant_id; ai: insight_tenant_id), so it is named per dataset here. # # Roles: tenant | entity | event_time | dimension | value | subject datasets: + # ---- git ---- - key: git_commits database: silver table: class_git_commits @@ -51,3 +60,156 @@ datasets: - {name: files_changed, role: value} - {name: lines_added, role: value} - {name: lines_removed, role: value} + # ---- ai (tenant column is insight_tenant_id) ---- + - key: ai_dev_usage + database: silver + table: class_ai_dev_usage + read_discipline: final + fields: + - {name: insight_tenant_id, role: tenant} + - {name: email, role: entity} + - {name: day, role: event_time} + - {name: tool, role: dimension} + - {name: source, role: dimension} + - {name: session_count, role: value} + - {name: conversation_count, role: value} + - {name: lines_added, role: value} + - {name: lines_removed, role: value} + - {name: tool_use_offered, role: value} + - {name: tool_use_accepted, role: value} + - {name: agent_sessions, role: value} + - {name: chat_requests, role: value} + - {name: cost_cents, role: value} + - key: ai_assistant_usage + database: silver + table: class_ai_assistant_usage + read_discipline: final + fields: + - {name: insight_tenant_id, role: tenant} + - {name: email, role: entity} + - {name: day, role: event_time} + - {name: tool, role: dimension} + - {name: surface, role: dimension} + - {name: source, role: dimension} + - {name: session_count, role: value} + - {name: conversation_count, role: value} + - {name: message_count, role: value} + - {name: action_count, role: value} + - {name: cost_cents, role: value} + - key: ai_api_usage + database: silver + table: class_ai_api_usage + read_discipline: final + fields: + - {name: insight_tenant_id, role: tenant} + - {name: email, role: entity} + - {name: day, role: event_time} + - {name: provider, role: dimension} + - {name: channel, role: dimension} + - {name: source, role: dimension} + - {name: input_tokens, role: value} + - {name: output_tokens, role: value} + - {name: cache_read_tokens, role: value} + - {name: cache_creation_tokens, role: value} + - {name: cost_amount, role: value} + - key: ai_overage + database: silver + table: class_ai_overage + read_discipline: final + fields: + - {name: insight_tenant_id, role: tenant} + - {name: email, role: entity} + - {name: period_month, role: event_time} + - {name: tool, role: dimension} + - {name: seat_tier, role: dimension} + - {name: source, role: dimension} + - {name: credit_limit_cents, role: value} + - {name: used_amount_cents, role: value} + - {name: overage_cents, role: value} + # ---- collab ---- + - key: collab_chat_activity + database: silver + table: class_collab_chat_activity + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: email, role: entity} + - {name: date, role: event_time} + - {name: data_source, role: dimension} + - {name: direct_messages, role: value} + - {name: group_chat_messages, role: value} + - {name: direct_and_group_messages, role: value} + - {name: total_chat_messages, role: value} + - {name: channel_posts, role: value} + - {name: channel_replies, role: value} + - key: collab_document_activity + database: silver + table: class_collab_document_activity + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: email, role: entity} + - {name: date, role: event_time} + - {name: product, role: dimension} + - {name: data_source, role: dimension} + - {name: viewed_or_edited_count, role: value} + - {name: shared_internally_count, role: value} + - {name: shared_externally_count, role: value} + - {name: visited_page_count, role: value} + - key: collab_email_activity + database: silver + table: class_collab_email_activity + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: email, role: entity} + - {name: date, role: event_time} + - {name: data_source, role: dimension} + - {name: sent_count, role: value} + - {name: received_count, role: value} + - {name: read_count, role: value} + - key: collab_meeting_activity + database: silver + table: class_collab_meeting_activity + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: email, role: entity} + - {name: date, role: event_time} + - {name: data_source, role: dimension} + - {name: calls_count, role: value} + - {name: meetings_organized, role: value} + - {name: meetings_attended, role: value} + - {name: adhoc_meetings_attended, role: value} + - {name: scheduled_meetings_attended, role: value} + - {name: audio_duration_seconds, role: value} + - {name: video_duration_seconds, role: value} + - {name: screen_share_duration_seconds, role: value} + # ---- wiki ---- + - key: wiki_activity + database: silver + table: class_wiki_activity + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: author_email, role: entity} + - {name: day, role: event_time} + - {name: source, role: dimension} + - {name: data_source, role: dimension} + - {name: pages_edited, role: value} + - {name: total_edits, role: value} + - {name: pages_created, role: value} + - key: wiki_pages + database: silver + table: class_wiki_pages + read_discipline: final + fields: + - {name: tenant_id, role: tenant} + - {name: author_email, role: entity} + - {name: created_at, role: event_time} + - {name: updated_at, role: event_time} + - {name: page_id, role: subject} + - {name: space_name, role: dimension} + - {name: status, role: dimension} + - {name: source, role: dimension} + - {name: version_count, role: value} diff --git a/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json b/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json index ed6bc1d52..e63b20539 100644 --- a/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json +++ b/src/backend/services/analytics/src/domain/field_catalog/types.snapshot.json @@ -1,4 +1,187 @@ { + "silver.class_ai_api_usage": { + "_version": "Int64", + "api_key_id": "Nullable(String)", + "cache_creation_tokens": "UInt64", + "cache_read_tokens": "UInt64", + "channel": "String", + "collected_at": "Nullable(DateTime64(3))", + "cost_amount": "Nullable(Decimal(18, 4))", + "cost_currency": "Nullable(String)", + "data_source": "String", + "day": "Nullable(Date)", + "email": "Nullable(String)", + "input_tokens": "UInt64", + "insight_tenant_id": "Nullable(String)", + "output_tokens": "UInt64", + "provider": "String", + "source": "String", + "source_id": "Nullable(String)", + "unique_key": "String", + "workspace_id": "Nullable(String)" + }, + "silver.class_ai_assistant_usage": { + "_version": "Int64", + "action_count": "Nullable(UInt32)", + "artifacts_created_count": "Nullable(UInt32)", + "collected_at": "Nullable(DateTime64(3))", + "connectors_used_count": "Nullable(UInt32)", + "conversation_count": "Nullable(UInt32)", + "cost_cents": "Nullable(UInt32)", + "data_source": "Nullable(String)", + "day": "Nullable(Date)", + "dispatch_turn_count": "Nullable(UInt32)", + "email": "Nullable(String)", + "files_uploaded_count": "Nullable(UInt32)", + "insight_tenant_id": "Nullable(String)", + "message_count": "Nullable(UInt32)", + "projects_created_count": "Nullable(UInt32)", + "projects_used_count": "Nullable(UInt32)", + "search_count": "Nullable(UInt32)", + "session_count": "Nullable(UInt32)", + "skills_used_count": "Nullable(UInt32)", + "source": "String", + "source_id": "Nullable(String)", + "surface": "String", + "surface_metrics_json": "Nullable(String)", + "thinking_message_count": "Nullable(UInt32)", + "tool": "String", + "unique_key": "String" + }, + "silver.class_ai_dev_usage": { + "_version": "Int64", + "agent_sessions": "Nullable(UInt32)", + "api_key_id": "Nullable(String)", + "chat_requests": "Nullable(UInt32)", + "collected_at": "Nullable(DateTime64(3))", + "commits_count": "Nullable(UInt32)", + "conversation_count": "Nullable(UInt32)", + "cost_cents": "Nullable(UInt32)", + "data_source": "Nullable(String)", + "day": "Nullable(Date)", + "email": "Nullable(String)", + "insight_tenant_id": "Nullable(String)", + "lines_added": "UInt32", + "lines_removed": "Nullable(UInt32)", + "prs_total_count": "Nullable(UInt32)", + "prs_with_cc_count": "Nullable(UInt32)", + "pull_requests_count": "Nullable(UInt32)", + "session_count": "UInt32", + "source": "String", + "source_id": "Nullable(String)", + "tool": "String", + "tool_action_breakdown_json": "Nullable(String)", + "tool_use_accepted": "Nullable(UInt32)", + "tool_use_offered": "Nullable(UInt32)", + "total_lines_added": "Nullable(UInt32)", + "total_lines_removed": "Nullable(UInt32)", + "unique_key": "String" + }, + "silver.class_ai_overage": { + "_version": "Int64", + "account_id": "Nullable(String)", + "collected_at": "Nullable(DateTime64(3))", + "credit_limit_cents": "Nullable(UInt32)", + "currency": "String", + "data_source": "Nullable(String)", + "email": "Nullable(String)", + "insight_tenant_id": "Nullable(String)", + "is_enabled": "Nullable(UInt8)", + "is_over_limit": "Nullable(UInt8)", + "overage_cents": "Nullable(UInt32)", + "overage_metrics_json": "String", + "period_month": "Date", + "seat_tier": "Nullable(String)", + "source": "String", + "source_id": "Nullable(String)", + "tool": "String", + "unique_key": "String", + "used_amount_cents": "UInt32" + }, + "silver.class_collab_chat_activity": { + "_version": "Int64", + "channel_posts": "Nullable(Int64)", + "channel_replies": "Nullable(Int64)", + "collected_at": "DateTime", + "data_source": "String", + "date": "Nullable(Date)", + "direct_and_group_messages": "Nullable(Int64)", + "direct_messages": "Nullable(Int64)", + "email": "Nullable(String)", + "group_chat_messages": "Nullable(Int64)", + "insight_source_id": "Nullable(String)", + "person_key": "Nullable(String)", + "report_period": "Nullable(String)", + "tenant_id": "Nullable(String)", + "total_chat_messages": "Int64", + "unique_key": "Nullable(FixedString(16))", + "urgent_messages": "Nullable(Int64)", + "user_id": "Nullable(String)", + "user_name": "Nullable(String)" + }, + "silver.class_collab_document_activity": { + "_version": "Int64", + "collected_at": "DateTime", + "data_source": "String", + "date": "Nullable(Date)", + "email": "Nullable(String)", + "insight_source_id": "Nullable(String)", + "person_key": "Nullable(String)", + "product": "String", + "report_period": "Nullable(String)", + "shared_externally_count": "Nullable(Decimal(38, 9))", + "shared_internally_count": "Nullable(Decimal(38, 9))", + "synced_count": "Nullable(Decimal(38, 9))", + "tenant_id": "Nullable(String)", + "unique_key": "Nullable(FixedString(16))", + "user_id": "Nullable(String)", + "user_name": "Nullable(String)", + "viewed_or_edited_count": "Nullable(Decimal(38, 9))", + "visited_page_count": "Nullable(Int64)" + }, + "silver.class_collab_email_activity": { + "_version": "Int64", + "collected_at": "DateTime", + "data_source": "String", + "date": "Nullable(Date)", + "email": "Nullable(String)", + "insight_source_id": "Nullable(String)", + "meetings_created": "Nullable(Decimal(38, 9))", + "meetings_interacted": "Nullable(Decimal(38, 9))", + "person_key": "Nullable(String)", + "read_count": "Nullable(Decimal(38, 9))", + "received_count": "Nullable(Decimal(38, 9))", + "report_period": "Nullable(String)", + "sent_count": "Nullable(Decimal(38, 9))", + "tenant_id": "Nullable(String)", + "unique_key": "Nullable(FixedString(16))", + "user_id": "Nullable(String)", + "user_name": "Nullable(String)" + }, + "silver.class_collab_meeting_activity": { + "_version": "Int64", + "adhoc_meetings_attended": "Nullable(Int64)", + "adhoc_meetings_organized": "Nullable(Int64)", + "audio_duration_seconds": "Nullable(Int64)", + "calls_count": "Nullable(Int64)", + "collected_at": "DateTime", + "data_source": "String", + "date": "Nullable(Date)", + "email": "Nullable(String)", + "insight_source_id": "Nullable(String)", + "meetings_attended": "Int64", + "meetings_organized": "Nullable(Int64)", + "person_key": "Nullable(String)", + "report_period": "Nullable(String)", + "scheduled_meetings_attended": "Nullable(Int64)", + "scheduled_meetings_organized": "Nullable(Int64)", + "screen_share_duration_seconds": "Nullable(Int64)", + "tenant_id": "Nullable(String)", + "unique_key": "Nullable(FixedString(16))", + "user_id": "Nullable(String)", + "user_name": "Nullable(String)", + "video_duration_seconds": "Nullable(Int64)" + }, "silver.class_git_commits": { "_airbyte_extracted_at": "DateTime64(3)", "_version": "Int64", @@ -46,5 +229,43 @@ "title": "String", "unique_key": "Nullable(String)", "updated_on": "Nullable(DateTime)" + }, + "silver.class_wiki_activity": { + "_version": "Int64", + "author_email": "Nullable(String)", + "author_id": "Nullable(String)", + "collected_at": "Nullable(DateTime64(3))", + "data_source": "String", + "day": "Nullable(Date)", + "pages_created": "UInt32", + "pages_edited": "UInt32", + "source": "String", + "source_id": "Nullable(String)", + "tenant_id": "Nullable(String)", + "total_edits": "UInt32", + "unique_key": "String" + }, + "silver.class_wiki_pages": { + "_version": "Int64", + "author_email": "Nullable(String)", + "author_id": "Nullable(String)", + "collected_at": "Nullable(DateTime64(3))", + "created_at": "Nullable(DateTime64(3))", + "data_source": "String", + "last_editor_email": "Nullable(String)", + "last_editor_id": "Nullable(String)", + "page_id": "Nullable(String)", + "parent_page_id": "Nullable(String)", + "source": "String", + "source_id": "Nullable(String)", + "space_id": "Nullable(String)", + "space_name": "Nullable(String)", + "space_url": "Nullable(String)", + "status": "Nullable(String)", + "tenant_id": "Nullable(String)", + "title": "Nullable(String)", + "unique_key": "Nullable(String)", + "updated_at": "Nullable(DateTime64(3))", + "version_count": "UInt32" } }