Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,15 @@ pub const BUILTIN_SOURCES: &[BuiltinSource] = &[
],
dimensions: &[],
},
BuiltinSource {
source: SourceSeed {
key: "wiki",
kind: SourceKind::ManagedObservation,
source_ref: "wiki_metric_observations",
},
measures: &["pages_created", "edits", "pages_edited", "comments"],
dimensions: &[],
},
];

pub const BUILTIN_METRICS: &[MetricSeed] = &[
Expand Down Expand Up @@ -1383,6 +1392,96 @@ pub const BUILTIN_METRICS: &[MetricSeed] = &[
}],
dimensions: &[],
},
MetricSeed {
metric_key: "wiki.pages_created",
source_key: "wiki",
label: "Pages created",
description: Some("Wiki pages authored"),
explanation: Some(
"Wiki pages the person created during the period, counted on the \
page's creation date.",
),
unit: Some("pages"),
format: MetricFormat::Integer,
direction: MetricDirection::HigherIsBetter,
entity_type: EntityType::Person,
computation: SeedComputation::Sum,
transform: None,
peer_cohort_key: Some(CohortKey::OrgUnit),
inputs: &[InputSeed {
input_role: MetricInputRole::Value,
measure_key: "pages_created",
}],
dimensions: &[],
},
MetricSeed {
metric_key: "wiki.edits",
source_key: "wiki",
label: "Page edits",
description: Some("Wiki edit sessions"),
explanation: Some(
"Logical wiki edits the person made during the period. Consecutive \
saves of the same page within a short window count as one edit, so \
autosaves do not inflate the number.",
),
unit: Some("edits"),
format: MetricFormat::Integer,
direction: MetricDirection::HigherIsBetter,
entity_type: EntityType::Person,
computation: SeedComputation::Sum,
transform: None,
peer_cohort_key: Some(CohortKey::OrgUnit),
inputs: &[InputSeed {
input_role: MetricInputRole::Value,
measure_key: "edits",
}],
dimensions: &[],
},
MetricSeed {
metric_key: "wiki.pages_edited",
source_key: "wiki",
label: "Pages edited",
description: Some("Distinct wiki pages edited"),
explanation: Some(
"Distinct wiki pages the person edited during the period, counted \
per day the page was touched.",
),
unit: Some("pages"),
format: MetricFormat::Integer,
direction: MetricDirection::HigherIsBetter,
entity_type: EntityType::Person,
computation: SeedComputation::Sum,
transform: None,
peer_cohort_key: Some(CohortKey::OrgUnit),
inputs: &[InputSeed {
input_role: MetricInputRole::Value,
measure_key: "pages_edited",
}],
dimensions: &[],
},
MetricSeed {
metric_key: "wiki.comments",
source_key: "wiki",
label: "Comments received",
description: Some("Comments on the person's wiki pages"),
explanation: Some(
"Comments and replies other people left on wiki pages the person \
authored — a signal of how much their documentation is read and \
discussed.",
),
unit: Some("comments"),
format: MetricFormat::Integer,
direction: MetricDirection::HigherIsBetter,
entity_type: EntityType::Person,
computation: SeedComputation::Sum,
transform: None,
peer_cohort_key: Some(CohortKey::OrgUnit),
inputs: &[InputSeed {
input_role: MetricInputRole::Value,
measure_key: "comments",
}],
dimensions: &[],
},
];

#[cfg(test)]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- Build-integrity check (untagged → error severity under `dbt build`).
-- Unified entity ids for persons are lowercased emails; the runtime and the
-- cohort view join on exact string equality, so an empty, mixed-case, or
-- non-email id (an unresolved wiki account leaking past the staging email
-- gate) silently drops the person from every surface.
SELECT
entity_id,
measure_key,
count() AS row_count
FROM {{ ref('wiki_metric_observations') }}
WHERE entity_id = ''
OR entity_id != lower(entity_id)
OR entity_id NOT LIKE '%@%'
GROUP BY entity_id, measure_key
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- Build-integrity check (untagged → error severity under `dbt build`).
-- Every wiki measure is a count (pages, edit sessions, distinct pages,
-- comments) — non-negative by construction. A negative value is a
-- regression in the gold model, not a data condition.
SELECT
measure_key,
count() AS row_count
FROM {{ ref('wiki_metric_observations') }}
WHERE value < 0
GROUP BY measure_key
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
-- Build-integrity check (untagged → error severity under `dbt build`).
-- The wiki family has no distinct-count measure, so the model is a single
-- value UNION branch that stamps subject_key = NULL on every row. A
-- non-NULL subject_key means a measure branch drifted into a distinct-count
-- shape — nothing downstream would count it, so it must never appear.
SELECT
measure_key,
countIf(subject_key IS NOT NULL) AS subject_rows
FROM {{ ref('wiki_metric_observations') }}
GROUP BY measure_key
HAVING subject_rows > 0
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Build-integrity check (untagged → error severity under `dbt build`).
-- All four wiki measures are day-grain sums: exactly one row per (tenant,
-- entity, date, measure, dimensions, subject). A duplicate means FINAL dedup
-- regressed on a class read or the engagement join fanned out, silently
-- inflating the sums.
SELECT
tenant_id,
entity_id,
metric_date,
measure_key,
dimensions,
subject_key,
count() AS row_count
FROM {{ ref('wiki_metric_observations') }}
GROUP BY tenant_id, entity_id, metric_date, measure_key, dimensions, subject_key
HAVING count() > 1
70 changes: 70 additions & 0 deletions src/ingestion/gold/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,76 @@ models:
tests:
- not_null

- name: wiki_metric_observations
description: >
Source measure observations for the unified metrics runtime, wiki
family. Day-grain sums over the wiki class contracts: page creation
from the page snapshot, edit sessions and distinct pages edited from
the per-author activity rollup, and comments received attributed to
the page author. No event or distinct-count measures, so subject_key
is NULL on every row and the model is a single value UNION branch.
The column set is the same published contract the analytics service
schema validator probes (`OBSERVATION_COLUMNS`); changing it requires
a coordinated backend change.
columns:
- name: tenant_id
description: "Tenant isolation field"
tests:
- not_null
- name: source_key
description: "Logical source key registered in the metric registry"
tests:
- not_null
- accepted_values:
arguments:
values: ['wiki']
- name: entity_type
description: "Measured entity type"
tests:
- not_null
- accepted_values:
arguments:
values: ['person']
- name: entity_id
description: >
Normalized entity identifier: the author's lowercased email as
resolved in the wiki staging models. Accounts that do not resolve
to an email are excluded upstream.
tests:
- not_null
- name: metric_date
description: >
Observation date: page creation date for pages_created, edit day
for edits/pages_edited, comment day for comments.
tests:
- not_null
- name: observed_at
description: "Reserved for point-in-time semantics. NULL for all wiki measures."
- name: measure_key
description: "Source measure key registered in the metric registry"
tests:
- not_null
- accepted_values:
arguments:
values:
- pages_created
- edits
- pages_edited
- comments
- name: value
description: >
Measure value. Rows are emitted only when the source provides a
value, so value is never NULL; the column stays Nullable(Float64)
in the published contract.
tests:
- not_null
- name: subject_key
description: "Unused by this family (no distinct-count measure); always NULL."
- name: dimensions
description: >
Array(Tuple(key, value, label)). Empty for every wiki measure — the
family carries no breakdown dimension.

- name: metric_entity_cohorts_current
description: >
Current cohort membership per entity for peer comparison. One row per
Expand Down
142 changes: 142 additions & 0 deletions src/ingestion/gold/wiki_metric_observations.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
{{ config(
materialized='table',
engine='MergeTree',
order_by=['source_key', 'measure_key', 'entity_id', 'metric_date'],
schema='insight',
alias='wiki_metric_observations',
tags=['gold']
) }}

-- Source measure observations for the unified metrics runtime, wiki family.
-- Reads the wiki class contracts only (class_wiki_pages, class_wiki_activity,
-- class_wiki_engagement); every measure is emitted through the shape macros
-- in macros/metric_observation_measures.sql. No dimensions: wiki sources
-- (Confluence, Outline) feed one undifferentiated family.
--
-- Materialized as a sorted table: the pipeline runs once per dbt build —
-- the only time the silver inputs can have changed — and the ordering key
-- mirrors the runtime's filter shape (source_key, measure_key, entity_id,
-- metric_date), so single-measure queries read index-pruned ranges.
--
-- Grain per measure:
-- day-grain sums (creation date, page author): pages_created (1/page —
-- the page object's own author/created_at are the
-- canonical creation facts; a version-derived proxy
-- undercounts imported pages)
-- day-grain sums (edit date, version author): edits (logical edit
-- sessions — autosave bursts collapsed in silver, see
-- class_wiki_activity), pages_edited (distinct pages
-- touched that day)
-- day-grain sums (comment date, page author): comments (engagement
-- RECEIVED on the person's pages — footer + inline +
-- replies; the commenter is deliberately not the entity,
-- see class_wiki_engagement's page-centric design note)
--
-- Attribution: entity_id = lower(author_email); only email-shaped keys pass.
-- Confluence resolves emails through the Jira directory join in staging and
-- yields NULL on tenants without Jira — those rows are excluded as
-- unmatchable rather than carried as dead entities (cohorts and API requests
-- address people by email). Outline resolves from its own user stream.
--
-- Memory shape: every class read keeps FINAL (ReplacingMergeTree dedup) over
-- a pruned column set. class_wiki_activity is already (author, day) grain,
-- so its sums are near-free. The single join in the model attributes
-- page-day comment rollups to the page author: engagement ⋈ pages on
-- (tenant_id, source_id, page_id) — source_id is part of the key so a
-- page_id colliding across two wiki instances of one tenant cannot fan out.
--
-- Peer measurability (who enters a metric's peer pool) is decided HERE, by
-- row emission — the runtime never fabricates zeros. All four measures are
-- engagement-gated: a row exists only where the source recorded authorship,
-- editing, or received comments. Rostered-but-inactive people take no
-- standing rather than dragging peer medians toward zero.

WITH
-- Page -> author attribution for the comment join. No created_at gate:
-- comment attribution needs only the page's author, so a page snapshot
-- without a creation timestamp still receives its comments.
pages AS (
SELECT
tenant_id,
source_id,
page_id,
lower(author_email) AS entity_id
FROM {{ ref('class_wiki_pages') }} FINAL
WHERE author_email LIKE '%@%'
),
-- Dated branch for pages_created; the creation date is the metric date, so
-- only here does a missing created_at exclude the page.
page_creations AS (
SELECT
tenant_id,
lower(author_email) AS entity_id,
toDate(created_at) AS metric_date,
CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions
FROM {{ ref('class_wiki_pages') }} FINAL
WHERE author_email LIKE '%@%'
AND created_at IS NOT NULL
),
activity AS (
SELECT
tenant_id,
lower(author_email) AS entity_id,
day AS metric_date,
total_edits,
pages_edited,
CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions
FROM {{ ref('class_wiki_activity') }} FINAL
WHERE author_email LIKE '%@%'
AND day IS NOT NULL
),
engagement AS (
SELECT
e.tenant_id AS tenant_id,
p.entity_id AS entity_id,
e.day AS metric_date,
e.total_comments AS total_comments,
CAST([] AS Array(Tuple(key String, value String, label Nullable(String)))) AS no_dimensions
FROM (
SELECT
tenant_id,
source_id,
page_id,
day,
total_comments
FROM {{ ref('class_wiki_engagement') }} FINAL
WHERE day IS NOT NULL
) AS e
INNER JOIN pages AS p
ON e.tenant_id = p.tenant_id
AND e.source_id = p.source_id
AND e.page_id = p.page_id
),
value_measures AS (
{{ sum_measure('pages_created', 'page_creations', '1', 'no_dimensions') }}

UNION ALL

{{ sum_measure('edits', 'activity', 'total_edits', 'no_dimensions') }}

UNION ALL

{{ sum_measure('pages_edited', 'activity', 'pages_edited', 'no_dimensions') }}

UNION ALL

{{ sum_measure('comments', 'engagement', 'total_comments', 'no_dimensions') }}
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
SELECT
assumeNotNull(tenant_id) AS tenant_id,
'wiki' AS source_key,
'person' AS entity_type,
assumeNotNull(entity_id) AS entity_id,
assumeNotNull(metric_date) AS metric_date,
CAST(NULL AS Nullable(DateTime64(3))) AS observed_at,
measure_key,
value,
CAST(NULL AS Nullable(String)) AS subject_key,
dimensions
FROM value_measures
WHERE tenant_id IS NOT NULL
AND entity_id IS NOT NULL
AND metric_date IS NOT NULL
Loading
Loading