diff --git a/Cargo.lock b/Cargo.lock index 00bbb5c4a..2295b137c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2656,6 +2656,7 @@ dependencies = [ "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", + "ourios-semconv", "ourios-telemetry", "serde_json", "tokio", @@ -2689,8 +2690,13 @@ dependencies = [ name = "ourios-miner" version = "0.0.0" dependencies = [ + "opentelemetry", + "opentelemetry_sdk", "ourios-core", + "ourios-semconv", + "ourios-telemetry", "sha2", + "tokio", ] [[package]] diff --git a/crates/ourios-core/Cargo.toml b/crates/ourios-core/Cargo.toml index ec7e2dea4..d6c90e40c 100644 --- a/crates/ourios-core/Cargo.toml +++ b/crates/ourios-core/Cargo.toml @@ -34,9 +34,13 @@ serde_json = { version = "1", default-features = false, features = ["std"] } # library crates depend on the lightweight API and resolve instruments # through `global::meter`; the heavy SDK + OTLP exporter live in # `ourios-telemetry`). Drives the alias map's -# `alias_assertions_total` / `alias_retractions_total` counters -# (§6.8 telemetry table). +# `ourios.miner.alias.assertions` / `ourios.miner.alias.retractions` +# counters (§6.8 telemetry table). opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } +# Generated dotted-`ourios.*` semconv name constants for the alias +# counters' metric names + `ourios.tenant` attribute key (§6.8). +# Zero-dep leaf crate. +ourios-semconv = { path = "../ourios-semconv" } [dev-dependencies] # `init_in_memory` collects the alias counters' exported metric stream diff --git a/crates/ourios-core/src/alias.rs b/crates/ourios-core/src/alias.rs index 027562022..049bffdff 100644 --- a/crates/ourios-core/src/alias.rs +++ b/crates/ourios-core/src/alias.rs @@ -41,6 +41,7 @@ use std::time::SystemTime; use opentelemetry::metrics::Counter; use opentelemetry::{KeyValue, global}; +use ourios_semconv as semconv; use crate::audit::{AuditEvent, AuditPayload, AuditSink}; use crate::tenant::TenantId; @@ -52,17 +53,6 @@ use crate::tenant::TenantId; /// input. pub const REASON_BYTE_LIMIT: usize = 256; -/// `alias_assertions_total` (RFC 0001 §6.8 telemetry table). Named with -/// the pre-redesign identifier the table pins — the dotted-`ourios.*` -/// semconv conversion is the deferred §6.8 redesign, not this slice. -const METRIC_ALIAS_ASSERTIONS_TOTAL: &str = "alias_assertions_total"; -/// `alias_retractions_total` (RFC 0001 §6.8 telemetry table). -const METRIC_ALIAS_RETRACTIONS_TOTAL: &str = "alias_retractions_total"; -/// The `tenant_id` data-point attribute key. Pre-redesign name per the -/// §6.8 table (the namespaced `ourios.tenant` key is the deferred -/// dotted-semconv redesign). -const ATTR_TENANT_ID: &str = "tenant_id"; - /// The operator / API principal that issued an alias assertion. /// /// `[§3.1]` "explicit": aliasing is never anonymous, so every assertion @@ -212,19 +202,20 @@ impl AliasMap { /// provider is installed, so constructing and recording is always /// safe). /// - /// `alias_assertions_total` / `alias_retractions_total` carry a - /// `tenant_id` attribute, so a per-tenant point appears on the first - /// real assertion / retraction (§6.8 collect-on-read) — they are not - /// zero-seeded, which would emit a spurious attribute-less series. + /// `ourios.miner.alias.assertions` / `ourios.miner.alias.retractions` + /// carry an `ourios.tenant` attribute, so a per-tenant point appears + /// on the first real assertion / retraction (§6.8 collect-on-read) — + /// they are not zero-seeded, which would emit a spurious + /// attribute-less series. #[must_use] pub fn new() -> Self { let meter = global::meter("ourios.miner"); let assertions_total = meter - .u64_counter(METRIC_ALIAS_ASSERTIONS_TOTAL) + .u64_counter(semconv::OURIOS_MINER_ALIAS_ASSERTIONS) .with_unit("{assertion}") .build(); let retractions_total = meter - .u64_counter(METRIC_ALIAS_RETRACTIONS_TOTAL) + .u64_counter(semconv::OURIOS_MINER_ALIAS_RETRACTIONS) .with_unit("{retraction}") .build(); Self { @@ -242,7 +233,7 @@ impl AliasMap { /// ≥ 2 distinct ids), emits an [`AuditPayload::AliasAsserted`] /// event through `sink`, folds the asserted set into the tenant's /// classes (union-on-overlap: any pre-existing class sharing a - /// member merges in), and increments `alias_assertions_total`. + /// member merges in), and increments `ourios.miner.alias.assertions`. /// `representative_id` is the operator's anchor id only — membership /// is the union, independent of which id was named the anchor. /// @@ -282,7 +273,10 @@ impl AliasMap { self.union_in(tenant, &asserted); self.assertions_total.add( 1, - &[KeyValue::new(ATTR_TENANT_ID, tenant.as_str().to_owned())], + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], ); Ok(()) } @@ -293,7 +287,7 @@ impl AliasMap { /// derived canonical. Emits an [`AuditPayload::AliasRetracted`] /// event through `sink` (with `representative_id = id` as the /// operator's anchor and empty `member_ids`), removes `id` from its - /// class, and increments `alias_retractions_total`. A class that + /// class, and increments `ourios.miner.alias.retractions`. A class that /// drops to a single member is no longer an alias set and is /// dropped — that lone id then resolves only to itself /// (RFC0001.16). The canonical re-derives as `min` of the remainder @@ -330,7 +324,10 @@ impl AliasMap { self.remove_id(tenant, id); self.retractions_total.add( 1, - &[KeyValue::new(ATTR_TENANT_ID, tenant.as_str().to_owned())], + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], ); Ok(()) } @@ -566,12 +563,13 @@ mod tests { assert_eq!(replayed.resolves(&t, 2), [2].into_iter().collect()); } - // RFC 0001 §6.8 telemetry table: `alias_assertions_total` / - // `alias_retractions_total` are mandatory counters with a - // `tenant_id` attribute. Collect the exported metric stream through - // an in-memory reader (no OTLP endpoint) and assert both surface, - // exactly as the compaction-metrics test does. `init_in_memory` - // installs the *global* provider, so this is a single-provider test. + // RFC 0001 §6.8 telemetry table: `ourios.miner.alias.assertions` / + // `ourios.miner.alias.retractions` are mandatory counters with an + // `ourios.tenant` attribute. Collect the exported metric stream + // through an in-memory reader (no OTLP endpoint) and assert both + // surface, exactly as the compaction-metrics test does. + // `init_in_memory` installs the *global* provider, so this is a + // single-provider test. #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn alias_counters_are_exported_with_tenant_attribute() { use opentelemetry_sdk::metrics::data::{ @@ -600,8 +598,8 @@ mod tests { .map(|m| m.name().to_string()) .collect(); for expected in [ - METRIC_ALIAS_ASSERTIONS_TOTAL, - METRIC_ALIAS_RETRACTIONS_TOTAL, + semconv::OURIOS_MINER_ALIAS_ASSERTIONS, + semconv::OURIOS_MINER_ALIAS_RETRACTIONS, ] { assert!( names.iter().any(|n| n == expected), @@ -621,17 +619,18 @@ mod tests { panic!("{name} should be a u64 sum"); }; sum.data_points().any(|dp| { - dp.attributes() - .any(|kv| kv.key.as_str() == ATTR_TENANT_ID && kv.value.as_str() == "acme") + dp.attributes().any(|kv| { + kv.key.as_str() == semconv::OURIOS_TENANT && kv.value.as_str() == "acme" + }) }) }; assert!( - tenant_attr_present(METRIC_ALIAS_ASSERTIONS_TOTAL), - "alias_assertions_total must carry the tenant_id attribute", + tenant_attr_present(semconv::OURIOS_MINER_ALIAS_ASSERTIONS), + "ourios.miner.alias.assertions must carry the ourios.tenant attribute", ); assert!( - tenant_attr_present(METRIC_ALIAS_RETRACTIONS_TOTAL), - "alias_retractions_total must carry the tenant_id attribute", + tenant_attr_present(semconv::OURIOS_MINER_ALIAS_RETRACTIONS), + "ourios.miner.alias.retractions must carry the ourios.tenant attribute", ); } } diff --git a/crates/ourios-core/src/audit.rs b/crates/ourios-core/src/audit.rs index b6a2c209c..009b8bbe4 100644 --- a/crates/ourios-core/src/audit.rs +++ b/crates/ourios-core/src/audit.rs @@ -268,9 +268,10 @@ impl AuditPayload { match self { Self::Template { change, .. } => change.counts_as_merge(), // Alias activity is counted separately via - // `alias_assertions_total` / `alias_retractions_total` - // (RFC 0001 §6.7); `merges_total` is reserved for the two - // structural widenings. + // `ourios.miner.alias.assertions` / + // `ourios.miner.alias.retractions` (RFC 0001 §6.7); + // `ourios.miner.merges` is reserved for the two structural + // widenings. Self::AliasAsserted { .. } | Self::AliasRetracted { .. } | Self::Compaction { .. } => { false } @@ -285,6 +286,19 @@ impl TemplateChange { pub fn counts_as_merge(&self) -> bool { matches!(self, Self::Widened { .. } | Self::TypeExpanded { .. }) } + + /// The canonical `event_type` string for this change — the + /// `ourios.miner.merges` `ourios.miner.template_change` attribute + /// (RFC 0001 §6.8) reads it. Mirrors [`AuditPayload::event_type`] + /// for the `Template` payload arm. + #[must_use] + pub fn event_type(&self) -> &'static str { + match self { + Self::Widened { .. } => EVENT_TYPE_TEMPLATE_WIDENED, + Self::TypeExpanded { .. } => EVENT_TYPE_TEMPLATE_TYPE_EXPANDED, + Self::RejectedDegenerate { .. } => EVENT_TYPE_TEMPLATE_WIDENING_REJECTED_DEGENERATE, + } + } } /// Type tag for a masked parameter slot. diff --git a/crates/ourios-miner/Cargo.toml b/crates/ourios-miner/Cargo.toml index 1aaeccf15..307d37a25 100644 --- a/crates/ourios-miner/Cargo.toml +++ b/crates/ourios-miner/Cargo.toml @@ -10,12 +10,30 @@ publish = false [dependencies] ourios-core = { path = "../ourios-core" } +# Generated dotted-`ourios.miner.*` metric + attribute name constants +# (weaver registry, `semconv/registry/`). RFC 0001 §6.8 names. +ourios-semconv = { path = "../ourios-semconv" } # RFC 0001 §6.5: `Param.value` for `ParamType::Overflow` carries a # `(length, sha256_prefix: [u8; 8])` marker. The default-features # = false build of `sha2` drops the asm backend and keeps the # pure-Rust path — sufficient for an 8-byte-prefix hash on a # per-param hot path, with no platform-specific build assumptions. sha2 = { version = "0.10", default-features = false } +# The OTel metrics **API** only (RFC 0001 §6.8 "Export architecture": +# library crates depend on the lightweight API and resolve instruments +# through `global::meter`; the heavy SDK + OTLP exporter live in +# `ourios-telemetry`). Drives the §6.8 miner instrument set in +# `crate::metrics`. +opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } + +[dev-dependencies] +# `init_in_memory` collects the §6.8 instrument stream through an +# in-memory reader (no OTLP endpoint) in the §3.1.2 / RFC0001.8 / +# H2.2 telemetry tests — same pattern as the alias + compaction +# metrics tests. +ourios-telemetry = { path = "../ourios-telemetry", features = ["testing"] } +tokio = { version = "1", default-features = false, features = ["rt", "rt-multi-thread", "macros"] } +opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "testing"] } [lints] workspace = true diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index 92b5d6464..9564e7b79 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -40,19 +40,13 @@ //! The eventual WAL-backed sink replaces the in-memory placeholder //! with the RFC §6.4 *ordering-plus-durability-barrier* contract. //! -//! # Not yet emitted +//! # Out of this crate / not yet wired //! -//! - Three-zone confidence + lossy-zone body retention (RFC §6.3, -//! `H1.2`). -//! - Type-expansion (`TemplateChange::TypeExpanded`); the variant -//! exists on [`TemplateChange`] but no widening path emits it yet -//! (`H5.2`). -//! - `reconstruct()` / `lossy_flag` semantics (RFC §6.6). -//! - Per-parameter 256 B overflow + `OVERFLOW` marker (RFC §6.5). -//! - Prometheus exposition for `merges_total` / -//! `parse_failures_total` / `template_count` (RFC §6.8). -//! - Parquet records for the data records themselves (those go +//! - Parquet records for the mined records themselves (those go //! through `ourios-parquet`). +//! - A WAL-backed audit/record sink (RFC §6.4); today an in-memory +//! placeholder stands in (see above). +//! - Snapshot + recovery of the cluster state (RFC §6.9). //! //! [`Tree`]: crate::tree::Tree //! [`AuditSink`]: ourios_core::audit::AuditSink @@ -73,6 +67,7 @@ use ourios_core::record::{BodyKind, MinedRecord, NoOpRecordSink, Param, RecordSi use ourios_core::tenant::TenantId; use crate::mask::mask; +use crate::metrics::{MinerMetrics, service_of}; use crate::sim_seq::sim_seq_owned; use crate::tokenize::tokenize; use crate::tree::{Leaf, OwnedToken, Tree}; @@ -158,8 +153,9 @@ pub struct MinerCluster { // §6.5 / §3.2 counter: per-parameter byte-limit overflow // events. Increments by the count of `Overflow`-tagged // [`Param`] entries on each emitted record. Read-side - // placeholder for the §6.8 `params_overflow_total` *counter* - // metric — the §3.2 `params_overflow_ratio` *gauge* is the + // placeholder for the §6.8 `ourios.miner.params.overflow` + // *counter* metric — the §3.2 + // `ourios.miner.params.overflow.utilization` *gauge* is the // derived rolling ratio that lands alongside it once total // emitted params is tracked. params_overflow_total: AtomicU64, @@ -170,6 +166,13 @@ pub struct MinerCluster { // assertions (wall-clock comparisons against `now()` flake // under NTP step / leap seconds / VM pause). clock: Box, + // RFC §6.8 OTel instrument set, resolved through the + // process-global `ourios.miner` meter. The atomic counters + // above remain the in-process read path for tests / accessors; + // these instruments are the exported telemetry surface (a + // no-op when no meter provider is installed). The two are kept + // in lockstep at the same emission sites. + metrics: MinerMetrics, } /// Per-tenant template store. @@ -261,6 +264,7 @@ impl MinerCluster { body_retentions_total: AtomicU64::new(0), params_overflow_total: AtomicU64::new(0), clock: Box::new(SystemClock::new()), + metrics: MinerMetrics::new(), } } @@ -430,9 +434,10 @@ impl MinerCluster { /// events per RFC §6.5. Increments by the count of /// `Overflow`-tagged [`Param`]s on each emitted record (so a /// record with two oversized params bumps the counter by 2). - /// Read-side placeholder for the §6.8 `params_overflow_total` - /// *counter* metric; the §3.2 `params_overflow_ratio` *gauge* - /// is the derived rolling ratio with the `> 0.01` per-service + /// Read-side placeholder for the §6.8 + /// `ourios.miner.params.overflow` *counter* metric; the §3.2 + /// `ourios.miner.params.overflow.utilization` *gauge* is the + /// derived rolling ratio with the `> 0.01` per-service /// alert threshold — both ship together once the exporter /// lands and are not this method's responsibility. #[must_use] @@ -453,7 +458,13 @@ impl MinerCluster { /// record's body must always carry the original line bytes /// so `reconstruct()`'s `Overflow` branch (RFC §6.6) has /// something to fall back to. - fn apply_overflow_retention(&self, rec: &mut MinedRecord, raw: &str) { + fn apply_overflow_retention( + &self, + record: &OtlpLogRecord, + service: Option<&str>, + rec: &mut MinedRecord, + raw: &str, + ) { let overflow_count = rec .params .iter() @@ -462,8 +473,11 @@ impl MinerCluster { if overflow_count > 0 { rec.body = Some(raw.to_string()); #[allow(clippy::cast_possible_truncation)] + let count = overflow_count as u64; self.params_overflow_total - .fetch_add(overflow_count as u64, Ordering::Relaxed); + .fetch_add(count, Ordering::Relaxed); + self.metrics + .record_overflow(&record.tenant_id, service, count); } } @@ -483,9 +497,12 @@ impl MinerCluster { /// `body_retentions_total` metric doc explicitly excludes, /// not the §6.3 lossy-zone retention the gauge is meant to /// surface. That path uses [`Self::record_tokenizer_failure`]. - fn record_parse_failure(&self) { + fn record_parse_failure(&self, record: &OtlpLogRecord, service: Option<&str>) { self.parse_failures_total.fetch_add(1, Ordering::Relaxed); self.body_retentions_total.fetch_add(1, Ordering::Relaxed); + self.metrics + .record_parse_failure(&record.tenant_id, service); + self.metrics.record_body_retention(&record.tenant_id); } /// Mark one tokenizer-failure event per RFC §6.6: increments @@ -498,8 +515,10 @@ impl MinerCluster { /// tokenizer failures here would inflate the ratio with /// events that aren't body-retention events in the /// gauge-contract sense. - fn record_tokenizer_failure(&self) { + fn record_tokenizer_failure(&self, record: &OtlpLogRecord, service: Option<&str>) { self.parse_failures_total.fetch_add(1, Ordering::Relaxed); + self.metrics + .record_parse_failure(&record.tenant_id, service); } /// Build the OTLP-envelope half of a `MinedRecord` from the @@ -554,7 +573,15 @@ impl MinerCluster { /// Hand one [`MinedRecord`] to the record sink. Centralised /// so a future "decorate every record with X" step has one /// site to change. - fn emit_record(&mut self, record: MinedRecord) { + /// + /// One emitted record is one ingested line, so this is also the + /// single site that observes the §6.8 per-line `confidence` + /// histogram + p50/p01 reservoir. The ratio-gauge denominators + /// are bumped earlier, at the top of [`Self::ingest`], so they + /// lead any numerator for the same line. + fn emit_record(&mut self, record: MinedRecord, service: Option<&str>) { + self.metrics + .record_line(&record.tenant_id, service, f64::from(record.confidence)); self.record_sink.emit(record); } } @@ -1085,7 +1112,24 @@ impl MinerCluster { /// On first sight of `record.tenant_id`, allocates a fresh /// per-tenant store. pub fn ingest(&mut self, record: &OtlpLogRecord) -> u64 { - match &record.body { + let started = std::time::Instant::now(); + // Resolve the source service once per ingest. It is + // constant for the record and feeds every §6.8 per-service + // instrument below; the hot-path helpers take the borrowed + // `Option<&str>` rather than re-scanning + re-allocating it. + // `None` when the source set no `service.name` — `ourios.service` + // is then omitted (recommended, not synthesized). + let service = service_of(&record.resource_attributes); + let service = service.as_deref(); + // Bump the per-line denominators before any per-line + // numerator (overflow / body-retention) for this line. The + // ratio-gauge callbacks may collect concurrently; denominator + // first keeps utilization ∈ [0, 1] at every collection point. + // Exactly one record is emitted per ingest, so this matches + // the `record_line` confidence observation one-to-one. + self.metrics + .record_line_denominator(&record.tenant_id, service); + let template_id = match &record.body { None => { // The wire delivered no body. Emit a single // record with `BodyKind::Absent` and the @@ -1095,12 +1139,23 @@ impl MinerCluster { // template, so reconstruction is not possible. let mut rec = Self::record_envelope(record, BodyKind::Absent); rec.lossy_flag = true; - self.emit_record(rec); + self.emit_record(rec, service); NO_TEMPLATE } - Some(Body::String(raw)) => self.ingest_string(record, raw), - Some(Body::Structured(av)) => self.ingest_structured(record, av), - } + Some(Body::String(raw)) => self.ingest_string(record, service, raw), + Some(Body::Structured(av)) => self.ingest_structured(record, service, av), + }; + // §6.8 `ourios.miner.duration` histogram (hot-path budget + // D1) and the `ourios.miner.template.count` observable-gauge + // mirror. Both read the post-ingest state, so they sit after + // the body fork. + self.metrics + .record_duration(&record.tenant_id, started.elapsed().as_secs_f64()); + // `usize` ≤ `u64` on every supported target; saturate + // rather than panic on the impossible overflow. + let count = u64::try_from(self.template_count(&record.tenant_id)).unwrap_or(u64::MAX); + self.metrics.set_template_count(&record.tenant_id, count); + template_id } /// `Body::String` path — RFC §6.2 steps 1–5 with widening. @@ -1115,7 +1170,7 @@ impl MinerCluster { // too_many_lines lint is silenced here rather than // fragmenting the per-step structure. #[allow(clippy::too_many_lines)] - fn ingest_string(&mut self, record: &OtlpLogRecord, raw: &str) -> u64 { + fn ingest_string(&mut self, record: &OtlpLogRecord, service: Option<&str>, raw: &str) -> u64 { // RFC §6.2 step 1 (H7.2): a tokenizer failure (today: // embedded NUL byte) routes the line to the parse-failure // path with `lossy_flag = true` and the original body @@ -1131,8 +1186,8 @@ impl MinerCluster { let mut rec = Self::record_envelope(record, BodyKind::String); rec.body = Some(raw.to_string()); rec.lossy_flag = true; - self.emit_record(rec); - self.record_tokenizer_failure(); + self.emit_record(rec, service); + self.record_tokenizer_failure(record, service); return NO_TEMPLATE; } }; @@ -1161,8 +1216,8 @@ impl MinerCluster { rec.separators = separators; rec.body = Some(raw.to_string()); rec.lossy_flag = true; - self.emit_record(rec); - self.record_parse_failure(); + self.emit_record(rec, service); + self.record_parse_failure(record, service); return NO_TEMPLATE; } @@ -1184,9 +1239,9 @@ impl MinerCluster { // §6.5: bump `params_overflow_total` if any param // exceeded the byte limit. Body retention is already // set above for this parse-failure path. - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); - self.record_parse_failure(); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); + self.record_parse_failure(record, service); return NO_TEMPLATE; } @@ -1221,8 +1276,8 @@ impl MinerCluster { rec.confidence = 1.0; // §6.5: force body retention on this fresh-leaf // record if any of its params overflowed. - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); new_id } Some(c) => { @@ -1237,6 +1292,7 @@ impl MinerCluster { // §6.5 capping. ConfidenceZone::Clean => self.attach_and_maybe_widen( record, + service, raw, &masked_strs, &masked.wildcard_positions, @@ -1255,6 +1311,7 @@ impl MinerCluster { // separately. ConfidenceZone::Lossy => { self.body_retentions_total.fetch_add(1, Ordering::Relaxed); + self.metrics.record_body_retention(&record.tenant_id); let new_id = self.create_new_leaf( record, &masked_strs, @@ -1274,8 +1331,8 @@ impl MinerCluster { // §6.5: bump `params_overflow_total` for // any overflow params (body is already // retained for the §6.3 reason). - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); new_id } // Parse failure: no template allocated. @@ -1289,9 +1346,9 @@ impl MinerCluster { // §6.5: bump `params_overflow_total` for // any overflow params (body is already // retained for the parse-failure reason). - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); - self.record_parse_failure(); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); + self.record_parse_failure(record, service); NO_TEMPLATE } } @@ -1463,6 +1520,7 @@ impl MinerCluster { fn attach_and_maybe_widen( &mut self, record: &OtlpLogRecord, + service: Option<&str>, raw: &str, masked_strs: &[&str], line_wildcard_positions: &[usize], @@ -1520,8 +1578,8 @@ impl MinerCluster { rec.confidence = 1.0; // §6.5: force body retention on this clean-reuse // record if any of its aligned params overflowed. - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); template_id } AttachPlan::Rejected { @@ -1559,9 +1617,9 @@ impl MinerCluster { // §6.5: bump `params_overflow_total` if the // line-ordered params contained any oversized // values (body already retained for §6.4). - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); - self.record_parse_failure(); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); + self.record_parse_failure(record, service); NO_TEMPLATE } AttachPlan::Mutated { @@ -1572,6 +1630,7 @@ impl MinerCluster { } => { for change in events { let counts_as_merge = change.counts_as_merge(); + let event_type = change.event_type(); self.audit_sink.emit(AuditEvent { tenant_id: record.tenant_id.clone(), timestamp: self.clock.now(), @@ -1584,6 +1643,7 @@ impl MinerCluster { }); if counts_as_merge { self.merges_total.fetch_add(1, Ordering::Relaxed); + self.metrics.record_merge(&record.tenant_id, event_type); } } let mut rec = Self::record_envelope(record, BodyKind::String); @@ -1595,8 +1655,8 @@ impl MinerCluster { // §6.5: force body retention on this widened / // type-expanded record if any of its aligned // params overflowed. - self.apply_overflow_retention(&mut rec, raw); - self.emit_record(rec); + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); template_id } } @@ -1611,6 +1671,7 @@ impl MinerCluster { fn ingest_structured( &mut self, record: &OtlpLogRecord, + service: Option<&str>, any_value: &ourios_core::otlp::AnyValue, ) -> u64 { let key = (record.severity_number, record.scope_name.clone()); @@ -1680,7 +1741,7 @@ impl MinerCluster { rec.template_version = 1; rec.confidence = 1.0; rec.body = Some(String::from_utf8(bytes).expect("serde_json emits valid UTF-8")); - self.emit_record(rec); + self.emit_record(rec, service); template_id } diff --git a/crates/ourios-miner/src/lib.rs b/crates/ourios-miner/src/lib.rs index 3299b5f23..1ebd67e6b 100644 --- a/crates/ourios-miner/src/lib.rs +++ b/crates/ourios-miner/src/lib.rs @@ -2,6 +2,7 @@ pub mod cluster; pub mod mask; +mod metrics; pub mod overflow; pub mod reconstruct; pub mod sim_seq; diff --git a/crates/ourios-miner/src/metrics.rs b/crates/ourios-miner/src/metrics.rs new file mode 100644 index 000000000..4530dc49d --- /dev/null +++ b/crates/ourios-miner/src/metrics.rs @@ -0,0 +1,703 @@ +//! RFC 0001 §6.8 telemetry instruments for the template miner. +//! +//! Instruments are resolved through the process-global meter +//! (`global::meter("ourios.miner")`) per the §6.8 *Export +//! architecture* API/SDK split: this library depends only on the +//! lightweight `opentelemetry` API crate; the SDK + OTLP exporter +//! live in `ourios-telemetry`. With no provider installed the +//! instrument `record` / `add` calls are cheap no-ops, so a +//! [`MinerMetrics`] is always safe to construct and drive. +//! +//! The in-process observable-gauge state ([`MinerMetricsState`]) is +//! *not* gated on whether a provider is installed: the per-line +//! denominator / reservoir updates take the state mutex and touch the +//! tally maps on every ingest regardless. That cost is unconditional +//! by design — the gauges are derived in-process (§6.8 / RFC0001.8), +//! so the state must be maintained to serve a collection that may +//! arrive at any time — but it is bounded: the hot-path update clones +//! a key only on first sight of a `(tenant, service)` pair (see +//! [`tally_mut`]) and the reservoir is capped at [`RESERVOIR_CAP`]. +//! +//! # Metric names and attributes +//! +//! Names (`ourios.miner.template.count`, `ourios.miner.merges`, …) +//! and data-point attribute keys (`ourios.tenant`, `ourios.service`, +//! `ourios.miner.template_change`) come from the generated +//! [`ourios_semconv`] constants — the dotted-`ourios.*` weaver +//! registry (`semconv/registry/`) alongside the compaction set +//! (RFC 0009 §3.6). +//! +//! # Sync vs. observable +//! +//! The §6.8 table fixes each instrument's kind. Counters and +//! histograms are **synchronous** — recorded at the hot-path site +//! that produces the measurement. The gauges are **observable**: +//! they read process state through a callback at collection time. +//! That state — per-`(tenant, service)` line tallies and a bounded +//! confidence reservoir, plus per-tenant template counts — lives in +//! [`MinerMetricsState`] behind a `Mutex`, shared with the +//! callbacks via `Arc`. +//! +//! # When instruments appear +//! +//! The mandatory §6.8 set is defined by the [`ourios_semconv`] +//! registry; each instrument is *registered* on the `ourios.miner` +//! meter at construction. `OTel`'s metric model is collect-on-read, +//! so an instrument contributes a data point on its **first real +//! measurement** — no synthetic zero-traffic points are emitted, so +//! every exported series carries the registry's `required` +//! attributes. + +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +use opentelemetry::metrics::{Counter, Histogram, Meter, ObservableGauge}; +use opentelemetry::{KeyValue, global}; + +use ourios_core::otlp::{KeyValue as OtlpKeyValue, any_value}; +use ourios_core::tenant::TenantId; +use ourios_semconv as semconv; + +/// Meter name per RFC 0001 §6.8 (`global::meter("ourios.miner")`). +const METER_NAME: &str = "ourios.miner"; + +/// Resource-attribute key the source service is read from. The +/// §6.8 `service` attribute is the *log's source* service (distinct +/// from Ourios's own `service.name` resource attribute), read from +/// the ingested record's `resource_attributes`. +const RESOURCE_SERVICE_NAME: &str = "service.name"; + +/// Confidence-reservoir window size. Bounded so a high-volume +/// `(tenant, service)` cannot grow the reservoir without limit +/// (§3.2's cardinality discipline applied to our own telemetry +/// state). +const RESERVOIR_CAP: usize = 1024; + +/// Read the source `service.name` from a record's +/// `resource_attributes`, returning `None` when it is absent, empty, +/// or non-string. The proto `KeyValue` carries +/// `value: Option`; only a non-empty `StringValue` is a +/// meaningful service identity. +/// +/// `ourios.service` is `recommended` (not `required`) in the semconv +/// registry: an absent source `service.name` means the attribute is +/// **omitted**, not synthesized to a sentinel. Synthesizing a value +/// would change the attribute's meaning and create a fake series that +/// could collide with a real service of that name. +#[must_use] +pub(crate) fn service_of(resource_attributes: &[OtlpKeyValue]) -> Option { + resource_attributes + .iter() + .find(|kv| kv.key == RESOURCE_SERVICE_NAME) + .and_then(|kv| kv.value.as_ref()) + .and_then(|av| av.value.as_ref()) + .and_then(|v| match v { + any_value::Value::StringValue(s) if !s.is_empty() => Some(s.clone()), + _ => None, + }) +} + +/// Lock the metrics state, recovering the guard if the mutex was poisoned +/// by an unrelated panic. Telemetry is best-effort: a poisoned lock must +/// never crash an ingest write or a collection callback. +fn lock_state(state: &Mutex) -> std::sync::MutexGuard<'_, MinerMetricsState> { + match state.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +/// Increment a per-tenant counter by one, cloning the tenant key only on +/// its first sight — the hot path (a tenant already in the map) reuses +/// the borrowed key via `get_mut`. +fn bump_tenant(map: &mut HashMap, tenant: &TenantId) { + if let Some(count) = map.get_mut(tenant) { + *count += 1; + } else { + map.insert(tenant.clone(), 1); + } +} + +/// Set a per-tenant value, cloning the tenant key only on first sight. +fn set_tenant(map: &mut HashMap, tenant: &TenantId, value: u64) { + if let Some(slot) = map.get_mut(tenant) { + *slot = value; + } else { + map.insert(tenant.clone(), value); + } +} + +/// Borrow the `(tenant, service)` tally for a mutable update, +/// allocating owned keys only on the first sight of the pair. +/// +/// The hot path — a `(tenant, service)` already in flight — clones +/// nothing: the outer probe is `get_mut(&TenantId)` (no clone) and the +/// `Some(name)` inner probe is `get_mut(name)` keyed on a borrowed +/// `&str`. Only the cold first-insert path owns a key (the prior code +/// cloned the tenant *and* the service on every line). A `None` service +/// — a line whose source carried no `service.name` — lands in the +/// dedicated [`ServiceTallies::no_service`] slot, attributed to the +/// tenant alone (`ourios.service` omitted, see [`service_attrs`]). +fn tally_mut<'a>( + by_service: &'a mut HashMap, + tenant: &TenantId, + service: Option<&str>, +) -> &'a mut ServiceTally { + // `contains_key` releases its borrow before the follow-up `get_mut` + // / `entry`, sidestepping the get-then-insert borrow-checker + // limitation while still cloning a key only on first insert. + if !by_service.contains_key(tenant) { + by_service.insert(tenant.clone(), ServiceTallies::default()); + } + let tallies = by_service + .get_mut(tenant) + .unwrap_or_else(|| unreachable!("inserted above when absent")); + match service { + None => &mut tallies.no_service, + Some(name) if tallies.by_name.contains_key(name) => tallies + .by_name + .get_mut(name) + .unwrap_or_else(|| unreachable!("contains_key was true")), + Some(name) => tallies.by_name.entry(name.to_owned()).or_default(), + } +} + +/// Per-tenant tallies, split so the common named-service path probes a +/// borrowed `&str` key and only the service-less lines share a slot. +#[derive(Default)] +struct ServiceTallies { + /// Tallies for lines carrying a non-empty `service.name`. + by_name: HashMap, + /// Tally for lines whose source set no `service.name`. These count + /// toward the tenant but emit no `ourios.service` attribute. A + /// tenant with only named services leaves this at its zero default; + /// the gauge callbacks gate on `lines > 0` so an unused slot emits + /// no point. + no_service: ServiceTally, +} + +impl ServiceTallies { + /// Yield each populated `(service, tally)` for a gauge callback: + /// every named service plus the service-less slot, the latter only + /// once it has seen a real line (`lines > 0`) so an unused slot + /// emits no point. + fn iter(&self) -> impl Iterator, &ServiceTally)> { + let named = self + .by_name + .iter() + .map(|(name, t)| (Some(name.as_str()), t)); + let unnamed = (self.no_service.lines > 0).then_some((None, &self.no_service)); + named.chain(unnamed) + } +} + +/// Per-`(tenant, service)` rolling tallies for the derived gauges. +/// +/// `lines` is the denominator for `ourios.miner.params.overflow.utilization`; +/// `overflow_lines` is its numerator. `confidence` is a bounded +/// reservoir over the same key, the source for the +/// `ourios.miner.confidence.p50` / `ourios.miner.confidence.p01` quantile gauges. +#[derive(Default)] +struct ServiceTally { + lines: u64, + overflow_lines: u64, + confidence: Reservoir, +} + +/// Process state the observable-gauge callbacks read at collection +/// time. Guarded by a `Mutex` because `OTel` may invoke a callback +/// from any thread on collection. +#[derive(Default)] +struct MinerMetricsState { + /// Per-tenant, per-service tallies driving the ratio + quantile + /// gauges. Nested (not a flat `(TenantId, _)` key) so the common + /// hot-path update borrows the tenant for the outer lookup + /// (`get_mut(tenant)`) and clones a key only on the first sight of + /// a `(tenant, service)` pair. Each tenant's [`ServiceTallies`] + /// splits a named-service map (`by_name`) from a single + /// `no_service` slot for lines whose source carried no + /// `service.name` — those still count toward the tenant, attributed + /// without a service (`ourios.service` omitted, see [`service_attrs`]). + by_service: HashMap, + /// Per-tenant template count, mirrored from the cluster so the + /// `ourios.miner.template.count` observable gauge can report it + /// without borrowing the cluster. + template_counts: HashMap, + /// Per-tenant body-retention numerator / line denominator for + /// the `ourios.miner.body_retention.utilization` gauge. + body_retentions: HashMap, + body_lines: HashMap, +} + +/// A bounded confidence reservoir: an exact quantile over a capped +/// FIFO window of recent samples. +/// +/// **Quantile mechanism (flagged decision).** RFC 0001 §6.8 does +/// not pin the quantile algorithm or window size. This is an exact +/// quantile over a bounded ring of the most recent +/// [`RESERVOIR_CAP`] samples per `(tenant, service)` — small, +/// allocation-bounded, and correct over its window, with no sketch +/// approximation error. The `confidence` histogram remains the +/// §6.8 source of truth; the two gauges are named in-process views +/// per RFC0001.8. A future contract change could instead make p50/p01 +/// backend-derived quantiles over the exported `confidence` histogram +/// (independent of the now-landed dotted-semconv naming); if so, this +/// in-process reservoir is replaced under that change's own review. +struct Reservoir { + samples: VecDeque, +} + +impl Default for Reservoir { + fn default() -> Self { + Self { + samples: VecDeque::with_capacity(RESERVOIR_CAP), + } + } +} + +impl Reservoir { + fn observe(&mut self, value: f64) { + if self.samples.len() == RESERVOIR_CAP { + self.samples.pop_front(); + } + self.samples.push_back(value); + } + + /// Copy the current sample window — O(n), no sort — so a collection + /// callback can take it under the metrics lock and compute the + /// quantile *after* releasing the lock. + fn snapshot(&self) -> Vec { + self.samples.iter().copied().collect() + } +} + +/// Exact nearest-rank `q`-quantile (`q` in `[0.0, 1.0]`) over a sample +/// window, or `None` when empty. Takes an owned `Vec` and sorts it so the +/// caller can run it OFF the metrics lock — collection must not block the +/// ingest hot path on the `O(n log n)` sort. +fn quantile_of(mut samples: Vec, q: f64) -> Option { + if samples.is_empty() { + return None; + } + // `total_cmp` is a total order over f64 (no panic; any stray NaN sorts + // to an end) — avoids `partial_cmp(...).expect(...)`. + samples.sort_by(f64::total_cmp); + let n = samples.len(); + // Nearest-rank: rank = ceil(q * n), clamped to [1, n]. + #[allow( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss + )] + let rank = (q * n as f64).ceil().max(1.0) as usize; + Some(samples[rank.min(n) - 1]) +} + +/// The miner's §6.8 instrument set plus the shared state its +/// observable gauges read. +/// +/// Constructed once per [`crate::cluster::MinerCluster`]. The +/// synchronous instruments are recorded at the hot-path sites; the +/// observable gauges are registered with callbacks over the shared +/// [`MinerMetricsState`] at construction and need no per-call +/// maintenance beyond updating that state. +pub(crate) struct MinerMetrics { + state: Arc>, + merges_total: Counter, + parse_failures_total: Counter, + params_overflow_total: Counter, + template_version_changes_total: Counter, + confidence: Histogram, + miner_duration: Histogram, + /// The observable gauges are held for the [`MinerMetrics`]'s + /// lifetime so their collection callbacks stay registered with + /// the meter — dropping a handle deregisters its callback, after + /// which the gauge would vanish from the exported stream. The + /// values are never read directly; the SDK invokes their + /// callbacks on collect. + _observable_gauges: ObservableGauges, +} + +/// The five §6.8 observable gauges, retained to keep their callbacks +/// registered (see [`MinerMetrics`]). `ourios.miner.template.count` +/// is the only `u64` gauge; the fractions and quantiles are `f64`. +struct ObservableGauges { + _template_count: ObservableGauge, + _body_retention_utilization: ObservableGauge, + _params_overflow_utilization: ObservableGauge, + _confidence_p50: ObservableGauge, + _confidence_p01: ObservableGauge, +} + +impl MinerMetrics { + /// Register every §6.8 instrument on the `ourios.miner` meter. + /// The mandatory set is defined by the [`ourios_semconv`] + /// registry; each instrument surfaces in the export on its first + /// real measurement (§3.1.2). + pub(crate) fn new() -> Self { + let meter = global::meter(METER_NAME); + let state = Arc::new(Mutex::new(MinerMetricsState::default())); + + let merges_total = meter + .u64_counter(semconv::OURIOS_MINER_MERGES) + .with_unit("{merge}") + .build(); + let parse_failures_total = meter + .u64_counter(semconv::OURIOS_MINER_PARSE_FAILURES) + .with_unit("{failure}") + .build(); + let params_overflow_total = meter + .u64_counter(semconv::OURIOS_MINER_PARAMS_OVERFLOW) + .with_unit("{overflow}") + .build(); + let template_version_changes_total = meter + .u64_counter(semconv::OURIOS_MINER_TEMPLATE_VERSION_CHANGES) + .with_unit("{change}") + .build(); + let confidence = meter + .f64_histogram(semconv::OURIOS_MINER_CONFIDENCE) + .with_unit("1") + .build(); + let miner_duration = meter + .f64_histogram(semconv::OURIOS_MINER_DURATION) + .with_unit("s") + .build(); + + let observable_gauges = Self::register_observable_gauges(&meter, &state); + + Self { + state, + merges_total, + parse_failures_total, + params_overflow_total, + template_version_changes_total, + confidence, + miner_duration, + _observable_gauges: observable_gauges, + } + } + + /// Register the §6.8 observable gauges + /// (`ourios.miner.template.count`, `…confidence.p50`, + /// `…confidence.p01`, `…body_retention.utilization`, + /// `…params.overflow.utilization`) with callbacks over the shared + /// state. A callback emits one data point per `(tenant, service)` + /// (or per tenant) present in the state — nothing until real + /// traffic populates it. + /// + /// Returns the gauge handles so the caller can retain them: a + /// dropped handle deregisters its callback (see + /// [`ObservableGauges`]). + fn register_observable_gauges( + meter: &Meter, + state: &Arc>, + ) -> ObservableGauges { + let s = Arc::clone(state); + let template_count = meter + .u64_observable_gauge(semconv::OURIOS_MINER_TEMPLATE_COUNT) + .with_unit("{template}") + .with_callback(move |obs| { + let st = lock_state(&s); + for (tenant, count) in &st.template_counts { + obs.observe( + *count, + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], + ); + } + }) + .build(); + + let s = Arc::clone(state); + let body_retention_utilization = meter + .f64_observable_gauge(semconv::OURIOS_MINER_BODY_RETENTION_UTILIZATION) + .with_unit("1") + .with_callback(move |obs| { + let st = lock_state(&s); + for (tenant, lines) in &st.body_lines { + let retained = st.body_retentions.get(tenant).copied().unwrap_or(0); + obs.observe( + ratio(retained, *lines), + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], + ); + } + }) + .build(); + + let s = Arc::clone(state); + let params_overflow_utilization = meter + .f64_observable_gauge(semconv::OURIOS_MINER_PARAMS_OVERFLOW_UTILIZATION) + .with_unit("1") + .with_callback(move |obs| { + let st = lock_state(&s); + for (tenant, tallies) in &st.by_service { + for (service, tally) in tallies.iter() { + obs.observe( + ratio(tally.overflow_lines, tally.lines), + &service_attrs(tenant, service), + ); + } + } + }) + .build(); + + let confidence_p50 = + Self::register_quantile_gauge(meter, state, semconv::OURIOS_MINER_CONFIDENCE_P50, 0.50); + let confidence_p01 = + Self::register_quantile_gauge(meter, state, semconv::OURIOS_MINER_CONFIDENCE_P01, 0.01); + + ObservableGauges { + _template_count: template_count, + _body_retention_utilization: body_retention_utilization, + _params_overflow_utilization: params_overflow_utilization, + _confidence_p50: confidence_p50, + _confidence_p01: confidence_p01, + } + } + + /// Register one confidence-quantile observable gauge + /// (`ourios.miner.confidence.p50` / `…p01`) over the + /// per-`(tenant, service)` reservoir, returning its handle. + fn register_quantile_gauge( + meter: &Meter, + state: &Arc>, + name: &'static str, + q: f64, + ) -> ObservableGauge { + let s = Arc::clone(state); + meter + .f64_observable_gauge(name) + .with_unit("1") + .with_callback(move |obs| { + // Snapshot the sample windows under the lock (cheap O(n) + // copy), then sort + compute the quantiles AFTER releasing + // it — collection must never block the ingest hot path on + // the O(n log n) sort. + let snapshots: Vec<(TenantId, Option, Vec)> = { + let st = lock_state(&s); + st.by_service + .iter() + .flat_map(|(tenant, tallies)| { + tallies.iter().map(move |(service, tally)| { + ( + tenant.clone(), + service.map(str::to_owned), + tally.confidence.snapshot(), + ) + }) + }) + .collect() + }; + for (tenant, service, samples) in snapshots { + if let Some(v) = quantile_of(samples, q) { + obs.observe(v, &service_attrs(&tenant, service.as_deref())); + } + } + }) + .build() + } + + /// Bump the per-`(tenant, service)` line denominator (for + /// `ourios.miner.params.overflow.utilization`) and the per-tenant + /// body-line denominator (for + /// `ourios.miner.body_retention.utilization`). + /// + /// **Ordering invariant.** This runs once, at the start of an + /// ingest, *before* any numerator bump (`record_overflow`, + /// `record_body_retention`) for the same line. The ratio gauges' + /// callbacks may collect concurrently from another thread; + /// incrementing the denominator first guarantees they never + /// observe `numerator > denominator` (utilization > 1) for a line + /// in flight. + pub(crate) fn record_line_denominator(&self, tenant: &TenantId, service: Option<&str>) { + let mut st = lock_state(&self.state); + bump_tenant(&mut st.body_lines, tenant); + tally_mut(&mut st.by_service, tenant, service).lines += 1; + } + + /// Observe one ingested line's `confidence` on both the §6.8 + /// histogram and the per-`(tenant, service)` reservoir feeding the + /// p50/p01 gauges. The line denominator is bumped separately and + /// earlier by [`Self::record_line_denominator`]. + pub(crate) fn record_line(&self, tenant: &TenantId, service: Option<&str>, confidence: f64) { + self.confidence + .record(confidence, &service_attrs(tenant, service)); + let mut st = lock_state(&self.state); + tally_mut(&mut st.by_service, tenant, service) + .confidence + .observe(confidence); + } + + /// Record the miner's per-line processing duration (§6.8 `ourios.miner.duration`). + pub(crate) fn record_duration(&self, tenant: &TenantId, seconds: f64) { + self.miner_duration.record( + seconds, + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], + ); + } + + /// Record `count` per-parameter overflow events on one line for + /// `(tenant, service)`: bumps the `params_overflow_total` + /// counter and the per-service overflow-line numerator (the + /// line is counted once toward the ratio regardless of how many + /// of its params overflowed). + pub(crate) fn record_overflow(&self, tenant: &TenantId, service: Option<&str>, count: u64) { + if count == 0 { + return; + } + self.params_overflow_total + .add(count, &service_attrs(tenant, service)); + let mut st = lock_state(&self.state); + tally_mut(&mut st.by_service, tenant, service).overflow_lines += 1; + } + + /// Record one parse-failure line (§6.8 `ourios.miner.parse_failures`). + pub(crate) fn record_parse_failure(&self, tenant: &TenantId, service: Option<&str>) { + self.parse_failures_total + .add(1, &service_attrs(tenant, service)); + } + + /// Record one body-retention event for the + /// `ourios.miner.body_retention.utilization` numerator (§6.3 retention paths). + pub(crate) fn record_body_retention(&self, tenant: &TenantId) { + let mut st = lock_state(&self.state); + bump_tenant(&mut st.body_retentions, tenant); + } + + /// Record one merge event (§6.8 `ourios.miner.merges`, + /// `ourios.miner.template_change` attribute) and one + /// `ourios.miner.template.version_changes` bump (every merge + /// advances `template_version` per §6.7 / H5). + pub(crate) fn record_merge(&self, tenant: &TenantId, event_type: &str) { + self.merges_total.add( + 1, + &[ + KeyValue::new(semconv::OURIOS_TENANT, tenant.as_str().to_owned()), + KeyValue::new(semconv::OURIOS_MINER_TEMPLATE_CHANGE, event_type.to_owned()), + ], + ); + self.template_version_changes_total.add( + 1, + &[KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )], + ); + } + + /// Mirror a tenant's current template count into the state the + /// `ourios.miner.template.count` observable gauge reads. + pub(crate) fn set_template_count(&self, tenant: &TenantId, count: u64) { + let mut st = lock_state(&self.state); + set_tenant(&mut st.template_counts, tenant, count); + } +} + +/// Data-point attributes for a `(tenant, service)` measurement. +/// +/// `ourios.tenant` is `required`, so it is always present. +/// `ourios.service` is `recommended`: it is emitted only when the +/// source carried a `service.name` (`Some`). A service-less line +/// (`None`) is attributed to the tenant alone — the point is **not** +/// dropped, and no synthetic service value is fabricated. +fn service_attrs(tenant: &TenantId, service: Option<&str>) -> Vec { + let mut attrs = Vec::with_capacity(1 + usize::from(service.is_some())); + attrs.push(KeyValue::new( + semconv::OURIOS_TENANT, + tenant.as_str().to_owned(), + )); + if let Some(name) = service { + attrs.push(KeyValue::new(semconv::OURIOS_SERVICE, name.to_owned())); + } + attrs +} + +/// `numerator / denominator`, or `0.0` when the denominator is zero +/// (a service that has seen no lines has a 0 ratio, not a NaN). +#[allow(clippy::cast_precision_loss)] +fn ratio(numerator: u64, denominator: u64) -> f64 { + if denominator == 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 + } +} + +#[cfg(test)] +mod tests { + use super::{Reservoir, ratio, service_of}; + use ourios_core::otlp::{AnyValue, KeyValue as OtlpKeyValue, any_value}; + + #[test] + fn reservoir_nearest_rank_quantiles() { + let mut r = Reservoir::default(); + for v in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] { + r.observe(v); + } + // Nearest-rank p50 over 10 samples = rank ceil(0.5*10)=5 → 5th + // smallest = 0.5; p01 = rank ceil(0.01*10)=1 → smallest = 0.1. + // Computed off-lock from a snapshot, as the gauge callback does. + assert!((super::quantile_of(r.snapshot(), 0.50).unwrap() - 0.5).abs() < 1e-9); + assert!((super::quantile_of(r.snapshot(), 0.01).unwrap() - 0.1).abs() < 1e-9); + } + + #[test] + fn reservoir_empty_is_none() { + assert!(super::quantile_of(Reservoir::default().snapshot(), 0.5).is_none()); + } + + #[test] + fn reservoir_is_bounded() { + let mut r = Reservoir::default(); + for i in 0..(super::RESERVOIR_CAP + 500) { + r.observe(f64::from(u32::try_from(i).unwrap())); + } + assert_eq!(r.samples.len(), super::RESERVOIR_CAP); + } + + #[test] + fn ratio_zero_denominator_is_zero() { + assert!((ratio(3, 0) - 0.0).abs() < f64::EPSILON); + assert!((ratio(1, 4) - 0.25).abs() < f64::EPSILON); + } + + #[test] + fn service_of_reads_service_name_string() { + let attrs = vec![OtlpKeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("checkout".to_string())), + }), + ..Default::default() + }]; + assert_eq!(service_of(&attrs).as_deref(), Some("checkout")); + } + + #[test] + fn service_of_absent_is_none() { + // `ourios.service` is `recommended`: an absent source + // `service.name` is omitted, not synthesized to a sentinel. + assert_eq!(service_of(&[]), None); + } + + #[test] + fn service_of_empty_string_is_none() { + let attrs = vec![OtlpKeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(String::new())), + }), + ..Default::default() + }]; + assert_eq!(service_of(&attrs), None); + } +} diff --git a/crates/ourios-miner/tests/hazards.rs b/crates/ourios-miner/tests/hazards.rs index 8c5d52e02..67eeff650 100644 --- a/crates/ourios-miner/tests/hazards.rs +++ b/crates/ourios-miner/tests/hazards.rs @@ -604,10 +604,120 @@ fn h2_1_per_tenant_byte_limit_override_honoured() { /// Scenario H2.2 — Per-service overflow rate above 1% raises an alert. /// See `docs/rfcs/0001-template-miner.md` §5. -#[test] -#[ignore = "RFC 0001 Red gate — implementation pending"] -fn h2_2_per_service_overflow_rate_above_one_percent_alerts() { - todo!("RFC 0001 §6.5"); +/// +/// "Alert" is the `ourios.miner.params.overflow.utilization` +/// gauge (attributes `ourios.tenant` / `ourios.service`) crossing +/// the documented `0.01` threshold (Ourios ships the +/// metric + the alert rule, not an alerting engine). Ingests a +/// per-service line stream whose overflow rate exceeds 1%, collects +/// the exported stream, and asserts the gauge for the over-1% +/// service is above `0.01` while a clean sibling service stays at +/// `0.0` — the per-service isolation H2.2 hinges on. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn h2_2_per_service_overflow_rate_above_one_percent_alerts() { + use opentelemetry_sdk::metrics::data::{ + AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics, + }; + + use ourios_core::config::MinerConfig; + use ourios_core::otlp::{AnyValue, Body, KeyValue, OtlpLogRecord, any_value}; + use ourios_core::tenant::TenantId; + use ourios_miner::cluster::MinerCluster; + + const ALERT_THRESHOLD: f64 = 0.01; + + // Arrange — in-memory provider, then the miner. + let (guard, exporter) = ourios_telemetry::init_in_memory("ourios-test"); + let mut cluster = MinerCluster::new(MinerConfig::default()); + let t = TenantId::new("acme"); + + let svc_attr = |service: &str| { + vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(service.to_string())), + }), + ..Default::default() + }] + }; + let make = |service: &str, text: String| OtlpLogRecord { + tenant_id: t.clone(), + resource_attributes: svc_attr(service), + body: Some(Body::String(text)), + ..Default::default() + }; + + // A param value over the 256 B limit (§6.5) forces an Overflow + // marker; each such line counts once toward the service's + // overflow numerator. + let big = "9".repeat(300); + + // `noisy`: 200 clean lines + 3 overflow lines → 3/203 ≈ 0.0148 + // > 1%. `quiet`: 100 clean lines, no overflow → 0.0. + for i in 0..200 { + cluster.ingest(&make("noisy", format!("user {i} logged in"))); + } + for _ in 0..3 { + cluster.ingest(&make("noisy", format!("user {big} logged in"))); + } + for i in 0..100 { + cluster.ingest(&make("quiet", format!("order {i} shipped"))); + } + guard.force_flush().expect("force_flush succeeds"); + + // Act — read the per-service ratio gauge. + let rms = exporter.get_finished_metrics().expect("metrics exported"); + let ratio_for = |service: &str| -> f64 { + let data = rms + .iter() + .flat_map(ResourceMetrics::scope_metrics) + .flat_map(ScopeMetrics::metrics) + .find(|m| m.name() == ourios_semconv::OURIOS_MINER_PARAMS_OVERFLOW_UTILIZATION) + .expect("ourios.miner.params.overflow.utilization missing from exported stream") + .data(); + let AggregatedMetrics::F64(MetricData::Gauge(gauge)) = data else { + panic!("ourios.miner.params.overflow.utilization should be an f64 gauge"); + }; + gauge + .data_points() + .find(|dp| { + let mut tenant_ok = false; + let mut service_ok = false; + for kv in dp.attributes() { + match kv.key.as_str() { + k if k == ourios_semconv::OURIOS_TENANT && kv.value.as_str() == "acme" => { + tenant_ok = true; + } + k if k == ourios_semconv::OURIOS_SERVICE + && kv.value.as_str() == service => + { + service_ok = true; + } + _ => {} + } + } + tenant_ok && service_ok + }) + .unwrap_or_else(|| { + panic!( + "ourios.miner.params.overflow.utilization missing the (acme, {service}) point" + ) + }) + .value() + }; + + // Assert — the over-1% service crosses the alert threshold; the + // clean sibling does not (per-service isolation). + let noisy = ratio_for("noisy"); + let quiet = ratio_for("quiet"); + assert!( + noisy > ALERT_THRESHOLD, + "noisy service overflow ratio {noisy} must exceed the {ALERT_THRESHOLD} alert threshold", + ); + assert!( + quiet <= ALERT_THRESHOLD, + "quiet service overflow ratio {quiet} must stay at/under the alert threshold", + ); } /// Scenario H5.1 — Wildcard widening increments `template_version` and emits `template_widened`. diff --git a/crates/ourios-miner/tests/invariants.rs b/crates/ourios-miner/tests/invariants.rs index 2ee48f0ef..4e05b802f 100644 --- a/crates/ourios-miner/tests/invariants.rs +++ b/crates/ourios-miner/tests/invariants.rs @@ -31,10 +31,114 @@ fn invariant_3_1_1_default_threshold_is_0_7() { /// Scenario §3.1.2 — Mandatory metric set is exposed. /// See `docs/rfcs/0001-template-miner.md` §5. -#[test] -#[ignore = "RFC 0001 Red gate — implementation pending"] -fn invariant_3_1_2_mandatory_metric_set_is_exposed() { - todo!("RFC 0001 §6.8"); +/// +/// Two-part, per the registry-based §3.1.2 decision: +/// (a) the mandatory set is §6.8's miner-instrument metrics — the +/// eleven `ourios_semconv` `ourios.miner.*` constants the miner +/// registers (the `OURIOS_MINER_ALIAS_*` counters are registered +/// by `AliasMap`, §6.7, and are out of this scenario's scope); +/// the registry is the source of truth; and +/// (b) every instrument is registered on the `ourios.miner` meter +/// and surfaces in the export on its **first real measurement** +/// — no synthetic zero-traffic points. A small representative +/// workload exercises every instrument (normal line, +/// template-widening near-duplicate, oversized-`param` line, +/// parse-failure line), then `force_flush` and assert each +/// mandatory name is present. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn invariant_3_1_2_mandatory_metric_set_is_exposed() { + use opentelemetry_sdk::metrics::data::{ResourceMetrics, ScopeMetrics}; + + use ourios_core::config::MinerConfig; + use ourios_core::otlp::{Body, OtlpLogRecord}; + use ourios_core::tenant::TenantId; + use ourios_miner::cluster::MinerCluster; + + // The mandatory §6.8 set, defined by the registry / `ourios-semconv`. + const MANDATORY: [&str; 11] = [ + ourios_semconv::OURIOS_MINER_TEMPLATE_COUNT, + ourios_semconv::OURIOS_MINER_MERGES, + ourios_semconv::OURIOS_MINER_CONFIDENCE, + ourios_semconv::OURIOS_MINER_CONFIDENCE_P50, + ourios_semconv::OURIOS_MINER_CONFIDENCE_P01, + ourios_semconv::OURIOS_MINER_BODY_RETENTION_UTILIZATION, + ourios_semconv::OURIOS_MINER_PARSE_FAILURES, + ourios_semconv::OURIOS_MINER_PARAMS_OVERFLOW, + ourios_semconv::OURIOS_MINER_PARAMS_OVERFLOW_UTILIZATION, + ourios_semconv::OURIOS_MINER_TEMPLATE_VERSION_CHANGES, + ourios_semconv::OURIOS_MINER_DURATION, + ]; + + // (a) Registry defines the set: the constants above are exactly + // the dotted `ourios.miner.*` names the registry pins. + assert_eq!( + MANDATORY, + [ + "ourios.miner.template.count", + "ourios.miner.merges", + "ourios.miner.confidence", + "ourios.miner.confidence.p50", + "ourios.miner.confidence.p01", + "ourios.miner.body_retention.utilization", + "ourios.miner.parse_failures", + "ourios.miner.params.overflow", + "ourios.miner.params.overflow.utilization", + "ourios.miner.template.version_changes", + "ourios.miner.duration", + ], + "the mandatory §6.8 set is the registry / ourios-semconv names", + ); + + // Arrange — in-memory provider, then the miner (so its + // instruments resolve against the global meter the provider + // installs). A param above the default 256-byte limit drives the + // §6.5 overflow path. + let (guard, exporter) = ourios_telemetry::init_in_memory("ourios-test"); + let mut cluster = MinerCluster::new(MinerConfig::default()); + let t = TenantId::new("acme"); + let make = |text: &str| OtlpLogRecord { + tenant_id: t.clone(), + body: Some(Body::String(text.to_string())), + ..Default::default() + }; + // A 300-digit token masks to a `` param (all ASCII digits) + // whose value exceeds the default 256-byte limit, so it overflows. + let oversized_num = "9".repeat(300); + + // Act — exercise every instrument with real data: + // confidence / confidence.p50 / .p01 / duration / template.count + // / params.overflow.utilization / body_retention.utilization + // — every ingested line feeds these (the gauges read the + // per-line denominators bumped on each ingest). + // merges + template.version_changes — the near-duplicate pair + // widens the first template (the differing literal token + // `alpha`/`beta` becomes a wildcard). + // params.overflow — the oversized `` param overflows and + // forces body retention. + // parse_failures + body_retention — the empty line is a §6.2 + // step-1 parse failure that retains body. + cluster.ingest(&make("user 42 logged in from host alpha")); + cluster.ingest(&make("user 99 logged in from host beta")); + cluster.ingest(&make(&format!("request id {oversized_num}"))); + cluster.ingest(&make("")); + guard.force_flush().expect("force_flush succeeds"); + + // Assert — every mandatory §6.8 metric name is in the stream, + // now backed by real data points carrying the required attributes. + let rms = exporter.get_finished_metrics().expect("metrics exported"); + let names: Vec = rms + .iter() + .flat_map(ResourceMetrics::scope_metrics) + .flat_map(ScopeMetrics::metrics) + .map(|m| m.name().to_string()) + .collect(); + + for expected in MANDATORY { + assert!( + names.iter().any(|n| n == expected), + "exported stream missing mandatory §6.8 metric `{expected}`, got {names:?}", + ); + } } /// Scenario §3.2.1 — Default per-parameter byte limit is 256. diff --git a/crates/ourios-miner/tests/rfc_internal.rs b/crates/ourios-miner/tests/rfc_internal.rs index 8e56176c7..d2987fed4 100644 --- a/crates/ourios-miner/tests/rfc_internal.rs +++ b/crates/ourios-miner/tests/rfc_internal.rs @@ -440,12 +440,141 @@ fn rfc0001_7_combined_widening_and_type_expansion_emits_two_events_in_order() { ); } -/// Scenario RFC0001.8 — `confidence_p50` and `confidence_p01` are emitted as gauges. +/// Scenario RFC0001.8 — `ourios.miner.confidence.p50` and +/// `ourios.miner.confidence.p01` are emitted as gauges. /// See `docs/rfcs/0001-template-miner.md` §5. -#[test] -#[ignore = "RFC 0001 Red gate — implementation pending"] -fn rfc0001_8_confidence_p50_and_p01_are_emitted_as_gauges() { - todo!("RFC 0001 §6.8"); +/// +/// Ingests a controlled confidence spread for one `(ourios.tenant, +/// ourios.service)`, collects the exported stream, and asserts the +/// `ourios.miner.confidence.p50` / `.p01` gauges are present for that +/// attribute pair with values equal to the nearest-rank quantile of +/// the same samples the `confidence` histogram saw (the in-process +/// reservoir per §6.8). +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn rfc0001_8_confidence_p50_and_p01_are_emitted_as_gauges() { + use opentelemetry_sdk::metrics::data::{ + AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics, + }; + + use ourios_core::config::MinerConfig; + use ourios_core::otlp::{AnyValue, Body, KeyValue, OtlpLogRecord, any_value}; + use ourios_core::record::SharedRecordSink; + use ourios_core::tenant::TenantId; + use ourios_miner::cluster::MinerCluster; + + // Nearest-rank quantile, mirroring `crate::metrics::Reservoir` + // (rank = ceil(q * n), clamped to [1, n]). The gauge reads the + // same per-line confidences the record sink captures, so + // recomputing here is the cross-check RFC0001.8 demands. + fn nearest_rank(samples: &[f64], q: f64) -> f64 { + let mut sorted = samples.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + #[allow( + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::cast_possible_truncation + )] + let rank = (q * sorted.len() as f64).ceil().max(1.0) as usize; + sorted[rank.min(sorted.len()) - 1] + } + + // Arrange — in-memory provider, then the miner with a record + // sink so the per-line confidences feeding the gauge are + // observable for an independent quantile cross-check. + let (guard, exporter) = ourios_telemetry::init_in_memory("ourios-test"); + let sink = SharedRecordSink::new(); + let mut cluster = + MinerCluster::new(MinerConfig::default()).with_record_sink(Box::new(sink.clone())); + let t = TenantId::new("acme"); + let service = "checkout"; + let svc_attr = vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(service.to_string())), + }), + ..Default::default() + }]; + let make = |text: &str| OtlpLogRecord { + tenant_id: t.clone(), + resource_attributes: svc_attr.clone(), + body: Some(Body::String(text.to_string())), + ..Default::default() + }; + + // Act — a mix of distinct shapes (fresh leaves, confidence + // sentinel 1.0) and a low-similarity line that lands in a zone + // with a sub-1.0 confidence, so the p50/p01 spread is + // non-degenerate. The exact per-line confidences are read back + // from the record sink rather than assumed. + cluster.ingest(&make("alpha beta gamma delta epsilon")); + cluster.ingest(&make("one two three four five")); + cluster.ingest(&make("red green blue cyan magenta")); + cluster.ingest(&make("alpha beta gamma rho sigma")); + cluster.ingest(&make("alpha beta phi rho sigma omega")); + guard.force_flush().expect("force_flush succeeds"); + + let samples: Vec = sink + .drain() + .iter() + .map(|r| f64::from(r.confidence)) + .collect(); + assert!(!samples.is_empty(), "the miner must emit a record per line"); + let expected_p50 = nearest_rank(&samples, 0.50); + let expected_p01 = nearest_rank(&samples, 0.01); + // Guard against a degenerate (all-equal) confidence stream so + // the quantile cross-check below is genuinely exercised: a + // p50 == p01 single-value distribution would pass trivially. + assert!( + (expected_p50 - expected_p01).abs() > 1e-9, + "test setup must produce a non-degenerate confidence spread (p50={expected_p50}, p01={expected_p01})", + ); + + // Assert — both gauges present for (ourios.tenant=acme, + // ourios.service=checkout) with the expected nearest-rank values. + let rms = exporter.get_finished_metrics().expect("metrics exported"); + let gauge_value = |name: &str| -> f64 { + let data = rms + .iter() + .flat_map(ResourceMetrics::scope_metrics) + .flat_map(ScopeMetrics::metrics) + .find(|m| m.name() == name) + .unwrap_or_else(|| panic!("{name} missing from exported stream")) + .data(); + let AggregatedMetrics::F64(MetricData::Gauge(gauge)) = data else { + panic!("{name} should be an f64 gauge"); + }; + gauge + .data_points() + .find(|dp| { + let mut tenant_ok = false; + let mut service_ok = false; + for kv in dp.attributes() { + match kv.key.as_str() { + k if k == ourios_semconv::OURIOS_TENANT && kv.value.as_str() == "acme" => { + tenant_ok = true; + } + k if k == ourios_semconv::OURIOS_SERVICE + && kv.value.as_str() == service => + { + service_ok = true; + } + _ => {} + } + } + tenant_ok && service_ok + }) + .unwrap_or_else(|| panic!("{name} missing the (acme, checkout) data point")) + .value() + }; + + assert!( + (gauge_value(ourios_semconv::OURIOS_MINER_CONFIDENCE_P50) - expected_p50).abs() < 1e-6, + "ourios.miner.confidence.p50 must match the in-process p50 quantile", + ); + assert!( + (gauge_value(ourios_semconv::OURIOS_MINER_CONFIDENCE_P01) - expected_p01).abs() < 1e-6, + "ourios.miner.confidence.p01 must match the in-process p01 quantile", + ); } /// Scenario RFC0001.9 — `body_kind = Structured` short-circuits to a structured-template id. diff --git a/crates/ourios-semconv/src/lib.rs b/crates/ourios-semconv/src/lib.rs index 4bbcd52b4..a3468ce82 100644 --- a/crates/ourios-semconv/src/lib.rs +++ b/crates/ourios-semconv/src/lib.rs @@ -43,6 +43,46 @@ pub const OURIOS_COMPACTION_ROWS: &str = "ourios.compaction.rows"; /// `ourios.compaction.sweeps` (counter, unit `{sweep}`). pub const OURIOS_COMPACTION_SWEEPS: &str = "ourios.compaction.sweeps"; +/// `ourios.miner.alias.assertions` (counter, unit `{assertion}`). +pub const OURIOS_MINER_ALIAS_ASSERTIONS: &str = "ourios.miner.alias.assertions"; + +/// `ourios.miner.alias.retractions` (counter, unit `{retraction}`). +pub const OURIOS_MINER_ALIAS_RETRACTIONS: &str = "ourios.miner.alias.retractions"; + +/// `ourios.miner.body_retention.utilization` (gauge, unit `1`). +pub const OURIOS_MINER_BODY_RETENTION_UTILIZATION: &str = "ourios.miner.body_retention.utilization"; + +/// `ourios.miner.confidence` (histogram, unit `1`). +pub const OURIOS_MINER_CONFIDENCE: &str = "ourios.miner.confidence"; + +/// `ourios.miner.confidence.p01` (gauge, unit `1`). +pub const OURIOS_MINER_CONFIDENCE_P01: &str = "ourios.miner.confidence.p01"; + +/// `ourios.miner.confidence.p50` (gauge, unit `1`). +pub const OURIOS_MINER_CONFIDENCE_P50: &str = "ourios.miner.confidence.p50"; + +/// `ourios.miner.duration` (histogram, unit `s`). +pub const OURIOS_MINER_DURATION: &str = "ourios.miner.duration"; + +/// `ourios.miner.merges` (counter, unit `{merge}`). +pub const OURIOS_MINER_MERGES: &str = "ourios.miner.merges"; + +/// `ourios.miner.params.overflow` (counter, unit `{overflow}`). +pub const OURIOS_MINER_PARAMS_OVERFLOW: &str = "ourios.miner.params.overflow"; + +/// `ourios.miner.params.overflow.utilization` (gauge, unit `1`). +pub const OURIOS_MINER_PARAMS_OVERFLOW_UTILIZATION: &str = + "ourios.miner.params.overflow.utilization"; + +/// `ourios.miner.parse_failures` (counter, unit `{failure}`). +pub const OURIOS_MINER_PARSE_FAILURES: &str = "ourios.miner.parse_failures"; + +/// `ourios.miner.template.count` (gauge, unit `{template}`). +pub const OURIOS_MINER_TEMPLATE_COUNT: &str = "ourios.miner.template.count"; + +/// `ourios.miner.template.version_changes` (counter, unit `{change}`). +pub const OURIOS_MINER_TEMPLATE_VERSION_CHANGES: &str = "ourios.miner.template.version_changes"; + /// `ourios.storage.parquet.file.size` (histogram, unit `By`). pub const OURIOS_STORAGE_PARQUET_FILE_SIZE: &str = "ourios.storage.parquet.file.size"; @@ -54,5 +94,11 @@ pub const OURIOS_COMPACTION_RESULT: &str = "ourios.compaction.result"; /// `ourios.io.direction` attribute key. pub const OURIOS_IO_DIRECTION: &str = "ourios.io.direction"; +/// `ourios.miner.template_change` attribute key. +pub const OURIOS_MINER_TEMPLATE_CHANGE: &str = "ourios.miner.template_change"; + +/// `ourios.service` attribute key. +pub const OURIOS_SERVICE: &str = "ourios.service"; + /// `ourios.tenant` attribute key. pub const OURIOS_TENANT: &str = "ourios.tenant"; diff --git a/crates/ourios-semconv/tests/exports.rs b/crates/ourios-semconv/tests/exports.rs index d3fab9622..526c4735b 100644 --- a/crates/ourios-semconv/tests/exports.rs +++ b/crates/ourios-semconv/tests/exports.rs @@ -20,3 +20,57 @@ fn metric_and_attribute_names_match_the_registry() { "ourios.compaction.result" ); } + +#[test] +fn miner_names_match_the_registry() { + // Arrange / Act / Assert — the RFC 0001 §6.8 miner metric set and + // its attributes, dotted-`ourios.miner.*` per the audited registry + // (`.utilization` for fraction-of-total gauges, `duration` not + // `latency`, no `_total` counter suffix). + assert_eq!( + ourios_semconv::OURIOS_MINER_TEMPLATE_COUNT, + "ourios.miner.template.count" + ); + assert_eq!(ourios_semconv::OURIOS_MINER_MERGES, "ourios.miner.merges"); + assert_eq!( + ourios_semconv::OURIOS_MINER_PARSE_FAILURES, + "ourios.miner.parse_failures" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_PARAMS_OVERFLOW, + "ourios.miner.params.overflow" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_TEMPLATE_VERSION_CHANGES, + "ourios.miner.template.version_changes" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_CONFIDENCE, + "ourios.miner.confidence" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_DURATION, + "ourios.miner.duration" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_CONFIDENCE_P50, + "ourios.miner.confidence.p50" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_CONFIDENCE_P01, + "ourios.miner.confidence.p01" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_BODY_RETENTION_UTILIZATION, + "ourios.miner.body_retention.utilization" + ); + assert_eq!( + ourios_semconv::OURIOS_MINER_PARAMS_OVERFLOW_UTILIZATION, + "ourios.miner.params.overflow.utilization" + ); + assert_eq!(ourios_semconv::OURIOS_SERVICE, "ourios.service"); + assert_eq!( + ourios_semconv::OURIOS_MINER_TEMPLATE_CHANGE, + "ourios.miner.template_change" + ); +} diff --git a/docs/rfcs/0001-template-miner.md b/docs/rfcs/0001-template-miner.md index 135b4948c..ed4e85f71 100644 --- a/docs/rfcs/0001-template-miner.md +++ b/docs/rfcs/0001-template-miner.md @@ -327,11 +327,13 @@ direction and the primary obligation lives in those other RFCs. > instead of the original value > - **And** the `body` column contains the original line bytes > regardless of `lossy_flag` -> - **And** `params_overflow_total{tenant_id, service}` increments +> - **And** `ourios.miner.params.overflow` +> (attributes `ourios.tenant`, `ourios.service`) increments > **Scenario H2.2 — Per-service overflow rate above 1% raises an alert** -> - **Given** the `params_overflow_ratio{tenant_id, service}` -> gauge for some service +> - **Given** the `ourios.miner.params.overflow.utilization` +> gauge (attributes `ourios.tenant`, `ourios.service`) +> for some service > - **When** the rolling rate exceeds `0.01` > - **Then** the documented alert rule fires (the rule ships > alongside §6.5's metric definition) @@ -405,16 +407,29 @@ direction and the primary obligation lives in those other RFCs. > - **Then** the effective threshold is `0.7` > **Scenario §3.1.2 — Mandatory metric set is exposed** -> - **Given** a running miner -> - **When** the miner's meter (`global::meter("ourios.miner")`) is -> collected via an SDK in-memory reader, at zero traffic +> - **Given** the mandatory set defined by §6.8's table — the +> `ourios.miner.*` metric instruments in the semconv registry +> (`semconv/registry/`, surfaced as generated `ourios_semconv` +> constants) that the miner registers on the `ourios.miner` meter +> when it is constructed (the `ourios.miner.alias.*` counters are +> registered separately by the alias map, §6.7, and are out of this +> scenario's scope) +> - **When** a small representative workload exercises every +> instrument (a normal line, a near-duplicate that widens a template, +> an oversized-`param` line, and a parse-failure line) and the meter +> is collected via an SDK in-memory reader > - **Then** the collected metric stream contains every metric named -> in §6.8's table (each present via its init-seeded data point) -> (`template_count`, `merges_total`, `confidence`, -> `confidence_p50`, `confidence_p01`, `body_retention_ratio`, -> `parse_failures_total`, `params_overflow_total`, -> `params_overflow_ratio`, `template_version_changes_total`, -> `miner_latency_seconds`) with the instrument kinds and +> in §6.8's table — each appearing on its **first real measurement**, +> carrying the registry's required attributes (no synthetic +> zero-traffic points) — +> (`ourios.miner.template.count`, `ourios.miner.merges`, +> `ourios.miner.confidence`, `ourios.miner.confidence.p50`, +> `ourios.miner.confidence.p01`, +> `ourios.miner.body_retention.utilization`, +> `ourios.miner.parse_failures`, `ourios.miner.params.overflow`, +> `ourios.miner.params.overflow.utilization`, +> `ourios.miner.template.version_changes`, +> `ourios.miner.duration`) with the instrument kinds and > attributes listed there > **Scenario §3.2.1 — Default per-parameter byte limit is 256** @@ -494,7 +509,7 @@ direction and the primary obligation lives in those other RFCs. > - **When** a line creates the first leaf at that node > - **Then** no event is appended to the audit stream for that > creation -> - **And** `template_count` increments to reflect the new leaf +> - **And** `ourios.miner.template.count` increments to reflect the new leaf > **Scenario RFC0001.2 — Degenerate-template guard rejects fully-wildcard widening** > - **Given** a leaf whose template, after a candidate widening, @@ -502,7 +517,7 @@ direction and the primary obligation lives in those other RFCs. > - **When** the candidate widening is attempted > - **Then** the widening is rejected > - **And** the line is treated as a parse failure -> (`confidence = 0`, body retained, `parse_failures_total` +> (`confidence = 0`, body retained, `ourios.miner.parse_failures` > increments) > - **And** an audit event with > `event_type = template_widening_rejected_degenerate` records @@ -556,22 +571,23 @@ direction and the primary obligation lives in those other RFCs. > followed by a `template_type_expanded` event for the type > expansion (in that order) -> **Scenario RFC0001.8 — confidence_p50 and confidence_p01 are emitted as gauges** -> - **Given** a running miner with a non-empty `confidence` -> histogram for some `(tenant_id, service)` +> **Scenario RFC0001.8 — ourios.miner.confidence.p50 and ourios.miner.confidence.p01 are emitted as gauges** +> - **Given** a running miner with a non-empty `ourios.miner.confidence` +> histogram for some `(ourios.tenant, ourios.service)` > - **When** the miner's meter is collected via an SDK in-memory reader -> - **Then** `confidence_p50` and `confidence_p01` (attributes -> `tenant_id`, `service`) are present as gauges +> - **Then** `ourios.miner.confidence.p50` and +> `ourios.miner.confidence.p01` (attributes `ourios.tenant`, +> `ourios.service`) are present as gauges > - **And** each value matches the corresponding quantile of the > same-attributed histogram (computed in-process on a short > ticker per §6.8) > -> *(Pending the §6.8 dotted-semconv redesign: this scenario may be -> superseded if `confidence_p50` / `confidence_p01` become +> *(The dotted-semconv rename landed in the 2026-06-08 amendment; the +> open fork is whether `confidence.p50` / `confidence.p01` become > backend-derived quantiles over the exported histogram rather than -> in-process gauges. The change is a contract change to the §3.1.2 -> mandatory set and is made under that redesign's own review, not -> the 2026-06-03 architecture amendment.)* +> in-process gauges. That is a contract change to the §3.1.2 mandatory +> set and would be made — possibly superseding this scenario — under +> its own review, not folded into the rename.)* > **Scenario RFC0001.9 — `body_kind = Structured` short-circuits to a structured-template id** > - **Given** an `OtlpLogRecord` whose `body` is @@ -1101,7 +1117,7 @@ For each ingested OTLP `LogRecord`: # reconstruction (§6.6) is byte-identical. # On failure (malformed UTF-8, embedded NUL, line longer # than max-line-bytes): emit a parse-failure record and - # increment parse_failures_total. Skip the rest. + # increment ourios.miner.parse_failures. Skip the rest. # Note: an empty-after-whitespace string (the AnyValue # carries `""` or only whitespace) is not a parse failure # — it has zero tokens and the miner short-circuits with @@ -1140,7 +1156,7 @@ For each ingested OTLP `LogRecord`: simSeq(L_masked, leaf.template) if candidate is None: # no leaves under parent yet; create one. Creation does not - # emit an audit event — `template_count` already reflects + # emit an audit event — `ourios.miner.template.count` already reflects # leaf allocation, and §6.4 reserves the audit stream for # widening events whose semantics need cross-referencing. leaf = new Leaf(template = L_masked) @@ -1171,7 +1187,7 @@ For each ingested OTLP `LogRecord`: old_version, new_version = candidate.version, positions_widened = new_wildcards.positions, ...) - merges_total.inc() + ourios.miner.merges.inc() # Build the params array. One entry per <*> in the (possibly # just-widened) template, in template order. For each slot: @@ -1220,11 +1236,11 @@ For each ingested OTLP `LogRecord`: confidence, lossy_flag = false, body = L_raw) # forced body retention - body_retention_ratio.observe(retained = true) + ourios.miner.body_retention.utilization.observe(retained = true) return # similarity < floor: parse failure - parse_failures_total.inc() + ourios.miner.parse_failures.inc() emit_failure_record(L_raw, reason = "no candidate above floor") ``` @@ -1233,7 +1249,7 @@ Branching invariants: - Step 0's structured short-circuit never enters the Drain mining steps (1–5). Structured-Body records do not widen, do not emit `template_widened` or `template_type_expanded` audit - events, do not contribute to `merges_total`, and never carry + events, do not contribute to `ourios.miner.merges`, and never carry `params`/`separators`. The structured branch's `[§3.1]` preservation is vacuous: no template merge happens, so no silent merge is possible. @@ -1264,8 +1280,8 @@ Branching invariants: `confidence = simSeq / threshold`. The ratio framing makes the decision boundary land at `confidence == 1.0` regardless of the -configured threshold, which gives `confidence_p50` and -`confidence_p01` (`[§3.1]` required metrics) a stable interpretation +configured threshold, which gives `ourios.miner.confidence.p50` and +`ourios.miner.confidence.p01` (`[§3.1]` required metrics) a stable interpretation across tenants with different thresholds: the p01 value tells you how close the *bottom 1% of attaches* are to the merge boundary. A collapsing p01 means many lines are barely passing — a tuning @@ -1282,7 +1298,7 @@ Three zones, with concrete defaults: reconstruction failure, not on lossy zone alone — the body is available either way). - `confidence < floor / threshold`: **parse failure**. - `parse_failures_total` increments; the line is written to a + `ourios.miner.parse_failures` increments; the line is written to a failure record with the original bytes intact. **Defaults.** `threshold = 0.7`, `floor = 0.4`. The threshold floor @@ -1324,10 +1340,11 @@ the schema: ``` `event_type` is the field §6.7's drift-detection query filters on. -`merges_total` increments on every event whose `event_type` is -`template_widened` or `template_type_expanded` (the two structural -widenings); `template_widening_rejected_degenerate` is recorded but -does not increment `merges_total`. The audit stream is written to +`ourios.miner.merges` increments on every event whose `event_type` +is `template_widened` or `template_type_expanded` (the two +structural widenings); `template_widening_rejected_degenerate` is +recorded but does not increment `ourios.miner.merges`. The audit +stream is written to the same WAL as the data records and ends up in a dedicated audit Parquet file per tenant per compaction window (schema in `ourios-parquet`'s RFC, not this one). @@ -1362,7 +1379,7 @@ stream cannot substantiate. template with zero non-wildcard tokens (the entire template becomes `<*> <*> … <*>`), the widening is rejected, the line is treated as a parse failure (`confidence = 0`, retain body, increment -`parse_failures_total`), and an audit event with `event_type = +`ourios.miner.parse_failures`), and an audit event with `event_type = template_widening_rejected_degenerate` records the rejection. A fully-wildcard template provides no parsing value and would swallow arbitrary lines. @@ -1393,11 +1410,11 @@ any param has `type_tag == OVERFLOW`. **Telemetry.** Two metrics for `[§3.2]` and hazard H2: -- `params_overflow_total` (counter, attributes `tenant_id`, - `service`): increments per overflow. -- `params_overflow_ratio` (gauge, attributes `tenant_id`, - `service`): rolling overflow rate. Alert at `> 0.01` per service - per `[§3.2]`. +- `ourios.miner.params.overflow` (counter, attributes + `ourios.tenant`, `ourios.service`): increments per overflow. +- `ourios.miner.params.overflow.utilization` (gauge, attributes + `ourios.tenant`, `ourios.service`): rolling overflow rate. Alert + at `> 0.01` per service per `[§3.2]`. ### 6.6 Body reconstruction `[§3.3]` @@ -1475,7 +1492,7 @@ A template's structural state changes over time as widenings (§6.4) and parameter-type expansions accrue. Each change increments `template_version` and emits an audit event of type `template_widened` or `template_type_expanded`. -`template_version_changes_total` counts these. +`ourios.miner.template.version_changes` counts these. **Two distinct cross-cutting questions.** "Same leaf, different structural snapshots" and "different leaves that mean the same @@ -1603,12 +1620,12 @@ is not acknowledged — there are no in-memory-only aliases. Unlike widenings, alias events are **not** emitted on the ingest hot path; they originate from an explicit operator/control-plane call, so they do not gate `attach` latency. They do **not** increment -`merges_total` (that counter is reserved for the two structural +`ourios.miner.merges` (that counter is reserved for the two structural widenings, §6.4); alias activity is counted separately as the -`alias_assertions_total` / `alias_retractions_total` counters -enumerated in §6.8's telemetry table (mandatory, `tenant_id` +`ourios.miner.alias.assertions` / `ourios.miner.alias.retractions` counters +enumerated in §6.8's telemetry table (mandatory, `ourios.tenant` attribute), keeping operator-driven aliasing first-class telemetry -per `[§3.1]` / §6.3 alongside `merges_total`. +per `[§3.1]` / §6.3 alongside `ourios.miner.merges`. *Materialization and storage.* The durable alias **event log** (the `alias_asserted` / `alias_retracted` stream above) is the source of @@ -1744,18 +1761,30 @@ exactly the H5 detection signal. > §5 note). This amendment fixes the *export architecture* and the > Prometheus-era terminology (registry → meter provider, scrape → > OTLP push, labels → attributes) throughout §§6.8–6.9 and the §5 -> scenarios. It deliberately does **not** rename the metrics: the -> identifiers in the table below stay as-is, pending a dedicated -> dotted-semconv redesign that converts them to the -> `ourios.miner.*` scheme, adds them to the `semconv/registry/` -> weaver registry alongside the compaction set (RFC 0009 §3.6), and -> resolves the instrument-type and quantile questions that are -> genuine contract changes to the §3.1.2 mandatory set — including -> whether `confidence_p50` / `confidence_p01` remain in-process -> gauges (RFC0001.8) or become collector-/backend-derived quantiles -> over the exported histogram, the OTLP-native idiom. That redesign -> moves the mandatory-set contract under its own review rather than -> riding this architecture change. +> scenarios. +> +> **Amendment 2026-06-08 (dotted-semconv migration — landed).** The +> metric and attribute names are now the dotted-`ourios.miner.*` +> scheme, defined in the `semconv/registry/` weaver registry +> alongside the compaction set (RFC 0009 §3.6) and consumed through +> the generated `ourios-semconv` constants. The table below lists the +> registry names. Instrument *kinds* are unchanged from the original +> set; the `confidence.p50` / `confidence.p01` gauges remain +> **in-process** views of the `confidence` histogram (RFC0001.8) — +> the question of whether they become collector-/backend-derived +> quantiles over the exported histogram is a genuine contract change +> to the §3.1.2 mandatory set and stays deferred to its own review, +> not folded into this rename. +> +> The registry names were audited against the OpenTelemetry +> semantic-conventions naming rules (the canonical +> `docs/general/{naming,metrics}.md` plus the weaver registry +> policies): counters drop the Prometheus `_total` suffix and take +> singular `{annotation}` units; the two fraction-of-total gauges use +> the conventional `.utilization` segment (unit `1`) rather than a +> bespoke `.ratio`; and the per-line elapsed-time histogram is +> `ourios.miner.duration` (UCUM `s`), the conventional segment for a +> discrete operation's elapsed time, not `latency`. #### Export architecture (OTel SDK + OTLP) @@ -1797,7 +1826,7 @@ the semantic conventions it MUST be set once on the provider's `Resource` and MUST NOT be repeated on individual data points. The per-measurement dimensions in the table below — among them -`tenant_id`, the originating **service** of the ingested logs, and +`ourios.tenant`, the originating **service** of the ingested logs, and per-metric dimensions like `event_type` — are **data-point attributes**. A single ingester multiplexes many tenants and many source services, and `[§3.1]` / `[§3.2]` require per-`(tenant, @@ -1806,50 +1835,56 @@ alert — which a single producer-level resource attribute could not provide. The `service` dimension here is the *log's source* service (the value §6.1's tenant derivation reads), **distinct from Ourios's own `service.name`** — it must not reuse that reserved resource key. -The table below shows the current attribute names (`tenant_id`, -`service`); like the metric names, they are converted to the -namespaced `ourios.*` dotted-semconv scheme by the deferred redesign -(which fixes the exact key for the source-service dimension) — not -here. +It is exported under the dedicated `ourios.service` attribute key; +the tenant dimension is `ourios.tenant` and the `merges` change-kind +is `ourios.miner.template_change` (registry enum +`widened` / `type_expanded`). All three are defined in +`semconv/registry/attributes.yaml` and consumed through the generated +`ourios-semconv` constants. + +The **mandatory set is defined by the semconv registry**: the +`ourios.miner.*` entries in `semconv/registry/`, surfaced as the +generated `ourios_semconv::OURIOS_MINER_*` constants, are the source +of truth for which metrics the miner must expose. Each is *registered* +as an instrument on the `ourios.miner` meter when the miner is +constructed. OTel's metric model is **collect-on-read**: a reader / exporter sees -the data points produced during a collection cycle, not a registry of -instruments, and a synchronous instrument that has recorded no -measurement contributes no data point. To preserve §3.1.2's -"full mandatory set is exposed" guarantee even at zero traffic, the -miner **emits an initial data point for each mandatory instrument at -init**, so every metric in the table below appears in the first -collection cycle regardless of traffic. The per-kind mechanism is an -implementation detail of the instrumentation slice: a zero `add()` -seeds additive instruments (counters / up-down-counters) without -distorting anything, and observable instruments report through their -callback; histograms are seeded so they surface **without** a -synthetic `record(0)` polluting their distribution. Verification is -therefore by *collecting the metric stream* (an SDK in-memory reader -in tests), not by enumerating registered instruments. - -The metrics enumerated in `[§3.1]` are mandatory. Full set (names and -instrument kinds pending the dotted-semconv redesign noted above; the -dimensions shown are exported as **attributes**, not labels): +the data points produced during a collection cycle, and an instrument +contributes a data point on its **first real measurement**. The miner +emits **no synthetic zero-traffic points** — so every exported series +carries the registry's `required` attributes, with no sentinel +attribute value to collide with a real tenant / service or violate the +`template_change` enum. §3.1.2 is verified by exercising every +instrument with a small representative workload and *collecting the +metric stream* (an SDK in-memory reader in tests): the registry pins +the mandatory set; the collection proves each instrument is registered +and emits real data under the required attributes. + +The metrics enumerated in `[§3.1]` are mandatory. Full set (the +dotted-`ourios.miner.*` registry names; the dimensions shown are +exported as **attributes**, not labels — `tenant` is `ourios.tenant`, +`service` is `ourios.service`): | Metric | Instrument kind | Attributes | Source invariant / hazard | |---|---|---|---| -| `template_count` | gauge | `tenant_id` | `[§3.1]` | -| `merges_total` | counter | `tenant_id`, `event_type` | `[§3.1]`, H1 | -| `alias_assertions_total` | counter | `tenant_id` | `[§3.1]`, §6.7, H5 | -| `alias_retractions_total` | counter | `tenant_id` | `[§3.1]`, §6.7, H5 | -| `confidence` | histogram | `tenant_id`, `service` | `[§3.1]`, §6.3 | -| `confidence_p50` | gauge | `tenant_id`, `service` | `[§3.1]` | -| `confidence_p01` | gauge | `tenant_id`, `service` | `[§3.1]` | -| `body_retention_ratio` | gauge | `tenant_id` | `[§3.1]`, `[§3.3]` | -| `parse_failures_total` | counter | `tenant_id`, `service` | `[§3.1]` | -| `params_overflow_total` | counter | `tenant_id`, `service` | `[§3.2]`, H2 | -| `params_overflow_ratio` | gauge | `tenant_id`, `service` | `[§3.2]`, H2 | -| `template_version_changes_total` | counter | `tenant_id` | `[§3.5]`, H5 | -| `miner_latency_seconds` | histogram | `tenant_id` | hot-path budget (D1) | - -**`confidence_p50` and `confidence_p01`.** The `confidence` -histogram is the source of truth; the two gauges are convenient +| `ourios.miner.template.count` | gauge | `tenant` | `[§3.1]` | +| `ourios.miner.merges` | counter | `tenant`, `template_change` | `[§3.1]`, H1 | +| `ourios.miner.alias.assertions` | counter | `tenant` | `[§3.1]`, §6.7, H5 | +| `ourios.miner.alias.retractions` | counter | `tenant` | `[§3.1]`, §6.7, H5 | +| `ourios.miner.confidence` | histogram | `tenant`, `service` | `[§3.1]`, §6.3 | +| `ourios.miner.confidence.p50` | gauge | `tenant`, `service` | `[§3.1]` | +| `ourios.miner.confidence.p01` | gauge | `tenant`, `service` | `[§3.1]` | +| `ourios.miner.body_retention.utilization` | gauge | `tenant` | `[§3.1]`, `[§3.3]` | +| `ourios.miner.parse_failures` | counter | `tenant`, `service` | `[§3.1]` | +| `ourios.miner.params.overflow` | counter | `tenant`, `service` | `[§3.2]`, H2 | +| `ourios.miner.params.overflow.utilization` | gauge | `tenant`, `service` | `[§3.2]`, H2 | +| `ourios.miner.template.version_changes` | counter | `tenant` | `[§3.5]`, H5 | +| `ourios.miner.duration` | histogram | `tenant` | hot-path budget (D1) | + +**`ourios.miner.confidence.p50` and `ourios.miner.confidence.p01`.** +The `ourios.miner.confidence` histogram is the source of truth; the +two gauges are convenient named views derived from it in-process. The miner recomputes them on a short ticker (default 10 s, configurable; the cost is one quantile evaluation over the histogram per tenant per service per @@ -1883,19 +1918,19 @@ Doing so naively would re-fire every counter increment, every histogram observation, and every gauge update for the entire replay window, polluting steady-state metrics for the post-restart horizon (a 10-minute replay on a high-volume tenant could shift -`merges_total` by orders of magnitude in a few seconds). The miner +`ourios.miner.merges` by orders of magnitude in a few seconds). The miner therefore runs in **replay mode** until the WAL cursor reaches the live tip: domain events are processed and tree state is mutated exactly as in live ingest, but updates to the §6.8 metrics are suppressed (counters do not increment, histograms do not observe, gauges retain their previous value or, if the miner has never -served live traffic, their zero / empty initialisation value). The -instruments' init-seeded data points keep the full §6.8 set visible to -the exporter during replay — suppressing the *update* path leaves each -metric at its seeded / last value, so §3.1.2's "full set exposed" -invariant holds across replay without recording replay-window -measurements. A single `wal_replay_progress` gauge -(attribute `tenant_id`, value: fraction of the tenant's replay +served live traffic, their zero / empty initialisation value). +Suppressing the *update* path means the replay window contributes no +data points — each instrument still surfaces on its first **live** +measurement once replay completes (§3.1.2's registry-defined set is +satisfied by instrument registration plus real post-replay traffic, +not by replay-window points). A single `wal_replay_progress` gauge +(attribute `ourios.tenant`, value: fraction of the tenant's replay window completed in `[0.0, 1.0]`) is exposed during replay so operators can see the cold-start curve and confirm replay finished. This metric is replay-only and is not part of the §3.1 mandatory @@ -2006,9 +2041,10 @@ resolves bidirectionally between RFC and tests. *Covers:* H7.1, H7.4, §3.3.1. - **Corpus tests** on `testdata/corpus/` (fixed, anonymised; see - `docs/benchmarks.md` §1): assert bounds on `template_count`, - `merges_total`, reconstruction accuracy, parameter overflow - rate. Regressions are build failures, not warnings. + `docs/benchmarks.md` §1): assert bounds on + `ourios.miner.template.count`, `ourios.miner.merges`, + reconstruction accuracy, parameter overflow rate. Regressions are + build failures, not warnings. *Covers:* H1.1 (login/logout corpus arm), H7.1 (corpus arm). - **Confidence calibration test**: on a labelled subset of the @@ -2070,13 +2106,17 @@ resolves bidirectionally between RFC and tests. §3.2.1 (default param byte limit = 256), §3.2.2 (limit > 1 KiB rejected). -- **Metric collection test**: collect the miner's meter - (`global::meter("ourios.miner")`) of a freshly-initialised miner via - an SDK in-memory reader and assert the collected stream contains - every §6.8 metric name (each present via its init-seeded data - point), with the instrument kinds and attributes in §6.8's table, - and that the `confidence_p50` / `confidence_p01` gauges track the - same-attributed `confidence` histogram quantiles. +- **Metric collection test**: assert the mandatory set equals the + generated `ourios_semconv::OURIOS_MINER_*` constants (the registry + is the source of truth), then construct a miner, ingest a small + representative workload that exercises every instrument, collect its + meter (`global::meter("ourios.miner")`) via an SDK in-memory reader, + and assert the collected stream contains every §6.8 metric name + (each appearing on its first real measurement, with the required + attributes), with the instrument kinds and attributes in §6.8's + table, and that the `ourios.miner.confidence.p50` / + `ourios.miner.confidence.p01` gauges track the same-attributed + `ourios.miner.confidence` histogram quantiles. *Covers:* §3.1.2, RFC0001.8. - **Data-model contract tests**: small unit tests against the diff --git a/semconv/registry/attributes.yaml b/semconv/registry/attributes.yaml index 1653a6caf..bbe1c31c2 100644 --- a/semconv/registry/attributes.yaml +++ b/semconv/registry/attributes.yaml @@ -9,6 +9,29 @@ groups: stability: development brief: The tenant the telemetry pertains to. examples: ["acme", "t-0042"] + - id: ourios.service + type: string + stability: development + brief: >- + The source service of the ingested logs (the log record's own + `service.name`). Distinct from Ourios's own `service.name` + resource attribute, which identifies the Ourios process. + examples: ["checkout", "billing"] + - id: ourios.miner.template_change + type: + members: + - id: widened + value: "template_widened" + stability: development + brief: A wildcard position was widened to absorb a new line. + - id: type_expanded + value: "template_type_expanded" + stability: development + brief: A slot's type set grew to admit a new parameter type. + stability: development + brief: >- + The kind of template change an `ourios.miner.merges` event + records. - id: ourios.io.direction type: members: diff --git a/semconv/registry/metrics.yaml b/semconv/registry/metrics.yaml index 285c3b6cf..3b13ba5ef 100644 --- a/semconv/registry/metrics.yaml +++ b/semconv/registry/metrics.yaml @@ -85,3 +85,168 @@ groups: attributes: - ref: ourios.tenant requirement_level: required + + # Template-miner metrics (RFC 0001 §6.8). + + - id: metric.ourios.miner.template.count + type: metric + metric_name: ourios.miner.template.count + stability: development + brief: Number of templates a tenant currently has (Drain-tree leaves plus structured-body templates). + instrument: gauge + unit: "{template}" + attributes: + - ref: ourios.tenant + requirement_level: required + + - id: metric.ourios.miner.merges + type: metric + metric_name: ourios.miner.merges + stability: development + brief: Template change events (wildcard widening, slot type-expansion). + instrument: counter + unit: "{merge}" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.miner.template_change + requirement_level: required + + - id: metric.ourios.miner.parse_failures + type: metric + metric_name: ourios.miner.parse_failures + stability: development + brief: Lines that failed to parse and fell back to body retention. + instrument: counter + unit: "{failure}" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.params.overflow + type: metric + metric_name: ourios.miner.params.overflow + stability: development + brief: Per-parameter byte-limit overflow events (§3.2). + instrument: counter + unit: "{overflow}" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.template.version_changes + type: metric + metric_name: ourios.miner.template.version_changes + stability: development + brief: Template-version increments (schema evolution, §3.5 / H5). + instrument: counter + unit: "{change}" + attributes: + - ref: ourios.tenant + requirement_level: required + + - id: metric.ourios.miner.confidence + type: metric + metric_name: ourios.miner.confidence + stability: development + brief: Per-line parse-confidence distribution. + instrument: histogram + unit: "1" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.duration + type: metric + metric_name: ourios.miner.duration + stability: development + brief: Per-line miner processing duration (hot-path budget D1). + instrument: histogram + unit: "s" + attributes: + - ref: ourios.tenant + requirement_level: required + + - id: metric.ourios.miner.confidence.p50 + type: metric + metric_name: ourios.miner.confidence.p50 + stability: development + brief: >- + Median parse confidence, an in-process named view of the + `ourios.miner.confidence` histogram (RFC 0001 §6.8, RFC0001.8). + instrument: gauge + unit: "1" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.confidence.p01 + type: metric + metric_name: ourios.miner.confidence.p01 + stability: development + brief: >- + 1st-percentile parse confidence, an in-process named view of the + `ourios.miner.confidence` histogram (RFC 0001 §6.8, RFC0001.8). + instrument: gauge + unit: "1" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.body_retention.utilization + type: metric + metric_name: ourios.miner.body_retention.utilization + stability: development + brief: Fraction of lines whose original body was retained on a §6.3 confidence-zone path (lossy zone or parse failure). + instrument: gauge + unit: "1" + attributes: + - ref: ourios.tenant + requirement_level: required + + - id: metric.ourios.miner.params.overflow.utilization + type: metric + metric_name: ourios.miner.params.overflow.utilization + stability: development + brief: >- + Fraction of lines with at least one overflowing parameter + (§3.2 / H2 per-service overflow alert). + instrument: gauge + unit: "1" + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.service + requirement_level: recommended + + - id: metric.ourios.miner.alias.assertions + type: metric + metric_name: ourios.miner.alias.assertions + stability: development + brief: Operator-driven template-alias assertions recorded. + instrument: counter + unit: "{assertion}" + attributes: + - ref: ourios.tenant + requirement_level: required + + - id: metric.ourios.miner.alias.retractions + type: metric + metric_name: ourios.miner.alias.retractions + stability: development + brief: Operator-driven template-alias retractions recorded. + instrument: counter + unit: "{retraction}" + attributes: + - ref: ourios.tenant + requirement_level: required