diff --git a/Cargo.lock b/Cargo.lock index 43387f5e..2898b32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3310,7 +3310,6 @@ dependencies = [ name = "ourios-ingester" version = "0.2.1" dependencies = [ - "async-stream", "axum", "flate2", "futures-core", diff --git a/crates/ourios-bench/src/c2.rs b/crates/ourios-bench/src/c2.rs index 0771f1b0..d70cf5a7 100644 --- a/crates/ourios-bench/src/c2.rs +++ b/crates/ourios-bench/src/c2.rs @@ -36,9 +36,22 @@ //! - **Count at 1 M lines**: the count at the sample whose //! 1-based line number is closest to `1_000_000`, floor //! tie-break. Defined only on corpora ≥ 1 M lines. -//! - **Convergence ratio** = `count_1m / SS`, in `(0, 1]`. -//! - **Pass**: `ratio ≥ 0.5` on a ≥ 1 M-line corpus; corpora -//! below 1 M lines abstain (`pass = None`). +//! - **Convergence ratio** = `count_1m / SS`, in `[0, 1]` when +//! defined — `0` when no template has been minted as of the +//! sample nearest the 1 M mark (`count_1m == 0`, `SS > 0`); +//! undefined (`None`) for a ≥ 1 M service that mints zero +//! templates at all (see the gate). +//! - **Pass** (per service, RFC 0006 §3.4.3 as amended for #444): +//! the gate is evaluated **per `service.name`**, since C2 is +//! defined over a single stable service. A corpus passes iff +//! every service with ≥ 1 M lines has `ratio ≥ 0.5` (a zero- +//! template ≥ 1 M service passes trivially — flat count); it +//! abstains (`pass = None`) only when no service reaches 1 M +//! lines. The whole-corpus [`crate::C2Result::convergence_ratio`] +//! is retained as a diagnostic — on a multi-service corpus it +//! conflates a noisy broker with clean application services +//! (`docs/benchmarks.md` §9.12). A single-service (or plain-text +//! ``) corpus collapses to that one service's verdict. use std::collections::BTreeMap; @@ -164,8 +177,9 @@ impl C2Accumulator { /// Attribute a line (and any template creation) to its service. /// The 1 M-line snapshot is taken at exactly the millionth line of - /// *that* service — within one line of the whole-corpus rule's - /// nearest-sample, sufficient for a diagnostic. + /// *that* service — the **gate** basis (RFC 0006 §3.4.3 as amended + /// for #444), and strictly more precise than the whole-corpus rule's + /// nearest-sample, which is now only the diagnostic ratio. fn attribute(&mut self, service: &str, created: bool) { // Cardinality guard: a known service (or a new one below the // cap) keeps its name; once the cap is hit, unseen services fold @@ -218,24 +232,30 @@ impl C2Accumulator { let template_count_at_end = self.curve.last().map_or(0, |s| s.template_count); let corpus_at_least_1m = self.total_lines >= ONE_MILLION; - let (template_count_at_1m_lines, convergence_ratio, pass) = if corpus_at_least_1m { - // Sample whose 1-based line number is closest to - // 1 M; on a tie the earlier (smaller `lines`) - // sample wins — the `(distance, lines)` key makes - // that the strict minimum. - let count_1m = self - .curve - .iter() - .min_by_key(|s| (s.lines.abs_diff(ONE_MILLION), s.lines)) - .map(|s| s.template_count); - let ratio = count_1m.and_then(|c| { - (template_count_at_end > 0).then(|| (c as f64) / (template_count_at_end as f64)) - }); - let pass = ratio.map(|r| r >= 0.5); - (count_1m, ratio, pass) - } else { - (None, None, None) - }; + // Whole-corpus convergence — a **diagnostic** now, not the gate + // (the gate is per-service below, RFC 0006 §3.4.3 as amended + // for #444): on a multi-service corpus a whole-corpus ratio + // conflates a noisy broker with clean application services + // (v8 §9.12). Undefined (both `None`) below 1 M lines *and* when + // the corpus mints zero templates (SS = 0, ratio 0/0): the count + // and the ratio stay a matched pair — both `Some` or both `None`, + // never mixed — which the report layer relies on (report.rs). + let (template_count_at_1m_lines, convergence_ratio) = + if corpus_at_least_1m && template_count_at_end > 0 { + // Sample whose 1-based line number is closest to + // 1 M; on a tie the earlier (smaller `lines`) + // sample wins — the `(distance, lines)` key makes + // that the strict minimum. + let count_1m = self + .curve + .iter() + .min_by_key(|s| (s.lines.abs_diff(ONE_MILLION), s.lines)) + .map(|s| s.template_count); + let ratio = count_1m.map(|c| (c as f64) / (template_count_at_end as f64)); + (count_1m, ratio) + } else { + (None, None) + }; // Per-service decomposition, largest service first. Each // service's gate follows §3.4.3 on its own line count; template @@ -245,10 +265,23 @@ impl C2Accumulator { .by_service .into_iter() .map(|(service_name, s)| { - let (at_1m, ratio, pass) = if s.lines >= ONE_MILLION && s.created > 0 { - let c = s.created_at_1m.unwrap_or(s.created); - let ratio = (c as f64) / (s.created as f64); - (Some(c), Some(ratio), Some(ratio >= 0.5)) + let (at_1m, ratio, pass) = if s.lines >= ONE_MILLION { + if s.created > 0 { + let c = s.created_at_1m.unwrap_or(s.created); + let ratio = (c as f64) / (s.created as f64); + (Some(c), Some(ratio), Some(ratio >= 0.5)) + } else { + // >= 1 M lines but zero templates minted (every line + // NO_TEMPLATE): the count is flat at zero, the strongest + // possible convergence — C2's falsifier is *linear* + // growth, so this passes trivially with an undefined + // ratio. Gated (`Some`), never abstaining, so + // `gate_pass`'s `None` keeps meaning "no service reached + // 1 M lines" (an all-NO_TEMPLATE service is a body- + // retention / parse-failure concern, caught by §3.1's + // counters — not a convergence failure). + (Some(0), None, Some(true)) + } } else { (None, None, None) }; @@ -268,6 +301,8 @@ impl C2Accumulator { .then(a.service_name.cmp(&b.service_name)) }); + let pass = gate_pass(&by_service); + C2Result { sample_cadence: self.cadence, total_lines: self.total_lines, @@ -283,6 +318,33 @@ impl C2Accumulator { } } +/// The per-service C2 **gate** (RFC 0006 §3.4.3 as amended for #444): +/// a corpus passes iff every service with ≥ 1 M lines passes its own +/// ratio ≥ 0.5. `finalize` sets `pass = Some(_)` for *every* ≥ 1 M +/// service (a zero-template service passes trivially — flat count), so +/// the fold below considers exactly the gated services (`s.pass.is_some()`) +/// and `None` means "no service reached 1 M lines", never a +/// silently-dropped ≥ 1 M service. A single-service +/// corpus — including the plain-text `` bucket — is gated on +/// that one service's ratio, measured at its **exact** millionth line +/// (`created_at_1m`). That reproduces the pre-#444 whole-corpus verdict +/// for every historical converged corpus (their ratio sits far from the +/// 0.5 boundary); it is not bit-identical to the whole-corpus +/// `convergence_ratio`, which is sampled at the nearest curve point and +/// is only a diagnostic. Only multi-service OTLP corpora change verdict. +fn gate_pass(by_service: &[PerServiceC2]) -> Option { + let mut verdict = None; + for s in by_service { + match s.pass { + // Any gated service that fails is decisive — short-circuit. + Some(false) => return Some(false), + Some(true) => verdict = Some(true), + None => {} + } + } + verdict +} + /// The record's `service.name` resource attribute, or a sentinel when /// absent. Borrowed — the caller copies into the map key only on a /// first sighting. @@ -342,21 +404,27 @@ mod tests { } /// A ≥ 1 M-line corpus with a bounded alphabet plateaus - /// immediately, so `count_1m == SS` → ratio 1.0 → pass. - /// Exercises the full ≥ 1 M gate math at scale without the - /// miner. + /// immediately, so `count_1m == SS` → whole-corpus diagnostic + /// ratio 1.0. Exercises the ≥ 1 M ratio math at scale without + /// the miner; the per-service *gate* abstains here (no + /// `service.name`), as the body notes. #[test] - fn stable_corpus_passes_the_gate() { + fn stable_curve_ratio_is_one() { + // `run_stable` drives `observe` (no `service.name`), so it + // exercises the whole-corpus ratio *diagnostic*; the per-service + // gate needs record input and is covered by `gate_pass_*` + + // the partition test. let r = run_stable(1_000_000, 8); assert!(r.corpus_at_least_1m); assert_eq!(r.template_count_at_end, 8); assert_eq!(r.template_count_at_1m_lines, Some(8)); assert_eq!(r.convergence_ratio, Some(1.0)); - assert_eq!(r.pass, Some(true)); + // No service data → the per-service gate abstains. + assert_eq!(r.pass, None); } - /// A corpus below 1 M lines abstains: no 1 M count, no - /// ratio, `pass = None`. + /// A corpus below 1 M lines has no 1 M count, no ratio; the gate + /// abstains. #[test] fn short_corpus_abstains() { let r = run_stable(10_000, 5); @@ -369,6 +437,33 @@ mod tests { assert_eq!(r.template_count_at_end, 5); } + /// The per-service gate fold: pass iff every ≥ 1 M service passes; + /// abstain when none are gated; a `<1 M` service (pass = None) does + /// not veto a passing sibling. + #[test] + fn gate_pass_folds_over_gated_services() { + let svc = |name: &str, pass: Option| PerServiceC2 { + service_name: name.to_string(), + lines: 0, + templates_created: 0, + templates_created_at_1m_lines: None, + convergence_ratio: None, + pass, + }; + // No gated service → abstain. + assert_eq!(gate_pass(&[svc("a", None), svc("b", None)]), None); + // All gated services pass → pass. + assert_eq!( + gate_pass(&[svc("a", Some(true)), svc("b", None), svc("c", Some(true))]), + Some(true) + ); + // One gated service fails → fail (even with passing siblings). + assert_eq!( + gate_pass(&[svc("a", Some(true)), svc("b", Some(false))]), + Some(false) + ); + } + /// A corpus whose template count is still climbing steeply /// at 1 M lines (no plateau) fails the gate: `count_1m` is /// far under half the end count. @@ -396,13 +491,11 @@ mod tests { let ratio = r.convergence_ratio.expect("ratio on ≥1M corpus"); assert!( ratio < 0.5, - "templates still climbing at 1 M → ratio {ratio} must be < 0.5", - ); - assert_eq!( - r.pass, - Some(false), - "a non-converged corpus must fail the C2 gate", + "templates still climbing at 1 M → whole-corpus ratio {ratio} must be < 0.5", ); + // `observe` has no service data, so the per-service gate abstains + // here — the fold's fail path is covered by `gate_pass_*`. + assert_eq!(r.pass, None); } /// A `MinedRecord` carrying just the two fields the per-service @@ -524,4 +617,38 @@ mod tests { "every line is attributed to some bucket", ); } + + /// A service that clears the 1 M-line floor but mints zero templates + /// (every line `NO_TEMPLATE`) is **gated** and passes trivially — its + /// count is flat at zero, the opposite of the linear growth C2 flags. + /// Regression for the fold (#451): such a service must not collapse to + /// `pass = None` and get silently dropped, which would leave `None` + /// meaning both "below 1 M lines" and "≥ 1 M but degenerate". + #[test] + fn zero_template_service_over_1m_passes_trivially() { + let quiet = rec(NO_TEMPLATE, "quiet-svc"); + let mut acc = C2Accumulator::new(ONE_MILLION); + for _ in 0..ONE_MILLION { + acc.record(&quiet); + } + let r = acc.finalize(); + let svc = r + .by_service + .iter() + .find(|s| s.service_name == "quiet-svc") + .expect("quiet-svc bucket"); + assert_eq!(svc.lines, ONE_MILLION); + assert_eq!(svc.templates_created, 0, "no template ever minted"); + assert_eq!(svc.convergence_ratio, None, "0/0 ratio is undefined"); + assert_eq!(svc.pass, Some(true), "flat count → trivial convergence"); + // Gated as a PASS, not folded away as an abstention. + assert_eq!(r.pass, Some(true)); + assert_eq!(r.template_count_at_end, 0); + // Whole-corpus diagnostic stays a matched pair (SS = 0 → both + // `None`, never the mixed `(Some(0), None)` the report layer + // rejects) even though the corpus cleared 1 M lines. + assert!(r.corpus_at_least_1m); + assert_eq!(r.template_count_at_1m_lines, None); + assert_eq!(r.convergence_ratio, None); + } } diff --git a/crates/ourios-bench/src/lib.rs b/crates/ourios-bench/src/lib.rs index 604f0185..b0e3c900 100644 --- a/crates/ourios-bench/src/lib.rs +++ b/crates/ourios-bench/src/lib.rs @@ -583,27 +583,40 @@ pub struct C1Mismatch { pub actual: String, } -/// §3.6 `c2` block (populated only when C2 ran). `pass` is -/// `None` when the corpus is `< 1 M lines` (§3.4.3 abstention). +/// §3.6 `c2` block (populated only when C2 ran). +/// +/// The gate is **per service** (RFC 0006 §3.4.3 as amended for #444): +/// `pass` is the fold over [`Self::by_service`] — `Some(true)` iff every +/// service with ≥ 1 M lines passes its own ratio ≥ 0.5 (a service that +/// mints zero templates over its ≥ 1 M lines passes *trivially* — SS = 0, +/// an undefined 0/0 ratio, a flat count being the strongest convergence), +/// `Some(false)` if any ≥ 1 M service with a defined ratio fails, `None` +/// when no service reaches 1 M lines. +/// The whole-corpus [`Self::convergence_ratio`] / +/// [`Self::template_count_at_1m_lines`] are retained as **diagnostics** +/// (on a multi-service corpus they conflate a noisy broker with clean +/// application services — v8 §9.12). #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct C2Result { pub sample_cadence: u64, pub total_lines: u64, + /// Whole-corpus diagnostic (not the gate — see the type doc). pub template_count_at_1m_lines: Option, pub template_count_at_end: u64, + /// Whole-corpus diagnostic (not the gate — see the type doc). pub convergence_ratio: Option, pub convergence_curve: Vec, + /// The per-service gate verdict (see the type doc). pub pass: Option, pub corpus_at_least_1m: bool, - /// Per-`service.name` convergence, largest service first. - /// **Diagnostic only** — the gate above is defined on the whole - /// corpus; this decomposition attributes it. On a multi-service - /// corpus (every OTel-Demo capture) a whole-corpus C2 conflates a - /// noisy broker with clean application services, so this surfaces - /// where non-convergence actually lives (v8 §9.12 / #444). A - /// plain-text corpus (no `service.name`) collapses to a single - /// `` bucket rather than being empty; empty only when C2 - /// did not run. + /// Per-`service.name` convergence, largest service first — the + /// **gate basis** (RFC 0006 §3.4.3 as amended for #444). On a + /// multi-service corpus (every OTel-Demo capture) a whole-corpus C2 + /// conflates a noisy broker with clean application services, so the + /// gate is evaluated here per service; this also surfaces where + /// non-convergence actually lives (v8 §9.12). A plain-text corpus + /// (no `service.name`) collapses to a single `` bucket + /// rather than being empty; empty only when C2 did not run. #[serde(default)] pub by_service: Vec, /// The distinct-`service.name` cap (`MAX_SERVICES`) was hit and diff --git a/crates/ourios-bench/src/main.rs b/crates/ourios-bench/src/main.rs index fc9584ca..351fc1a5 100644 --- a/crates/ourios-bench/src/main.rs +++ b/crates/ourios-bench/src/main.rs @@ -315,38 +315,38 @@ fn print_summary(results: &ourios_bench::ResultsFile) { ); } if let Some(c2) = &results.c2 { - // `pass = None` is the §3.4.3 abstention (corpus - // < 1 M lines) — surface it as ABSTAIN, not a silent - // omission. (C2 isn't computed yet; this line is ready - // for when it lands.) + // The gate is per service (RFC 0006 §3.4.3, #444). `None` is the + // abstention — no service reaches 1 M lines. let verdict = match c2.pass { Some(true) => "PASS", Some(false) => "FAIL", - None => "ABSTAIN (corpus < 1 M lines)", + None => "ABSTAIN (no service ≥ 1 M lines)", }; let ratio = c2 .convergence_ratio .map_or_else(|| "n/a".to_string(), |r| format!("{r:.3}")); + // The verdict is per-service; the whole-corpus ratio is a + // diagnostic, so it is labelled as such to avoid reading as the + // gate (on a multi-service corpus the two can disagree). println!( - " C2 convergence: ratio {ratio} (end template count {}, sample cadence {}) — {verdict}", + " C2 convergence (per-service gate): {verdict} \ + — whole-corpus ratio {ratio} (diagnostic; end templates {}, cadence {})", c2.template_count_at_end, c2.sample_cadence, ); - // Per-service decomposition (diagnostic) — printed whenever the - // corpus resolves to more than one bucket (distinct `service.name` - // values plus any ``/``), since a whole-corpus - // ratio then conflates a noisy broker with clean application - // services (v8 §9.12 / #444). - if c2.by_service.len() > 1 { - println!(" C2 by service (diagnostic; creations sum to the end count):"); + // Per-service breakdown — the gate basis, so it is printed + // whenever any service bucket exists (including a single-service + // or plain-text `` corpus): the whole-corpus line above + // is only the diagnostic, and the operator needs the per-service + // measurement to reproduce the verdict. + if !c2.by_service.is_empty() { + println!(" C2 by service (the gate; creations sum to the end count):"); for svc in &c2.by_service { - let per = match (svc.convergence_ratio, svc.pass) { - (Some(r), Some(true)) => format!("ratio {r:.3} PASS"), - (Some(r), Some(false)) => format!("ratio {r:.3} FAIL"), - _ => "abstain (< 1 M lines)".to_string(), - }; println!( - " {:<24} {:>10} lines, {:>7} created — {per}", - svc.service_name, svc.lines, svc.templates_created, + " {:<24} {:>10} lines, {:>7} created — {}", + svc.service_name, + svc.lines, + svc.templates_created, + per_service_status(svc), ); } if c2.services_truncated { @@ -356,10 +356,51 @@ fn print_summary(results: &ourios_bench::ResultsFile) { } } +/// One-line per-service C2 status for the CLI breakdown. The gate is the +/// per-service ratio; a zero-template ≥ 1 M service (SS = 0) reads as a +/// trivial pass with no ratio, and a service below 1 M lines abstains. +fn per_service_status(svc: &ourios_bench::PerServiceC2) -> String { + match (svc.convergence_ratio, svc.pass) { + (Some(r), Some(true)) => format!("ratio {r:.3} PASS"), + (Some(r), Some(false)) => format!("ratio {r:.3} FAIL"), + (None, Some(true)) => "0 templates, converged PASS".to_string(), + (_, Some(false)) => "FAIL".to_string(), + (_, None) => "abstain (< 1 M lines)".to_string(), + } +} + #[cfg(test)] mod tests { use super::*; + /// The per-service CLI status covers every gate state: a defined + /// ratio that passes/fails, the SS = 0 trivial pass (no ratio), and a + /// sub-1M abstention. + #[test] + fn per_service_status_covers_gate_states() { + let mk = |ratio, pass| ourios_bench::PerServiceC2 { + service_name: "s".to_string(), + lines: 1_000_000, + templates_created: 5, + templates_created_at_1m_lines: None, + convergence_ratio: ratio, + pass, + }; + assert_eq!( + per_service_status(&mk(Some(0.9), Some(true))), + "ratio 0.900 PASS" + ); + assert_eq!( + per_service_status(&mk(Some(0.2), Some(false))), + "ratio 0.200 FAIL" + ); + assert_eq!( + per_service_status(&mk(None, Some(true))), + "0 templates, converged PASS" + ); + assert_eq!(per_service_status(&mk(None, None)), "abstain (< 1 M lines)"); + } + /// RFC0006.5 — `--hardware-kind` is required unless /// `--allow-unknown-hardware`. clap rejects the bare /// invocation at parse time, before any measurement runs. diff --git a/crates/ourios-bench/src/report.rs b/crates/ourios-bench/src/report.rs index 9c9ebd23..740a1c11 100644 --- a/crates/ourios-bench/src/report.rs +++ b/crates/ourios-bench/src/report.rs @@ -206,22 +206,35 @@ pub fn update_status_section(md: &str, results: &ResultsFile) -> Result 0), both `None` + // otherwise (corpus < 1 M lines, or a ≥ 1 M corpus that mints zero + // templates — SS = 0, a 0/0 ratio). Any other combination is a + // corrupt `ResultsFile` — a mixed pair, or a ≥ 1 M / SS > 0 corpus + // with no measurement — and is surfaced rather than rendered as a + // self-contradictory row. let measurement = match (c2.template_count_at_1m_lines, c2.convergence_ratio) { (Some(count_1m), Some(ratio)) => { + // Whole-corpus ratio — a diagnostic since #444; the gate + // verdict (Verdict column) is per-service, so a + // multi-service corpus can show a sub-0.5 ratio here with a + // PASS verdict. Label it so the two aren't conflated. format!( - "ratio {ratio:.3} (count@1M {count_1m} / SS {})", + "whole-corpus ratio {ratio:.3} (count@1M {count_1m} / SS {}) — diagnostic; verdict is per-service", c2.template_count_at_end, ) } - (None, None) => format!("n/a (SS {}, corpus < 1 M lines)", c2.template_count_at_end), + (None, None) if c2.corpus_at_least_1m && c2.template_count_at_end == 0 => { + "n/a (SS 0 — no templates mined)".to_string() + } + (None, None) if !c2.corpus_at_least_1m => { + format!("n/a (SS {}, corpus < 1 M lines)", c2.template_count_at_end) + } _ => { return Err(BenchError::Report { detail: "C2 result is inconsistent: convergence_ratio and \ - template_count_at_1m_lines must both be set or both absent \ + template_count_at_1m_lines must both be set exactly when the \ + corpus is ≥ 1 M lines with SS > 0, and both absent otherwise \ (§3.4.3)" .to_string(), }); @@ -748,6 +761,62 @@ mod tests { assert!(matches!(err, BenchError::Report { .. }), "got {err:?}"); } + /// A ≥ 1 M corpus that mints zero templates (SS = 0) is a *consistent* + /// `(None, None)` diagnostic pair, not the inconsistent state above: + /// it renders an "SS 0 — no templates mined" measurement (distinct + /// from the sub-1M "corpus < 1 M lines" abstention) with the + /// per-service verdict, and must not error (#444/#451). + #[test] + fn zero_template_corpus_renders_no_templates_mined() { + let mut r = sample_results(); + r.c2 = Some(crate::C2Result { + sample_cadence: 1000, + total_lines: 1_000_000, + template_count_at_1m_lines: None, + template_count_at_end: 0, + convergence_ratio: None, + convergence_curve: Vec::new(), + pass: Some(true), + corpus_at_least_1m: true, + by_service: Vec::new(), + services_truncated: false, + }); + let md = update_status_section(&md_with_status(), &r).expect("SS=0 corpus must render"); + assert!(md.contains("| C2 |"), "C2 row present"); + assert!( + md.contains("no templates mined"), + "SS=0 ≥1M renders the no-templates measurement, not < 1 M lines: {md}", + ); + assert!( + !md.contains("corpus < 1 M lines"), + "not the sub-1M abstention" + ); + } + + /// A ≥ 1 M corpus with `SS > 0` but no convergence measurement + /// (`(None, None)`) is impossible from `finalize` → a corrupt + /// `ResultsFile`. It must error, not render a self-contradictory + /// "SS 42 — no templates mined" row (#451). + #[test] + fn absent_measurement_on_nonempty_1m_corpus_errors() { + let mut r = sample_results(); + r.c2 = Some(crate::C2Result { + sample_cadence: 1000, + total_lines: 1_000_000, + template_count_at_1m_lines: None, + template_count_at_end: 42, // SS > 0 … + convergence_ratio: None, // … but no measurement + convergence_curve: Vec::new(), + pass: Some(true), + corpus_at_least_1m: true, + by_service: Vec::new(), + services_truncated: false, + }); + let err = update_status_section(&md_with_status(), &r) + .expect_err("≥1M with SS>0 but no measurement must error"); + assert!(matches!(err, BenchError::Report { .. }), "got {err:?}"); + } + /// A malformed `timestamp` (not RFC3339-shaped) errors /// rather than rendering an empty "updated " date. #[test] diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 4f03ac97..5f686ee6 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -259,6 +259,14 @@ every run. - **Bar**: must-win. - **Metric**: template count as a function of lines ingested, on a corpus from a single stable service. +- **Grain (amended for #444, 2026-07-10)**: because the metric is + defined *per stable service*, the gate is evaluated **per + `service.name`** on a multi-service corpus, not on the whole corpus. + A corpus passes iff every service with ≥ 1 M lines converges; a + single-service (or plain-text ``) corpus is gated on that + one service's exact-millionth-line ratio, reproducing the + pre-amendment verdict for historical converged corpora. The + whole-corpus ratio is retained as a diagnostic. See RFC 0006 §3.4.3. - **Target**: template count grows **sub-linearly** and plateaus within **2×** of its steady-state value by 1 M lines. Steady-state value is corpus-specific but is on the order of 10²–10⁴ templates @@ -1102,11 +1110,17 @@ bodies retained, and C1 = 1.000000 over the remaining 4,948,579 rows — the honesty contract holds at 4.9 M rows through failure-mode churn. -**C2 — template-count convergence (bar: ratio ≥ 0.5 at 1 M lines): -FAIL on the whole corpus — attributed.** Ratio **0.199**, end -template count **14,631** (sample cadence 4,833). The per-service -decomposition (splitting the corpus on `service.name` and re-running -the gates per service) localises the failure completely: +**C2 — template-count convergence (bar: ratio ≥ 0.5 at 1 M lines, +evaluated per service since #444): PASS.** Under the per-service gate +(RFC 0006 §3.4.3, amended 2026-07-10) the corpus passes: the only +service that clears the 1 M-line evaluation floor is **cart**, which +converges at ratio **1.000** with two templates. Every other service +abstains for want of volume; the whole-corpus ratio (**0.199**, end +template count **14,631**, sample cadence 4,833) is retained below as +a diagnostic — it is a category error to grade a multi-service corpus +as one Drain stream (§3.4.3 rationale). The per-service decomposition +(splitting on `service.name` and re-running the gates per service) +localises the whole-corpus fragmentation completely: | service | lines | end templates | C2 | |---|---|---|---| @@ -1116,9 +1130,14 @@ the gates per service) localises the failure completely: | ad | 486,726 | 3 | abstain (< 1 M) | | **kafka** | **136,790** | **14,608** | abstain (< 1 M) | -Every application service converges essentially perfectly — cart -passes the formal gate at 2.76 M lines with **two** templates. The -kafka broker mints 14,608 templates on 2.8 % of the lines. Mechanism +The gate folds over the gated services (those ≥ 1 M lines): cart is +the sole such service and it passes, so the corpus passes. cart clears +the formal gate at 2.76 M lines with **two** templates; the smaller +services abstain below the 1 M-line floor, so they are not graded — +though their *observed* counts (1–17 templates over 0.5–1.0 M lines) +sit at the same near-flat convergence. The kafka broker, also +abstaining, is the outlier: it mints 14,608 templates on 2.8 % of the +lines. Mechanism (measured): kafka's cleaner logs emit **3-token lines whose third token is a unique offset-bearing path** (`Deleted log /tmp/kafka-logs/…/00000000000000000429.log.deleted.`, @@ -1131,16 +1150,17 @@ length-aware thresholding vs. accept-and-scope-C2-per-service — an RFC-level pillar #2 decision); the safety story held throughout (bounded memory per RFC 0023, per-service C1 perfect). -The per-service decomposition is now a **first-class bench diagnostic** -(`ourios-bench --gates c2` prints it whenever a corpus resolves to more -than one bucket — distinct `service.name` values plus any -``/``); template creation is a globally-monotonic +The per-service decomposition is now the **first-class bench gate** +(`ourios-bench --gates c2` prints it whenever any service bucket exists +— distinct `service.name` values plus any ``/``, so a +single-service or plain-text corpus shows its one gated row too); +template creation is a globally-monotonic event attributed to the minting service, so per-service creations partition the whole-corpus count exactly (2 + 17 + 1 + 3 + 14,608 = -14,631) in `O(services)` memory — no per-service id set. The gate -itself is unchanged (whole-corpus); the breakdown is additive, so -option 3 of #444 ("scope C2 per service") can be evaluated on real -numbers without a code change first. +14,631) in `O(services)` memory — no per-service id set. As of #444 +(option 3) this decomposition **is** the gate: C2 is evaluated per +service and folds over the services that clear the 1 M-line floor, +with the whole-corpus ratio kept as a diagnostic (RFC 0006 §3.4.3). **What the fragmentation actually costs — B2 pricing (indicative, local M-series).** Running the B2 windowed query on the fragmented diff --git a/docs/rfcs/0006-bench-harness.md b/docs/rfcs/0006-bench-harness.md index 0702d62f..6e41d080 100644 --- a/docs/rfcs/0006-bench-harness.md +++ b/docs/rfcs/0006-bench-harness.md @@ -455,13 +455,64 @@ Pinned definitions: line, zero-indexed). When two samples are equidistant, the earlier one wins (floor tie-break). Defined only on corpora of `≥ 1_000_000` lines. -- **Convergence ratio**: `count_at_1m / SS`. By monotonicity, - this lives in `(0.0, 1.0]`. -- **Pass condition** (gate): `convergence_ratio ≥ 0.5` on a - corpus of `≥ 1_000_000` lines. This is the "within 2× of - SS by 1 M lines" rule. Corpora smaller than 1 M lines are - recorded as `c2.pass = null` (insufficient data); the §9 - row notes the corpus size and the gate is not asserted. +- **Convergence ratio**: `count_at_1m / SS`, defined only when + `SS > 0`. By monotonicity (`count_at_1m ≤ SS`) it is `≤ 1.0`; + it is `0.0` when no template has been minted as of the sample + nearest the 1 M-line mark (`count_at_1m == 0`, `SS > 0`) — + `count_at_1m` is that nearest sample, not the exact millionth + line — so the defined range is `[0.0, 1.0]`. It is **undefined** + (`null`, paired with a `null` `count_at_1m`) when `SS == 0` — a + ≥ 1 M corpus that mints no templates at all, a `0/0` ratio. +- **Pass condition** (gate) — **per service** (amended for + #444, maintainer-approved 2026-07-10): C2 is defined over + "a corpus from a single **stable service**", so on a + multi-service corpus the gate is evaluated **per + `service.name`**, not on the whole corpus. Each service's + ratio is `count_at_1m / SS` over *that service's* lines, with + `count_at_1m` taken at that service's **exact** millionth line + (not the whole-corpus nearest-sample; template creation is a + globally-monotonic event attributed to the minting service, so + per-service creations partition the whole-corpus template count + exactly). A corpus **passes** iff every service with + `≥ 1_000_000` lines has ratio `≥ 0.5` — with one exception: a + service that mints **zero** templates over its ≥ 1 M lines + (`SS == 0`, an undefined `0/0` ratio) passes *trivially*, since + a flat-zero count is the strongest possible convergence (C2's + falsifier is *linear* growth; an all-`NO_TEMPLATE` service is a + body-retention / parse-failure concern, caught by §3.1's + counters, not a convergence failure). It **fails** if any ≥ 1 M + service has a defined ratio below `0.5`; it **abstains** + (`c2.pass = null`) when no service reaches 1 M lines. A + single-service corpus — including the plain-text `` + bucket (no `service.name`) — is gated on that one service's + ratio, measured at its **exact** millionth line. That + reproduces the pre-amendment whole-corpus verdict for every + historical converged corpus (whose ratio sits far from the + 0.5 boundary); it is not bit-identical to the whole-corpus + `convergence_ratio`, which is sampled at the nearest curve + point (cadence granularity) and is now only a diagnostic. + Only multi-service OTLP corpora change verdict. **Rationale**: + running one whole-corpus ratio over a multi-service capture + (e.g. the OTel-Demo) is a category error — it conflates a + noisy infra service (a broker emitting high-cardinality + offset/path tokens) with clean application services, so the + whole-corpus number fails even when every application service + converges perfectly (v8 §9.12). The whole-corpus + `convergence_ratio` is retained as a **diagnostic** (the + `by_service` breakdown is the gate basis). Note: token-level + polishing of high-cardinality infra logs is an OTel Collector + concern (a `transform`/`redaction` processor upstream), not + the miner's — consistent with "format parsing is the + Collector's job". **Cardinality cap**: the decomposition holds + at most `MAX_SERVICES = 1024` distinct `service.name` buckets + (an O(services) memory guard mirroring §3.2); beyond that, + further services fold into one `` bucket and + `c2.services_truncated` is set. A real OTLP capture carries + tens of services, so the cap is not expected to bind; if it + does, the folded `` bucket mixes services and its + per-service ratio is no longer strictly single-service — + `services_truncated` flags that the run should be re-scoped + (the truncation is surfaced, never silent). - **Plateau-detection diagnostic** (not a gate): the curve is "plateaued" at the sample where the trailing `K = 64` samples all lie within `± 5%` of the SS. The diagnostic is @@ -470,9 +521,12 @@ Pinned definitions: RFC — the gate is the 2× rule above. Reported as: `template_count_at_1m_lines` (integer; `null` for -corpora < 1 M lines), `template_count_at_end` (integer; -this is SS), `convergence_ratio` (three-decimal float; `null` -for short corpora), `pass` (bool or `null`), +corpora < 1 M lines **or** a ≥ 1 M corpus with `SS == 0`), +`template_count_at_end` (integer; this is SS), +`convergence_ratio` (three-decimal float; `null` under the same +two conditions). These two form a matched pair — both `null` or +both set, never mixed — which the report layer relies on +(`report.rs` errors on a mixed pair). `pass` (bool or `null`), `corpus_at_least_1m` (bool). v1 records the convergence curve in the results JSON (as