Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

200 changes: 163 additions & 37 deletions crates/ourios-bench/src/c2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,22 @@
//! - **Count at 1 M lines**: the count at the sample whose
//! 1-based line number is closest to `1_000_000`, floor
//! tie-break. Defined only on corpora ≥ 1 M lines.
//! - **Convergence ratio** = `count_1m / SS`, in `(0, 1]`.
//! - **Pass**: `ratio ≥ 0.5` on a ≥ 1 M-line corpus; corpora
//! below 1 M lines abstain (`pass = None`).
//! - **Convergence ratio** = `count_1m / SS`, in `[0, 1]` when
//! defined — `0` when no template has been minted as of the
//! sample nearest the 1 M mark (`count_1m == 0`, `SS > 0`);
//! undefined (`None`) for a ≥ 1 M service that mints zero
//! templates at all (see the gate).
//! - **Pass** (per service, RFC 0006 §3.4.3 as amended for #444):
//! the gate is evaluated **per `service.name`**, since C2 is
//! defined over a single stable service. A corpus passes iff
//! every service with ≥ 1 M lines has `ratio ≥ 0.5` (a zero-
//! template ≥ 1 M service passes trivially — flat count); it
//! abstains (`pass = None`) only when no service reaches 1 M
//! lines. The whole-corpus [`crate::C2Result::convergence_ratio`]
//! is retained as a diagnostic — on a multi-service corpus it
//! conflates a noisy broker with clean application services
//! (`docs/benchmarks.md` §9.12). A single-service (or plain-text
//! `<unknown>`) corpus collapses to that one service's verdict.

use std::collections::BTreeMap;

Expand Down Expand Up @@ -164,8 +177,9 @@ impl C2Accumulator {

/// Attribute a line (and any template creation) to its service.
/// The 1 M-line snapshot is taken at exactly the millionth line of
/// *that* service — within one line of the whole-corpus rule's
/// nearest-sample, sufficient for a diagnostic.
/// *that* service — the **gate** basis (RFC 0006 §3.4.3 as amended
/// for #444), and strictly more precise than the whole-corpus rule's
/// nearest-sample, which is now only the diagnostic ratio.
fn attribute(&mut self, service: &str, created: bool) {
// Cardinality guard: a known service (or a new one below the
// cap) keeps its name; once the cap is hit, unseen services fold
Expand Down Expand Up @@ -218,24 +232,30 @@ impl C2Accumulator {
let template_count_at_end = self.curve.last().map_or(0, |s| s.template_count);
let corpus_at_least_1m = self.total_lines >= ONE_MILLION;

let (template_count_at_1m_lines, convergence_ratio, pass) = if corpus_at_least_1m {
// Sample whose 1-based line number is closest to
// 1 M; on a tie the earlier (smaller `lines`)
// sample wins — the `(distance, lines)` key makes
// that the strict minimum.
let count_1m = self
.curve
.iter()
.min_by_key(|s| (s.lines.abs_diff(ONE_MILLION), s.lines))
.map(|s| s.template_count);
let ratio = count_1m.and_then(|c| {
(template_count_at_end > 0).then(|| (c as f64) / (template_count_at_end as f64))
});
let pass = ratio.map(|r| r >= 0.5);
(count_1m, ratio, pass)
} else {
(None, None, None)
};
// Whole-corpus convergence — a **diagnostic** now, not the gate
// (the gate is per-service below, RFC 0006 §3.4.3 as amended
// for #444): on a multi-service corpus a whole-corpus ratio
// conflates a noisy broker with clean application services
// (v8 §9.12). Undefined (both `None`) below 1 M lines *and* when
// the corpus mints zero templates (SS = 0, ratio 0/0): the count
// and the ratio stay a matched pair — both `Some` or both `None`,
// never mixed — which the report layer relies on (report.rs).
let (template_count_at_1m_lines, convergence_ratio) =
if corpus_at_least_1m && template_count_at_end > 0 {
// Sample whose 1-based line number is closest to
// 1 M; on a tie the earlier (smaller `lines`)
// sample wins — the `(distance, lines)` key makes
// that the strict minimum.
let count_1m = self
.curve
.iter()
.min_by_key(|s| (s.lines.abs_diff(ONE_MILLION), s.lines))
.map(|s| s.template_count);
let ratio = count_1m.map(|c| (c as f64) / (template_count_at_end as f64));
(count_1m, ratio)
} else {
(None, None)
};

// Per-service decomposition, largest service first. Each
// service's gate follows §3.4.3 on its own line count; template
Expand All @@ -245,10 +265,23 @@ impl C2Accumulator {
.by_service
.into_iter()
.map(|(service_name, s)| {
let (at_1m, ratio, pass) = if s.lines >= ONE_MILLION && s.created > 0 {
let c = s.created_at_1m.unwrap_or(s.created);
let ratio = (c as f64) / (s.created as f64);
(Some(c), Some(ratio), Some(ratio >= 0.5))
let (at_1m, ratio, pass) = if s.lines >= ONE_MILLION {
if s.created > 0 {
let c = s.created_at_1m.unwrap_or(s.created);
let ratio = (c as f64) / (s.created as f64);
(Some(c), Some(ratio), Some(ratio >= 0.5))
} else {
// >= 1 M lines but zero templates minted (every line
// NO_TEMPLATE): the count is flat at zero, the strongest
// possible convergence — C2's falsifier is *linear*
// growth, so this passes trivially with an undefined
// ratio. Gated (`Some`), never abstaining, so
// `gate_pass`'s `None` keeps meaning "no service reached
// 1 M lines" (an all-NO_TEMPLATE service is a body-
// retention / parse-failure concern, caught by §3.1's
// counters — not a convergence failure).
(Some(0), None, Some(true))
}
} else {
(None, None, None)
};
Expand All @@ -268,6 +301,8 @@ impl C2Accumulator {
.then(a.service_name.cmp(&b.service_name))
});

let pass = gate_pass(&by_service);
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved

C2Result {
sample_cadence: self.cadence,
total_lines: self.total_lines,
Expand All @@ -283,6 +318,33 @@ impl C2Accumulator {
}
}

/// The per-service C2 **gate** (RFC 0006 §3.4.3 as amended for #444):
/// a corpus passes iff every service with ≥ 1 M lines passes its own
/// ratio ≥ 0.5. `finalize` sets `pass = Some(_)` for *every* ≥ 1 M
/// service (a zero-template service passes trivially — flat count), so
/// the fold below considers exactly the gated services (`s.pass.is_some()`)
/// and `None` means "no service reached 1 M lines", never a
/// silently-dropped ≥ 1 M service. A single-service
/// corpus — including the plain-text `<unknown>` bucket — is gated on
/// that one service's ratio, measured at its **exact** millionth line
/// (`created_at_1m`). That reproduces the pre-#444 whole-corpus verdict
/// for every historical converged corpus (their ratio sits far from the
/// 0.5 boundary); it is not bit-identical to the whole-corpus
/// `convergence_ratio`, which is sampled at the nearest curve point and
/// is only a diagnostic. Only multi-service OTLP corpora change verdict.
fn gate_pass(by_service: &[PerServiceC2]) -> Option<bool> {
let mut verdict = None;
for s in by_service {
match s.pass {
// Any gated service that fails is decisive — short-circuit.
Some(false) => return Some(false),
Some(true) => verdict = Some(true),
None => {}
}
}
verdict
}
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved

/// The record's `service.name` resource attribute, or a sentinel when
/// absent. Borrowed — the caller copies into the map key only on a
/// first sighting.
Expand Down Expand Up @@ -346,17 +408,22 @@ mod tests {
/// Exercises the full ≥ 1 M gate math at scale without the
/// miner.
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
#[test]
fn stable_corpus_passes_the_gate() {
fn stable_curve_ratio_is_one() {
// `run_stable` drives `observe` (no `service.name`), so it
// exercises the whole-corpus ratio *diagnostic*; the per-service
// gate needs record input and is covered by `gate_pass_*` +
// the partition test.
let r = run_stable(1_000_000, 8);
assert!(r.corpus_at_least_1m);
assert_eq!(r.template_count_at_end, 8);
assert_eq!(r.template_count_at_1m_lines, Some(8));
assert_eq!(r.convergence_ratio, Some(1.0));
assert_eq!(r.pass, Some(true));
// No service data → the per-service gate abstains.
assert_eq!(r.pass, None);
}

/// A corpus below 1 M lines abstains: no 1 M count, no
/// ratio, `pass = None`.
/// A corpus below 1 M lines has no 1 M count, no ratio; the gate
/// abstains.
#[test]
fn short_corpus_abstains() {
let r = run_stable(10_000, 5);
Expand All @@ -369,6 +436,33 @@ mod tests {
assert_eq!(r.template_count_at_end, 5);
}

/// The per-service gate fold: pass iff every ≥ 1 M service passes;
/// abstain when none are gated; a `<1 M` service (pass = None) does
/// not veto a passing sibling.
#[test]
fn gate_pass_folds_over_gated_services() {
let svc = |name: &str, pass: Option<bool>| PerServiceC2 {
service_name: name.to_string(),
lines: 0,
templates_created: 0,
templates_created_at_1m_lines: None,
convergence_ratio: None,
pass,
};
// No gated service → abstain.
assert_eq!(gate_pass(&[svc("a", None), svc("b", None)]), None);
// All gated services pass → pass.
assert_eq!(
gate_pass(&[svc("a", Some(true)), svc("b", None), svc("c", Some(true))]),
Some(true)
);
// One gated service fails → fail (even with passing siblings).
assert_eq!(
gate_pass(&[svc("a", Some(true)), svc("b", Some(false))]),
Some(false)
);
}

/// A corpus whose template count is still climbing steeply
/// at 1 M lines (no plateau) fails the gate: `count_1m` is
/// far under half the end count.
Expand Down Expand Up @@ -396,13 +490,11 @@ mod tests {
let ratio = r.convergence_ratio.expect("ratio on ≥1M corpus");
assert!(
ratio < 0.5,
"templates still climbing at 1 M → ratio {ratio} must be < 0.5",
);
assert_eq!(
r.pass,
Some(false),
"a non-converged corpus must fail the C2 gate",
"templates still climbing at 1 M → whole-corpus ratio {ratio} must be < 0.5",
);
// `observe` has no service data, so the per-service gate abstains
// here — the fold's fail path is covered by `gate_pass_*`.
assert_eq!(r.pass, None);
}

/// A `MinedRecord` carrying just the two fields the per-service
Expand Down Expand Up @@ -524,4 +616,38 @@ mod tests {
"every line is attributed to some bucket",
);
}

/// A service that clears the 1 M-line floor but mints zero templates
/// (every line `NO_TEMPLATE`) is **gated** and passes trivially — its
/// count is flat at zero, the opposite of the linear growth C2 flags.
/// Regression for the fold (#451): such a service must not collapse to
/// `pass = None` and get silently dropped, which would leave `None`
/// meaning both "below 1 M lines" and "≥ 1 M but degenerate".
#[test]
fn zero_template_service_over_1m_passes_trivially() {
let quiet = rec(NO_TEMPLATE, "quiet-svc");
let mut acc = C2Accumulator::new(ONE_MILLION);
for _ in 0..ONE_MILLION {
acc.record(&quiet);
}
let r = acc.finalize();
let svc = r
.by_service
.iter()
.find(|s| s.service_name == "quiet-svc")
.expect("quiet-svc bucket");
assert_eq!(svc.lines, ONE_MILLION);
assert_eq!(svc.templates_created, 0, "no template ever minted");
assert_eq!(svc.convergence_ratio, None, "0/0 ratio is undefined");
assert_eq!(svc.pass, Some(true), "flat count → trivial convergence");
// Gated as a PASS, not folded away as an abstention.
assert_eq!(r.pass, Some(true));
assert_eq!(r.template_count_at_end, 0);
// Whole-corpus diagnostic stays a matched pair (SS = 0 → both
// `None`, never the mixed `(Some(0), None)` the report layer
// rejects) even though the corpus cleared 1 M lines.
assert!(r.corpus_at_least_1m);
assert_eq!(r.template_count_at_1m_lines, None);
assert_eq!(r.convergence_ratio, None);
}
}
32 changes: 21 additions & 11 deletions crates/ourios-bench/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -583,27 +583,37 @@ pub struct C1Mismatch {
pub actual: String,
}

/// §3.6 `c2` block (populated only when C2 ran). `pass` is
/// `None` when the corpus is `< 1 M lines` (§3.4.3 abstention).
/// §3.6 `c2` block (populated only when C2 ran).
///
/// The gate is **per service** (RFC 0006 §3.4.3 as amended for #444):
/// `pass` is the fold over [`Self::by_service`] — `Some(true)` iff every
/// service with ≥ 1 M lines passes its own ratio ≥ 0.5, `Some(false)` if
/// any ≥ 1 M service fails, `None` when no service reaches 1 M lines.
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
/// The whole-corpus [`Self::convergence_ratio`] /
/// [`Self::template_count_at_1m_lines`] are retained as **diagnostics**
/// (on a multi-service corpus they conflate a noisy broker with clean
/// application services — v8 §9.12).
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct C2Result {
pub sample_cadence: u64,
pub total_lines: u64,
/// Whole-corpus diagnostic (not the gate — see the type doc).
pub template_count_at_1m_lines: Option<u64>,
pub template_count_at_end: u64,
/// Whole-corpus diagnostic (not the gate — see the type doc).
pub convergence_ratio: Option<f64>,
pub convergence_curve: Vec<ConvergenceSample>,
/// The per-service gate verdict (see the type doc).
pub pass: Option<bool>,
pub corpus_at_least_1m: bool,
/// Per-`service.name` convergence, largest service first.
/// **Diagnostic only** — the gate above is defined on the whole
/// corpus; this decomposition attributes it. On a multi-service
/// corpus (every OTel-Demo capture) a whole-corpus C2 conflates a
/// noisy broker with clean application services, so this surfaces
/// where non-convergence actually lives (v8 §9.12 / #444). A
/// plain-text corpus (no `service.name`) collapses to a single
/// `<unknown>` bucket rather than being empty; empty only when C2
/// did not run.
/// Per-`service.name` convergence, largest service first — the
/// **gate basis** (RFC 0006 §3.4.3 as amended for #444). On a
/// multi-service corpus (every OTel-Demo capture) a whole-corpus C2
/// conflates a noisy broker with clean application services, so the
/// gate is evaluated here per service; this also surfaces where
/// non-convergence actually lives (v8 §9.12). A plain-text corpus
/// (no `service.name`) collapses to a single `<unknown>` bucket
/// rather than being empty; empty only when C2 did not run.
#[serde(default)]
pub by_service: Vec<PerServiceC2>,
/// The distinct-`service.name` cap (`MAX_SERVICES`) was hit and
Expand Down
26 changes: 14 additions & 12 deletions crates/ourios-bench/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -315,34 +315,36 @@ fn print_summary(results: &ourios_bench::ResultsFile) {
);
}
if let Some(c2) = &results.c2 {
// `pass = None` is the §3.4.3 abstention (corpus
// < 1 M lines) — surface it as ABSTAIN, not a silent
// omission. (C2 isn't computed yet; this line is ready
// for when it lands.)
// The gate is per service (RFC 0006 §3.4.3, #444). `None` is the
// abstention — no service reaches 1 M lines.
let verdict = match c2.pass {
Some(true) => "PASS",
Some(false) => "FAIL",
None => "ABSTAIN (corpus < 1 M lines)",
None => "ABSTAIN (no service ≥ 1 M lines)",
};
let ratio = c2
.convergence_ratio
.map_or_else(|| "n/a".to_string(), |r| format!("{r:.3}"));
// The verdict is per-service; the whole-corpus ratio is a
// diagnostic, so it is labelled as such to avoid reading as the
// gate (on a multi-service corpus the two can disagree).
println!(
" C2 convergence: ratio {ratio} (end template count {}, sample cadence {}) — {verdict}",
" C2 convergence (per-service gate): {verdict} \
— whole-corpus ratio {ratio} (diagnostic; end templates {}, cadence {})",
c2.template_count_at_end, c2.sample_cadence,
);
// Per-service decomposition (diagnostic) — printed whenever the
// Per-service breakdown — the gate basis. Printed whenever the
// corpus resolves to more than one bucket (distinct `service.name`
// values plus any `<unknown>`/`<other>`), since a whole-corpus
// ratio then conflates a noisy broker with clean application
// services (v8 §9.12 / #444).
// values plus any `<unknown>`/`<other>`).
if c2.by_service.len() > 1 {
println!(" C2 by service (diagnostic; creations sum to the end count):");
println!(" C2 by service (the gate; creations sum to the end count):");
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
for svc in &c2.by_service {
let per = match (svc.convergence_ratio, svc.pass) {
(Some(r), Some(true)) => format!("ratio {r:.3} PASS"),
(Some(r), Some(false)) => format!("ratio {r:.3} FAIL"),
_ => "abstain (< 1 M lines)".to_string(),
(None, Some(true)) => "0 templates, converged PASS".to_string(),
(_, Some(false)) => "FAIL".to_string(),
(_, None) => "abstain (< 1 M lines)".to_string(),
};
println!(
" {:<24} {:>10} lines, {:>7} created — {per}",
Expand Down
Loading