diff --git a/crates/ourios-core/src/config.rs b/crates/ourios-core/src/config.rs index ae4cd5e27..90cf004a2 100644 --- a/crates/ourios-core/src/config.rs +++ b/crates/ourios-core/src/config.rs @@ -36,6 +36,9 @@ use std::fmt; /// | [`similarity_floor`](Self::similarity_floor) | `0.4` | `(0, threshold]` | §3.1 — bounds the §6.3 lossy zone; body retention in that zone is invariant | /// | [`prefix_depth`](Self::prefix_depth) | `2` | `0..=8` | §3.1 — affects tree quality, not safety | /// | [`param_byte_limit`](Self::param_byte_limit) | `256` | `1..=1024` | §3.2 — bounds cardinality; overflow spilling is invariant | +/// | [`max_node_children`](Self::max_node_children) | `100` | `1..` | §3.2 — bounds tree width (RFC 0023 §3.1) | +/// | [`max_templates`](Self::max_templates) | `20_000` | `1..` | §3.2 — bounds tree size; overflow goes parse-failure, never merge (RFC 0023 §3.1) | +/// | [`max_line_tokens`](Self::max_line_tokens) | `512` | `1..` | §3.2 — bounds stored-template width (RFC 0023 §3.1) | /// /// The struct is `Clone + Copy + 'static`, so the cluster can /// hold a default and a per-tenant override map without @@ -102,6 +105,29 @@ pub struct MinerConfig { /// the type rejects nonsensical "tens of thousands" values at /// the API surface. pub prefix_depth: u8, + + /// RFC 0023 §3.1 bound 1 — cap on an internal prefix node's + /// distinct-token children. When a node is full, unseen tokens + /// route through its `<*>` wildcard child instead of minting a + /// new branch (attach below it stays similarity-gated — routing + /// is not merging). Default `100`, Drain3's default. + pub max_node_children: u16, + + /// RFC 0023 §3.1 bound 2 — per-tenant ceiling on Drain-tree + /// leaves. At the ceiling, would-mint lines take the §6.3 + /// parse-failure path (body retained, counted) instead of + /// minting; existing leaves keep widening. Default `20_000` — + /// an order of magnitude above what healthy corpora mine to, + /// far below the unbounded growth that OOMs a 32 GiB host on + /// shape-diverse input (the RFC 0023 §1 finding). + pub max_templates: u32, + + /// RFC 0023 §3.1 bound 3 — lines tokenizing past this take the + /// §6.3 parse-failure path with the body retained, bounding + /// stored-template token width. Default `512`. The type keeps + /// every accepted line inside the RFC 0001 §6.4 `u16` audit + /// position width. + pub max_line_tokens: u16, } /// Per-`CLAUDE.md` §3.2: the project ceiling on configurable @@ -140,6 +166,11 @@ pub enum MinerConfigError { /// The supplied prefix depth exceeds the RFC 0001 §6.1 /// ceiling of [`PREFIX_DEPTH_CEILING`]. PrefixDepthTooLarge(u8), + /// A supplied RFC 0023 §3.1 bound (`max_node_children`, + /// `max_templates`, or `max_line_tokens`) is zero — a zero + /// bound would divert every line to parse-failure at startup. + /// Carries the field name. + BoundZero(&'static str), } impl fmt::Display for MinerConfigError { @@ -173,6 +204,12 @@ impl fmt::Display for MinerConfigError { "prefix_depth exceeds the RFC 0001 §6.1 ceiling of {PREFIX_DEPTH_CEILING} (got {v})", ) } + Self::BoundZero(field) => { + write!( + f, + "{field} must be non-zero (RFC 0023 §3.1: a zero bound would divert every line to parse-failure)", + ) + } } } } @@ -189,6 +226,9 @@ impl Default for MinerConfig { similarity_floor: 0.4, param_byte_limit: 256, prefix_depth: 2, + max_node_children: 100, + max_templates: 20_000, + max_line_tokens: 512, } } } @@ -264,9 +304,12 @@ impl MinerConfig { similarity_floor: floor, param_byte_limit: byte_limit, // Picks up `MinerConfig::default()`'s prefix_depth (2, - // per the Drain paper). Override with - // [`Self::with_prefix_depth`]. + // per the Drain paper) and the RFC 0023 bounds. + // Override with the `with_*` builders. prefix_depth: Self::default().prefix_depth, + max_node_children: Self::default().max_node_children, + max_templates: Self::default().max_templates, + max_line_tokens: Self::default().max_line_tokens, }) } @@ -285,4 +328,49 @@ impl MinerConfig { self.prefix_depth = depth; Ok(self) } + + /// Return a copy of `self` with + /// [`max_node_children`][Self::max_node_children] replaced + /// (RFC 0023 §3.1 bound 1). + /// + /// # Errors + /// + /// [`MinerConfigError::BoundZero`] when `cap` is zero. + pub fn with_max_node_children(mut self, cap: u16) -> Result { + if cap == 0 { + return Err(MinerConfigError::BoundZero("max_node_children")); + } + self.max_node_children = cap; + Ok(self) + } + + /// Return a copy of `self` with + /// [`max_templates`][Self::max_templates] replaced (RFC 0023 + /// §3.1 bound 2). + /// + /// # Errors + /// + /// [`MinerConfigError::BoundZero`] when `ceiling` is zero. + pub fn with_max_templates(mut self, ceiling: u32) -> Result { + if ceiling == 0 { + return Err(MinerConfigError::BoundZero("max_templates")); + } + self.max_templates = ceiling; + Ok(self) + } + + /// Return a copy of `self` with + /// [`max_line_tokens`][Self::max_line_tokens] replaced + /// (RFC 0023 §3.1 bound 3). + /// + /// # Errors + /// + /// [`MinerConfigError::BoundZero`] when `cap` is zero. + pub fn with_max_line_tokens(mut self, cap: u16) -> Result { + if cap == 0 { + return Err(MinerConfigError::BoundZero("max_line_tokens")); + } + self.max_line_tokens = cap; + Ok(self) + } } diff --git a/crates/ourios-ingester/tests/rfc0023_overflow_roundtrip.rs b/crates/ourios-ingester/tests/rfc0023_overflow_roundtrip.rs index 5c25ac8f5..eb591a000 100644 --- a/crates/ourios-ingester/tests/rfc0023_overflow_roundtrip.rs +++ b/crates/ourios-ingester/tests/rfc0023_overflow_roundtrip.rs @@ -1,16 +1,60 @@ //! RFC 0023 §5 — RFC0023.2: ceiling-overflow lines stay stored and //! searchable. The miner-level scenarios (`.1/.3/.4/.5/.6`) live in //! `crates/ourios-miner/tests/rfc0023_bounded_memory.rs`; this one -//! crosses the ingest path (miner overflow → record sink → Parquet → -//! body read-back), which needs both crates. +//! crosses the ingest path (miner overflow → Parquet → body +//! read-back), which needs both crates. + +use ourios_core::config::MinerConfig; +use ourios_core::otlp::{Body, OtlpLogRecord}; +use ourios_core::record::SharedRecordSink; +use ourios_core::tenant::TenantId; +use ourios_miner::cluster::{MinerCluster, NO_TEMPLATE}; +use ourios_parquet::{DEFAULT_ZSTD_LEVEL, Reader, encode_records_to_parquet}; /// Scenario RFC0023.2 — overflow lines stay stored and searchable. /// See `docs/rfcs/0023-bounded-template-memory.md` §5. #[test] -#[ignore = "RFC0023.2 stub — implemented in the miner-bounds green slice (as its ingest-path integration arm)"] fn rfc0023_2_overflow_bodies_round_trip_bit_identically() { - todo!( - "RFC0023.2 — ceiling-overflow lines written through the record sink \ - round-trip bit-identically through the Parquet body column" + let tenant = TenantId::new("overflow"); + let sink = SharedRecordSink::new(); + let config = MinerConfig::default() + .with_max_templates(1) + .expect("non-zero ceiling"); + let mut cluster = MinerCluster::new(config).with_record_sink(Box::new(sink.clone())); + + // Distinct shapes: the first mints (and fills the ceiling), the + // second diverts to the parse-failure path with its body retained. + let overflow_line = "gamma rejected request from peer five with cause umbrella"; + for text in ["alpha started", overflow_line] { + cluster.ingest(&OtlpLogRecord { + tenant_id: tenant.clone(), + body: Some(Body::String(text.to_string())), + time_unix_nano: 1_775_127_480_000_000_000, + ..Default::default() + }); + } + let mined = sink.drain(); + assert_eq!(mined.len(), 2); + assert_eq!(mined[1].template_id, NO_TEMPLATE, "second line diverted"); + + // Through the RFC 0005 writer and back: the overflow body is + // bit-identical in the Parquet body column. + let bytes = encode_records_to_parquet(&mined, DEFAULT_ZSTD_LEVEL).expect("encode"); + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join("overflow.parquet"); + std::fs::write(&path, bytes).expect("write parquet"); + let rows = Reader::open_file(&path) + .expect("open parquet") + .read_all() + .expect("read rows"); + assert_eq!(rows.len(), 2); + let overflow_row = rows + .iter() + .find(|r| r.template_id == NO_TEMPLATE) + .expect("the diverted row is stored"); + assert_eq!( + overflow_row.body.as_deref(), + Some(overflow_line), + "overflow body round-trips bit-identically", ); } diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index bc784966a..84f56f0d3 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -210,6 +210,11 @@ struct TenantState { tree: Tree, structured_templates: HashMap<(u8, Option), u64>, template_count: usize, + /// Drain-tree leaves only (excludes structured-template + /// entries) — the quantity RFC 0023 §3.1's `max_templates` + /// ceiling bounds. Incremented exactly where a leaf is pushed; + /// rebuilt from `leaves.len()` on snapshot restore. + leaf_count: usize, /// Effective [`MinerConfig`] for this tenant, captured at /// lazy allocation time. Resolves to the per-tenant override /// (set via [`MinerCluster::with_tenant_config`]) if one @@ -227,6 +232,7 @@ impl TenantState { tree: Tree::new(), structured_templates: HashMap::new(), template_count: 0, + leaf_count: 0, config, } } @@ -1228,28 +1234,15 @@ impl MinerCluster { return NO_TEMPLATE; } - // Cap the line's token count at the audit-event's - // position width (RFC §6.4 pins `positions_widened: - // Vec`). Lines exceeding this can't be widened - // safely — emitting an audit with a truncated set would - // be the silent-merge bug `[CLAUDE.md §3.1]` exists to - // prevent. ≥65 536 tokens in a single log line is - // pathological; the §6.2 step 1 "line longer than - // max-line-bytes" parse-failure path. Retain body per - // §6.3. - if masked_strs.len() > u16::MAX as usize { - let mut rec = Self::record_envelope(record, BodyKind::String); - rec.separators = separators; - rec.params = params; - rec.body = Some(raw.to_string()); - rec.lossy_flag = true; - // §6.5: bump `params_overflow_total` if any param - // exceeded the byte limit. Body retention is already - // set above for this parse-failure path. - self.apply_overflow_retention(record, service, &mut rec, raw); - self.emit_record(rec, service); - self.record_parse_failure(record, service); - return NO_TEMPLATE; + // RFC 0023 §3.1 bound 3: lines tokenizing past + // `max_line_tokens` take the parse-failure path with the + // body retained, bounding stored-template token width. The + // config type (`u16`) also keeps every accepted line inside + // the RFC §6.4 audit position width (`positions_widened: + // Vec`), whose violation would otherwise be the + // silent-merge bug `[CLAUDE.md §3.1]` exists to prevent. + if masked_strs.len() > usize::from(effective_config.max_line_tokens) { + return self.emit_string_parse_failure(record, service, raw, separators, params); } // Phase 1 — read-only candidate selection. RFC §6.2 step @@ -1269,6 +1262,14 @@ impl MinerCluster { // into the lossy zone against, and no template to // declare a parse failure against. None => { + // RFC 0023 §3.1 bound 2: at the per-tenant leaf + // ceiling the mint diverts to the §6.3 + // parse-failure path — body retained, counted, + // never force-merged. + if self.at_template_ceiling(&record.tenant_id, effective_config.max_templates) { + return self + .emit_string_parse_failure(record, service, raw, separators, params); + } let new_id = self.create_new_leaf( record, raw, @@ -1318,6 +1319,18 @@ impl MinerCluster { // here; `record_parse_failure` covers the // parse-failure-zone path separately. ConfidenceZone::Lossy => { + // RFC 0023 §3.1 bound 2 — same divert as + // the no-candidate arm: the §6.3 lossy zone + // mints rather than force-merges into the + // too-weak candidate, so at the ceiling it + // must fail parse, not merge. + if self + .at_template_ceiling(&record.tenant_id, effective_config.max_templates) + { + return self.emit_string_parse_failure( + record, service, raw, separators, params, + ); + } self.body_retentions_total.fetch_add(1, Ordering::Relaxed); self.metrics.record_body_retention(&record.tenant_id); let new_id = self.create_new_leaf( @@ -1347,24 +1360,48 @@ impl MinerCluster { // Parse failure: no template allocated. // Both counters bump via the shared helper. ConfidenceZone::ParseFailure => { - let mut rec = Self::record_envelope(record, BodyKind::String); - rec.separators = separators; - rec.params = params; - rec.body = Some(raw.to_string()); - rec.lossy_flag = true; - // §6.5: bump `params_overflow_total` for - // any overflow params (body is already - // retained for the parse-failure reason). - self.apply_overflow_retention(record, service, &mut rec, raw); - self.emit_record(rec, service); - self.record_parse_failure(record, service); - NO_TEMPLATE + self.emit_string_parse_failure(record, service, raw, separators, params) } } } } } + /// Emit the §6.3 parse-failure record for a string line — the + /// shared exit for the below-floor zone, the §6.4 + /// degenerate-widening rejection, the RFC 0023 §3.1 long-line + /// guard, and the RFC 0023 template-ceiling diverts: no + /// template, body retained verbatim, lossy-flagged, counted. + fn emit_string_parse_failure( + &mut self, + record: &OtlpLogRecord, + service: Option<&str>, + raw: &str, + separators: Vec, + params: Vec, + ) -> u64 { + let mut rec = Self::record_envelope(record, BodyKind::String); + rec.separators = separators; + rec.params = params; + rec.body = Some(raw.to_string()); + rec.lossy_flag = true; + // §6.5: bump `params_overflow_total` for any overflow params + // (body is already retained for the parse-failure reason). + self.apply_overflow_retention(record, service, &mut rec, raw); + self.emit_record(rec, service); + self.record_parse_failure(record, service); + NO_TEMPLATE + } + + /// RFC 0023 §3.1 bound 2 — whether the tenant's Drain-tree + /// leaf count has reached the configured ceiling. An unseen + /// tenant is trivially below it. + fn at_template_ceiling(&self, tenant: &TenantId, max_templates: u32) -> bool { + self.tenants + .get(tenant) + .is_some_and(|s| s.leaf_count >= max_templates as usize) + } + /// RFC §6.2 step 4 — find the best-matching leaf in the /// `(severity, scope, length, prefix)` bucket, or `None` if /// the tenant is unseen, the prefix path doesn't exist, or @@ -1379,9 +1416,11 @@ impl MinerCluster { // Per-tenant `prefix_depth` (RFC 0004 §3.4) — read from // the captured `state.config` rather than the cluster // default. - let parent = state - .tree - .descend(masked_strs, state.config.prefix_depth as usize)?; + let parent = state.tree.descend( + masked_strs, + state.config.prefix_depth as usize, + usize::from(state.config.max_node_children), + )?; let mut best: Option = None; for (leaf_idx, leaf) in parent.leaves.iter().enumerate() { @@ -1467,9 +1506,11 @@ impl MinerCluster { .tenants .entry(record.tenant_id.clone()) .or_insert_with(|| TenantState::new(effective_config)); - let parent = state - .tree - .descend_mut(masked_strs, state.config.prefix_depth as usize); + let parent = state.tree.descend_mut( + masked_strs, + state.config.prefix_depth as usize, + usize::from(state.config.max_node_children), + ); // Build the leaf template: Wildcard at every mask-emitted // position, Fixed at every other. `wildcard_positions` is // ascending (single forward pass over the tokens) so we @@ -1506,8 +1547,10 @@ impl MinerCluster { }); // Maintain the TenantState::template_count cache invariant — // every fresh allocation under `state` is mirrored here so - // `MinerCluster::template_count` can stay O(1). + // `MinerCluster::template_count` can stay O(1). `leaf_count` + // mirrors tree leaves only (the RFC 0023 ceiling's basis). state.template_count += 1; + state.leaf_count += 1; created_template }; // RFC 0017 §3.1 — audit the leaf's initial (version 1) creation so a @@ -1581,9 +1624,11 @@ impl MinerCluster { .tenants .get_mut(&record.tenant_id) .expect("tenant present: find_best_candidate returned Some(...)"); - let parent = state - .tree - .descend_mut(masked_strs, state.config.prefix_depth as usize); + let (depth, fanout) = ( + state.config.prefix_depth as usize, + usize::from(state.config.max_node_children), + ); + let parent = state.tree.descend_mut(masked_strs, depth, fanout); let leaf = &mut parent.leaves[candidate.leaf_idx]; plan_attach( leaf, @@ -1642,22 +1687,10 @@ impl MinerCluster { }, }); // §6.4 treats degenerate widening as a parse - // failure that retains body. `lossy_flag = true` - // so reconstruct surfaces the retained body and - // ignores `params`; the line-ordered fallback is - // fine here. - let mut rec = Self::record_envelope(record, BodyKind::String); - rec.separators = separators; - rec.params = params; - rec.body = Some(raw.to_string()); - rec.lossy_flag = true; - // §6.5: bump `params_overflow_total` if the - // line-ordered params contained any oversized - // values (body already retained for §6.4). - self.apply_overflow_retention(record, service, &mut rec, raw); - self.emit_record(rec, service); - self.record_parse_failure(record, service); - NO_TEMPLATE + // failure that retains body (the line-ordered + // params fallback is fine — reconstruct ignores + // `params` on the lossy path). + self.emit_string_parse_failure(record, service, raw, separators, params) } AttachPlan::Mutated { template_id, @@ -1925,6 +1958,7 @@ impl MinerCluster { } let config = self.effective_config(tenant_id); let prefix_depth = usize::from(config.prefix_depth); + let max_node_children = config.max_node_children; let mut tenant = TenantState::new(config); // Ids are unique cluster-wide and the structured map keys on @@ -1982,7 +2016,10 @@ impl MinerCluster { } } - let parent = tenant.tree.descend_mut(&masked, prefix_depth); + let parent = + tenant + .tree + .descend_mut(&masked, prefix_depth, usize::from(max_node_children)); parent.leaves.push(Leaf { template, template_id: record.template_id, @@ -2015,7 +2052,10 @@ impl MinerCluster { } // Mirror live ingest's cache invariant: every fresh // allocation — tree leaf or structured-map entry — counts. + // `leaf_count` (the RFC 0023 ceiling's basis) rebuilds from + // the tree leaves alone. tenant.template_count = state.leaves.len() + state.structured_templates.len(); + tenant.leaf_count = state.leaves.len(); // The id allocator is cluster-wide; without this bump a // post-restore allocation would collide with a restored id. diff --git a/crates/ourios-miner/src/tree.rs b/crates/ourios-miner/src/tree.rs index cafc205d9..22f45998f 100644 --- a/crates/ourios-miner/src/tree.rs +++ b/crates/ourios-miner/src/tree.rs @@ -182,7 +182,12 @@ impl Tree { /// If `masked` is empty. The miner's tokenize → mask path /// guarantees at least one token before any tree call; an /// empty input here means a caller bypassed that path. - pub fn descend_mut(&mut self, masked: &[&str], prefix_depth: usize) -> &mut PrefixNode { + pub fn descend_mut( + &mut self, + masked: &[&str], + prefix_depth: usize, + max_node_children: usize, + ) -> &mut PrefixNode { assert!( !masked.is_empty(), "descend_mut precondition: masked must be non-empty", @@ -196,7 +201,7 @@ impl Tree { let walk_depth = prefix_depth.min(length); let path = &masked[..walk_depth]; - descend_recursively(&mut length_node.root, path) + descend_recursively(&mut length_node.root, path, max_node_children) } /// Read-only counterpart to [`Tree::descend_mut`]. Returns the @@ -213,7 +218,12 @@ impl Tree { /// /// If `masked` is empty (same precondition as [`Tree::descend_mut`]). #[must_use] - pub fn descend(&self, masked: &[&str], prefix_depth: usize) -> Option<&PrefixNode> { + pub fn descend( + &self, + masked: &[&str], + prefix_depth: usize, + max_node_children: usize, + ) -> Option<&PrefixNode> { assert!( !masked.is_empty(), "descend precondition: masked must be non-empty", @@ -225,7 +235,7 @@ impl Tree { let walk_depth = prefix_depth.min(length); let path = &masked[..walk_depth]; - descend_immutable(&length_node.root, path) + descend_immutable(&length_node.root, path, max_node_children) } /// Total leaf count across the tree. Suitable for the @@ -252,13 +262,45 @@ impl Tree { } } -fn descend_immutable<'t>(node: &'t PrefixNode, path: &[&str]) -> Option<&'t PrefixNode> { +/// Reserved child key for RFC 0023 §3.1 overflow routing: when a +/// node's keyed children reach `max_node_children`, unseen tokens +/// descend through this child instead of minting a new branch. The +/// string matches the leaf-template wildcard rendering +/// ([`format_template`]); a literal log token reading `"<*>"` merely +/// shares the routing bucket — the same accepted collision class the +/// mask tags (`""` etc.) already document — and attach below the +/// child stays similarity-gated, so routing is never merging. +pub(crate) const WILDCARD_CHILD: &str = "<*>"; + +/// A node's keyed (non-wildcard) child count — the quantity +/// `max_node_children` bounds. The wildcard child is routing +/// infrastructure, not a keyed branch, so it doesn't count. +fn keyed_children(node: &PrefixNode) -> usize { + node.children.len() - usize::from(node.children.contains_key(WILDCARD_CHILD)) +} + +fn descend_immutable<'t>( + node: &'t PrefixNode, + path: &[&str], + max_node_children: usize, +) -> Option<&'t PrefixNode> { if path.is_empty() { return Some(node); } let (head, tail) = path.split_first().expect("non-empty by guard above"); - let child = node.children.get(*head)?; - descend_immutable(child, tail) + match node.children.get(*head) { + Some(child) => descend_immutable(child, tail, max_node_children), + // RFC 0023 §3.1 bound 1, read side: an unseen token at a full + // node was (or would be) wildcard-routed on the write side, so + // candidate selection must look there too. Children only grow, + // so a token found keyed was inserted keyed — the two sides + // can never disagree on a token's route. + None if keyed_children(node) >= max_node_children => { + let child = node.children.get(WILDCARD_CHILD)?; + descend_immutable(child, tail, max_node_children) + } + None => None, + } } fn count_leaves(node: &PrefixNode) -> usize { @@ -289,13 +331,30 @@ fn collect_leaves_recursive<'t>(node: &'t PrefixNode, out: &mut Vec<&'t Leaf>) { /// owned key, so the lookup-then-insert is the standard workaround /// on stable; the second `get_mut` is one extra hash, never an /// extra allocation. -fn descend_recursively<'t>(node: &'t mut PrefixNode, path: &[&str]) -> &'t mut PrefixNode { +fn descend_recursively<'t>( + node: &'t mut PrefixNode, + path: &[&str], + max_node_children: usize, +) -> &'t mut PrefixNode { if path.is_empty() { return node; } let (head, tail) = path.split_first().expect("non-empty by guard above"); if !node.children.contains_key(*head) { + // RFC 0023 §3.1 bound 1: a full node routes unseen tokens + // through the wildcard child rather than minting a branch. + if keyed_children(node) >= max_node_children { + if !node.children.contains_key(WILDCARD_CHILD) { + node.children + .insert(WILDCARD_CHILD.to_string(), PrefixNode::default()); + } + let child = node + .children + .get_mut(WILDCARD_CHILD) + .expect("just inserted above if missing"); + return descend_recursively(child, tail, max_node_children); + } node.children .insert((*head).to_string(), PrefixNode::default()); } @@ -303,7 +362,7 @@ fn descend_recursively<'t>(node: &'t mut PrefixNode, path: &[&str]) -> &'t mut P .children .get_mut(*head) .expect("just inserted above if missing"); - descend_recursively(child, tail) + descend_recursively(child, tail, max_node_children) } /// Render a template to its canonical string form: tokens joined by a @@ -356,6 +415,95 @@ pub fn parse_template(canonical: &str) -> Vec { mod tests { use super::*; + /// RFC 0023 §3.1 bound 1, tree level: keyed children never exceed + /// the cap, overflow tokens route through [`WILDCARD_CHILD`], and + /// the read-side `descend` finds what the write side routed there. + #[test] + fn full_node_routes_overflow_tokens_through_the_wildcard_child() { + let mut tree = Tree::new(); + // Cap 2 at prefix depth 1: three distinct first tokens on + // same-length lines — the third must not mint a third branch. + let _ = tree.descend_mut(&["alpha", "x"], 1, 2); + let _ = tree.descend_mut(&["beta", "x"], 1, 2); + { + let overflow = tree.descend_mut(&["gamma", "x"], 1, 2); + overflow.leaves.push(Leaf { + template: vec![ + OwnedToken::Fixed("gamma".to_owned()), + OwnedToken::Fixed("x".to_owned()), + ], + template_id: 7, + template_version: 1, + severity_number: 9, + scope_name: None, + slot_types: Vec::new(), + }); + } + + // The length-2 bucket's root has exactly the cap's keyed + // children plus the wildcard child. + let root = &tree.by_length[&2].root; + assert_eq!(keyed_children(root), 2, "keyed children capped"); + assert!( + root.children.contains_key(WILDCARD_CHILD), + "overflow minted the wildcard child", + ); + assert!( + !root.children.contains_key("gamma"), + "the overflow token did not mint a keyed branch", + ); + + // Read side finds the wildcard-routed leaf... + let found = tree + .descend(&["gamma", "x"], 1, 2) + .expect("read-side routing reaches the wildcard child"); + assert_eq!(found.leaves.len(), 1); + assert_eq!(found.leaves[0].template_id, 7); + // ...and a fourth unseen token shares the same route. + let shared = tree + .descend(&["delta", "x"], 1, 2) + .expect("every overflow token shares the wildcard route"); + assert_eq!(shared.leaves.len(), 1, "same bucket as gamma's leaf"); + } + + /// Property: for any token stream and any cap ≥ 1, no node's + /// keyed child count ever exceeds the cap (the RFC 0023 §3.1 + /// bounded-fanout invariant), and every descend lands somewhere. + #[test] + fn prop_keyed_children_never_exceed_the_cap() { + fn assert_capped(node: &PrefixNode, cap: usize) { + assert!( + keyed_children(node) <= cap, + "keyed children {} exceed cap {cap}", + keyed_children(node), + ); + for child in node.children.values() { + assert_capped(child, cap); + } + } + + let mut runner = proptest::test_runner::TestRunner::default(); + runner + .run( + &( + proptest::collection::vec(proptest::collection::vec("[a-e]{1,3}", 1..5), 1..40), + 1usize..5, + ), + |(lines, cap)| { + let mut tree = Tree::new(); + for line in &lines { + let masked: Vec<&str> = line.iter().map(String::as_str).collect(); + let _ = tree.descend_mut(&masked, 2, cap); + } + for length_node in tree.by_length.values() { + assert_capped(&length_node.root, cap); + } + Ok(()) + }, + ) + .expect("bounded-fanout property holds"); + } + #[test] fn parse_template_inverts_format_template() { // The canonical encode/decode pair must round-trip so the read-time @@ -419,7 +567,7 @@ mod tests { assert!(tree.by_length.is_empty()); // Act - let _ = tree.descend_mut(&["user", "logged", "in"], DEFAULT_PREFIX_DEPTH); + let _ = tree.descend_mut(&["user", "logged", "in"], DEFAULT_PREFIX_DEPTH, 100); // Assert — exactly one length bucket allocated, for N=3. assert_eq!(tree.by_length.len(), 1); @@ -432,8 +580,8 @@ mod tests { let mut tree = Tree::new(); // Act - let _ = tree.descend_mut(&["user", "logged", "in"], DEFAULT_PREFIX_DEPTH); - let _ = tree.descend_mut(&["GET", "/api", "200"], DEFAULT_PREFIX_DEPTH); + let _ = tree.descend_mut(&["user", "logged", "in"], DEFAULT_PREFIX_DEPTH, 100); + let _ = tree.descend_mut(&["GET", "/api", "200"], DEFAULT_PREFIX_DEPTH, 100); // Assert — same length bucket, distinct first-prefix // children under it. @@ -450,8 +598,8 @@ mod tests { let mut tree = Tree::new(); // Act - let _ = tree.descend_mut(&["GET", "/home"], DEFAULT_PREFIX_DEPTH); - let _ = tree.descend_mut(&["user", "42", "logged", "in"], DEFAULT_PREFIX_DEPTH); + let _ = tree.descend_mut(&["GET", "/home"], DEFAULT_PREFIX_DEPTH, 100); + let _ = tree.descend_mut(&["user", "42", "logged", "in"], DEFAULT_PREFIX_DEPTH, 100); // Assert — distinct length buckets, no cross-pollination. assert_eq!(tree.by_length.len(), 2); @@ -472,7 +620,7 @@ mod tests { // leaf-list is the same allocation (so a leaf pushed by // call 1 is visible to call 2). let leaves_addr_a = { - let parent = tree.descend_mut(&line_a, DEFAULT_PREFIX_DEPTH); + let parent = tree.descend_mut(&line_a, DEFAULT_PREFIX_DEPTH, 100); parent.leaves.push(Leaf { template: [OwnedToken::Fixed("marker".to_string())].into(), template_id: 99, @@ -483,7 +631,7 @@ mod tests { }); std::ptr::from_ref(&parent.leaves) }; - let parent_b = tree.descend_mut(&line_b, DEFAULT_PREFIX_DEPTH); + let parent_b = tree.descend_mut(&line_b, DEFAULT_PREFIX_DEPTH, 100); let leaves_addr_b = std::ptr::from_ref(&parent_b.leaves); // Assert — same parent (pointer identity on the leaves @@ -501,7 +649,7 @@ mod tests { let mut tree = Tree::new(); // Act - let _ = tree.descend_mut(&["hello"], 2); + let _ = tree.descend_mut(&["hello"], 2, 100); // Assert — single length bucket for N=1, single child // ("hello") under its root, no further descent. @@ -528,8 +676,8 @@ mod tests { let mut tree = Tree::new(); // Act - let _ = tree.descend_mut(&["a", "b"], 0); - let _ = tree.descend_mut(&["x", "y"], 0); + let _ = tree.descend_mut(&["a", "b"], 0, 100); + let _ = tree.descend_mut(&["x", "y"], 0, 100); // Assert — same length bucket, no children under its // root (no prefix-token branching), both lines land at @@ -546,7 +694,7 @@ mod tests { let masked: [&str; 0] = []; // Act + Assert - let _ = tree.descend_mut(&masked, DEFAULT_PREFIX_DEPTH); + let _ = tree.descend_mut(&masked, DEFAULT_PREFIX_DEPTH, 100); } // OwnedToken / Token round-trip @@ -608,7 +756,7 @@ mod tests { let tree = Tree::new(); // Act - let r = tree.descend(&["a", "b"], DEFAULT_PREFIX_DEPTH); + let r = tree.descend(&["a", "b"], DEFAULT_PREFIX_DEPTH, 100); // Assert assert!(r.is_none()); @@ -620,10 +768,10 @@ mod tests { // first-prefix token. The length bucket exists; the // prefix child does not. let mut tree = Tree::new(); - let _ = tree.descend_mut(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH); + let _ = tree.descend_mut(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH, 100); // Act — same length 3, different first-prefix token. - let r = tree.descend(&["GET", "/home", "200"], DEFAULT_PREFIX_DEPTH); + let r = tree.descend(&["GET", "/home", "200"], DEFAULT_PREFIX_DEPTH, 100); // Assert assert!(r.is_none()); @@ -634,7 +782,7 @@ mod tests { // Arrange — materialise a path, then look it up. let mut tree = Tree::new(); { - let parent = tree.descend_mut(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH); + let parent = tree.descend_mut(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH, 100); parent.leaves.push(Leaf { template: [ OwnedToken::Fixed("user".to_string()), @@ -652,7 +800,7 @@ mod tests { // Act let parent = tree - .descend(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH) + .descend(&["user", "42", "logged"], DEFAULT_PREFIX_DEPTH, 100) .expect("path was built"); // Assert — same leaf observable via immutable descend. @@ -668,7 +816,7 @@ mod tests { let masked: [&str; 0] = []; // Act + Assert - let _ = tree.descend(&masked, DEFAULT_PREFIX_DEPTH); + let _ = tree.descend(&masked, DEFAULT_PREFIX_DEPTH, 100); } // leaf_count + collect_leaves @@ -692,7 +840,7 @@ mod tests { // - length 3, prefix "user 42" — 2 leaves (same parent) let mut tree = Tree::new(); { - let p = tree.descend_mut(&["GET", "/home"], DEFAULT_PREFIX_DEPTH); + let p = tree.descend_mut(&["GET", "/home"], DEFAULT_PREFIX_DEPTH, 100); p.leaves.push(Leaf { template: [ OwnedToken::Fixed("GET".to_string()), @@ -707,7 +855,7 @@ mod tests { }); } { - let p = tree.descend_mut(&["user", "42", "in"], DEFAULT_PREFIX_DEPTH); + let p = tree.descend_mut(&["user", "42", "in"], DEFAULT_PREFIX_DEPTH, 100); p.leaves.push(Leaf { template: [ OwnedToken::Fixed("user".to_string()), @@ -748,7 +896,7 @@ mod tests { // Arrange — two leaves in different length buckets. let mut tree = Tree::new(); { - let p = tree.descend_mut(&["a", "b"], DEFAULT_PREFIX_DEPTH); + let p = tree.descend_mut(&["a", "b"], DEFAULT_PREFIX_DEPTH, 100); p.leaves.push(Leaf { template: [ OwnedToken::Fixed("a".to_string()), @@ -763,7 +911,7 @@ mod tests { }); } { - let p = tree.descend_mut(&["x", "y", "z"], DEFAULT_PREFIX_DEPTH); + let p = tree.descend_mut(&["x", "y", "z"], DEFAULT_PREFIX_DEPTH, 100); p.leaves.push(Leaf { template: [ OwnedToken::Fixed("x".to_string()), diff --git a/crates/ourios-miner/tests/rfc0004_configuration_policy.rs b/crates/ourios-miner/tests/rfc0004_configuration_policy.rs index 207614def..441ed9505 100644 --- a/crates/ourios-miner/tests/rfc0004_configuration_policy.rs +++ b/crates/ourios-miner/tests/rfc0004_configuration_policy.rs @@ -172,5 +172,12 @@ fn rfc0004_3_no_invariant_breaking_field_exists_on_miner_config() { similarity_floor: _, param_byte_limit: _, prefix_depth: _, + // RFC 0023 §3.1 bounds — classified as tunables *inside* + // the invariants: overflow at any of the three diverts to + // the §6.3 parse-failure path (body retained, counted), + // never force-merges (§3.1) and never drops data. + max_node_children: _, + max_templates: _, + max_line_tokens: _, } = cfg; } diff --git a/crates/ourios-miner/tests/rfc0023_bounded_memory.rs b/crates/ourios-miner/tests/rfc0023_bounded_memory.rs index fa484eed2..6afa127f2 100644 --- a/crates/ourios-miner/tests/rfc0023_bounded_memory.rs +++ b/crates/ourios-miner/tests/rfc0023_bounded_memory.rs @@ -1,65 +1,259 @@ //! RFC 0023 §5 — bounded template memory (RFC 0001 amendment). //! -//! Scenarios RFC0023.1/.3/.4/.5/.6 live here (miner-level bounds + -//! corpus invariance + telemetry). RFC0023.2 (overflow bodies -//! round-trip through the Parquet body column) is an ingest-path -//! integration and lives in +//! Scenarios RFC0023.1/.3/.4/.5 live here (miner-level bounds + +//! corpus invariance); RFC0023.6 (telemetry) stays stubbed for the +//! telemetry green slice. RFC0023.2 (overflow bodies round-trip +//! through the Parquet body column) is an ingest-path integration +//! and lives in //! `crates/ourios-ingester/tests/rfc0023_overflow_roundtrip.rs`. //! RFC0023.7 (the 16 GiB `HDFS_v2` scale rerun under 8 GiB peak RSS) //! is a bench-hardware criterion discharged by the //! `docs/benchmarks.md` §9 record, not a `cargo test` — the runner //! lives in the maintainer's `scratch/baseline/` tooling. -//! -//! Stubs are `#[ignore]`d so the default run stays green while the -//! RFC is red; each names the green slice that discharges it. + +use ourios_core::config::MinerConfig; +use ourios_core::otlp::{Body, OtlpLogRecord}; +use ourios_core::record::SharedRecordSink; +use ourios_core::tenant::TenantId; +use ourios_miner::cluster::{MinerCluster, NO_TEMPLATE}; + +fn record(tenant: &TenantId, text: &str) -> OtlpLogRecord { + OtlpLogRecord { + tenant_id: tenant.clone(), + body: Some(Body::String(text.to_string())), + ..Default::default() + } +} + +/// The tenant's template set as canonical `(rendered_template, id)` +/// pairs, sorted — the identity oracle RFC0023.1/.5 compare on. +/// Rendering goes through the miner's own [`format_template`] +/// (`ourios_miner::tree`), the stable canonical form, not `Debug`. +fn template_set(cluster: &MinerCluster, tenant: &TenantId) -> Vec<(String, u64)> { + let mut set: Vec<(String, u64)> = cluster + .templates_for(tenant) + .into_iter() + .map(|leaf| { + ( + ourios_miner::tree::format_template(&leaf.template), + leaf.template_id, + ) + }) + .collect(); + set.sort(); + set +} + +/// RFC 0023 §3.1 — a zero bound is a startup configuration error +/// (`BoundZero` names the field), never a runtime divert-everything +/// state. Companion to the RFC 0004 validation suite in +/// `tests/invariants.rs`. +#[test] +fn zero_bounds_are_rejected_at_startup() { + use ourios_core::config::MinerConfigError; + assert_eq!( + MinerConfig::default().with_max_node_children(0), + Err(MinerConfigError::BoundZero("max_node_children")), + ); + assert_eq!( + MinerConfig::default().with_max_templates(0), + Err(MinerConfigError::BoundZero("max_templates")), + ); + assert_eq!( + MinerConfig::default().with_max_line_tokens(0), + Err(MinerConfigError::BoundZero("max_line_tokens")), + ); +} /// Scenario RFC0023.1 — the ceiling holds and never merges. /// See `docs/rfcs/0023-bounded-template-memory.md` §5. #[test] -#[ignore = "RFC0023.1 stub — implemented in the miner-bounds green slice"] fn rfc0023_1_template_ceiling_holds_and_never_merges() { - todo!( - "RFC0023.1 — with a small max_templates, the tenant's template count \ - plateaus at the ceiling, every would-mint line takes the \ - parse-failure path with its body retained, and no overflow line is \ - attached to any existing template" - ); + // Six structurally distinct lines (distinct token counts ⇒ + // distinct length buckets ⇒ every one would mint uncapped). + let lines = [ + "alpha started", + "beta accepted request now", + "gamma rejected request from peer five", + "delta flushed segment to disk at offset nine", + "epsilon compacted partition into one file at generation two", + "zeta rotated wal segment after threshold breach on node three today", + ]; + let tenant = TenantId::new("capped"); + + let ceiling = 3u32; + let sink = SharedRecordSink::new(); + let config = MinerConfig::default() + .with_max_templates(ceiling) + .expect("non-zero ceiling"); + let mut capped = MinerCluster::new(config).with_record_sink(Box::new(sink.clone())); + let mut uncapped = MinerCluster::new(MinerConfig::default()); + + let mut capped_ids = Vec::new(); + for line in lines { + capped_ids.push(capped.ingest(&record(&tenant, line))); + uncapped.ingest(&record(&tenant, line)); + } + + // The count plateaus at the ceiling… + assert_eq!(capped.templates_for(&tenant).len(), ceiling as usize); + // …every would-mint line diverted to parse-failure (no template, + // body retained, lossy-flagged), never attached to an existing + // template. + let emitted = sink.drain(); + assert_eq!(emitted.len(), lines.len()); + for (i, (line, rec)) in lines.iter().zip(&emitted).enumerate() { + if i < ceiling as usize { + assert_ne!(rec.template_id, NO_TEMPLATE, "line {i} minted"); + } else { + assert_eq!(capped_ids[i], NO_TEMPLATE, "line {i} diverted"); + assert_eq!(rec.template_id, NO_TEMPLATE, "line {i} carries no template"); + assert!(rec.lossy_flag, "line {i} is lossy-flagged"); + assert_eq!(rec.body.as_deref(), Some(*line), "line {i} body retained"); + } + } + // The capped set is exactly the uncapped run truncated at the + // ceiling — the ceiling stopped growth without disturbing what + // was already mined. + let uncapped_first: Vec<_> = template_set(&uncapped, &tenant) + .into_iter() + .filter(|(_, id)| capped_ids.contains(id)) + .collect(); + assert_eq!(template_set(&capped, &tenant), uncapped_first); } /// Scenario RFC0023.3 — node fan-out caps via wildcard routing. /// See `docs/rfcs/0023-bounded-template-memory.md` §5. #[test] -#[ignore = "RFC0023.3 stub — implemented in the miner-bounds green slice"] fn rfc0023_3_node_fanout_caps_via_wildcard_routing() { - todo!( - "RFC0023.3 — a prefix level presenting more than max_node_children \ - distinct tokens never exceeds the cap; later tokens route through \ - the wildcard child and attach stays threshold-gated (a below-floor \ - line still fails parse rather than merging)" + let tenant = TenantId::new("fanout"); + let config = MinerConfig::default() + .with_max_node_children(2) + .expect("non-zero cap"); + let mut cluster = MinerCluster::new(config); + + // Four distinct first tokens at one prefix level (same token + // count ⇒ one length bucket). The first two branch keyed; the + // rest route through the wildcard child — and still mint their + // own leaves (routing is not merging). + let first = cluster.ingest(&record(&tenant, "alpha handler finished cleanly")); + let second = cluster.ingest(&record(&tenant, "beta handler finished cleanly")); + let third = cluster.ingest(&record(&tenant, "gamma worker crashed hard")); + let fourth = cluster.ingest(&record(&tenant, "delta poller idled quietly")); + let minted: std::collections::HashSet = [first, second, third, fourth].into(); + assert_eq!(minted.len(), 4, "all four lines mint distinct templates"); + assert!(!minted.contains(&NO_TEMPLATE)); + + // Attach through the wildcard route: an exact repeat of a + // wildcard-routed line reuses its leaf (read-side routing finds + // the same bucket the write side used). + let repeat = cluster.ingest(&record(&tenant, "gamma worker crashed hard")); + assert_eq!(repeat, third, "wildcard-routed repeat attaches cleanly"); + + // Attach below the wildcard child stays threshold-gated: a + // below-floor line landing in gamma's exact bucket (unseen first + // token ⇒ wildcard route; second token "worker" ⇒ gamma's + // prefix path; same length) matches gamma's leaf at 1/4 < the + // 0.4 floor and must fail parse — not merge, not mint. + let templates_before = cluster.templates_for(&tenant).len(); + let unrelated = cluster.ingest(&record(&tenant, "omega worker jumped nowhere")); + assert_eq!( + unrelated, NO_TEMPLATE, + "below-floor under the wildcard child fails parse rather than merging", + ); + assert_eq!( + cluster.templates_for(&tenant).len(), + templates_before, + "the parse failure minted nothing", ); } /// Scenario RFC0023.4 — the long-line guard. /// See `docs/rfcs/0023-bounded-template-memory.md` §5. #[test] -#[ignore = "RFC0023.4 stub — implemented in the miner-bounds green slice"] fn rfc0023_4_long_lines_fail_parse_with_body_retained() { - todo!( - "RFC0023.4 — a line tokenizing past max_line_tokens takes the \ - parse-failure path, its body round-trips bit-identically, and no \ - template of that width exists in the tree" + let tenant = TenantId::new("longline"); + let sink = SharedRecordSink::new(); + let config = MinerConfig::default() + .with_max_line_tokens(4) + .expect("non-zero cap"); + let mut cluster = MinerCluster::new(config).with_record_sink(Box::new(sink.clone())); + + let long = "one two three four five"; + let id = cluster.ingest(&record(&tenant, long)); + assert_eq!( + id, NO_TEMPLATE, + "over-cap line takes the parse-failure path" ); + let mut emitted = sink.drain(); + assert_eq!(emitted.len(), 1, "exactly one record per ingested line"); + let rec = emitted.pop().expect("asserted non-empty above"); + assert!(rec.lossy_flag); + assert_eq!( + rec.body.as_deref(), + Some(long), + "body retained bit-identically" + ); + assert!( + cluster.templates_for(&tenant).is_empty(), + "no template of that width exists in the tree", + ); + + // At the cap exactly, the line mines normally. + let ok = cluster.ingest(&record(&tenant, "one two three four")); + assert_ne!(ok, NO_TEMPLATE); } /// Scenario RFC0023.5 — defaults are invisible on healthy corpora. /// See `docs/rfcs/0023-bounded-template-memory.md` §5. +/// +/// The seed corpus (`testdata/corpus/`) mines to an identical +/// template set under the default bounds and under bounds set to +/// their type maxima (the closest expressible "unbounded" run). +/// The full-strength oracle stays the corpus/C1/C2 suites in CI, +/// which now run under the defaults. #[test] -#[ignore = "RFC0023.5 stub — implemented in the miner-bounds green slice"] fn rfc0023_5_default_bounds_are_invisible_on_healthy_corpora() { - todo!( - "RFC0023.5 — under default bounds the seed-corpus template set is \ - identical to an unbounded run (the corpus/C1/C2 suites remain the \ - full-strength oracle in CI)" + use std::io::BufRead; + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root is two levels above CARGO_MANIFEST_DIR") + .join("testdata/corpus"); + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .expect("seed corpus dir") + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "txt")) + .collect(); + paths.sort(); + assert!(!paths.is_empty(), "seed corpus has *.txt files"); + + let tenant = TenantId::new("seed"); + let mut defaults = MinerCluster::new(MinerConfig::default()); + let unbounded_config = MinerConfig::default() + .with_max_node_children(u16::MAX) + .and_then(|c| c.with_max_templates(u32::MAX)) + .and_then(|c| c.with_max_line_tokens(u16::MAX)) + .expect("maxima are non-zero"); + let mut unbounded = MinerCluster::new(unbounded_config); + + for path in &paths { + let file = std::fs::File::open(path).expect("open corpus file"); + for line in std::io::BufReader::new(file).lines() { + let line = line.expect("read line"); + if line.is_empty() { + continue; + } + defaults.ingest(&record(&tenant, &line)); + unbounded.ingest(&record(&tenant, &line)); + } + } + + assert_eq!( + template_set(&defaults, &tenant), + template_set(&unbounded, &tenant), + "default bounds must not change what a healthy corpus mines to", ); }