diff --git a/crates/ourios-core/src/audit.rs b/crates/ourios-core/src/audit.rs index 88da7a38..036d3db5 100644 --- a/crates/ourios-core/src/audit.rs +++ b/crates/ourios-core/src/audit.rs @@ -35,6 +35,23 @@ use crate::tenant::TenantId; /// compiler enforces those per-variant contracts. #[derive(Debug, Clone, PartialEq, Eq)] pub enum TemplateChange { + /// A new leaf was allocated — the template's initial (version 1) + /// creation (RFC 0017 §3.1). Audited so a read-time template + /// registry can recover the v1 tokens once the originating rows + /// age out; without it, v1 rows would have no derivable tokens. + /// + /// Carries **only** the initial tokens. A leaf is *always* born at + /// version 1, so the variant deliberately omits a `new_version` field: + /// the v1 invariant is made unrepresentable rather than carried-and- + /// validated (there is no way to construct a creation at any other + /// version). The on-disk row still stores `new_version = 1` (the + /// writer supplies it) with `old_version` / `old_template` left `NULL` + /// — the RFC 0005 §3.7 "not applicable" sentinel (no prior template). + Created { + /// Canonical-form template at creation (the initial tokens, + /// literals + `<*>`), the same encoding `Widened` carries. + new_template: String, + }, /// An existing template gained one or more wildcard slots /// because a clean attach would otherwise mismatch positions /// (RFC §6.2 step 5). @@ -221,6 +238,10 @@ pub const EVENT_KIND_COMPACTION: u8 = 3; pub const EVENT_KIND_ALIAS_ASSERTED: u8 = 4; /// See [`EVENT_KIND_TEMPLATE_WIDENED`]. RFC 0001 §6.7 alias write path. pub const EVENT_KIND_ALIAS_RETRACTED: u8 = 5; +/// See [`EVENT_KIND_TEMPLATE_WIDENED`]. RFC 0017 §3.1 leaf-creation audit — +/// an **append-only** addition (next free ordinal, no renumber): old readers +/// surface it via the [`AuditPayload::Unknown`] tolerance path (RFC 0005 §3.7). +pub const EVENT_KIND_TEMPLATE_CREATED: u8 = 6; /// Canonical `event_type` strings paired with the ordinals above /// (RFC 0005 §3.7 / RFC 0001 §6.4 / RFC 0009 §3.6). @@ -236,6 +257,14 @@ pub const EVENT_TYPE_COMPACTION: &str = "compaction"; pub const EVENT_TYPE_ALIAS_ASSERTED: &str = "alias_asserted"; /// See [`EVENT_TYPE_TEMPLATE_WIDENED`]. RFC 0001 §6.7 alias write path. pub const EVENT_TYPE_ALIAS_RETRACTED: &str = "alias_retracted"; +/// See [`EVENT_TYPE_TEMPLATE_WIDENED`]. RFC 0017 §3.1 leaf-creation audit. +pub const EVENT_TYPE_TEMPLATE_CREATED: &str = "template_created"; + +/// The `template_version` a leaf is born at (RFC 0017 §3.1). The +/// [`TemplateChange::Created`] variant omits a version field — the invariant +/// is unrepresentable — so the writer supplies this for the on-disk +/// `new_version` column and the read-time registry keys creation rows by it. +pub const TEMPLATE_INITIAL_VERSION: u32 = 1; impl AuditPayload { /// The stable `event_kind` ordinal for this payload (RFC 0005 @@ -245,6 +274,7 @@ impl AuditPayload { pub fn event_kind(&self) -> u8 { match self { Self::Template { change, .. } => match change { + TemplateChange::Created { .. } => EVENT_KIND_TEMPLATE_CREATED, TemplateChange::Widened { .. } => EVENT_KIND_TEMPLATE_WIDENED, TemplateChange::TypeExpanded { .. } => EVENT_KIND_TEMPLATE_TYPE_EXPANDED, TemplateChange::RejectedDegenerate { .. } => { @@ -266,6 +296,7 @@ impl AuditPayload { pub fn event_type(&self) -> &str { match self { Self::Template { change, .. } => match change { + TemplateChange::Created { .. } => EVENT_TYPE_TEMPLATE_CREATED, TemplateChange::Widened { .. } => EVENT_TYPE_TEMPLATE_WIDENED, TemplateChange::TypeExpanded { .. } => EVENT_TYPE_TEMPLATE_TYPE_EXPANDED, TemplateChange::RejectedDegenerate { .. } => { @@ -314,6 +345,7 @@ impl TemplateChange { #[must_use] pub fn event_type(&self) -> &'static str { match self { + Self::Created { .. } => EVENT_TYPE_TEMPLATE_CREATED, Self::Widened { .. } => EVENT_TYPE_TEMPLATE_WIDENED, Self::TypeExpanded { .. } => EVENT_TYPE_TEMPLATE_TYPE_EXPANDED, Self::RejectedDegenerate { .. } => EVENT_TYPE_TEMPLATE_WIDENING_REJECTED_DEGENERATE, diff --git a/crates/ourios-core/tests/rfc0017_audit.rs b/crates/ourios-core/tests/rfc0017_audit.rs index 087231d2..49019307 100644 --- a/crates/ourios-core/tests/rfc0017_audit.rs +++ b/crates/ourios-core/tests/rfc0017_audit.rs @@ -1,25 +1,75 @@ //! RFC 0017 — read-time template registry & query-row rendering, the //! audit-schema arm of scenario `.1`. //! -//! **Status: `red`.** Failing stub driving the `green` implementation: it -//! encodes the audit-contract half of RFC 0017 §5 scenario .1 (the new -//! `template_created` `event_kind`/`event_type` is an append-only addition — -//! ordinal `6`, no existing ordinal renumbered) and currently `todo!()`s. It -//! is `#[ignore]`d so the default `cargo test` (and CI) stays green until the -//! `green` slice lands `TemplateChange::Created`; `green` replaces the body -//! with the real assertions and removes the `#[ignore]`. +//! Asserts the `template_created` audit event is an **append-only** addition: +//! a new `event_kind` ordinal `6` paired with the `event_type` string +//! `template_created`, with every existing ordinal (`0`–`5`) unchanged +//! (RFC 0005 §3.7), and that a `Created` payload derives the new +//! kind/type and does not count as a merge. //! //! See `docs/rfcs/0017-template-registry-query-rendering.md` §3.1 / §5 / §6. -/// Scenario RFC0017.1 (audit-schema arm) — the `template_created` event is an -/// append-only audit addition: a new `event_kind` ordinal `6` paired with the -/// `event_type` string `template_created`, with every existing ordinal (`0`–`5`) -/// left unchanged (RFC 0005 §3.7 append-only rule). +use std::time::SystemTime; + +use ourios_core::audit::{ + AuditEvent, AuditPayload, EVENT_KIND_ALIAS_ASSERTED, EVENT_KIND_ALIAS_RETRACTED, + EVENT_KIND_COMPACTION, EVENT_KIND_TEMPLATE_CREATED, EVENT_KIND_TEMPLATE_TYPE_EXPANDED, + EVENT_KIND_TEMPLATE_WIDENED, EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE, + EVENT_TYPE_TEMPLATE_CREATED, TemplateChange, hash_triggering_line, +}; +use ourios_core::tenant::TenantId; + +/// Scenario RFC0017.1 (audit-schema arm) — `template_created` is an +/// append-only audit addition: ordinal `6` / `event_type = "template_created"`, +/// existing ordinals `0`–`5` unchanged (RFC 0005 §3.7). /// See `docs/rfcs/0017-template-registry-query-rendering.md` §5. #[test] -#[ignore = "RFC0017.1 — red until TemplateChange::Created + event_kind 6 land (green)"] fn rfc0017_1_template_created_is_append_only_audit_addition() { - todo!( - "RFC0017.1: template_created = event_kind ordinal 6 / event_type \"template_created\", existing ordinals unchanged" - ) + // The new ordinal is the next free value, and the existing ordinals + // are untouched (the RFC 0005 §3.7 append-only rule — no renumber, so + // old readers are unaffected). + assert_eq!(EVENT_KIND_TEMPLATE_CREATED, 6); + assert_eq!(EVENT_TYPE_TEMPLATE_CREATED, "template_created"); + assert_eq!(EVENT_KIND_TEMPLATE_WIDENED, 0); + assert_eq!(EVENT_KIND_TEMPLATE_TYPE_EXPANDED, 1); + assert_eq!(EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE, 2); + assert_eq!(EVENT_KIND_COMPACTION, 3); + assert_eq!(EVENT_KIND_ALIAS_ASSERTED, 4); + assert_eq!(EVENT_KIND_ALIAS_RETRACTED, 5); + + // All seven ordinals are distinct — no collision with the new one. + let mut ordinals = [ + EVENT_KIND_TEMPLATE_WIDENED, + EVENT_KIND_TEMPLATE_TYPE_EXPANDED, + EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE, + EVENT_KIND_COMPACTION, + EVENT_KIND_ALIAS_ASSERTED, + EVENT_KIND_ALIAS_RETRACTED, + EVENT_KIND_TEMPLATE_CREATED, + ]; + let count = ordinals.len(); + ordinals.sort_unstable(); + let mut deduped = ordinals.to_vec(); + deduped.dedup(); + assert_eq!(deduped.len(), count, "event_kind ordinals must be distinct"); + + // A `Created` payload derives the new kind/type and is not a merge. + let event = AuditEvent { + tenant_id: TenantId::new("tenant-x"), + timestamp: SystemTime::UNIX_EPOCH, + payload: AuditPayload::Template { + template_id: 7, + triggering_line_hash: hash_triggering_line(b"user 42 logged in"), + triggering_line_sample: Some("user 42 logged in".to_owned()), + change: TemplateChange::Created { + new_template: "user <*> logged in".to_owned(), + }, + }, + }; + assert_eq!(event.payload.event_kind(), EVENT_KIND_TEMPLATE_CREATED); + assert_eq!(event.payload.event_type(), EVENT_TYPE_TEMPLATE_CREATED); + assert!( + !event.payload.counts_as_merge(), + "leaf creation is not a merge", + ); } diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index 448fb1e4..89ab5a0a 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -15,7 +15,8 @@ //! tree on `Body::String` records: //! //! - **No candidate** in the `(severity, scope, length, prefix)` -//! bucket → fresh leaf, no audit event (RFC0001.1). +//! bucket → fresh leaf, emitting a `TemplateChange::Created` audit +//! event (`event_type` `template_created`, RFC 0017 §3.1; not a merge). //! - **Best candidate has `sim_seq == 1.0`** → clean attach to the //! existing leaf, no widening, no audit. //! - **Best candidate has `threshold ≤ sim_seq < 1.0`** → compute @@ -1270,6 +1271,7 @@ impl MinerCluster { None => { let new_id = self.create_new_leaf( record, + raw, &masked_strs, &masked.wildcard_positions, &masked.typed_params, @@ -1310,16 +1312,17 @@ impl MinerCluster { ), // Lossy: new leaf rather than force-merge // into a too-weak candidate (RFC §6.2 step - // 5b). Body retained; no audit event - // (no widening happened). The retention - // counter bumps here; `record_parse_failure` - // covers the parse-failure-zone path - // separately. + // 5b). Body retained; no *widening* event, but + // `create_new_leaf` audits the leaf's creation + // (RFC 0017 §3.1). The retention counter bumps + // here; `record_parse_failure` covers the + // parse-failure-zone path separately. ConfidenceZone::Lossy => { self.body_retentions_total.fetch_add(1, Ordering::Relaxed); self.metrics.record_body_retention(&record.tenant_id); let new_id = self.create_new_leaf( record, + raw, &masked_strs, &masked.wildcard_positions, &masked.typed_params, @@ -1429,13 +1432,16 @@ impl MinerCluster { /// .type_tag}` for the k-th masked position, recording the /// type observed at that slot's first sight. /// - /// RFC0001.1: this path **does not** emit an audit event — - /// `template_count` already reflects the allocation and - /// `merges_total` is reserved for widening / type-expansion - /// events on existing leaves. + /// RFC 0017 §3.1: this path emits a `TemplateChange::Created` audit + /// event so a read-time registry can recover the leaf's v1 tokens. It + /// is **not** a merge — `template_count` reflects the allocation and + /// `merges_total` stays reserved for widening / type-expansion events + /// on existing leaves. (Supersedes the original RFC0001.1 + /// "creation emits nothing" contract.) fn create_new_leaf( &mut self, record: &OtlpLogRecord, + raw: &str, masked_strs: &[&str], line_wildcard_positions: &[usize], line_typed_params: &[crate::mask::TypedParam<'_>], @@ -1453,48 +1459,73 @@ impl MinerCluster { // can't reach back to `self.tenant_overrides` while the // map is borrowed mutably. let effective_config = self.effective_config(&record.tenant_id); - let state = self - .tenants - .entry(record.tenant_id.clone()) - .or_insert_with(|| TenantState::new(effective_config)); - let parent = state - .tree - .descend_mut(masked_strs, state.config.prefix_depth as usize); - // Build the leaf template: Wildcard at every mask-emitted - // position, Fixed at every other. `wildcard_positions` is - // ascending (single forward pass over the tokens) so we - // can walk both arrays in lockstep without allocating a - // membership set. - let mut new_template = Vec::with_capacity(masked_strs.len()); - let mut wp_iter = line_wildcard_positions.iter().copied().peekable(); - for (p, s) in masked_strs.iter().enumerate() { - if wp_iter.peek().copied() == Some(p) { - new_template.push(OwnedToken::Wildcard); - wp_iter.next(); - } else { - new_template.push(OwnedToken::Fixed((*s).to_string())); + // Scope the `self.tenants` borrow so it is released before the + // audit emit below (which borrows `self.audit_sink`); the leaf's + // canonical template string is computed inside and handed out. + let created_template = { + let state = self + .tenants + .entry(record.tenant_id.clone()) + .or_insert_with(|| TenantState::new(effective_config)); + let parent = state + .tree + .descend_mut(masked_strs, state.config.prefix_depth as usize); + // Build the leaf template: Wildcard at every mask-emitted + // position, Fixed at every other. `wildcard_positions` is + // ascending (single forward pass over the tokens) so we + // can walk both arrays in lockstep without allocating a + // membership set. + let mut new_template = Vec::with_capacity(masked_strs.len()); + let mut wp_iter = line_wildcard_positions.iter().copied().peekable(); + for (p, s) in masked_strs.iter().enumerate() { + if wp_iter.peek().copied() == Some(p) { + new_template.push(OwnedToken::Wildcard); + wp_iter.next(); + } else { + new_template.push(OwnedToken::Fixed((*s).to_string())); + } } - } - debug_assert!( - wp_iter.peek().is_none(), - "every wildcard_position must land within masked_strs.len()", - ); - let slot_types: Vec = line_typed_params - .iter() - .map(|tp| SlotTypes::singleton(tp.type_tag)) - .collect(); - parent.leaves.push(Leaf { - template: new_template, - template_id: new_id, - template_version: 1, - severity_number: record.severity_number, - scope_name: record.scope_name.clone(), - slot_types, + debug_assert!( + wp_iter.peek().is_none(), + "every wildcard_position must land within masked_strs.len()", + ); + let slot_types: Vec = line_typed_params + .iter() + .map(|tp| SlotTypes::singleton(tp.type_tag)) + .collect(); + // Canonical form for the audit event, taken before `new_template` + // is moved into the leaf. + let created_template = format_template(&new_template); + parent.leaves.push(Leaf { + template: new_template, + template_id: new_id, + template_version: 1, + severity_number: record.severity_number, + scope_name: record.scope_name.clone(), + slot_types, + }); + // Maintain the TenantState::template_count cache invariant — + // every fresh allocation under `state` is mirrored here so + // `MinerCluster::template_count` can stay O(1). + state.template_count += 1; + created_template + }; + // RFC 0017 §3.1 — audit the leaf's initial (version 1) creation so a + // read-time template registry can recover the v1 tokens once the + // originating rows age out. Same WAL-before-ack path as the widening + // events; not a merge, so it does not bump `merges_total`. + self.audit_sink.emit(AuditEvent { + tenant_id: record.tenant_id.clone(), + timestamp: self.clock.now(), + payload: AuditPayload::Template { + template_id: new_id, + triggering_line_hash: hash_triggering_line(raw.as_bytes()), + triggering_line_sample: Some(sample_first_256_bytes(raw)), + change: TemplateChange::Created { + new_template: created_template, + }, + }, }); - // Maintain the TenantState::template_count cache invariant — - // every fresh allocation under `state` is mirrored here so - // `MinerCluster::template_count` can stay O(1). - state.template_count += 1; new_id } @@ -2273,6 +2304,30 @@ mod tests { (cluster, sink) } + /// Drain the sink and return only the template *changes* a widening / + /// type-expansion / rejection test asserts on, dropping the per-leaf + /// `Created` events RFC 0017 §3.1 emits on every allocation. Leaf + /// creation is audited now (so a read-time registry can recover v1 + /// tokens), but its correctness is covered by + /// `fresh_leaf_emits_created_event` and the RFC0017.1 acceptance test; + /// filtering here keeps each widening test decoupled from how many + /// leaves the scenario happens to allocate rather than re-asserting the + /// creation count in every one. + fn drain_changes(sink: &SharedAuditSink) -> Vec { + sink.drain() + .into_iter() + .filter(|e| { + !matches!( + &e.payload, + AuditPayload::Template { + change: TemplateChange::Created { .. }, + .. + } + ) + }) + .collect() + } + // ---------- existing String-body behaviour preserved ---------- #[test] @@ -2663,7 +2718,7 @@ mod tests { assert_eq!(cluster.template_count(&t), 1); assert_eq!(cluster.merges_total(), 1); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1); let AuditPayload::Template { template_id, @@ -2698,21 +2753,38 @@ mod tests { } #[test] - fn fresh_leaf_does_not_emit_audit_event() { - // RFC0001.1 — leaf allocation is reflected in - // `template_count`, but the audit stream is reserved for - // widening events. Verifies both: - // - `merges_total` stays 0 - // - the sink stays empty + fn fresh_leaf_emits_created_event() { + // RFC 0017 §3.1 overturns the former "fresh leaf emits nothing" + // contract: leaf allocation now emits a `template_created` audit + // event (so a read-time registry can recover v1 tokens), while + // still NOT counting as a merge. Two distinct fresh leaves → + // exactly two `Created` events, `merges_total` still 0. let (mut cluster, sink) = cluster_with_observable_sink(); let t = TenantId::new("tenant-x"); - let _ = cluster.ingest(&string_record(&t, "user 42 logged in")); - let _ = cluster.ingest(&string_record(&t, "GET /home 200")); + let id_a = cluster.ingest(&string_record(&t, "user 42 logged in")); + let id_b = cluster.ingest(&string_record(&t, "GET /home 200")); assert_eq!(cluster.template_count(&t), 2); - assert_eq!(cluster.merges_total(), 0); - assert!(sink.is_empty()); + assert_eq!(cluster.merges_total(), 0, "creation is not a merge"); + + let events = sink.drain(); + assert_eq!(events.len(), 2, "one Created event per fresh leaf"); + for (event, id) in events.iter().zip([id_a, id_b]) { + let AuditPayload::Template { + template_id, + change: TemplateChange::Created { new_template }, + .. + } = &event.payload + else { + panic!("expected Template/Created, got {:?}", event.payload); + }; + assert_eq!(*template_id, id); + assert!( + !new_template.is_empty(), + "creation carries the initial tokens", + ); + } } #[test] @@ -2741,7 +2813,7 @@ mod tests { assert_eq!(id1, id2); assert_eq!(cluster.merges_total(), 0); - assert!(sink.is_empty()); + assert!(drain_changes(&sink).is_empty()); let templates = cluster.templates_for(&t); assert_eq!(templates.len(), 1); @@ -2779,7 +2851,7 @@ mod tests { let _ = cluster.ingest(&string_record(&t, "user 42 logged in from 10.0.0.1")); let _ = cluster.ingest(&string_record(&t, "user 42 logged out from 10.0.0.1")); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1); let AuditPayload::Template { change: @@ -2818,7 +2890,7 @@ mod tests { assert_eq!(cluster.template_count(&t), 1); assert_eq!(cluster.merges_total(), 2); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 2); let AuditPayload::Template { change: @@ -2873,9 +2945,10 @@ mod tests { // Mask emits at positions 1 (``) and 5 (``). let _ = cluster.ingest(&string_record(&t, "user 42 logged in from 10.0.0.1")); - // Fresh-leaf creation does NOT emit an audit event - // (RFC0001.1) — even with non-empty slot_types. - assert!(sink.is_empty()); + // Fresh-leaf creation emits a `Created` event now (RFC 0017 §3.1), + // but no *widening / type-expansion* — even with non-empty + // slot_types. `drain_changes` filters the Created event out. + assert!(drain_changes(&sink).is_empty()); let templates = cluster.templates_for(&t); assert_eq!(templates.len(), 1); @@ -2933,7 +3006,7 @@ mod tests { let _ = cluster.ingest(&string_record(&t, "user 42 logged in")); let _ = cluster.ingest(&string_record(&t, "user 42 logged out")); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1, "literal widening: one event only"); assert!(matches!( events[0].payload, @@ -2989,7 +3062,7 @@ mod tests { "user logged 550e8400-e29b-41d4-a716-446655440000 in", )); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1, "single TemplateTypeExpanded, no widening"); let AuditPayload::Template { change: @@ -3043,7 +3116,7 @@ mod tests { let _ = cluster.ingest(&string_record(&t, "user logged 99 in")); assert!( - sink.is_empty(), + drain_changes(&sink).is_empty(), "known type at typed wildcard must not emit", ); let templates = cluster.templates_for(&t); @@ -3073,7 +3146,7 @@ mod tests { // leaf has a Wildcard with slot_types[0] = {Str}. let _ = cluster.ingest(&string_record(&t, "user logged at hour 13")); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1, "exactly one TemplateTypeExpanded"); let AuditPayload::Template { change: @@ -3156,7 +3229,7 @@ mod tests { // Expansion fires at ordinal 1, adding Num. let _ = cluster.ingest(&string_record(&t, "user logged at minute 13")); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 2, "combined widening + type expansion"); let AuditPayload::Template { change: @@ -3277,7 +3350,7 @@ mod tests { "GET /home 550e8400-e29b-41d4-a716-446655440000 ok", )); - let events = sink.drain(); + let events = drain_changes(&sink); assert!( !events.is_empty(), "§3.1: mask-tag type change at a tree-routed wildcard slot must audit", @@ -3335,7 +3408,7 @@ mod tests { let _ = cluster.ingest(&string_record(&t, "user logged at hour ")); assert!( - sink.is_empty(), + drain_changes(&sink).is_empty(), "literal `` must be Str (already in slot's set), not a spurious Num expansion", ); let templates = cluster.templates_for(&t); @@ -3389,9 +3462,11 @@ mod tests { "leaf `Fixed(\"\")` (literal) must not absorb a real mask-emit ``", ); assert_eq!(cluster.template_count(&t), 2); - // No widening fired (the Lossy zone created a new leaf - // rather than widening). No audit events. - assert!(audit_sink.is_empty()); + // No widening fired (the Lossy zone created a new leaf rather than + // widening). The two leaf creations each emit a `Created` event + // (RFC 0017 §3.1), which `drain_changes` filters out — so there are + // no widening / type-expansion / rejection events. + assert!(drain_changes(&audit_sink).is_empty()); // The §6.3 lossy zone bumped body_retentions for L2 (and // retained its body on the emitted record). assert_eq!(cluster.body_retentions_total(), 1); @@ -3491,7 +3566,7 @@ mod tests { let l2 = "user 42 logged out from 10.0.0.1"; let _ = cluster.ingest(&string_record(&t, l2)); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1); let AuditPayload::Template { triggering_line_hash, @@ -3521,7 +3596,7 @@ mod tests { assert_ne!(id1, id2); assert_eq!(cluster.template_count(&t), 2); assert_eq!(cluster.merges_total(), 0); - assert!(sink.is_empty()); + assert!(drain_changes(&sink).is_empty()); } #[test] @@ -3579,7 +3654,7 @@ mod tests { "§6.4 says degenerate-rejected lines retain body", ); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 2); assert!( matches!( @@ -3714,7 +3789,7 @@ mod tests { assert_ne!(id_a, id_b, "leaves are distinct after L2"); assert_eq!(cluster.template_count(&t), 2); assert!( - sink.is_empty(), + drain_changes(&sink).is_empty(), "L2 fell into the lossy zone → fresh leaf, no widening", ); assert_eq!( @@ -3730,7 +3805,7 @@ mod tests { "best-candidate selection must pick the higher-similarity leaf", ); assert_eq!(cluster.merges_total(), 1); - let events = sink.drain(); + let events = drain_changes(&sink); assert_eq!(events.len(), 1); let AuditPayload::Template { template_id, @@ -4043,7 +4118,10 @@ mod tests { assert_eq!(cluster.body_retentions_total(), 1); assert_eq!(cluster.merges_total(), 0); assert_eq!(cluster.parse_failures_total(), 0); - assert!(sink.is_empty(), "lossy attach emits no audit event"); + assert!( + drain_changes(&sink).is_empty(), + "lossy attach emits no audit event" + ); } #[test] @@ -4087,7 +4165,10 @@ mod tests { "RFC §6.3: parse failure retains body too", ); assert_eq!(cluster.merges_total(), 0); - assert!(sink.is_empty(), "parse failure emits no audit event"); + assert!( + drain_changes(&sink).is_empty(), + "parse failure emits no audit event" + ); } #[test] diff --git a/crates/ourios-miner/tests/hazards.rs b/crates/ourios-miner/tests/hazards.rs index f85c73b1..2d527316 100644 --- a/crates/ourios-miner/tests/hazards.rs +++ b/crates/ourios-miner/tests/hazards.rs @@ -219,8 +219,22 @@ fn h1_3_every_widening_emits_an_audit_event() { let _ = cluster.ingest(&make("user 42 logged in from 10.0.0.1")); let _ = cluster.ingest(&make("user 42 logged out from 10.0.0.1")); - // Assert — exactly one audit event, schema fields populated. - let events = sink.drain(); + // Assert — exactly one *widening* event, schema fields populated. + // Leaf creation is audited separately now (a leading `Created` event, + // RFC 0017 §3.1), so filter to the widening this scenario exercises. + let events: Vec<_> = sink + .drain() + .into_iter() + .filter(|e| { + !matches!( + &e.payload, + AuditPayload::Template { + change: TemplateChange::Created { .. }, + .. + } + ) + }) + .collect(); assert_eq!(events.len(), 1, "exactly one widening occurred"); let e = &events[0]; assert_eq!(e.tenant_id, t); @@ -748,8 +762,21 @@ fn h5_1_wildcard_widening_increments_version_and_emits_template_widened() { let _ = cluster.ingest(&make("user 42 logged in from 10.0.0.1")); let _ = cluster.ingest(&make("user 42 logged out from 10.0.0.1")); - // Assert - let events = sink.drain(); + // Assert — filter out the leading `Created` event (RFC 0017 §3.1 + // audits leaf creation); this scenario asserts the widening. + let events: Vec<_> = sink + .drain() + .into_iter() + .filter(|e| { + !matches!( + &e.payload, + AuditPayload::Template { + change: TemplateChange::Created { .. }, + .. + } + ) + }) + .collect(); assert_eq!(events.len(), 1); let AuditPayload::Template { change: diff --git a/crates/ourios-miner/tests/rfc0017_template_created.rs b/crates/ourios-miner/tests/rfc0017_template_created.rs index 2097a561..2af031e7 100644 --- a/crates/ourios-miner/tests/rfc0017_template_created.rs +++ b/crates/ourios-miner/tests/rfc0017_template_created.rs @@ -1,25 +1,55 @@ //! RFC 0017 — read-time template registry & query-row rendering, the //! miner-emit arm of scenario `.1`. //! -//! **Status: `red`.** Failing stub driving the `green` implementation: it -//! encodes the miner half of RFC 0017 §5 scenario .1 (a new leaf's allocation -//! emits a `template_created` audit event carrying its initial tokens, on the -//! WAL-before-ack path) and currently `todo!()`s. It is `#[ignore]`d so the -//! default `cargo test` (and CI) stays green until the `green` slice lands the -//! emit; `green` replaces the body with the real assertion and removes the -//! `#[ignore]`. +//! Asserts that allocating a new leaf emits a `template_created` audit event +//! carrying the leaf's `template_id` and the initial tokens — on the same +//! `AuditSink` (WAL-before-ack) path as the existing template events. The +//! `Created` variant carries no version field (a leaf is always born at v1, +//! made unrepresentable); the on-disk row stores `new_version = 1`. //! //! See `docs/rfcs/0017-template-registry-query-rendering.md` §3.1 / §5 / §6. -/// Scenario RFC0017.1 (miner-emit arm) — allocating a new leaf emits a -/// `template_created` audit event carrying `(template_id, new_version = 1, -/// new_template = the initial tokens)`, with `old_template`/`old_version` left -/// `NULL`, on the same WAL-before-ack path as the existing template events. +use ourios_core::audit::{AuditPayload, SharedAuditSink, TemplateChange}; +use ourios_core::config::MinerConfig; +use ourios_core::otlp::{Body, OtlpLogRecord}; +use ourios_core::tenant::TenantId; +use ourios_miner::cluster::MinerCluster; + +/// Scenario RFC0017.1 (miner-emit arm) — a new leaf's allocation emits a +/// `template_created` event with `(template_id, new_version = 1, +/// new_template = the initial tokens)`. /// See `docs/rfcs/0017-template-registry-query-rendering.md` §5. #[test] -#[ignore = "RFC0017.1 — red until the miner emits template_created on leaf creation (green)"] fn rfc0017_1_new_leaf_emits_template_created() { - todo!( - "RFC0017.1: first leaf allocation emits template_created with new_version=1 + initial tokens" - ) + let sink = SharedAuditSink::new(); + let mut cluster = MinerCluster::with_audit_sink(MinerConfig::default(), Box::new(sink.clone())); + let t = TenantId::new("tenant-a"); + + // "user 42 logged in" masks `` at position 1, so the leaf's + // canonical template is "user <*> logged in". + let id = cluster.ingest(&OtlpLogRecord { + tenant_id: t.clone(), + body: Some(Body::String("user 42 logged in".to_owned())), + ..Default::default() + }); + + // The event is emitted through the same `AuditSink` as every other + // template event (the WAL-before-ack path once the WAL sink lands). + let events = sink.drain(); + assert_eq!(events.len(), 1, "fresh leaf emits exactly one audit event"); + let AuditPayload::Template { + template_id, + change: TemplateChange::Created { new_template }, + .. + } = &events[0].payload + else { + panic!("expected Template/Created, got {:?}", events[0].payload); + }; + assert_eq!(*template_id, id, "event names the allocated template_id"); + // The variant carries no version (a leaf is always born at v1, made + // unrepresentable-if-otherwise); the on-disk row stores new_version = 1. + assert_eq!( + new_template, "user <*> logged in", + "creation carries the initial tokens", + ); } diff --git a/crates/ourios-miner/tests/rfc_internal.rs b/crates/ourios-miner/tests/rfc_internal.rs index 99797045..74bfaa85 100644 --- a/crates/ourios-miner/tests/rfc_internal.rs +++ b/crates/ourios-miner/tests/rfc_internal.rs @@ -10,20 +10,24 @@ //! stub runs `cargo test -- --ignored` and watches the //! `todo!()` panic. See `docs/verification.md` §3. -/// Scenario RFC0001.1 — Fresh-leaf creation does not emit an audit event. -/// See `docs/rfcs/0001-template-miner.md` §5. +/// Scenario RFC0001.1 — **superseded by RFC 0017 §3.1.** The original +/// RFC0001.1 contract was "fresh-leaf creation does not emit an audit +/// event". RFC 0017 §3.1 overturns it: a read-time template registry must +/// be able to recover a leaf's version-1 tokens after the originating rows +/// age out, so leaf creation now emits a `template_created` audit event. +/// This test asserts the amended contract — still *not* a merge. +/// See `docs/rfcs/0017-template-registry-query-rendering.md` §3.1 / §5 +/// (and `docs/rfcs/0001-template-miner.md` §5 for the original scenario). #[test] -fn rfc0001_1_fresh_leaf_creation_does_not_emit_audit_event() { - use ourios_core::audit::SharedAuditSink; +fn rfc0001_1_fresh_leaf_creation_emits_template_created() { + use ourios_core::audit::{AuditPayload, SharedAuditSink, TemplateChange}; use ourios_core::config::MinerConfig; use ourios_core::otlp::{Body, OtlpLogRecord}; use ourios_core::tenant::TenantId; use ourios_miner::cluster::MinerCluster; // Arrange — ingest two structurally distinct lines, both - // creating fresh leaves. Per §6.2 step 4, fresh-leaf - // creation does not emit an audit event; the audit stream - // is reserved for widening events. + // creating fresh leaves. let sink = SharedAuditSink::new(); let mut cluster = MinerCluster::with_audit_sink(MinerConfig::default(), Box::new(sink.clone())); let t = TenantId::new("tenant-x"); @@ -37,13 +41,22 @@ fn rfc0001_1_fresh_leaf_creation_does_not_emit_audit_event() { let _ = cluster.ingest(&make("user 42 logged in")); let _ = cluster.ingest(&make("GET /home 200")); - // Assert — both lines created fresh leaves; the sink - // remains empty. + // Assert — both lines created fresh leaves, each emitting one + // `template_created` event; creation is not a merge. assert_eq!(cluster.template_count(&t), 2); - assert_eq!(cluster.merges_total(), 0); + assert_eq!(cluster.merges_total(), 0, "creation is not a merge"); + + let events = sink.drain(); + assert_eq!(events.len(), 2, "one template_created event per fresh leaf"); assert!( - sink.is_empty(), - "fresh-leaf creation must not emit audit events", + events.iter().all(|e| matches!( + &e.payload, + AuditPayload::Template { + change: TemplateChange::Created { .. }, + .. + } + )), + "every fresh leaf emits a version-1 template_created event", ); } @@ -117,7 +130,21 @@ fn rfc0001_2_degenerate_template_guard_rejects_fully_wildcard_widening() { "L3's rejection is a parse failure", ); - let events = sink.drain(); + // Filter out the leading `Created` events (RFC 0017 §3.1 audits leaf + // creation); this scenario asserts the widening + rejection pair. + let events: Vec<_> = sink + .drain() + .into_iter() + .filter(|e| { + !matches!( + &e.payload, + AuditPayload::Template { + change: TemplateChange::Created { .. }, + .. + } + ) + }) + .collect(); assert_eq!(events.len(), 2); assert!( matches!( diff --git a/crates/ourios-parquet/src/audit_reader.rs b/crates/ourios-parquet/src/audit_reader.rs index 8baa2cd2..5bf4e730 100644 --- a/crates/ourios-parquet/src/audit_reader.rs +++ b/crates/ourios-parquet/src/audit_reader.rs @@ -43,7 +43,7 @@ use parquet::errors::ParquetError; use crate::audit_columns; use crate::audit_record_batch::{ EVENT_KIND_ALIAS_ASSERTED, EVENT_KIND_ALIAS_RETRACTED, EVENT_KIND_COMPACTION, - EVENT_KIND_TEMPLATE_TYPE_EXPANDED, EVENT_KIND_TEMPLATE_WIDENED, + EVENT_KIND_TEMPLATE_CREATED, EVENT_KIND_TEMPLATE_TYPE_EXPANDED, EVENT_KIND_TEMPLATE_WIDENED, EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE, }; use crate::audit_writer::{audit_partition_matches, derive_audit_partition}; @@ -327,7 +327,8 @@ fn batch_to_audit_events( let file_row = row_offset + i; let ts = decode_timestamp(timestamp[i], file_row)?; let payload = match event_kind[i] { - EVENT_KIND_TEMPLATE_WIDENED + EVENT_KIND_TEMPLATE_CREATED + | EVENT_KIND_TEMPLATE_WIDENED | EVENT_KIND_TEMPLATE_TYPE_EXPANDED | EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE => { let cols = TemplateColumns { @@ -539,6 +540,18 @@ fn decode_template_change( i: usize, file_row: usize, ) -> Result { + // Creation has no prior template, so its `old_*` columns are NULL + // (RFC 0017 §3.1) — handle it before `require_at` on `old_version` / + // `old_template`, which would (correctly) reject those NULLs for the + // widening kinds. The variant omits a version (a leaf is always born at + // v1), so the on-disk `new_version` (canonically `1`) is not read back + // into it — the v1 contract is structural, not a decoded value. + if cols.event_kind == EVENT_KIND_TEMPLATE_CREATED { + return Ok(TemplateChange::Created { + new_template: require_at(cols.new_template, i, audit_columns::NEW_TEMPLATE, file_row)?, + }); + } + let old_version = require_at(cols.old_version, i, audit_columns::OLD_VERSION, file_row)?; let old_template = require_at(cols.old_template, i, audit_columns::OLD_TEMPLATE, file_row)?; diff --git a/crates/ourios-parquet/src/audit_record_batch.rs b/crates/ourios-parquet/src/audit_record_batch.rs index 260b2466..bb040cf0 100644 --- a/crates/ourios-parquet/src/audit_record_batch.rs +++ b/crates/ourios-parquet/src/audit_record_batch.rs @@ -57,7 +57,9 @@ use arrow_array::builder::{ }; use arrow_array::{ArrayRef, RecordBatch}; use arrow_schema::{ArrowError, DataType, Field}; -use ourios_core::audit::{AuditEvent, AuditPayload, ParamType, SlotExpansion, TemplateChange}; +use ourios_core::audit::{ + AuditEvent, AuditPayload, ParamType, SlotExpansion, TEMPLATE_INITIAL_VERSION, TemplateChange, +}; use crate::audit_schema; @@ -67,10 +69,11 @@ use crate::audit_schema; /// existing call sites resolve them at their established path. pub use ourios_core::audit::{ EVENT_KIND_ALIAS_ASSERTED, EVENT_KIND_ALIAS_RETRACTED, EVENT_KIND_COMPACTION, - EVENT_KIND_TEMPLATE_TYPE_EXPANDED, EVENT_KIND_TEMPLATE_WIDENED, + EVENT_KIND_TEMPLATE_CREATED, EVENT_KIND_TEMPLATE_TYPE_EXPANDED, EVENT_KIND_TEMPLATE_WIDENED, EVENT_KIND_TEMPLATE_WIDENING_REJECTED_DEGENERATE, EVENT_TYPE_ALIAS_ASSERTED, - EVENT_TYPE_ALIAS_RETRACTED, EVENT_TYPE_COMPACTION, EVENT_TYPE_TEMPLATE_TYPE_EXPANDED, - EVENT_TYPE_TEMPLATE_WIDENED, EVENT_TYPE_TEMPLATE_WIDENING_REJECTED_DEGENERATE, + EVENT_TYPE_ALIAS_RETRACTED, EVENT_TYPE_COMPACTION, EVENT_TYPE_TEMPLATE_CREATED, + EVENT_TYPE_TEMPLATE_TYPE_EXPANDED, EVENT_TYPE_TEMPLATE_WIDENED, + EVENT_TYPE_TEMPLATE_WIDENING_REJECTED_DEGENERATE, }; /// Build an Arrow `RecordBatch` matching [`audit_schema`] from a @@ -385,6 +388,20 @@ impl Builders { /// template, positions/slots, and reason columns. fn append_template_change(&mut self, change: &TemplateChange) -> Result<(), AuditBatchError> { match change { + TemplateChange::Created { new_template } => { + // RFC 0017 §3.1 — creation has no prior template: the + // `old_*` columns are NULL (the "not applicable" sentinel), + // not a copy of the new template. The variant omits a + // version (a leaf is always born at v1), so the on-disk + // `new_version` is the canonical initial version. + self.old_version.append_null(); + self.new_version.append_value(TEMPLATE_INITIAL_VERSION); + self.old_template.append_null(); + self.new_template.append_value(new_template); + append_positions(&mut self.positions_widened, &[]); + append_slots(&mut self.slots_expanded, &[]); + self.reason.append_null(); + } TemplateChange::Widened { old_version, new_version, diff --git a/crates/ourios-parquet/tests/audit_round_trip.rs b/crates/ourios-parquet/tests/audit_round_trip.rs index 48b0bf62..6ab04bb4 100644 --- a/crates/ourios-parquet/tests/audit_round_trip.rs +++ b/crates/ourios-parquet/tests/audit_round_trip.rs @@ -191,6 +191,42 @@ fn rfc0005_7_audit_round_trip_one_of_each_variant() { } } +/// Scenario RFC0017.1 (storage round-trip) — the `template_created` variant +/// round-trips through the audit file series. Full `AuditEvent` equality +/// confirms `new_template` survives and the reader reconstructs `Created` +/// (which carries no version — a leaf is always v1 — and no `old_*`, +/// reflecting the NULL "not applicable" columns; the on-disk `new_version` +/// is the canonical `1`, RFC 0017 §3.1) — pins the writer/reader arms. +#[test] +fn rfc0017_1_template_created_round_trips() { + let bucket = TempDir::new().unwrap(); + let event = template_event( + "acme", + 11, + Some("user 42 logged in"), + 1_775_127_480, + TemplateChange::Created { + new_template: "user <*> logged in".to_string(), + }, + ); + let partition = audit_partition_for(&event); + + let mut writer = AuditWriter::open(bucket.path(), partition.clone()).expect("open"); + writer + .append_events(std::slice::from_ref(&event)) + .expect("append"); + let written = writer.close().expect("close"); + + let reader = AuditReader::open_partition(&written.path, partition).expect("open_partition"); + let round_tripped = reader.read_all().expect("read_all"); + + assert_eq!(round_tripped.len(), 1); + assert_eq!( + round_tripped[0], event, + "template_created round-trips with full AuditEvent equality", + ); +} + /// RFC0005.7 sub-test — audit files land under /// `audit/tenant_id=…/year/month/day/.parquet`. There /// is NO `hour=HH` segment (the audit partitioning is one axis diff --git a/docs/rfcs/0017-template-registry-query-rendering.md b/docs/rfcs/0017-template-registry-query-rendering.md index a5a9531e..d6f0cc6c 100644 --- a/docs/rfcs/0017-template-registry-query-rendering.md +++ b/docs/rfcs/0017-template-registry-query-rendering.md @@ -21,11 +21,16 @@ plus the body (rendered for string bodies, returned as structure for time, so this RFC builds a **read-time template registry** (`(template_id, template_version) → tokens`) by folding the tenant's audit stream — and, because a template's initial creation is unaudited today, -**amends the audit contract to emit a `TemplateCreated` event** on leaf +**amends the audit contract to emit a `template_created` event** on leaf creation. This delivers the typed-row payload RFC 0007 §4.1 specifies but the engine never built, and is the prerequisite for RFC 0016's endpoint to return actual logs. +This **amends RFC 0001**: scenario RFC0001.1 ("fresh-leaf creation does not +emit an audit event") is superseded — leaf creation now emits a +`template_created` event (§3.1). It remains a non-merge (`merges_total` +unchanged), so RFC 0001's merge-counting contract is untouched. + ## 2. Motivation A query returns `QueryResult { rows: u64, stats }` today — a count, no @@ -47,12 +52,12 @@ designed away" — the manifest fork #94/#147). So the registry should be blocker: derivation is only correct if the audit stream records **every** template version's tokens. It records widening (`new_template`) and type-expansion, but **not a template's initial (version 1) creation** — -so v1 rows have no derivable tokens. Closing that gap (a `TemplateCreated` +so v1 rows have no derivable tokens. Closing that gap (a `template_created` audit event) makes the registry complete and the rendering correct. ## 3. Proposed design -### 3.1 The audit gap → a `TemplateCreated` event +### 3.1 The audit gap → a `template_created` event When the miner allocates a new leaf it assigns a `template_id` / `template_version = 1` but emits **no audit event**; the first event for @@ -72,9 +77,16 @@ old readers are unaffected and §3.5 migration holds). It reuses the existing audit columns: `new_template` = the initial tokens, `new_version = 1`, and `old_template`/`old_version` left **`NULL`** — the OPTIONAL "not applicable to this event kind" sentinel per RFC 0005 §3.7 -(no prior template), not a zero/empty value. The miner emits it at leaf creation, on the same WAL-before-ack -path as the existing template events, so by the time a v1 row reaches -Parquet its `TemplateCreated` event is durable. +(no prior template), not a zero/empty value. The in-memory +`TemplateChange::Created` variant carries **only** `new_template`: a leaf is +always born at version 1, so rather than carry-and-validate a `new_version` +field the invariant is made *unrepresentable* (there is no way to construct +a creation at another version). The writer supplies the canonical +`new_version = 1` for the on-disk column (`TEMPLATE_INITIAL_VERSION`); the +reader does not read it back into the variant. The miner emits it at leaf +creation, on the same WAL-before-ack path as the existing template events, +so by the time a v1 row reaches Parquet its `template_created` event is +durable. ### 3.2 `derive_template_registry` — fold the audit stream @@ -214,7 +226,7 @@ Rendering is bounded to the returned (`limit`-capped) rows. ## 4. Alternatives considered **Derive ≥v2, reconstruct v1 from a surviving row.** Skip the -`TemplateCreated` event; if a v1 token set is missing, recover it from any +`template_created` event; if a v1 token set is missing, recover it from any still-present v1 row's shape. Rejected — fragile and lossy: once every v1 row of a template is compacted/retention-expired, its tokens are unrecoverable, so a later query over an older file that *does* reference @@ -366,7 +378,7 @@ is greppable (`docs/verification.md` §2). - [ ] **Registry memory bound** — for tenants with very large template counts, is the per-query in-memory registry acceptable, or does it need a cap / lazy per-`(id,version)` lookup? -- [ ] **`TemplateCreated` payload** — does it also carry `slot_types` +- [ ] **`template_created` payload** — does it also carry `slot_types` (like `TypeExpanded`), or just tokens? (Leaning tokens-only for v1; slot types are derivable / not needed for `render`.) - [x] **Structured-body rendering** — *resolved* (§3.3 / §3.4): the OTLP