diff --git a/Cargo.lock b/Cargo.lock index 2295b137c..499cc9a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2695,6 +2695,8 @@ dependencies = [ "ourios-core", "ourios-semconv", "ourios-telemetry", + "serde", + "serde_json", "sha2", "tokio", ] diff --git a/crates/ourios-miner/Cargo.toml b/crates/ourios-miner/Cargo.toml index 307d37a25..59e81a58a 100644 --- a/crates/ourios-miner/Cargo.toml +++ b/crates/ourios-miner/Cargo.toml @@ -25,6 +25,12 @@ sha2 = { version = "0.10", default-features = false } # `ourios-telemetry`). Drives the §6.8 miner instrument set in # `crate::metrics`. opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } +# RFC 0001 §6.9 snapshot payload codec. The leading version byte (byte 0) +# keeps the codec a v1 implementation detail behind the format framing; +# serde derive + serde_json are the workspace-default serialization stack +# (same pins as ourios-core / ourios-parquet / ourios-querier). +serde = { version = "1", default-features = false, features = ["std", "derive"] } +serde_json = { version = "1", default-features = false, features = ["std"] } [dev-dependencies] # `init_in_memory` collects the §6.8 instrument stream through an diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index 9564e7b79..7e8b4903b 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -1783,6 +1783,74 @@ impl MinerCluster { .collect() }) } + + /// Capture one tenant's full template state as a serialisable + /// [`SnapshotState`](crate::snapshot::SnapshotState) per RFC 0001 + /// §6.9. Returns an empty state (no leaves, no structured + /// templates) for an unseen tenant. + /// + /// This is the producer side of the §6.9 snapshot format: it + /// captures every `Body::String` leaf (template tokens, + /// `template_id`, `template_version`, the `(severity_number, + /// scope_name)` template key, and per-slot `slot_types`) plus the + /// §6.2 step-0 structured-template-id map. `wal_high_water` is the + /// caller's to supply — the cluster does not track WAL offsets — + /// so it is left `None` here; the snapshot writer fills it from + /// the WAL at the segment-rotation boundary it snapshots on. + #[must_use] + pub fn snapshot_state(&self, tenant_id: &TenantId) -> crate::snapshot::SnapshotState { + use crate::snapshot::{ + LeafRecord, SnapshotState, StructuredTemplateRecord, TokenRecord, + slot_types_vec_to_record, + }; + + let Some(state) = self.tenants.get(tenant_id) else { + return SnapshotState { + leaves: Vec::new(), + structured_templates: Vec::new(), + wal_high_water: None, + }; + }; + + // `collect_leaves` and the `structured_templates` map both iterate + // in `HashMap` order, which varies across runs — sort by the + // cluster-unique `template_id` so the serialized snapshot is + // byte-deterministic (no spurious churn between snapshots of an + // unchanged tree). + let mut leaves: Vec = state + .tree + .collect_leaves() + .into_iter() + .map(|leaf| LeafRecord { + template: leaf.template.iter().map(TokenRecord::from).collect(), + template_id: leaf.template_id, + template_version: leaf.template_version, + severity_number: leaf.severity_number, + scope_name: leaf.scope_name.clone(), + slot_types: slot_types_vec_to_record(&leaf.slot_types), + }) + .collect(); + leaves.sort_by_key(|leaf| leaf.template_id); + + let mut structured_templates: Vec = state + .structured_templates + .iter() + .map( + |((severity_number, scope_name), template_id)| StructuredTemplateRecord { + severity_number: *severity_number, + scope_name: scope_name.clone(), + template_id: *template_id, + }, + ) + .collect(); + structured_templates.sort_by_key(|record| record.template_id); + + SnapshotState { + leaves, + structured_templates, + wal_high_water: None, + } + } } /// Read-only view of a single leaf surfaced by @@ -2000,6 +2068,60 @@ mod tests { assert_eq!(cluster.merges_total(), 0); } + #[test] + fn snapshot_state_orders_records_by_template_id() { + // The tree and the structured-template map both iterate in + // `HashMap` order; `snapshot_state` sorts by the cluster-unique + // `template_id` so the serialized snapshot is byte-deterministic. + let mut cluster = MinerCluster::new(MinerConfig::default()); + let t = TenantId::new("tenant-x"); + for line in [ + "GET /home 200", + "user 42 logged in", + "cache evicted 5 keys", + "disk usage high", + ] { + let _ = cluster.ingest(&string_record(&t, line)); + } + // Distinct (severity, scope) keys populate the structured-template + // map so its ordering is exercised too. + for (severity, scope) in [(9, Some("lib.a")), (5, Some("lib.b")), (13, None)] { + let _ = cluster.ingest(&structured_record(&t, severity, scope)); + } + + let state = cluster.snapshot_state(&t); + + assert!(state.leaves.len() >= 2, "needs multiple leaves to order"); + assert!( + state + .leaves + .windows(2) + .all(|w| w[0].template_id <= w[1].template_id), + "snapshot leaves must be sorted by template_id, got {:?}", + state + .leaves + .iter() + .map(|l| l.template_id) + .collect::>(), + ); + assert!( + state.structured_templates.len() >= 2, + "needs multiple structured templates to order", + ); + assert!( + state + .structured_templates + .windows(2) + .all(|w| w[0].template_id <= w[1].template_id), + "structured templates must be sorted by template_id, got {:?}", + state + .structured_templates + .iter() + .map(|s| s.template_id) + .collect::>(), + ); + } + #[test] fn template_count_is_zero_for_unseen_tenant() { let cluster = MinerCluster::new(MinerConfig::default()); diff --git a/crates/ourios-miner/src/lib.rs b/crates/ourios-miner/src/lib.rs index 1ebd67e6b..5b18aac2a 100644 --- a/crates/ourios-miner/src/lib.rs +++ b/crates/ourios-miner/src/lib.rs @@ -6,5 +6,6 @@ mod metrics; pub mod overflow; pub mod reconstruct; pub mod sim_seq; +pub mod snapshot; pub mod tokenize; pub mod tree; diff --git a/crates/ourios-miner/src/snapshot.rs b/crates/ourios-miner/src/snapshot.rs new file mode 100644 index 000000000..e13bc2b41 --- /dev/null +++ b/crates/ourios-miner/src/snapshot.rs @@ -0,0 +1,515 @@ +//! Per-tenant template-tree snapshot format + v1 recovery +//! (RFC 0001 §6.9, §3.5.1 / §3.5.2). +//! +//! A snapshot is a **rebuildable recovery-acceleration cache, not +//! durable state** — the WAL is the durable truth +//! (`CLAUDE.md §3.4`). It exists only to shorten cold-start replay; a lost, +//! absent, or corrupt snapshot is never a data-loss event, it +//! degrades to a full WAL replay. +//! +//! # Format +//! +//! Byte 0 is the snapshot format version ([`SNAPSHOT_VERSION`]); +//! the remaining bytes are that version's serialised payload. The +//! reader dispatches on byte 0 — the version byte is what makes +//! format evolution safe (RFC 0001 §6.9 *Migration*). The concrete +//! payload codec (here: `serde_json` over [`SnapshotState`]) is an +//! implementation detail *behind* the version byte; a future +//! version may change it without changing the framing. +//! +//! The payload captures the per-tenant state needed to reconstruct +//! the miner: the tree leaves (template token sequence, +//! `template_id`, `template_version`, the `(severity_number, +//! scope_name)` template key, and per-slot `slot_types`), the +//! structured-template-id map (§6.2 step-0 short-circuit), and the +//! WAL high-water mark. +//! +//! # v1 recovery — rebuild from a full replay; do not restore yet +//! +//! [`recover`] **ignores the snapshot payload and rebuilds the tree +//! from a full WAL replay in both branches.** The known-version +//! branch does **not** restore. This is a correctness constraint, +//! not a simplification: the restore-then-replay-the-tail path of +//! RFC 0001 §6.9 step (2) needs the RFC 0008 §6.7 checkpoint / +//! replay-from-offset API (`Wal::checkpoint`), which is an RFC 0008 +//! red-gate stub. Restoring a tree from a snapshot and *then* +//! replaying the full WAL — the only replay available without +//! offset support — would **double-apply** every frame the snapshot +//! already captured, corrupting the tree. So until offset-resume +//! lands, recovery discards the snapshot payload and rebuilds from +//! the WAL. What lands now is the snapshot *format* (the leading +//! version byte and the recorded high-water mark) and the +//! version-dispatch + WAL-fallback contract; the restore path is +//! switched on, with no format change, once RFC 0008 §6.7 lands. + +use ourios_core::audit::{ParamType, SlotTypes}; +use serde::{Deserialize, Serialize}; + +use crate::tree::OwnedToken; + +/// Snapshot format version written as byte 0 of every artefact +/// (RFC 0001 §6.9, §3.5.1). [`load_snapshot`] dispatches on this: +/// a matching byte 0 deserialises the payload; any other value is +/// a [`SnapshotError::UnknownVersion`] that recovery treats as +/// "discard and full-replay" (§3.5.2). +pub const SNAPSHOT_VERSION: u8 = 1; + +/// One tenant's full snapshot payload (the bytes after the version +/// byte) — the per-tenant state a restore would rebuild the miner +/// from. +/// +/// `leaves` and `structured_templates` are `Vec`s (not maps) so the +/// serialised form is order-deterministic for a given build order. +/// v1 recovery does **not** restore from a snapshot (it full-replays +/// the WAL); the restore path that rebuilds the in-memory `HashMap`s +/// from these lands with RFC 0008 §6.7 offset-resume. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SnapshotState { + /// Every `Body::String` leaf in the tenant's tree. + pub leaves: Vec, + /// The §6.2 step-0 structured-template-id map: each + /// `(severity_number, scope_name)` tuple and the `template_id` + /// allocated on its first observation. The + /// `BodyKind::Structured` discriminator is implicit from the + /// map's identity (RFC 0001 §6.1). + pub structured_templates: Vec, + /// WAL high-water mark this snapshot's tree state reflects, or + /// `None` if no offset was recorded. Captured so a future + /// optimisation can resume replay from here (RFC 0008 §6.7) + /// rather than from the start of the log; v1 does not use it. + pub wal_high_water: Option, +} + +/// Serialisable mirror of one tree [`crate::tree::Leaf`]. Carries +/// the §6.1 template-key fields and per-slot type sets — without +/// `(severity_number, scope_name)` two records sharing masked +/// tokens but differing in severity / scope would silently coalesce +/// on restore (H1.4 / H1.5). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeafRecord { + pub template: Vec, + pub template_id: u64, + pub template_version: u32, + pub severity_number: u8, + pub scope_name: Option, + /// One entry per `TokenRecord::Wildcard` in `template`, in + /// wildcard-slot ordinal order. Each is the set of observed + /// `ParamType`s for that slot (RFC 0001 §6.1). + pub slot_types: Vec>, +} + +/// One `(severity_number, scope_name) → template_id` entry of the +/// structured-template-id map. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StructuredTemplateRecord { + pub severity_number: u8, + pub scope_name: Option, + pub template_id: u64, +} + +/// Serialisable mirror of [`crate::tree::OwnedToken`]. The tree type +/// is kept serde-free (it is on the ingest hot path and its derive +/// surface is the algorithm's, not the codec's); this view exists +/// purely so the snapshot codec lives entirely inside this module. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum TokenRecord { + Fixed(String), + Wildcard, +} + +impl From<&OwnedToken> for TokenRecord { + fn from(t: &OwnedToken) -> Self { + match t { + OwnedToken::Fixed(s) => Self::Fixed(s.clone()), + OwnedToken::Wildcard => Self::Wildcard, + } + } +} + +impl From<&TokenRecord> for OwnedToken { + fn from(t: &TokenRecord) -> Self { + match t { + TokenRecord::Fixed(s) => Self::Fixed(s.clone()), + TokenRecord::Wildcard => Self::Wildcard, + } + } +} + +/// Serialisable mirror of [`ourios_core::audit::ParamType`]. The +/// core type has no serde derive and a private bit layout; this +/// view keeps the snapshot codec self-contained and stable against +/// the bitset representation. `Unknown(i32)` is carried verbatim so +/// a reader-side catch-all ordinal round-trips. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ParamTypeRecord { + Ip, + Uuid, + Num, + Hex, + Ts, + Path, + Str, + Overflow, + Unknown(i32), +} + +impl From for ParamTypeRecord { + fn from(t: ParamType) -> Self { + match t { + ParamType::Ip => Self::Ip, + ParamType::Uuid => Self::Uuid, + ParamType::Num => Self::Num, + ParamType::Hex => Self::Hex, + ParamType::Ts => Self::Ts, + ParamType::Path => Self::Path, + ParamType::Str => Self::Str, + ParamType::Overflow => Self::Overflow, + ParamType::Unknown(n) => Self::Unknown(n), + } + } +} + +impl From for ParamType { + fn from(t: ParamTypeRecord) -> Self { + match t { + ParamTypeRecord::Ip => Self::Ip, + ParamTypeRecord::Uuid => Self::Uuid, + ParamTypeRecord::Num => Self::Num, + ParamTypeRecord::Hex => Self::Hex, + ParamTypeRecord::Ts => Self::Ts, + ParamTypeRecord::Path => Self::Path, + ParamTypeRecord::Str => Self::Str, + ParamTypeRecord::Overflow => Self::Overflow, + ParamTypeRecord::Unknown(n) => Self::Unknown(n), + } + } +} + +/// Serialise one [`SlotTypes`] bitset as the ordered set of its +/// members. The core type's byte layout is private (no `bits()` +/// accessor), so the stable view is its `iter()` order — which is +/// also the canonical `ParamType` declaration order, so two equal +/// sets always serialise identically. +fn slot_types_to_record(s: SlotTypes) -> Vec { + s.iter().map(ParamTypeRecord::from).collect() +} + +/// WAL high-water mark, mirroring `ourios_wal::WalOffset`'s +/// `(segment: Uuid, byte)` shape without depending on `ourios-wal` +/// or `uuid` from the miner crate. The segment id is carried as its +/// textual form; the miner only needs to record and round-trip it, +/// not to compare WAL offsets, so a string is sufficient. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WalHighWater { + pub segment: String, + pub byte: u64, +} + +/// Errors from [`load_snapshot`]. Enum-carried states (no panicking +/// accessors on the recovery path): every malformed input is a +/// typed variant the caller dispatches on, and [`recover`] maps all +/// of them to the same "discard, full-replay" outcome. +#[derive(Debug)] +#[non_exhaustive] +pub enum SnapshotError { + /// Byte 0 is a version this build does not understand. Recovery + /// rejects the artefact and falls back to full WAL replay + /// (§3.5.2) rather than misinterpreting the payload bytes. + UnknownVersion(u8), + /// Byte 0 matched [`SNAPSHOT_VERSION`] but the payload did not + /// deserialise. Carries the decoder's message for diagnostics. + Corrupt(String), + /// The artefact was empty — no version byte to dispatch on. + Empty, + /// The payload failed to encode in [`snapshot`]. Not expected for + /// the plain mirror types, but surfaced rather than silently + /// writing a truncated artefact every reader would reject. + Serialize(String), +} + +impl std::fmt::Display for SnapshotError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnknownVersion(v) => { + write!(f, "unknown snapshot version byte {v:#04x}") + } + Self::Corrupt(detail) => write!(f, "corrupt snapshot payload: {detail}"), + Self::Empty => f.write_str("empty snapshot artefact (no version byte)"), + Self::Serialize(detail) => write!(f, "snapshot payload failed to encode: {detail}"), + } + } +} + +impl std::error::Error for SnapshotError {} + +/// Serialise one tenant's [`SnapshotState`] into the wire artefact: +/// `[SNAPSHOT_VERSION][payload]` (RFC 0001 §6.9, §3.5.1). Byte 0 is +/// always [`SNAPSHOT_VERSION`]. +/// +/// # Errors +/// +/// Returns [`SnapshotError::Serialize`] if the payload fails to +/// encode. The mirror types are plain owned data so this is not +/// expected in practice, but surfacing it beats writing a truncated +/// `[SNAPSHOT_VERSION]` artefact that every reader would reject as +/// corrupt, forcing perpetual full replays. +pub fn snapshot(state: &SnapshotState) -> Result, SnapshotError> { + let payload = serde_json::to_vec(state).map_err(|e| SnapshotError::Serialize(e.to_string()))?; + let mut out = Vec::with_capacity(payload.len() + 1); + out.push(SNAPSHOT_VERSION); + out.extend_from_slice(&payload); + Ok(out) +} + +/// Read the version byte and, when it matches [`SNAPSHOT_VERSION`], +/// deserialise the payload (RFC 0001 §6.9 recovery step). Does +/// **not** decide whether to use the result — [`recover`] owns the +/// v1 "discard and full-replay regardless" policy. This function is +/// the version-dispatch surface §3.5.2 exercises. +/// +/// # Errors +/// +/// - [`SnapshotError::Empty`] when `bytes` is empty. +/// - [`SnapshotError::UnknownVersion`] when byte 0 is not +/// [`SNAPSHOT_VERSION`]. +/// - [`SnapshotError::Corrupt`] when byte 0 matches but the payload +/// does not deserialise. +pub fn load_snapshot(bytes: &[u8]) -> Result { + match bytes.split_first() { + None => Err(SnapshotError::Empty), + Some((&SNAPSHOT_VERSION, payload)) => { + serde_json::from_slice(payload).map_err(|e| SnapshotError::Corrupt(e.to_string())) + } + Some((&other, _)) => Err(SnapshotError::UnknownVersion(other)), + } +} + +/// Outcome of a v1 recovery: which fallback path produced the tree, +/// for snapshot-load telemetry (RFC 0001 §6.9 *Snapshot-load +/// telemetry*). The tree itself always comes from `rebuild` in v1; +/// this only records *why* the snapshot did not short-circuit it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RecoveryOutcome { + /// No snapshot artefact was supplied (cold start, or the cache + /// file was absent). + NoSnapshot, + /// A snapshot was supplied and its version byte was known. In + /// v1 the payload is still discarded and the tree is rebuilt + /// from the WAL (the restore path is gated on RFC 0008 §6.7). + KnownVersionDiscarded, + /// A snapshot was supplied but its version byte was unknown or + /// its payload was corrupt; it was discarded (§3.5.2). + UnknownOrCorruptDiscarded, +} + +/// Recover one tenant's tree state on ingester restart (RFC 0001 +/// §6.9 recovery algorithm). +/// +/// **v1 behaviour: always rebuild from a full WAL replay; never +/// restore from the snapshot.** `rebuild` is the caller-supplied +/// full-replay closure — in production the ingester's +/// `Wal::replay`-driven path that re-runs `OtlpBatch` frames back +/// through the miner from the start of the log. This function still +/// calls [`load_snapshot`] to exercise and observe the +/// version-dispatch (known → [`RecoveryOutcome::KnownVersionDiscarded`], +/// unknown / corrupt / empty → [`RecoveryOutcome::UnknownOrCorruptDiscarded`]), +/// but the dispatch result is **not** used to skip the replay. +/// +/// The miner crate deliberately does not depend on `ourios-wal`: +/// the `OtlpBatch`-decode + tenant-fan-out + miner-ingest pipeline +/// the full replay drives lives in the ingester (`ourios-ingester`), +/// and the snapshot *format* + version-dispatch + WAL-fallback +/// *decision* are what this slice owns. Passing the replay in as a +/// closure keeps both on the correct side of the crate boundary. +/// +/// The known-version restore path (deserialise the payload, restore +/// the tree, then replay only the WAL tail from `wal_high_water`) +/// activates — with no format change — once the RFC 0008 §6.7 +/// offset-resume API lands. Restoring here and *then* full-replaying +/// would double-apply every captured frame and corrupt the tree, so +/// v1 must discard the payload in both branches. +pub fn recover(snapshot_bytes: Option<&[u8]>, rebuild: F) -> (T, RecoveryOutcome) +where + F: FnOnce() -> T, +{ + let outcome = match snapshot_bytes { + None => RecoveryOutcome::NoSnapshot, + Some(bytes) => match load_snapshot(bytes) { + Ok(_state) => RecoveryOutcome::KnownVersionDiscarded, + Err(_e) => RecoveryOutcome::UnknownOrCorruptDiscarded, + }, + }; + (rebuild(), outcome) +} + +/// Convert a [`SlotTypes`] vector (the leaf's per-slot type sets) +/// into the serialisable form. +#[must_use] +pub(crate) fn slot_types_vec_to_record(slot_types: &[SlotTypes]) -> Vec> { + slot_types + .iter() + .copied() + .map(slot_types_to_record) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_state() -> SnapshotState { + SnapshotState { + leaves: vec![ + LeafRecord { + template: vec![ + TokenRecord::Fixed("user".to_string()), + TokenRecord::Wildcard, + TokenRecord::Fixed("logged".to_string()), + TokenRecord::Fixed("in".to_string()), + ], + template_id: 1, + template_version: 2, + severity_number: 9, + scope_name: Some("lib.auth".to_string()), + slot_types: vec![vec![ParamTypeRecord::Num, ParamTypeRecord::Str]], + }, + LeafRecord { + template: vec![ + TokenRecord::Fixed("GET".to_string()), + TokenRecord::Fixed("/home".to_string()), + ], + template_id: 2, + template_version: 1, + severity_number: 0, + scope_name: None, + slot_types: vec![], + }, + ], + structured_templates: vec![StructuredTemplateRecord { + severity_number: 17, + scope_name: Some("lib.payments".to_string()), + template_id: 3, + }], + wal_high_water: Some(WalHighWater { + segment: "0190b3c8-1a2b-7c3d-9e4f-50607080a0b0".to_string(), + byte: 4096, + }), + } + } + + #[test] + fn snapshot_leading_byte_is_version() { + // Arrange + let state = sample_state(); + + // Act + let bytes = snapshot(&state).expect("snapshot encodes"); + + // Assert — §3.5.1: byte 0 is the format version. + assert_eq!(bytes[0], SNAPSHOT_VERSION); + } + + #[test] + fn snapshot_round_trips_to_equal_state() { + // Arrange + let state = sample_state(); + + // Act + let bytes = snapshot(&state).expect("snapshot encodes"); + let restored = load_snapshot(&bytes).expect("known version deserialises"); + + // Assert — the format deserialises to an equal state. + assert_eq!(restored, state); + } + + #[test] + fn load_snapshot_rejects_unknown_version() { + // Arrange — a valid snapshot with byte 0 corrupted to an + // unknown version. + let mut bytes = snapshot(&sample_state()).expect("snapshot encodes"); + bytes[0] = 0xFF; + + // Act + let err = load_snapshot(&bytes).expect_err("unknown version must error"); + + // Assert + assert!(matches!(err, SnapshotError::UnknownVersion(0xFF))); + } + + #[test] + fn load_snapshot_rejects_empty_input() { + // Arrange + Act + let err = load_snapshot(&[]).expect_err("empty input must error"); + + // Assert + assert!(matches!(err, SnapshotError::Empty)); + } + + #[test] + fn load_snapshot_rejects_corrupt_payload() { + // Arrange — correct version byte, garbage payload. + let bytes = [SNAPSHOT_VERSION, 0x7B, 0x21, 0x21]; + + // Act + let err = load_snapshot(&bytes).expect_err("corrupt payload must error"); + + // Assert + assert!(matches!(err, SnapshotError::Corrupt(_))); + } + + #[test] + fn recover_with_no_snapshot_rebuilds_and_reports_no_snapshot() { + // Arrange + Act — v1 always rebuilds via the closure. + let (tree, outcome) = recover(None, || 42u32); + + // Assert + assert_eq!(tree, 42); + assert_eq!(outcome, RecoveryOutcome::NoSnapshot); + } + + #[test] + fn recover_with_known_version_still_rebuilds_from_closure() { + // Arrange — a well-formed snapshot. v1 must NOT restore from + // it: the rebuild closure's value is what comes back, and the + // outcome records that the known-version payload was discarded. + let bytes = snapshot(&sample_state()).expect("snapshot encodes"); + + // Act + let (tree, outcome) = recover(Some(&bytes), || 7u32); + + // Assert — rebuilt value, not anything derived from the snapshot. + assert_eq!(tree, 7); + assert_eq!(outcome, RecoveryOutcome::KnownVersionDiscarded); + } + + #[test] + fn recover_with_unknown_version_discards_and_rebuilds() { + // Arrange — unknown version byte. + let mut bytes = snapshot(&sample_state()).expect("snapshot encodes"); + bytes[0] = 0xFF; + + // Act + let (tree, outcome) = recover(Some(&bytes), || 9u32); + + // Assert — the stale snapshot is discarded; the tree comes + // from the rebuild closure (the WAL, in production). + assert_eq!(tree, 9); + assert_eq!(outcome, RecoveryOutcome::UnknownOrCorruptDiscarded); + } + + #[test] + fn slot_types_to_record_captures_every_member() { + // The serialisable form lists each member of the set (in + // `SlotTypes::iter` / canonical `ParamType` order). The + // restore-side inverse lands with the recovery restore path + // (RFC 0008 §6.7); here we pin the forward encoding `snapshot` + // relies on. + let record = + slot_types_to_record(SlotTypes::singleton(ParamType::Num).insert(ParamType::Str)); + + // Exact order, not just membership — the byte-determinism of the + // snapshot rests on `SlotTypes::iter`'s canonical order. + assert_eq!(record, vec![ParamTypeRecord::Num, ParamTypeRecord::Str]); + } +} diff --git a/crates/ourios-miner/tests/invariants.rs b/crates/ourios-miner/tests/invariants.rs index 9800445e0..c2e0bba6e 100644 --- a/crates/ourios-miner/tests/invariants.rs +++ b/crates/ourios-miner/tests/invariants.rs @@ -307,17 +307,132 @@ fn invariant_3_3_1_separators_captured_on_every_tokenization() { /// Scenario §3.5.1 — Snapshot format carries a leading version byte. /// See `docs/rfcs/0001-template-miner.md` §5. #[test] -#[ignore = "RFC 0001 Red gate — implementation pending"] fn invariant_3_5_1_snapshot_format_carries_leading_version_byte() { - todo!("RFC 0001 §6.9"); + use ourios_core::config::MinerConfig; + use ourios_core::otlp::{Body, OtlpLogRecord}; + use ourios_core::tenant::TenantId; + use ourios_miner::cluster::MinerCluster; + use ourios_miner::snapshot::{SNAPSHOT_VERSION, snapshot}; + + // Arrange — a tenant tree holding a couple of templates. + let mut cluster = MinerCluster::new(MinerConfig::default()); + let tenant = TenantId::new("tenant-snapshot"); + for text in ["user 42 logged in", "GET /home 200"] { + cluster.ingest(&OtlpLogRecord { + tenant_id: tenant.clone(), + body: Some(Body::String(text.to_string())), + ..Default::default() + }); + } + let state = cluster.snapshot_state(&tenant); + assert!( + !state.leaves.is_empty(), + "precondition: the tenant must hold at least one template", + ); + + // Act + let bytes = snapshot(&state).expect("snapshot encodes"); + + // Assert — byte 0 is the snapshot format version. + assert_eq!( + bytes[0], SNAPSHOT_VERSION, + "snapshot byte 0 must be the format version, got {:#04x}", + bytes[0], + ); } /// Scenario §3.5.2 — Unknown snapshot version triggers full WAL replay. /// See `docs/rfcs/0001-template-miner.md` §5. +/// +/// v1 recovery (RFC 0001 §6.9 *v1 scope*) rebuilds the tree from a +/// full `Wal::replay()` in **both** branches; here the WAL is stood +/// in for by re-ingesting the frames the WAL holds into a fresh +/// cluster (the `OtlpBatch`-decode + miner-ingest pipeline that the +/// real replay drives lives in `ourios-ingester`). The stale +/// snapshot is deliberately built from *different* templates than +/// the WAL frames, so "recovered == full WAL replay, NOT the stale +/// snapshot" is a real assertion. #[test] -#[ignore = "RFC 0001 Red gate — implementation pending"] fn invariant_3_5_2_unknown_snapshot_version_triggers_wal_replay() { - todo!("RFC 0001 §6.9"); + use ourios_core::config::MinerConfig; + use ourios_core::otlp::{Body, OtlpLogRecord}; + use ourios_core::tenant::TenantId; + use ourios_miner::cluster::MinerCluster; + use ourios_miner::snapshot::{ + RecoveryOutcome, SnapshotError, load_snapshot, recover, snapshot, + }; + + let tenant = TenantId::new("tenant-recovery"); + let rec = |text: &str| OtlpLogRecord { + tenant_id: tenant.clone(), + body: Some(Body::String(text.to_string())), + ..Default::default() + }; + + // The frames the WAL durably holds (RFC 0001 §3.4): the source + // of truth recovery must rebuild from. + let wal_frames = ["order 7 shipped", "order 9 shipped"]; + + // Arrange — a *stale* snapshot whose tree is unrelated to the + // WAL frames, with its version byte set to an unknown value. + let stale_snapshot = { + let mut other = MinerCluster::new(MinerConfig::default()); + other.ingest(&rec("login user 1 ok")); + let mut bytes = snapshot(&other.snapshot_state(&tenant)).expect("snapshot encodes"); + bytes[0] = 0xFF; + bytes + }; + + // Act 1 — the unknown version is rejected by the loader. + let load_err = load_snapshot(&stale_snapshot).expect_err("unknown version must be rejected"); + + // Assert 1 — §3.5.2: rejected as an unknown version. + assert!( + matches!(load_err, SnapshotError::UnknownVersion(0xFF)), + "unknown version byte must surface UnknownVersion(0xFF), got {load_err:?}", + ); + + // Act 2 — recover from the stale snapshot + the WAL. The rebuild + // closure is the full-replay stand-in: it re-ingests every WAL + // frame into a fresh cluster and returns the resulting tree state. + let (recovered, outcome) = recover(Some(&stale_snapshot), || { + let mut replayed = MinerCluster::new(MinerConfig::default()); + for text in wal_frames { + replayed.ingest(&rec(text)); + } + replayed.snapshot_state(&tenant) + }); + + // The tree a full WAL replay alone would produce — the truth. + let from_wal_only = { + let mut replayed = MinerCluster::new(MinerConfig::default()); + for text in wal_frames { + replayed.ingest(&rec(text)); + } + replayed.snapshot_state(&tenant) + }; + + // Assert 2 — the stale snapshot was discarded and the tree came + // from the WAL, not the snapshot. + assert_eq!( + outcome, + RecoveryOutcome::UnknownOrCorruptDiscarded, + "an unknown-version snapshot must be discarded", + ); + assert_eq!( + recovered, from_wal_only, + "recovered tree must equal the full WAL replay", + ); + assert_ne!( + recovered, + load_snapshot(&{ + let mut valid = stale_snapshot.clone(); + valid[0] = ourios_miner::snapshot::SNAPSHOT_VERSION; + valid + }) + .expect("re-versioned stale payload deserialises"), + "recovered tree must NOT be the stale snapshot's content", + ); } /// Scenario §3.7.1 — Tenants' template trees never cross-pollinate.