diff --git a/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs b/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs index 52f7fe093..8d0b585b5 100644 --- a/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs +++ b/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs @@ -196,6 +196,33 @@ fn seed_and_compact( .file } +/// The same one-hour corpus [`seed_and_compact`] builds, but as a +/// single flat Vec in ingest (round-robin) order — the shape a +/// partition has *before* compaction: one unsorted ingest-side file, no +/// `sorting_columns`, no per-service/-time clustering. Every `(service, +/// time)` key matches the compacted store's, so the same window query +/// returns the identical answer on both. +fn all_records(plan: &[(&str, u64)], body_len: usize, grid_ns: u64) -> Vec { + let max_rows = plan.iter().map(|&(_, rows)| rows).max().unwrap_or(0); + let total = usize::try_from(plan.iter().map(|&(_, rows)| rows).sum::()).expect("fits"); + let mut records: Vec = Vec::with_capacity(total); + let mut id: u64 = 0; + for i in 0..max_rows { + for &(service, rows) in plan { + if i < rows { + id += 1; + records.push(rec( + service, + HOUR10_START + i * grid_ns, + id, + payload(id, body_len), + )); + } + } + } + records +} + /// The queried service's bytes within the window: the compressed sizes /// of the row groups a `service == target AND time ∈ window` scan can /// NOT prune (target within the group's `service.name` min/max AND the @@ -355,3 +382,212 @@ async fn rfc0036_2_window_materialization_bound() { r.stats, ); } + +/// Read `file_path`'s footer and run `query` against the store at +/// `bucket_root`, returning `(row_group_total, window_survivors, +/// survivor_bytes, result)`. The survivor set and bytes are the RFC 0036 §9 +/// materialization term ([`window_service_bytes`]); `result` carries the live +/// `row_groups_scanned` that cross-checks the footer prediction. +async fn measure_window( + bucket_root: &std::path::Path, + file_path: &std::path::Path, + query: &ourios_querier::dsl::Query, + target: &str, + window: &std::ops::Range, +) -> (u64, Vec, u64, ourios_querier::QueryResult) { + let meta = ParquetRecordBatchReaderBuilder::try_new(File::open(file_path).expect("open file")) + .expect("read footer") + .metadata() + .clone(); + let total = u64::try_from(meta.num_row_groups()).expect("count fits u64"); + let (survivors, bytes) = window_service_bytes(&meta, target, window); + let result = Querier::new(bucket_root) + .run_query( + query, + &TenantId::new("a"), + NOW, + DEFAULT_WINDOW_NS, + Some(&ourios_core::alias::AliasMap::new()), + ) + .await + .expect("run_query"); + (total, survivors, bytes, result) +} + +/// RFC0036.2 — the before/after materialization measurement (the "cheap +/// A" for RFC 0036, per the maintainer decision recorded in the RFC's +/// §9). The v8 comparative harness builds one ingest file per partition, +/// so `compact_partition` no-ops there and RFC 0036's sort never runs — +/// the baseline comparative run was byte-identical pre/post-0036, so the +/// before/after must be measured in-repo on a genuinely-compacted store. +/// +/// The **same** synthetic multi-service hour is materialised two ways +/// and hit with the same L6-shape window query (one service, a narrow +/// time range): +/// +/// - **Before** — the uncompacted ingest-side store: every row in one +/// flat, unsorted file at the 128 MiB [`ROW_GROUP_FLUSH_BYTES`] +/// rotation, no `sorting_columns`. Every row group spans the whole +/// hour and every service, so nothing prunes: the window query +/// materialises the *entire* file (`scanned == total`). +/// - **After** — the compacted, `sorting_columns`-declaring store +/// ([`seed_and_compact`], 32 MiB [`COMPACTED_ROW_GROUP_FLUSH_BYTES`] +/// row groups). The §3.1 clustering gives each row group tight +/// `service.name`/`time` statistics, so the same query prunes away most +/// of its groups (`scanned < total`). +/// +/// The win is measured in **materialization bytes** — the RFC 0036 §9 +/// diagnostic: "the column chunks of the row groups that survive pruning." +/// For each store that is the summed compressed size of the row groups a +/// `service == TARGET AND time ∈ window` scan can NOT prune (`window_service_bytes`, +/// the same footer computation the bound test trusts); the live query's +/// `row_groups_scanned` cross-checks that the engine prunes exactly as the +/// footer predicts (before scans the whole file, after prunes to a subset). +/// Row-group *counts* are not the honest axis: the ingest side rotates at +/// 128 MiB and compaction at 32 MiB (RFC 0036 §3.3), so the compacted store +/// deliberately holds more, smaller groups (and, at these sub-threshold +/// sizes, a physically larger file — the smaller groups compress a little +/// worse). The window still materialises fewer bytes on the compacted store +/// because it prunes to a small survivor subset, so the assertion is +/// `after_bytes < before_bytes` with **identical `rows`** (pruning changed +/// the IO, not the answer). The full comparative bytes-vs-Loki arm (total +/// minus the RFC 0033 ~188 KB registry floor) stays the deferred +/// `baseline-8vcpu-32gib` bench arm; this is the in-repo evidence the win is +/// real. +#[tokio::test] +async fn rfc0036_2_materialization_before_after() { + const BODY_LEN: usize = 4096; + const TARGET: &str = "svc-m"; + // A k=100-row window (1 s at the 10 ms grid), matching the RFC 0036 §9 + // materialization illustration — a genuinely selective L6 "window browse", + // narrow enough that the compacted survivor set collapses to the single + // boundary row group holding svc-m's earliest rows. + const WINDOW_NS: u64 = 1_000_000_000; // 1 s + const GRID_NS: u64 = 10_000_000; // 10 ms between a service's rows + let plan: [(&str, u64); 3] = [("svc-a", 20_000), (TARGET, 18_500), ("svc-z", 20_000)]; + + let src = + format!("service == \"{TARGET}\" | range(2026-04-02T10:00:00Z, 2026-04-02T10:00:01Z)"); + let query = ourios_querier::dsl::parse(&src).expect("parse"); + let part = PartitionKey::derive(&rec(TARGET, HOUR10_START, 1, String::new())) + .expect("derive partition"); + + let window = HOUR10_START..HOUR10_START + WINDOW_NS; + + // --- Before: one unsorted ingest-side file, no manifest (glob path). The + // materialization term (RFC 0036 §9) is the compressed column chunks of + // the row groups a `service == TARGET AND time ∈ window` scan can NOT + // prune; on the unsorted ingest file every row group spans the whole hour + // and every service, so none prune — the survivors are the *whole file*. --- + let before_bucket = tempfile::TempDir::new().expect("temp"); + let before_store = Store::local(before_bucket.path()).expect("local store"); + write_input(&before_store, &part, &all_records(&plan, BODY_LEN, GRID_NS)); + let before_dir = part.data_path(before_bucket.path()); + let mut before_files: Vec = std::fs::read_dir(&before_dir) + .expect("read partition dir") + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "parquet")) + .collect(); + // One `write_input` ⇒ exactly one ingest file; assert it so a stray + // file can't silently make the measurement pick the wrong parquet. + assert_eq!( + before_files.len(), + 1, + "the uncompacted store must hold exactly one ingest file, found {before_files:?}", + ); + let before_path = before_files.pop().expect("one ingest file"); + let (before_total, before_survivors, before_bytes, before) = + measure_window(before_bucket.path(), &before_path, &query, TARGET, &window).await; + // The "before materialises the whole file" claim below rests on the + // uncompacted ingest file being a single row group (this corpus is + // ~100 MB encoded, under the 128 MiB ingest rotation). Make that + // explicit: if the writer ever flushed multiple time-ordered groups + // here, time-pruning could kick in and this test would fail in a + // confusing way — assert the assumption so it fails loudly instead. + assert_eq!( + before_total, 1, + "the uncompacted before-store is expected to be a single row group \ + (the whole-file-materialisation baseline); got {before_total}", + ); + + // --- After: the compacted, sorted store (built exactly as the bound test). + // Same survivor computation: the §3.1 clustering gives each row group tight + // `service.name`/`time` statistics, so the survivors are a contiguous + // minority of the file's groups. --- + let after_bucket = tempfile::TempDir::new().expect("temp"); + let after_store = Store::local(after_bucket.path()).expect("local store"); + let committed = seed_and_compact(&after_store, &part, &plan, BODY_LEN, GRID_NS); + let after_path = part.data_path(after_bucket.path()).join(&committed); + let (after_total, after_survivors, after_bytes, after) = + measure_window(after_bucket.path(), &after_path, &query, TARGET, &window).await; + + // Integer ×100 ratio (avoids a float cast under clippy::pedantic); read + // as `N.NNx`, e.g. 198 ⇒ before materialises 1.98× the after bytes. + // `u128` so the ×100 can't overflow if the fixture ever grows. + let win_x100 = u128::from(before_bytes) * 100 / u128::from(after_bytes.max(1)); + eprintln!( + "RFC0036.2 before/after: uncompacted survivors {}/{} groups, {} bytes \ + (query scanned {}); compacted survivors {}/{} groups, {} bytes (query \ + scanned {}); rows {} (identical); materialization-bytes win {}.{:02}x", + before_survivors.len(), + before_total, + before_bytes, + before.stats.row_groups_scanned, + after_survivors.len(), + after_total, + after_bytes, + after.stats.row_groups_scanned, + after.rows, + win_x100 / 100, + win_x100 % 100, + ); + + // Identical answer — the layout change is transparent to the result. + assert_eq!( + before.rows, after.rows, + "before/after must return the same rows (correctness preserved)", + ); + let expected_rows = WINDOW_NS.div_ceil(GRID_NS); + assert_eq!( + after.rows, expected_rows, + "the window selects k target rows" + ); + + // Before materialises the whole file: unsorted, every row group survives + // pruning (footer), and the live query confirms it scans them all. + assert_eq!( + u64::try_from(before_survivors.len()).expect("fits"), + before_total, + "uncompacted store has no prunable row group — the whole file is the \ + window's materialization set (survivors {before_survivors:?} of {before_total})", + ); + assert_eq!( + before.stats.row_groups_scanned, before_total, + "uncompacted query scans the whole file (no pruning); stats={:?}", + before.stats, + ); + // After prunes: the survivor set is a strict minority of the compacted + // file's groups, and the live query confirms it skips the rest. + assert!( + !after_survivors.is_empty() + && u64::try_from(after_survivors.len()).expect("fits") < after_total, + "compacted store must prune to a subset (survivors {after_survivors:?} of {after_total})", + ); + assert!( + after.stats.row_groups_scanned < after_total, + "compacted query must prune (scanned {} of {after_total} groups); stats={:?}", + after.stats.row_groups_scanned, + after.stats, + ); + // The materialization win, measured: the same window's survivor set is + // strictly fewer compressed bytes on the compacted store than on the + // uncompacted whole-file scan. Bytes — not row-group *count* — is the + // honest axis: compaction rotates at 32 MiB vs the ingest side's 128 MiB + // (§3.3), so the compacted store deliberately holds more, smaller groups. + assert!( + after_bytes < before_bytes, + "compaction materialises fewer bytes for the same window \ + (after {after_bytes} < before {before_bytes})", + ); +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 948f11452..926e40b11 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -2027,7 +2027,83 @@ declared) is pinned deterministically in `ourios-parquet`'s `rfc0036_3_compaction_properties_preserved` (tests/it) and the merge proptests; the §3.2 peak-memory bound (forced-spill residency = one input + F×batch, never whole-partition) in -`rfc0036_3_forced_spill_peak_is_one_input_not_whole_partition`. These +`rfc0036_3_forced_spill_peak_far_below_whole_partition`. These wall-clock figures are the indicative stamp; the authoritative sorted D2/D3 awaits the `validated` baseline rerun (RFC0036.5's frozen-gate pass will accompany it). + +### 9.26 Results — 2026-07-22 (authoritative, `baseline-8vcpu-32gib`) — RFC 0036 comparative rerun: no regression + the single-file-harness finding + +**Purpose.** RFC0036.5's deferred `validated` item: rerun the frozen +RFC 0031 comparative dispatch at post-RFC-0036 `main` (HEAD `5e5aa66`) +on baseline hardware, and confirm the frozen gates still pass on the +sorted-compaction code. The v8 corpus (frozen `corpus/otel-demo-v8` +release), one Loki container + one replay, same methodology as §9.24. + +**Hardware / run.** Fresh `ccx33` (8 dedicated vCPU EPYC-Milan / +32 GiB = `baseline-8vcpu-32gib`), release build, Docker for the Loki +testcontainer, box deleted on exit. ~75 min/attempt. + +**Result — every measured frozen gate passes, in §9.24's band (no +regression):** + +| gate | this run (post-0036) | §9.24 (pre-0036) | verdict | +|---|---|---|---| +| L1 template storage (PRIMARY, margin 10) | 99.86× / 99.35× | 97.82× | PASS | +| L2 severity processed (PRIMARY, margin 10) | 43.53× / 42.12× | 38.37× | PASS | +| L2 storage floor (11/10) | 1.40× / 1.36× | 1.239× | PASS | +| L4 frequency processed (PRIMARY, margin 10) | 85.41× / 85.52× | 85.14× | PASS | +| L4 storage floor (11/10) | 3.62× / 3.63× | 3.60× | PASS | +| L6 k=2000 latency floor (factor 3) | 4.09 / 3.98 | 4.341 | PASS | + +**The finding — RFC 0036 is not exercised by this harness, and the +run proves it.** Every `ourios_bytes_read` came back **byte-for-byte +identical to §9.24** (L1 1,032,727; L2 2,223,173; L4 47,995,205; +L6 k=100 1,931,911; L6 k=2000 4,202,473). `build_comparative_store` +writes exactly **one ingest file per partition** (`append_record` keeps +one `Writer` per partition), so `compact_partition` no-ops (it needs +≥ 2 files) and RFC 0036's compaction-time sort never runs. The +comparative store is therefore identical pre/post-0036 — which is why +the gates pass unchanged (no regression) *and* why this harness cannot +measure RFC 0036's window-materialization win. Making it accrue +multiple files per partition then compact would re-base RFC 0031's +frozen-gate store; that is deferred as **future harness work**, not a +`validated` blocker (the RFC0036.2 bytes channel is a §2.2 diagnostic, +not a gate). + +**Two orthogonal noise items** (neither an RFC 0036 layout regression): +attempt 1 measured L3 cleanly (21.6× storage) but landed the L6 k=100 +latency floor at 0.311 < 0.333 (marginal tiny-query noise; attempt 2 +passed it at 0.364); attempt 2 hit the known Loki L3 flicker (0/9 rows +before timeout — the §9.13 class, #490). One justified retry per the +retry-once rule; no third. + +### 9.27 Results — 2026-07-22 (in-repo, deterministic) — RFC0036.2 window-materialization before/after + +**Purpose.** The RFC0036.2 materialization diagnostic that §9.26's +harness cannot produce, measured directly on a genuinely-compacted +store. `ourios-querier`'s `rfc0036_2_materialization_before_after` +(tests/it) runs the identical L6 k=100 window query (1 s at the 10 ms +grid, one service) against two stores built from the same synthetic +multi-service hour, and reads the **materialization bytes** — the +compressed column chunks of the row groups a `service = target ∧ time ∈ +window` scan cannot prune — from each file's footer. + +| store | survivors / groups | materialization bytes | query scanned | +|---|---|---|---| +| **before** — one unsorted ingest file (128 MiB groups) | 1 / 1 | 100,520,155 | 1 (the whole file) | +| **after** — compacted, §3.1-sorted (32 MiB groups) | 2 / 6 | 70,018,075 | 2 | + +**Reading.** Identical 100-row answer from both; the sort takes the +window from materialising the **whole file** (the single-row-group +unsorted ingest parquet — no row group prunes) to a contiguous minority +(2 of 6 groups) — a **1.43× +materialization-bytes win**. Modest by design, and honestly so: the +compacted file is physically ~2× larger (six 32 MiB groups compress a +little worse than one 128 MiB group — the RFC 0036 §3.3 pruning- +granularity-over-bytes trade), and §2.2's ~188 KB registry floor is +exactly why the **gate** is the scanned-row-group bound (enforced by +`rfc0036_2_window_materialization_bound`), not a bytes ratio. `count`- +scan `stats.bytes_read` is *not* the materialization term — it reads +only the small filter columns and points the wrong way; the footer +survivor-chunk sum is the RFC §9 metric. diff --git a/docs/rfcs/0036-write-side-layout.md b/docs/rfcs/0036-write-side-layout.md index 442fb2ca2..64531c0b8 100644 --- a/docs/rfcs/0036-write-side-layout.md +++ b/docs/rfcs/0036-write-side-layout.md @@ -1,7 +1,7 @@ --- rfc: 0036 title: Write-side layout — compacted-partition clustering and row-group sizing -status: green +status: validated author: Jens Holdgaard Pedersen drafting-assistance: Claude created: 2026-07-21 @@ -11,8 +11,9 @@ superseded-by: — # RFC 0036 — Write-side layout -> **Status note.** **`green`** (2026-07-21) — all five §5 scenarios -> discharged. **RFC0036.1** (footer inspection — compacted threshold, +> **Status note.** **`validated`** (2026-07-22) — all five §5 scenarios +> green, plus the comparative evidence below. `accepted` is a maintainer +> flip. **RFC0036.1** (footer inspection — compacted threshold, > `sorting_columns`, per-group service min/max — plus the §6 merge > property), **RFC0036.3** (D3 file-band + forced-spill memory bound), > **RFC0036.4** (shuffled-listing byte-identity rebuild), and @@ -30,13 +31,34 @@ superseded-by: — > L6-scanned-bytes-vs-L1/L3 curve deferred to the `ourios-bench` > harness); the remaining §7 questions stay open as noted there. > -> **Deferred to `validated`.** The comparative arm of RFC0036.2 and -> RFC0036.5 — the L6-shape pair on the v8 corpus through the RFC 0031 -> dispatch, the before/after materialization-**bytes** §9 diagnostic, -> and the frozen-gate rerun confirming L1/L3/L4 stay inside the -> Loki-wobble band — is a paid `baseline-8vcpu-32gib` measurement, not -> a CI gate (RFC 0031 §5 declined to gate the L6 storage channel; §2.2). -> `validated` needs that baseline run. +> **`validated` evidence (2026-07-22).** The deferred comparative arm +> resolved in two parts, both recorded in `docs/benchmarks.md`: +> - **RFC0036.5 no-regression (§9.26, authoritative +> `baseline-8vcpu-32gib`, HEAD `5e5aa66`).** The frozen RFC 0031 +> dispatch reran on post-RFC-0036 `main`: every measured frozen gate +> passes in §9.24's band (L1 ~99×, L2 ~43× + floor 1.40, L4 ~85× + +> floor 3.62, L6 k=2000 latency 4.0). **Finding:** the comparative +> harness writes one ingest file per partition, so `compact_partition` +> no-ops and RFC 0036's sort never runs there — every `ourios_bytes_read` +> is byte-identical to §9.24. So the baseline run proves *no regression* +> but structurally cannot show RFC 0036's win; a multi-file-per-partition +> harness that re-bases the frozen-gate store is **future work**, not a +> `validated` blocker (RFC0036.2 bytes is a §2.2 diagnostic, not a gate). +> - **RFC0036.2 materialization before/after (§9.27, in-repo, +> deterministic).** Measured directly on a genuinely-compacted store: +> the sort takes a one-service window from materialising the whole file +> (unsorted: 1/1 groups, 100.5 MB — no row group prunes) to a contiguous +> minority (sorted: +> 2/6 groups, 70.0 MB) for the identical answer — a **1.43×** +> materialization-bytes win, modest and honest (the compacted file is +> ~2× larger on disk; §2.2's registry floor is why the *gate* is the +> scanned-row-group bound, which `rfc0036_2_window_materialization_bound` +> enforces). +> +> The scanned-row-group **gate** was always the load-bearing acceptance +> criterion (RFC0036.2 §5, green in-repo); the v8-corpus comparative +> materialization number remains desirable future work behind the harness +> change above. > > **How to read this document.** This is the write-side layout lever > that `docs/benchmarks.md` §9.13 named and §9.24 left "parked on its