Skip to content
215 changes: 215 additions & 0 deletions crates/ourios-querier/tests/it/rfc0036_window_materialization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,32 @@ fn seed_and_compact(
.file
}

/// The same one-hour corpus [`seed_and_compact`] builds, but as a
/// single flat Vec in ingest (round-robin) order — the shape a
/// partition has *before* compaction: one unsorted ingest-side file, no
/// `sorting_columns`, no per-service/-time clustering. Every `(service,
/// time)` key matches the compacted store's, so the same window query
/// returns the identical answer on both.
fn all_records(plan: &[(&str, u64)], body_len: usize, grid_ns: u64) -> Vec<MinedRecord> {
let max_rows = plan.iter().map(|&(_, rows)| rows).max().unwrap_or(0);
let mut records: Vec<MinedRecord> = Vec::new();
let mut id: u64 = 0;
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
for i in 0..max_rows {
for &(service, rows) in plan {
if i < rows {
id += 1;
records.push(rec(
service,
HOUR10_START + i * grid_ns,
id,
payload(id, body_len),
));
}
}
}
records
}

/// The queried service's bytes within the window: the compressed sizes
/// of the row groups a `service == target AND time ∈ window` scan can
/// NOT prune (target within the group's `service.name` min/max AND the
Expand Down Expand Up @@ -355,3 +381,192 @@ async fn rfc0036_2_window_materialization_bound() {
r.stats,
);
}

/// Read `file_path`'s footer and run `query` against the store at
/// `bucket_root`, returning `(row_group_total, window_survivors,
/// survivor_bytes, result)`. The survivor set and bytes are the RFC 0036 §9
/// materialization term ([`window_service_bytes`]); `result` carries the live
/// `row_groups_scanned` that cross-checks the footer prediction.
async fn measure_window(
bucket_root: &std::path::Path,
file_path: &std::path::Path,
query: &ourios_querier::dsl::Query,
target: &str,
window: &std::ops::Range<u64>,
) -> (u64, Vec<usize>, u64, ourios_querier::QueryResult) {
let meta = ParquetRecordBatchReaderBuilder::try_new(File::open(file_path).expect("open file"))
.expect("read footer")
.metadata()
.clone();
let total = u64::try_from(meta.num_row_groups()).expect("count fits u64");
let (survivors, bytes) = window_service_bytes(&meta, target, window);
let result = Querier::new(bucket_root)
.run_query(
query,
&TenantId::new("a"),
NOW,
DEFAULT_WINDOW_NS,
Some(&ourios_core::alias::AliasMap::new()),
)
.await
.expect("run_query");
(total, survivors, bytes, result)
}

/// RFC0036.2 — the before/after materialization measurement (the "cheap
/// A" for RFC 0036, per the maintainer decision recorded in the RFC's
/// §9). The v8 comparative harness builds one ingest file per partition,
/// so `compact_partition` no-ops there and RFC 0036's sort never runs —
/// the baseline comparative run was byte-identical pre/post-0036, so the
/// before/after must be measured in-repo on a genuinely-compacted store.
///
/// The **same** synthetic multi-service hour is materialised two ways
/// and hit with the same L6-shape window query (one service, a narrow
/// time range):
///
/// - **Before** — the uncompacted ingest-side store: every row in one
/// flat, unsorted file at the 128 MiB [`ROW_GROUP_FLUSH_BYTES`]
/// rotation, no `sorting_columns`. Every row group spans the whole
/// hour and every service, so nothing prunes: the window query
/// materialises the *entire* file (`scanned == total`).
/// - **After** — the compacted, `sorting_columns`-declaring store
/// ([`seed_and_compact`], 32 MiB [`COMPACTED_ROW_GROUP_FLUSH_BYTES`]
/// row groups). The §3.1 clustering gives each row group tight
/// `service.name`/`time` statistics, so the same query prunes away most
/// of its groups (`scanned < total`).
///
/// The win is measured in **materialization bytes** — the RFC 0036 §9
/// diagnostic: "the column chunks of the row groups that survive pruning."
/// For each store that is the summed compressed size of the row groups a
/// `service == TARGET AND time ∈ window` scan can NOT prune (`window_service_bytes`,
/// the same footer computation the bound test trusts); the live query's
/// `row_groups_scanned` cross-checks that the engine prunes exactly as the
/// footer predicts (before scans the whole file, after prunes to a subset).
/// Row-group *counts* are not the honest axis: the ingest side rotates at
/// 128 MiB and compaction at 32 MiB (RFC 0036 §3.3), so the compacted store
/// deliberately holds more, smaller groups (and, at these sub-threshold
/// sizes, a physically larger file — the smaller groups compress a little
/// worse). The window still materialises fewer bytes on the compacted store
/// because it prunes to a small survivor subset, so the assertion is
/// `after_bytes < before_bytes` with **identical `rows`** (pruning changed
/// the IO, not the answer). The full comparative bytes-vs-Loki arm (total
/// minus the RFC 0033 ~188 KB registry floor) stays the deferred
/// `baseline-8vcpu-32gib` bench arm; this is the in-repo evidence the win is
/// real.
#[tokio::test]
async fn rfc0036_2_materialization_before_after() {
const BODY_LEN: usize = 4096;
const TARGET: &str = "svc-m";
// A k=100-row window (1 s at the 10 ms grid), matching the RFC 0036 §9
// materialization illustration — a genuinely selective L6 "window browse",
// narrow enough that the compacted survivor set collapses to the single
// boundary row group holding svc-m's earliest rows.
const WINDOW_NS: u64 = 1_000_000_000; // 1 s
const GRID_NS: u64 = 10_000_000; // 10 ms between a service's rows
let plan: [(&str, u64); 3] = [("svc-a", 20_000), (TARGET, 18_500), ("svc-z", 20_000)];

let src =
format!("service == \"{TARGET}\" | range(2026-04-02T10:00:00Z, 2026-04-02T10:00:01Z)");
let query = ourios_querier::dsl::parse(&src).expect("parse");
let part = PartitionKey::derive(&rec(TARGET, HOUR10_START, 1, String::new()))
.expect("derive partition");

let window = HOUR10_START..HOUR10_START + WINDOW_NS;

// --- Before: one unsorted ingest-side file, no manifest (glob path). The
// materialization term (RFC 0036 §9) is the compressed column chunks of
// the row groups a `service == TARGET AND time ∈ window` scan can NOT
// prune; on the unsorted ingest file every row group spans the whole hour
// and every service, so none prune — the survivors are the *whole file*. ---
let before_bucket = tempfile::TempDir::new().expect("temp");
let before_store = Store::local(before_bucket.path()).expect("local store");
write_input(&before_store, &part, &all_records(&plan, BODY_LEN, GRID_NS));
let before_dir = part.data_path(before_bucket.path());
let before_path = std::fs::read_dir(&before_dir)
.expect("read partition dir")
.filter_map(Result::ok)
.map(|e| e.path())
.find(|p| p.extension().is_some_and(|x| x == "parquet"))
.expect("one committed ingest file");
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
let (before_total, before_survivors, before_bytes, before) =
measure_window(before_bucket.path(), &before_path, &query, TARGET, &window).await;

Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
// --- After: the compacted, sorted store (built exactly as the bound test).
// Same survivor computation: the §3.1 clustering gives each row group tight
// `service.name`/`time` statistics, so the survivors are a contiguous
// minority of the file's groups. ---
let after_bucket = tempfile::TempDir::new().expect("temp");
let after_store = Store::local(after_bucket.path()).expect("local store");
let committed = seed_and_compact(&after_store, &part, &plan, BODY_LEN, GRID_NS);
let after_path = part.data_path(after_bucket.path()).join(&committed);
let (after_total, after_survivors, after_bytes, after) =
measure_window(after_bucket.path(), &after_path, &query, TARGET, &window).await;

// Integer ×100 ratio (avoids a float cast under clippy::pedantic); read
// as `N.NNx`, e.g. 198 ⇒ before materialises 1.98× the after bytes.
let win_x100 = before_bytes * 100 / after_bytes.max(1);
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
eprintln!(
"RFC0036.2 before/after: uncompacted survivors {}/{} groups, {} bytes \
(query scanned {}); compacted survivors {}/{} groups, {} bytes (query \
scanned {}); rows {} (identical); materialization-bytes win {}.{:02}x",
before_survivors.len(),
before_total,
before_bytes,
before.stats.row_groups_scanned,
after_survivors.len(),
after_total,
after_bytes,
after.stats.row_groups_scanned,
after.rows,
win_x100 / 100,
win_x100 % 100,
);

// Identical answer — the layout change is transparent to the result.
assert_eq!(
before.rows, after.rows,
"before/after must return the same rows (correctness preserved)",
);
let expected_rows = WINDOW_NS.div_ceil(GRID_NS);
assert_eq!(
after.rows, expected_rows,
"the window selects k target rows"
);

// Before materialises the whole file: unsorted, every row group survives
// pruning (footer), and the live query confirms it scans them all.
assert_eq!(
u64::try_from(before_survivors.len()).expect("fits"),
before_total,
"uncompacted store has no prunable row group — the whole file is the \
window's materialization set (survivors {before_survivors:?} of {before_total})",
);
assert_eq!(
before.stats.row_groups_scanned, before_total,
"uncompacted query scans the whole file (no pruning); stats={:?}",
before.stats,
);
// After prunes: the survivor set is a strict minority of the compacted
// file's groups, and the live query confirms it skips the rest.
assert!(
!after_survivors.is_empty()
&& u64::try_from(after_survivors.len()).expect("fits") < after_total,
"compacted store must prune to a subset (survivors {after_survivors:?} of {after_total})",
);
assert!(
after.stats.row_groups_scanned < after_total,
"compacted query must prune (scanned {} of {after_total} groups); stats={:?}",
after.stats.row_groups_scanned,
after.stats,
);
// The materialization win, measured: the same window's survivor set is
// strictly fewer compressed bytes on the compacted store than on the
// uncompacted whole-file scan. Bytes — not row-group *count* — is the
// honest axis: compaction rotates at 32 MiB vs the ingest side's 128 MiB
// (§3.3), so the compacted store deliberately holds more, smaller groups.
assert!(
after_bytes < before_bytes,
"compaction materialises fewer bytes for the same window \
(after {after_bytes} < before {before_bytes})",
);
}
77 changes: 76 additions & 1 deletion docs/benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -2027,7 +2027,82 @@ declared) is pinned deterministically in `ourios-parquet`'s
`rfc0036_3_compaction_properties_preserved` (tests/it) and the merge
proptests; the §3.2 peak-memory bound (forced-spill residency = one
input + F×batch, never whole-partition) in
`rfc0036_3_forced_spill_peak_is_one_input_not_whole_partition`. These
`rfc0036_3_forced_spill_peak_far_below_whole_partition`. These
wall-clock figures are the indicative stamp; the authoritative sorted
D2/D3 awaits the `validated` baseline rerun (RFC0036.5's frozen-gate
pass will accompany it).

### 9.26 Results — 2026-07-22 (authoritative, `baseline-8vcpu-32gib`) — RFC 0036 comparative rerun: no regression + the single-file-harness finding

**Purpose.** RFC0036.5's deferred `validated` item: rerun the frozen
RFC 0031 comparative dispatch at post-RFC-0036 `main` (HEAD `5e5aa66`)
on baseline hardware, and confirm the frozen gates still pass on the
sorted-compaction code. The v8 corpus (frozen `corpus/otel-demo-v8`
release), one Loki container + one replay, same methodology as §9.24.

**Hardware / run.** Fresh `ccx33` (8 dedicated vCPU EPYC-Milan /
32 GiB = `baseline-8vcpu-32gib`), release build, Docker for the Loki
testcontainer, box deleted on exit. ~75 min/attempt.

**Result — every measured frozen gate passes, in §9.24's band (no
regression):**

| gate | this run (post-0036) | §9.24 (pre-0036) | verdict |
|---|---|---|---|
| L1 template storage (PRIMARY, margin 10) | 99.86× / 99.35× | 97.82× | PASS |
| L2 severity processed (PRIMARY, margin 10) | 43.53× / 42.12× | 38.37× | PASS |
| L2 storage floor (11/10) | 1.40× / 1.36× | 1.239× | PASS |
| L4 frequency processed (PRIMARY, margin 10) | 85.41× / 85.52× | 85.14× | PASS |
| L4 storage floor (11/10) | 3.62× / 3.63× | 3.60× | PASS |
| L6 k=2000 latency floor (factor 3) | 4.09 / 3.98 | 4.341 | PASS |

**The finding — RFC 0036 is not exercised by this harness, and the
run proves it.** Every `ourios_bytes_read` came back **byte-for-byte
identical to §9.24** (L1 1,032,727; L2 2,223,173; L4 47,995,205;
L6 k=100 1,931,911; L6 k=2000 4,202,473). `build_comparative_store`
writes exactly **one ingest file per partition** (`append_record` keeps
one `Writer` per partition), so `compact_partition` no-ops (it needs
≥ 2 files) and RFC 0036's compaction-time sort never runs. The
comparative store is therefore identical pre/post-0036 — which is why
the gates pass unchanged (no regression) *and* why this harness cannot
measure RFC 0036's window-materialization win. Making it accrue
multiple files per partition then compact would re-base RFC 0031's
frozen-gate store; that is deferred as **future harness work**, not a
`validated` blocker (the RFC0036.2 bytes channel is a §2.2 diagnostic,
not a gate).

**Two orthogonal noise items** (neither an RFC 0036 layout regression):
attempt 1 measured L3 cleanly (21.6× storage) but landed the L6 k=100
latency floor at 0.311 < 0.333 (marginal tiny-query noise; attempt 2
passed it at 0.364); attempt 2 hit the known Loki L3 flicker (0/9 rows
before timeout — the §9.13 class, #490). One justified retry per the
retry-once rule; no third.

### 9.27 Results — 2026-07-22 (in-repo, deterministic) — RFC0036.2 window-materialization before/after

**Purpose.** The RFC0036.2 materialization diagnostic that §9.26's
harness cannot produce, measured directly on a genuinely-compacted
store. `ourios-querier`'s `rfc0036_2_materialization_before_after`
(tests/it) runs the identical L6 k=100 window query (1 s at the 10 ms
grid, one service) against two stores built from the same synthetic
multi-service hour, and reads the **materialization bytes** — the
compressed column chunks of the row groups a `service = target ∧ time ∈
window` scan cannot prune — from each file's footer.

| store | survivors / groups | materialization bytes | query scanned |
|---|---|---|---|
| **before** — one unsorted ingest file (128 MiB groups) | 1 / 1 | 100,520,155 | 1 (the whole file) |
| **after** — compacted, §3.1-sorted (32 MiB groups) | 2 / 6 | 70,018,075 | 2 |

**Reading.** Identical 100-row answer from both; the sort takes the
window from materialising the **whole hour** (no row group prunes on
an unsorted file) to a contiguous minority (2 of 6 groups) — a **1.43×
materialization-bytes win**. Modest by design, and honestly so: the
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
compacted file is physically ~2× larger (six 32 MiB groups compress a
little worse than one 128 MiB group — the RFC 0036 §3.3 pruning-
granularity-over-bytes trade), and §2.2's ~188 KB registry floor is
exactly why the **gate** is the scanned-row-group bound (enforced by
`rfc0036_2_window_materialization_bound`), not a bytes ratio. `count`-
scan `stats.bytes_read` is *not* the materialization term — it reads
only the small filter columns and points the wrong way; the footer
survivor-chunk sum is the RFC §9 metric.
41 changes: 31 additions & 10 deletions docs/rfcs/0036-write-side-layout.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
rfc: 0036
title: Write-side layout — compacted-partition clustering and row-group sizing
status: green
status: validated
author: Jens Holdgaard Pedersen <jens@holdgaard.org>
drafting-assistance: Claude
created: 2026-07-21
Expand All @@ -11,8 +11,9 @@ superseded-by: —

# RFC 0036 — Write-side layout

> **Status note.** **`green`** (2026-07-21) — all five §5 scenarios
> discharged. **RFC0036.1** (footer inspection — compacted threshold,
> **Status note.** **`validated`** (2026-07-22) — all five §5 scenarios
> green, plus the comparative evidence below. `accepted` is a maintainer
> flip. **RFC0036.1** (footer inspection — compacted threshold,
> `sorting_columns`, per-group service min/max — plus the §6 merge
> property), **RFC0036.3** (D3 file-band + forced-spill memory bound),
> **RFC0036.4** (shuffled-listing byte-identity rebuild), and
Expand All @@ -30,13 +31,33 @@ superseded-by: —
> L6-scanned-bytes-vs-L1/L3 curve deferred to the `ourios-bench`
> harness); the remaining §7 questions stay open as noted there.
>
> **Deferred to `validated`.** The comparative arm of RFC0036.2 and
> RFC0036.5 — the L6-shape pair on the v8 corpus through the RFC 0031
> dispatch, the before/after materialization-**bytes** §9 diagnostic,
> and the frozen-gate rerun confirming L1/L3/L4 stay inside the
> Loki-wobble band — is a paid `baseline-8vcpu-32gib` measurement, not
> a CI gate (RFC 0031 §5 declined to gate the L6 storage channel; §2.2).
> `validated` needs that baseline run.
> **`validated` evidence (2026-07-22).** The deferred comparative arm
> resolved in two parts, both recorded in `docs/benchmarks.md`:
> - **RFC0036.5 no-regression (§9.26, authoritative
> `baseline-8vcpu-32gib`, HEAD `5e5aa66`).** The frozen RFC 0031
> dispatch reran on post-RFC-0036 `main`: every measured frozen gate
> passes in §9.24's band (L1 ~99×, L2 ~43× + floor 1.40, L4 ~85× +
> floor 3.62, L6 k=2000 latency 4.0). **Finding:** the comparative
> harness writes one ingest file per partition, so `compact_partition`
> no-ops and RFC 0036's sort never runs there — every `ourios_bytes_read`
> is byte-identical to §9.24. So the baseline run proves *no regression*
> but structurally cannot show RFC 0036's win; a multi-file-per-partition
> harness that re-bases the frozen-gate store is **future work**, not a
> `validated` blocker (RFC0036.2 bytes is a §2.2 diagnostic, not a gate).
> - **RFC0036.2 materialization before/after (§9.27, in-repo,
> deterministic).** Measured directly on a genuinely-compacted store:
> the sort takes a one-service window from materialising the whole hour
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
> (unsorted: 1/1 groups, 100.5 MB) to a contiguous minority (sorted:
> 2/6 groups, 70.0 MB) for the identical answer — a **1.43×**
> materialization-bytes win, modest and honest (the compacted file is
> ~2× larger on disk; §2.2's registry floor is why the *gate* is the
> scanned-row-group bound, which `rfc0036_2_window_materialization_bound`
> enforces).
>
> The scanned-row-group **gate** was always the load-bearing acceptance
> criterion (RFC0036.2 §5, green in-repo); the v8-corpus comparative
> materialization number remains desirable future work behind the harness
> change above.
>
> **How to read this document.** This is the write-side layout lever
> that `docs/benchmarks.md` §9.13 named and §9.24 left "parked on its
Expand Down
Loading