From 7f48f3b7bf3e0371497e6a765b1b268f165030d0 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 21 Jul 2026 21:46:47 +0200 Subject: [PATCH 1/2] =?UTF-8?q?test(rfc0036):=20red=20=E2=80=94=20all=20fi?= =?UTF-8?q?ve=20=C2=A75=20stubs=20land,=20status=20specified=E2=86=92red?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five #[ignore]d stubs, one per §5 acceptance scenario (RFC0036.1–.5); default runs stay green (1201 passed, 0 failed, 42 ignored across the workspace; stubs skipped) and force-running the ignored stubs fails all five on todo!(). Each doc comment carries the §5 anchor, each todo!() paraphrases its scenario's Given/When/Then, and each #[ignore] literal names the green slice that discharges it: sorted-compaction (.1), pruning (.2), compaction-properties (.3), determinism (.4), compat (.5). Placement follows §6's mapping: RFC0036.1/.3/.4/.5 in crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs — the machinery they gate (the §3.2 sort-run merge, the §3.3 compacted threshold, the §3.4 sorting_columns declaration) is compaction.rs/writer.rs code — and RFC0036.2's in-repo slice (the synthetic-hour scanned-count bound) in crates/ourios-querier/tests/it/rfc0036_window_materialization.rs, beside the existing RFC 0016 scanned/pruned counter assertions (execution.rs). The .2 comparative arm is a harness/measurement concern through the ourios-bench RFC 0031 dispatch, not a CI stub — the same treatment RFC 0033's red gave its .6 (b49758ee). RFC status flips specified→red with a status note recording the stub placement, the design-review go (2026-07-21), and the §7 at-red decisions (threshold sweep, run format, fan-in cap F, D2 band) deferred to the green implementation where they are measured. fmt clean; workspace clippy -D warnings clean; cargo test --all-features green; cargo doc --workspace --no-deps --all-features clean; mdbook builds. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WQY9wfrfRggqSpMLH8Xj3Y --- crates/ourios-parquet/tests/it/main.rs | 1 + .../tests/it/rfc0036_write_side_layout.rs | 84 +++++++++++++++++++ crates/ourios-querier/tests/it/main.rs | 1 + .../it/rfc0036_window_materialization.rs | 37 ++++++++ docs/rfcs/0036-write-side-layout.md | 21 +++-- 5 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs create mode 100644 crates/ourios-querier/tests/it/rfc0036_window_materialization.rs diff --git a/crates/ourios-parquet/tests/it/main.rs b/crates/ourios-parquet/tests/it/main.rs index b46667318..d4fa96f03 100644 --- a/crates/ourios-parquet/tests/it/main.rs +++ b/crates/ourios-parquet/tests/it/main.rs @@ -20,6 +20,7 @@ mod rfc0021_arrow_upgrade; mod rfc0022_promoted_columns; mod rfc0024_properties; mod rfc0025_absent_body; +mod rfc0036_write_side_layout; mod round_trip; mod row_vs_path_validation; mod schema_pin; diff --git a/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs b/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs new file mode 100644 index 000000000..af1ad58c4 --- /dev/null +++ b/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs @@ -0,0 +1,84 @@ +//! RFC 0036 §5 — write-side layout, the four compaction-side scenarios. +//! +//! Stubs are `#[ignore]`d so the default run stays green while the +//! RFC is red; each names the green slice that discharges it. +//! +//! Placement note: RFC0036.1/.3/.4/.5 live here because the machinery +//! they gate — the §3.2 sort-run merge, the §3.3 compacted row-group +//! threshold, and the §3.4 `sorting_columns` declaration — is +//! `compaction.rs`/`writer.rs` code (RFC 0036 §6). RFC0036.2's +//! in-repo slice (the synthetic-hour scanned-count bound via the +//! RFC 0016 counters) lives with the querier counter assertions in +//! `ourios-querier/tests/it/rfc0036_window_materialization.rs`. + +/// Scenario RFC0036.1 — compacted layout (clustering + sizing + +/// declaration). See `docs/rfcs/0036-write-side-layout.md` §5. +#[test] +#[ignore = "RFC0036.1 stub — implemented in the sorted-compaction green slice (run formation + merge + writer properties)"] +fn rfc0036_1_compacted_layout() { + todo!( + "RFC0036.1 — partition with ≥ 2 input files whose rows span \ + multiple promoted service.name values and interleaved times, \ + compacted: footer inspection of the consolidated file shows \ + row groups rotated at the configured compacted threshold \ + (each uncompressed size ≤ threshold + one sub-batch's bounded \ + overshoot), sorting_columns declared as the §3.1 keys 1–2 on \ + every row group, and per-row-group service.name min/max \ + spanning at most a boundary pair of services; decoding yields \ + rows in §3.1 key order with the row multiset equal to the \ + inputs' union — plus the §6 merge proptest: arbitrary \ + service/time/duplicate-key mixes ⇒ output multiset equals \ + input union, §3.1-sorted, equal-key rows in tie-break order" + ); +} + +/// Scenario RFC0036.3 — compaction properties preserved (D2 / D3 / +/// memory). See `docs/rfcs/0036-write-side-layout.md` §5. +#[test] +#[ignore = "RFC0036.3 stub — implemented in the compaction-properties green slice (D2 band + memory bound)"] +fn rfc0036_3_compaction_properties_preserved() { + todo!( + "RFC0036.3 — §9.7-scale compaction workload (band-scale \ + partition, tens of input files) run through the sorted \ + compaction: D3 holds unchanged (one output file per \ + partition, inside the 256 MiB – 2 GiB band, < 5% of live \ + files below 128 MiB), D2 throughput stays within the band \ + set from a first measurement and still ≫ the per-partition \ + seal rate; a memory-bound test shows peak decoded-row \ + residency of the order of one input file (phase 1) and \ + F × batch (phase 2) — never whole-partition residency" + ); +} + +/// Scenario RFC0036.4 — determinism (the harness's contract). +/// See `docs/rfcs/0036-write-side-layout.md` §5. +#[test] +#[ignore = "RFC0036.4 stub — implemented in the determinism green slice (rebuild differential)"] +fn rfc0036_4_rebuild_byte_identity() { + todo!( + "RFC0036.4 — the same set of input files (same bytes, same \ + names) compacted twice, the second run with the store fake \ + returning listings in a shuffled order: the two consolidated \ + outputs are byte-identical (a file hash is the correct \ + assertion here — byte identity is exactly the property \ + claimed), preserving the §9.13 determinism property the \ + comparative ledger depends on" + ); +} + +/// Scenario RFC0036.5 — no read-path or schema regression. +/// See `docs/rfcs/0036-write-side-layout.md` §5. +#[test] +#[ignore = "RFC0036.5 stub — implemented in the compat green slice (pre-RFC fixture reads + B1/B2 + frozen-gate rerun)"] +fn rfc0036_5_no_read_path_or_schema_regression() { + todo!( + "RFC0036.5 — stores built before and after the change: B1/B2 \ + and the frozen RFC 0031 comparative gates run against the \ + post-change store and every frozen gate still passes, with \ + the L1/L3/L4 pairs not degraded beyond the documented \ + Loki-wobble band and query results identical row-sets; a \ + pre-RFC-0036 fixture file (no sorting_columns, 128 MiB row \ + groups) reads without error or special-casing — no migration \ + exists because none is needed (CLAUDE.md §3.5)" + ); +} diff --git a/crates/ourios-querier/tests/it/main.rs b/crates/ourios-querier/tests/it/main.rs index d2f7c21f4..d19b38942 100644 --- a/crates/ourios-querier/tests/it/main.rs +++ b/crates/ourios-querier/tests/it/main.rs @@ -32,3 +32,4 @@ mod rfc0024_properties; mod rfc0025_rendering; mod rfc0031_single_pass; mod rfc0033_cached_template_map; +mod rfc0036_window_materialization; diff --git a/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs b/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs new file mode 100644 index 000000000..b8a2b9422 --- /dev/null +++ b/crates/ourios-querier/tests/it/rfc0036_window_materialization.rs @@ -0,0 +1,37 @@ +//! RFC 0036 §5 — window-query materialization (RFC0036.2), the +//! in-repo slice. +//! +//! The stub is `#[ignore]`d so the default run stays green while the +//! RFC is red; it names the green slice that discharges it. +//! +//! Placement note: the RFC 0016 scanned/pruned row-group counts this +//! scenario bounds are asserted from this harness (`execution.rs`), so +//! the synthetic-hour CI slice co-locates with them (RFC 0036 §6). The +//! full comparative arm — the L6-shape pair on the v8 corpus through +//! the RFC 0031 dispatch and the before/after §9 bytes diagnostic — is +//! a harness/measurement concern in `ourios-bench`, not a CI stub, +//! matching how RFC 0033 handled its `.6` comparative arm. The other +//! four scenarios live with the compaction code in +//! `ourios-parquet/tests/it/rfc0036_write_side_layout.rs`. + +/// Scenario RFC0036.2 — window-query materialization (the point). +/// See `docs/rfcs/0036-write-side-layout.md` §5. +#[test] +#[ignore = "RFC0036.2 stub — implemented in the pruning green slice (synthetic-hour scanned-count bound; the comparative arm runs via the ourios-bench RFC 0031 harness)"] +fn rfc0036_2_window_materialization_bound() { + todo!( + "RFC0036.2 — compacted store built from a synthetic v8-shape \ + hour (many services, promoted service.name); the L6-shape \ + query (one service, k-row time window) runs: the RFC 0016 \ + scanned/pruned counts show row groups scanned \ + ≤ ceil(B_sw / T) + 2, where B_sw is the queried service's \ + bytes within the window (measurable from the compacted \ + file's footer) and T the configured row-group threshold — \ + the groups that hold the answer plus at most two boundary \ + groups, not the whole hour; the before/after materialization \ + bytes (total minus the registry acquisition) are measured on \ + the comparative corpus and published in the §9 series as the \ + storage-channel diagnostic (the gate is the scanned bound, \ + not a bytes ratio — §2.2)" + ); +} diff --git a/docs/rfcs/0036-write-side-layout.md b/docs/rfcs/0036-write-side-layout.md index aa90ea86d..195e10d7b 100644 --- a/docs/rfcs/0036-write-side-layout.md +++ b/docs/rfcs/0036-write-side-layout.md @@ -1,7 +1,7 @@ --- rfc: 0036 title: Write-side layout — compacted-partition clustering and row-group sizing -status: specified +status: red author: Jens Holdgaard Pedersen drafting-assistance: Claude created: 2026-07-21 @@ -11,13 +11,18 @@ superseded-by: — # RFC 0036 — Write-side layout -> **Status note.** **`specified`** at draft: §§1–8 are written, -> including the §5 acceptance criteria and §6 testing strategy, and -> every hazard this RFC touches (H4; `CLAUDE.md` §3.5, §3.6) carries a -> scenario. Per `docs/rfcs/README.md`, `specified` additionally -> requires review to confirm the criteria are testable in principle — -> this PR's review round is that confirmation; nothing in the codebase -> changes until `red` → `green`. +> **Status note.** **`red`** (2026-07-21). The five §5 stubs are +> landed `#[ignore]`d, each failing on `todo!()` when force-run: +> RFC0036.1/.3/.4/.5 in +> `crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs` with +> the compaction machinery they gate, and RFC0036.2's in-repo slice in +> `crates/ourios-querier/tests/it/rfc0036_window_materialization.rs` +> beside the RFC 0016 counter assertions (its comparative arm runs +> through the `ourios-bench` RFC 0031 harness, not a CI stub). Design +> review is done — maintainer go, 2026-07-21. The §7 decisions marked +> "at `red`" (the threshold sweep, the run format, the fan-in cap F, +> the D2 band) are deferred to the green implementation, where they +> are measured rather than guessed. > > **How to read this document.** This is the write-side layout lever > that `docs/benchmarks.md` §9.13 named and §9.24 left "parked on its From 38aaa772201b5b0af58f6ddc76c0fca50b214e03 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 21 Jul 2026 21:53:46 +0200 Subject: [PATCH 2/2] test(rfc0036): keep scenario doc-comment titles on one line for greppability Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WQY9wfrfRggqSpMLH8Xj3Y --- .../ourios-parquet/tests/it/rfc0036_write_side_layout.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs b/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs index af1ad58c4..de69daed5 100644 --- a/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs +++ b/crates/ourios-parquet/tests/it/rfc0036_write_side_layout.rs @@ -11,8 +11,8 @@ //! RFC 0016 counters) lives with the querier counter assertions in //! `ourios-querier/tests/it/rfc0036_window_materialization.rs`. -/// Scenario RFC0036.1 — compacted layout (clustering + sizing + -/// declaration). See `docs/rfcs/0036-write-side-layout.md` §5. +/// Scenario RFC0036.1 — compacted layout (clustering + sizing + declaration). +/// See `docs/rfcs/0036-write-side-layout.md` §5. #[test] #[ignore = "RFC0036.1 stub — implemented in the sorted-compaction green slice (run formation + merge + writer properties)"] fn rfc0036_1_compacted_layout() { @@ -32,8 +32,8 @@ fn rfc0036_1_compacted_layout() { ); } -/// Scenario RFC0036.3 — compaction properties preserved (D2 / D3 / -/// memory). See `docs/rfcs/0036-write-side-layout.md` §5. +/// Scenario RFC0036.3 — compaction properties preserved (D2 / D3 / memory). +/// See `docs/rfcs/0036-write-side-layout.md` §5. #[test] #[ignore = "RFC0036.3 stub — implemented in the compaction-properties green slice (D2 band + memory bound)"] fn rfc0036_3_compaction_properties_preserved() {