From f1f65ce25cb8f9a5f6c2ed5024297222d86476f6 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Wed, 17 Jun 2026 18:02:39 +0200 Subject: [PATCH 1/2] =?UTF-8?q?test(ingester):=20rfc0014=20red=20=E2=80=94?= =?UTF-8?q?=20=C2=A75=20ingest-write-path=20stubs=20+=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advance RFC 0014 specified → red: six `#[ignore]`d acceptance stubs (RFC0014.1–.6) in crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs, each carrying the verification.md §2.3 doc format (`/// Scenario — …` + `/// See …§5.`) with `todo!()` bodies, so the scenario↔test mapping exists and CI stays green while the sink is built. `green` replaces each body with a real assertion against the buffering `ParquetRecordSink` and wires it into the miner in place of `NoOpRecordSink`. Stubs live in `ourios-ingester` (the ingest pipeline's home: miner emit + WAL rotation hook + ourios-parquet deps); placement of the buffer-trigger scenarios may shift next to the sink at `green`, while RFC0014.5 (crash recovery) extends the RFC 0008 WAL harness here. Co-Authored-By: Claude Opus 4.8 --- .../tests/rfc0014_ingest_write_path.rs | 72 +++++++++++++++++++ docs/rfcs/0014-ingest-write-path.md | 9 ++- 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs diff --git a/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs b/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs new file mode 100644 index 000000000..f1321ba6d --- /dev/null +++ b/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs @@ -0,0 +1,72 @@ +//! RFC 0014 — ingest write path (record sink + flush policy) acceptance +//! scenarios (§5). +//! +//! **Status: `red`.** These are the failing stubs that drive the `green` +//! implementation: each encodes one RFC0014.§5 scenario and currently +//! `todo!()`s. They are `#[ignore]`d so the default `cargo test` (and CI) +//! stays green while the buffering `ParquetRecordSink` is built — `green` +//! replaces each body with a real assertion against the sink (per-partition +//! buffers; hybrid size + age + WAL-rotation flush; hard byte-ceiling with +//! blocking backpressure) and removes the `#[ignore]`. +//! +//! Placement may shift at `green`: RFC0014.5 (crash recovery) extends the RFC +//! 0008 WAL harness here in `ourios-ingester`; the buffer-trigger scenarios +//! (.1–.4, .6) may move next to the sink wherever it lands. +//! +//! See `docs/rfcs/0014-ingest-write-path.md` §5/§6. + +/// Scenario RFC0014.1 — Size trigger: the emit that crosses the size target +/// flushes the partition to one right-sized Parquet object. +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.1 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_1_size_trigger() { + todo!("RFC0014.1: a partition flushes on the emit that crosses the size target") +} + +/// Scenario RFC0014.2 — Age trigger: a sub-target low-volume partition flushes +/// on the next batch-window tick once its oldest record reaches `max_buffer_age`. +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.2 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_2_age_trigger() { + todo!("RFC0014.2: low-volume partition flushes on age") +} + +/// Scenario RFC0014.3 — Rotation force-flush: a WAL segment rotation flushes +/// every partition (including sub-threshold ones); nothing un-flushed predates +/// the sealed segment. +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.3 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_3_rotation_force_flush() { + todo!("RFC0014.3: rotation flushes every partition") +} + +/// Scenario RFC0014.4 — Bounded memory: the sink early-flushes under pressure +/// and, at the hard ceiling, `emit` blocks so buffered bytes never exceed it. +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.4 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_4_bounded_memory() { + todo!("RFC0014.4: hard ceiling, never exceeded") +} + +/// Scenario RFC0014.5 — No acknowledged-data loss: a crash with a non-empty +/// buffer loses nothing — WAL replay re-mines every un-flushed acknowledged +/// record (`CLAUDE.md` §3.4). +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.5 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_5_no_acknowledged_data_loss() { + todo!("RFC0014.5: crash mid-buffer loses no acknowledged data (WAL replay)") +} + +/// Scenario RFC0014.6 — Tenant isolation: a flush produces an object holding +/// only one tenant's rows; no buffer or flush crosses tenants (`CLAUDE.md` §3.7). +/// See `docs/rfcs/0014-ingest-write-path.md` §5. +#[test] +#[ignore = "RFC0014.6 — red until the ParquetRecordSink + flush policy land (green)"] +fn rfc0014_6_tenant_isolation() { + todo!("RFC0014.6: no cross-tenant buffer or flush") +} diff --git a/docs/rfcs/0014-ingest-write-path.md b/docs/rfcs/0014-ingest-write-path.md index 8671a7beb..493b0809e 100644 --- a/docs/rfcs/0014-ingest-write-path.md +++ b/docs/rfcs/0014-ingest-write-path.md @@ -1,7 +1,7 @@ --- rfc: 0014 title: Ingest write path — record sink and flush policy -status: specified +status: red author: Jens Holdgaard Pedersen drafting-assistance: Claude created: 2026-06-17 @@ -11,7 +11,7 @@ superseded-by: — # RFC 0014 — Ingest write path: record sink and flush policy -> **Status note.** **`specified`** (2026-06-17). The conspicuous gap in the +> **Status note.** **`red`** (2026-06-17). The conspicuous gap in the > ingest stack: today the miner (RFC 0001) emits each mined `MinedRecord` > into a `RecordSink`, and **production wires `NoOpRecordSink` — the records > are dropped.** Every other layer is built and tested (OTLP → WAL → miner; @@ -35,6 +35,11 @@ superseded-by: — > than exceed it (§3.4, RFC0014.4). The remaining §7 questions (defaults, > early-flush victim, rotation-hook surface, size estimation) are tuning / > implementation detail, decided across the `red`/`green` PRs. +> +> **`red`** lands the six `#[ignore]`d acceptance stubs (RFC0014.1–.6) in +> `crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs` (CI stays green). +> `green` builds the buffering `ParquetRecordSink` and wires it into the miner +> in place of `NoOpRecordSink`. ## 1. Summary From eecd3c876883c10f49dd9ea37f8e75c31f00c4b3 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Wed, 17 Jun 2026 18:12:01 +0200 Subject: [PATCH 2/2] =?UTF-8?q?test(ingester):=20fix=20RFC=200014=20=C2=A7?= =?UTF-8?q?5=20ref=20typo=20in=20stub=20module=20doc=20(copilot)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs b/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs index f1321ba6d..d238f0a7f 100644 --- a/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs +++ b/crates/ourios-ingester/tests/rfc0014_ingest_write_path.rs @@ -2,7 +2,7 @@ //! scenarios (§5). //! //! **Status: `red`.** These are the failing stubs that drive the `green` -//! implementation: each encodes one RFC0014.§5 scenario and currently +//! implementation: each encodes one RFC 0014 §5 scenario and currently //! `todo!()`s. They are `#[ignore]`d so the default `cargo test` (and CI) //! stays green while the buffering `ParquetRecordSink` is built — `green` //! replaces each body with a real assertion against the sink (per-partition