From a687151b3057e109aed722484759914812f58c08 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 6 Jul 2026 00:06:20 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(parquet)!:=20rfc=200025=20green=20a=20?= =?UTF-8?q?=E2=80=94=20body=5Fkind=20ordinal=202=20for=20absent=20bodies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RFC 0005 §3.2 amendment: BodyKind::Absent writes ordinal 2 with a NULL body cell; the reader accepts it; UnsupportedAbsentBody is retired (the writer's remaining loud rejection is timestamp overflow). The miner drops lossy_flag on absent rows — absence is not loss, reconstruction renders nothing exactly (RFC 0025 §3.1). The RFC 0024 P1 pinned-rejection arm flips to a round-trip. Discharges RFC0025.1 and RFC0025.2 (committed pre-amendment fixture generated with the two-ordinal writer). BREAKING CHANGE: files containing body_kind ordinal 2 are unreadable by pre-amendment readers (RFC0005.14 unknown-ordinal rejection); upgrade readers before writers. Co-Authored-By: Claude Fable 5 --- crates/ourios-miner/src/cluster.rs | 13 +- crates/ourios-parquet/src/reader.rs | 6 +- crates/ourios-parquet/src/record_batch.rs | 61 +++--- .../tests/rfc0024_properties.rs | 22 +-- .../tests/rfc0025_absent_body.rs | 185 ++++++++++++++++-- testdata/rfc0025/pre-amendment.parquet | Bin 0 -> 8222 bytes 6 files changed, 217 insertions(+), 70 deletions(-) create mode 100644 testdata/rfc0025/pre-amendment.parquet diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index 6e784963e..92bf70743 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -1153,10 +1153,10 @@ impl MinerCluster { // record with `BodyKind::Absent` and the // template-id sentinel; tokenize/mask didn't // run, so there's no separator / param info to - // carry. `lossy_flag = true` because there is no - // template, so reconstruction is not possible. - let mut rec = Self::record_envelope(record, BodyKind::Absent); - rec.lossy_flag = true; + // carry. `lossy_flag = false` per RFC 0025 §3.1: + // absence is not loss — reconstruction is defined + // and total (it renders nothing). + let rec = Self::record_envelope(record, BodyKind::Absent); self.emit_record(rec, service); NO_TEMPLATE } @@ -4331,7 +4331,10 @@ mod tests { assert_eq!(rec.tenant_id, t); assert_eq!(rec.template_id, NO_TEMPLATE); assert_eq!(rec.body_kind, BodyKind::Absent); - assert!(rec.lossy_flag, "Body::None records are lossy (no template)"); + assert!( + !rec.lossy_flag, + "absence is not loss (RFC 0025 §3.1) — reconstruction renders nothing, exactly" + ); assert!(rec.separators.is_empty()); assert!(rec.params.is_empty()); assert!(rec.body.is_none()); diff --git a/crates/ourios-parquet/src/reader.rs b/crates/ourios-parquet/src/reader.rs index 7bd043e23..c29235cb1 100644 --- a/crates/ourios-parquet/src/reader.rs +++ b/crates/ourios-parquet/src/reader.rs @@ -682,9 +682,13 @@ fn decode_body_kind(ord: u8) -> Result { match ord { 0 => Ok(BodyKind::String), 1 => Ok(BodyKind::Structured), + 2 => Ok(BodyKind::Absent), other => Err(ReaderError::Conversion { column: columns::BODY_KIND, - detail: format!("unknown ordinal {other} (RFC 0005 §3.2 pins 0=String, 1=Structured)"), + detail: format!( + "unknown ordinal {other} (RFC 0005 §3.2 pins 0=String, 1=Structured, \ + 2=Absent per RFC 0025)" + ), }), } } diff --git a/crates/ourios-parquet/src/record_batch.rs b/crates/ourios-parquet/src/record_batch.rs index 85fcde09d..b9cdb9ccb 100644 --- a/crates/ourios-parquet/src/record_batch.rs +++ b/crates/ourios-parquet/src/record_batch.rs @@ -166,16 +166,6 @@ pub enum BatchError { count: usize, source: ourios_core::otlp::canonical::CanonicalJsonError, }, - /// A record carried [`BodyKind::Absent`] (the in-memory - /// "wire delivered no body" variant). RFC 0005 §3.2's - /// `body_kind` column pins exactly two ordinals (`0 = String, - /// 1 = Structured`); silently mapping `Absent` to one of - /// them would misclassify wire-absent rows. Until a future - /// RFC 0005 amendment either adds a third ordinal or adds a - /// separate `body_present` boolean column, the writer - /// rejects these records rather than corrupting the - /// `body_kind` semantics. - UnsupportedAbsentBody, /// A clean-attach `body_kind = String` record had too few /// `separators` entries to satisfy the RFC 0005 §3.2 /// invariant ("`tokens.len() + 1` elements when @@ -226,13 +216,6 @@ impl fmt::Display for BatchError { `with-serde` derives are infallible on every spec-compliant `AnyValue`; \ this means an `opentelemetry-proto` upgrade broke that contract)", ), - Self::UnsupportedAbsentBody => write!( - f, - "record carries BodyKind::Absent (wire-absent body), which RFC 0005 §3.2's \ - body_kind column does not yet encode (the column pins ordinals 0=String, \ - 1=Structured); a future RFC 0005 amendment is required to represent this \ - in the schema", - ), Self::InvalidSeparatorsForString { expected_at_least, actual, @@ -259,7 +242,6 @@ impl std::error::Error for BatchError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { Self::TimestampOverflow { .. } - | Self::UnsupportedAbsentBody | Self::InvalidSeparatorsForString { .. } | Self::MissingBodyForLossyString => None, Self::AttributeEncode { source, .. } => Some(source), @@ -421,7 +403,7 @@ impl Builders { self.flags.append_value(r.flags); append_option_str(&mut self.event_name, r.event_name.as_deref()); - self.body_kind.append_value(body_kind_ordinal(r.body_kind)?); + self.body_kind.append_value(body_kind_ordinal(r.body_kind)); // RFC 0005 §3.3: when `body_kind = Structured`, the // body column carries Ourios-canonical JSON — the bytes // the miner has already encoded via @@ -534,16 +516,14 @@ fn append_option_str(b: &mut StringBuilder, v: Option<&str>) { } /// Map an in-memory [`BodyKind`] to the §3.2 on-disk `body_kind` -/// ordinal. The schema pins exactly two ordinals (`0 = String, -/// 1 = Structured`); `BodyKind::Absent` has no on-disk -/// representation today and the writer rejects records carrying -/// it via [`BatchError::UnsupportedAbsentBody`] rather than -/// silently misclassifying them. -fn body_kind_ordinal(k: BodyKind) -> Result { +/// ordinal (`0 = String, 1 = Structured, 2 = Absent` — the third +/// ordinal is the RFC 0025 §3.1 amendment; wire-absent rows carry a +/// `NULL` body cell). +fn body_kind_ordinal(k: BodyKind) -> u8 { match k { - BodyKind::String => Ok(0), - BodyKind::Structured => Ok(1), - BodyKind::Absent => Err(BatchError::UnsupportedAbsentBody), + BodyKind::String => 0, + BodyKind::Structured => 1, + BodyKind::Absent => 2, } } @@ -840,18 +820,27 @@ mod tests { assert_eq!(stored.value(0), canonical.as_bytes()); } - /// `BodyKind::Absent` is not representable in the §3.2 - /// `body_kind` column today (the ordinals pin to - /// `0 = String, 1 = Structured`). The writer rejects such - /// records rather than silently lumping them with String. + /// RFC 0025 §3.1: `BodyKind::Absent` writes ordinal 2 with a + /// `NULL` body cell — the contract change that retired the old + /// `UnsupportedAbsentBody` rejection (approved via RFC 0025; + /// wire-absent bodies are spec-legal and must persist). #[test] - fn absent_body_kind_returns_unsupported_error() { + fn absent_body_kind_writes_ordinal_two_with_null_body() { let mut rec = empty_record(); rec.body_kind = BodyKind::Absent; - let err = mined_records_to_batch(&[rec]).expect_err("Absent body must error"); + rec.body = None; + let batch = mined_records_to_batch(&[rec]).expect("Absent body must write"); + let kind_idx = batch.schema().index_of(crate::columns::BODY_KIND).unwrap(); + let kinds = batch + .column(kind_idx) + .as_any() + .downcast_ref::() + .expect("body_kind is UInt8"); + assert_eq!(kinds.value(0), 2, "Absent is ordinal 2 (RFC 0025 §3.1)"); + let body_idx = batch.schema().index_of(crate::columns::BODY).unwrap(); assert!( - matches!(err, BatchError::UnsupportedAbsentBody), - "expected UnsupportedAbsentBody, got {err:?}", + batch.column(body_idx).is_null(0), + "the body cell is NULL for Absent rows", ); } diff --git a/crates/ourios-parquet/tests/rfc0024_properties.rs b/crates/ourios-parquet/tests/rfc0024_properties.rs index 158d90414..53b21bfa9 100644 --- a/crates/ourios-parquet/tests/rfc0024_properties.rs +++ b/crates/ourios-parquet/tests/rfc0024_properties.rs @@ -78,7 +78,8 @@ fn fail(what: &str, e: impl std::fmt::Display) -> TestCaseError { /// Mine `batch`, store every writable row, read it back, and assert /// fidelity. Adversarial timestamps can exceed `i64::MAX`, which the /// writer *rejects by contract* (RFC 0005 §3.2 timestamp overflow) — -/// those rows are asserted rejected, everything else must round-trip. +/// those rows are asserted rejected, everything else (absent bodies +/// included, per RFC 0025) must round-trip. fn assert_round_trip(batch: &[OtlpLogRecord]) -> Result<(), TestCaseError> { let sink = SharedRecordSink::new(); let mut cluster = @@ -97,24 +98,17 @@ fn assert_round_trip(batch: &[OtlpLogRecord]) -> Result<(), TestCaseError> { let mut groups: Vec<(PartitionKey, Vec<(usize, MinedRecord)>)> = Vec::new(); let mut writable = vec![false; batch.len()]; for (i, mined) in emitted.into_iter().enumerate() { - let absent_body = mined.body_kind == ourios_core::record::BodyKind::Absent; let ts_overflow = !fits_i64(mined.time_unix_nano) || mined.observed_time_unix_nano.is_some_and(|t| !fits_i64(t)); - if absent_body || ts_overflow { - // - Absent body: KNOWN GAP (#362, found by this suite) — - // no §3.2 on-disk representation until the RFC 0005 - // amendment lands; this arm then turns into a - // round-trip. - // - Timestamp overflow: the §3.2 u64→i64 contract, on - // *either* timestamp column. - // Both must be loud rejections, never silent drops. + if ts_overflow { + // Timestamp overflow: the §3.2 u64→i64 contract, on + // *either* timestamp column — the writer's one remaining + // documented loud rejection. (Absent bodies round-trip + // since RFC 0025 §3.1 gave them ordinal 2.) let err = mined_records_to_batch(std::slice::from_ref(&mined)) .expect_err("the writer must reject this record, not silently map it"); prop_assert!( - matches!( - err, - BatchError::UnsupportedAbsentBody | BatchError::TimestampOverflow { .. } - ), + matches!(err, BatchError::TimestampOverflow { .. }), "record {}: rejected for an undocumented reason: {}", i, err diff --git a/crates/ourios-parquet/tests/rfc0025_absent_body.rs b/crates/ourios-parquet/tests/rfc0025_absent_body.rs index 3dc5dc0f9..0eaebbacf 100644 --- a/crates/ourios-parquet/tests/rfc0025_absent_body.rs +++ b/crates/ourios-parquet/tests/rfc0025_absent_body.rs @@ -4,30 +4,187 @@ //! quarantine (`.4`/`.5`) in //! `crates/ourios-ingester/tests/rfc0025_quarantine.rs`. //! -//! Stubs are `#[ignore]`d so the default run stays green while the -//! RFC is red; each names the green slice that discharges it. +//! The pre-amendment file is the committed +//! `testdata/rfc0025/pre-amendment.parquet` fixture (the RFC 0021 §6 +//! committed-fixture discipline), generated by the pre-RFC 0025 +//! writer — two ordinals only, no `Absent` rows. + +use ourios_core::audit::ParamType; +use ourios_core::record::{BodyKind, MinedRecord, Param}; +use ourios_core::tenant::TenantId; +use ourios_parquet::{PartitionKey, Reader, Writer}; +use tempfile::TempDir; + +/// 2026-04-02T10:58:00Z — the fixture baseline instant. +const TS0: u64 = 1_775_127_480_000_000_000; + +fn repo_root() -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root") + .to_path_buf() +} + +fn fixture_path() -> std::path::PathBuf { + repo_root().join("testdata/rfc0025/pre-amendment.parquet") +} + +/// A minimal record of each pre-amendment kind, deterministic so the +/// fixture and the parity assertion agree forever. +fn pre_records() -> Vec { + vec![ + // Clean-attach String row (reconstructable shape). + MinedRecord { + tenant_id: TenantId::new("rfc0025"), + template_id: 11, + template_version: 1, + severity_number: 9, + severity_text: Some("INFO".to_string()), + scope_name: Some("lib.fixture".to_string()), + scope_version: None, + scope_attributes: Vec::new(), + resource_schema_url: None, + scope_schema_url: None, + time_unix_nano: TS0, + observed_time_unix_nano: None, + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0, + event_name: None, + body_kind: BodyKind::String, + params: vec![Param { + type_tag: ParamType::Num, + value: "7".to_string(), + }], + separators: vec![String::new(), " ".to_string()], + body: None, + confidence: 1.0, + lossy_flag: false, + }, + // Structured row (canonical-JSON body bytes). + MinedRecord { + tenant_id: TenantId::new("rfc0025"), + template_id: 12, + template_version: 1, + severity_number: 9, + severity_text: None, + scope_name: None, + scope_version: None, + scope_attributes: Vec::new(), + resource_schema_url: None, + scope_schema_url: None, + time_unix_nano: TS0 + 1_000, + observed_time_unix_nano: None, + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0, + event_name: None, + body_kind: BodyKind::Structured, + params: Vec::new(), + separators: Vec::new(), + body: Some(r#"{"intValue":"42"}"#.to_string()), + confidence: 1.0, + lossy_flag: true, + }, + ] +} + +/// An absent-body record as the miner emits it post-RFC 0025: +/// `NO_TEMPLATE`, empty params/separators, `body = None`, and — +/// per §3.1 — `lossy_flag = false` (absence reconstructs exactly, +/// to nothing). +fn absent_record(ts_offset: u64) -> MinedRecord { + MinedRecord { + body_kind: BodyKind::Absent, + template_id: 0, + template_version: 0, + time_unix_nano: TS0 + ts_offset, + body: None, + params: Vec::new(), + separators: Vec::new(), + confidence: 0.0, + lossy_flag: false, + ..pre_records().remove(0) + } +} /// Scenario RFC0025.1 — absent bodies round-trip. /// See `docs/rfcs/0025-absent-body-representation.md` §5. #[test] -#[ignore = "RFC0025.1 stub — implemented in the schema green slice"] fn rfc0025_1_absent_bodies_round_trip() { - todo!( - "RFC0025.1 — a BodyKind::Absent record writes under body_kind \ - ordinal 2 with a NULL body cell and reads back with every \ - RFC 0005 §3.2 column intact; the RFC 0024 P1 pinned-rejection \ - arm for absent bodies flips to a round-trip assertion" - ); + let bucket = TempDir::new().expect("temp dir"); + let originals = vec![ + pre_records().remove(0), + absent_record(2_000), + absent_record(3_000), + ]; + + let partition = PartitionKey::derive(&originals[0]).expect("derive"); + let mut writer = Writer::open(bucket.path(), partition.clone()).expect("open writer"); + writer.append_records(&originals).expect("append"); + let written = writer.close().expect("close"); + + let reader = Reader::open_partition(&written.path, partition).expect("open_partition"); + let round_tripped = reader.read_all().expect("read_all"); + + assert_eq!(round_tripped, originals, "every RFC 0005 §3.2 column"); + for r in round_tripped + .iter() + .filter(|r| r.body_kind == BodyKind::Absent) + { + assert_eq!(r.body, None, "the body cell is NULL for Absent rows"); + assert!(!r.lossy_flag, "absence is not loss (RFC 0025 §3.1)"); + } } /// Scenario RFC0025.2 — old files unaffected. /// See `docs/rfcs/0025-absent-body-representation.md` §5. #[test] -#[ignore = "RFC0025.2 stub — implemented in the schema green slice"] fn rfc0025_2_old_files_unaffected() { - todo!( - "RFC0025.2 — a pre-amendment committed fixture reads identically \ - under the amended reader (the RFC 0021 §6 committed-fixture \ - parity discipline)" + let bucket = TempDir::new().expect("temp dir"); + let partition = PartitionKey::derive(&pre_records()[0]).expect("derive"); + let dir = partition.data_path(bucket.path()); + std::fs::create_dir_all(&dir).expect("mkdir partition"); + std::fs::copy(fixture_path(), dir.join("0-pre-amendment.parquet")).expect( + "committed fixture missing — regenerate via the ignored rfc0025_fixture test \ + (pre-amendment writer only)", + ); + + let reader = Reader::open_partition(&dir.join("0-pre-amendment.parquet"), partition) + .expect("open_partition"); + let rows = reader.read_all().expect("read_all"); + assert_eq!( + rows, + pre_records(), + "pre-amendment files read identically under the amended reader", ); } + +/// Regenerates `testdata/rfc0025/pre-amendment.parquet`. Generated +/// once with the **pre-RFC 0025** writer (two `body_kind` ordinals) +/// and committed; kept for provenance — the rows deliberately avoid +/// `Absent`, so the output is stable across the amendment. Run +/// manually (`cargo test -p ourios-parquet --test rfc0025_absent_body \ +/// -- --ignored rfc0025_fixture`); never in CI. +#[test] +#[ignore = "fixture generator — run manually, commit the output"] +fn rfc0025_fixture() { + let out_dir = fixture_path(); + let out_dir = out_dir.parent().expect("fixture parent"); + std::fs::create_dir_all(out_dir).expect("mkdir testdata/rfc0025"); + + let scratch = TempDir::new().expect("temp dir"); + let partition = PartitionKey::derive(&pre_records()[0]).expect("derive"); + let mut writer = Writer::open(scratch.path(), partition).expect("open writer"); + writer.append_records(&pre_records()).expect("append"); + let written = writer.close().expect("close"); + std::fs::copy(&written.path, fixture_path()).expect("install fixture"); + eprintln!("fixture written to {}", fixture_path().display()); +} diff --git a/testdata/rfc0025/pre-amendment.parquet b/testdata/rfc0025/pre-amendment.parquet new file mode 100644 index 0000000000000000000000000000000000000000..48f31ecd7d078e103a9e40d37fc38468b25554f1 GIT binary patch literal 8222 zcmcgyZD?E989tH~TTYzBadJ+jL`_{cxui=M%W~|*!EJM8S(Y6|wcSKtrFHdj6)V1y zTz&X!A7rH)p_I_=*C@q5N(m#Bvcc9Lqd!6^V}w#devA!D$Bipoy`Pe0 zCzP>EeD6Kyeb4iLzvo<$S$TelySU&dqTI<@A7dx}F!=BOJt@Yz7-Q8!&hHP5aE3h3 zz^#{e@=l$}c>p+EFxk%oQFxfEdw<5*9-!gXqdgcvbaGECM2D9#M~4#@yZPZJ4O~wG z1=j8;nxSaKL~uhbl+OH>kGp^Vp69D!?FkG${Aep@=MZB~V0NLDXZUlibn!c#?%0CY zUg7SCDU1#rXtL8SqkP7uW8oJ-3m~`p2su#84hKU=9p(Nefc@*5v(;kuRH3+5Z&Z~> z!eP8G3XEpZ(AzNH$NO6q=o&NJgbW`3gH}KFWHg4CUPe7%dU>000u2Fn$|ML50CE7(QhBkN6`2CIZ@97Tg70E@q!qji*~3K#z%RiH2_Ecrl$y%O{+aExYgzoX7ZRD}3KCjRfcbo(?Cec?JU6$B#wfVdhZxG|=Hf zGx`Awa`hQ7#@;kAT}hsk?j2bDX4>1!SaZp>=NY^9Ja9Pqp;omVP;OWeJ0{Hz1?6&3 z_xipu{Qs^y@AENc{4vLQ#vS1?9o@D{vz=)tXNx6vA5mDz1|vcv0v6%aeGa6K)7YMPwwRy-|1C)8wNBL!L1 zPc2z4Y?3fdy%MIKPfL(Rv|^8~{dqn15lS~0+baZ$ku-c5mbKw*GX-9I#KksFs&Si9 znBas1d2UTRzTrG^M+$jY^L+f6e%`^)>#sLl{8jz+{aq&SJDofPvtHrGmAAU|k-*WrZXP5W3HZ{@qZa4=ZWH3( zZe1mUcyo^_@s9O+yT@d|xz~<*bDwF!{eAo@2^c`(clMh`Z1(ab5g=uJ;-dq6-Wpx+ zBaM;pSsxz-0=|BH&~DX-hwKtJxqWm)m?MbKcLR~0Oto5BU8v<2l~RU}!UlDmdAjRL zIa97L6!XVf6v`R4G??8}S4zujrmo<;FZ#j3^Kc4-LckDIB>k-PLI1@%>cQW$Ma4v zOmx+jGi4g1pj`zubEVdV3RZyi5IegS$lIN*?7d2t_=1eI0yN&s@Qjae8*G;69FwyK4#p)hpGuP?TT> z7~^h$+MTPE3&p%r&MC05V}0p`_? z0!}=yK={}=^f4y>{sceF5Y(lE+AE7-^WwTgy62GIa3Hd|%UN4r8*Fh`@K>04qn``$ zmWMw_5}dk(Ed@$qW<*)~#3{YyL@BgkYTITTbSB}|j>-4r3HlYT^CFxt| zxV6oC%)M8DF~;U=&7wFTcnA^;yz^U>^T05`kjFiPU(wXzES51I*Vv0k~Xm8!c9jMCayoped6w6 z*a_@eZ|jVNt%|oD(ycD(%TC0$pJJMhG;JjbUDEp~=@&=%e3PW+W=q1NVp)3EEq&AF z(;~d3rWl>sn7ECIe?6jyx<|_#TaPBRb0R8z=^lrKhcgPEQ|#FXe!$QHPE*vjZRmuE zRdl~wy46kLgXlr{@ZQFHCLTNq;rq{#mhf3E%xZCj9TQdQ-frpFyAj)_@D?AS-Zutx zz4vLRo4W@=VU@hOXPYXyH2`XT;L}~FuQxUubNQX^mfyzZ_l)I6)6upk(?_>^q&IsW z#YcZZvH`f4vJ zh)3TYf_VKYQ1G+IO$Dh%(LPF8NpV%Wzi%8W&9W$3Z$;o7W~H{N^YhXliO(jxPk17C)6fJ}A9@@S(L$Ur|i_{um_koug*jX(tk^ zcUUJJ!p8#n@Q`%v(8CJkF5*5oYW7VHcgwNK1l{D)KOPGDHG*zF(E{53xMd>W5Ym0# zUS09wWBe7LFpB#6ae)66;l*aJa7k|QXK@HWhCDs;{QL`vbDA?eU-b>*wK^A(WqA(% z@VX$&tMbGZc?PH8Da%2charq*c^XGBWjtX}gPDn?An+#S6uhra$#O^z0%Bgi4D`7$ zjB^O{8j6WrnS{wn9EWin17>+51!Cp2d=XgZU@TAKcnHN@K(Rr;4EQyemcw(nG=)n? z;YE((EqEI5Fs0u^r%$d74uW8~0En*1Fam2=X!>LU=(r#1g+AqxyzuaafL>KPNH*x*h+3HUVvvMofuk74#(HC#1>pS z39jB=AK8!kPNJ$deUihmtC^{IJrzg&R^`$xFs+d&Lw+zgXV@ppnb|7PQDAT*6&PDh z$Hx})k@@vZVrD2m72hCLE^MH_&GyJC15cJ`hz@xm%h{>1joh$WNSD?Y(;#M=md^MGvS-{Y#?s$hAG$#P#GFF?A-^^t z-Y2I~KLWM%iz!IgQs0kn#6OMl2QLoJZi(-tnJ@B04)L%i(qeH1`7PBZaCT~DB^^-pg%)gGwd)ER1NBM!KTH{^2XYWU9KQ@H zM6J86UDFi5w*Id}P7SW>V&Fmf9f3U}8LUA)wB``f6S_ZI{X?6M6_06S6PiMO!KLst z&IcrqP7RPF_1x4N><{rmCNXFFA2{0LTmHGM6%;{1pVd{T{)hUP!i{`*Y?=C>{Oq&L s Date: Mon, 6 Jul 2026 00:20:11 +0200 Subject: [PATCH 2/2] fix(parquet): reject absent-with-bytes + correct the fixture's structured lossy flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NonNullBodyForAbsent guards the §3.1 NULL-cell contract's other direction (with a unit test); the fixture's structured row now carries lossy_flag = false per RFC 0001 §6.1 (regenerated). Co-Authored-By: Claude Fable 5 --- crates/ourios-parquet/src/record_batch.rs | 33 +++++++++++++++++- .../tests/rfc0025_absent_body.rs | 5 ++- testdata/rfc0025/pre-amendment.parquet | Bin 8222 -> 8222 bytes 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/ourios-parquet/src/record_batch.rs b/crates/ourios-parquet/src/record_batch.rs index b9cdb9ccb..925eb0731 100644 --- a/crates/ourios-parquet/src/record_batch.rs +++ b/crates/ourios-parquet/src/record_batch.rs @@ -166,6 +166,12 @@ pub enum BatchError { count: usize, source: ourios_core::otlp::canonical::CanonicalJsonError, }, + /// A `body_kind = Absent` record carried body bytes. RFC 0025 + /// §3.1 pins the on-disk contract as ordinal 2 **with a `NULL` + /// `body` cell**; silently writing the bytes would smuggle an + /// undefined state into the schema, so the writer rejects the + /// producer bug loudly. + NonNullBodyForAbsent, /// A clean-attach `body_kind = String` record had too few /// `separators` entries to satisfy the RFC 0005 §3.2 /// invariant ("`tokens.len() + 1` elements when @@ -233,6 +239,12 @@ impl fmt::Display for BatchError { reconstruction path returns the retained body verbatim — without one, the \ record is unreconstructable on read", ), + Self::NonNullBodyForAbsent => write!( + f, + "body_kind = Absent record carries body bytes, but RFC 0025 §3.1 pins the \ + Absent on-disk contract as a NULL body cell — a producer bug, rejected \ + rather than written", + ), Self::Arrow(e) => write!(f, "arrow rejected RecordBatch: {e}"), } } @@ -243,7 +255,8 @@ impl std::error::Error for BatchError { match self { Self::TimestampOverflow { .. } | Self::InvalidSeparatorsForString { .. } - | Self::MissingBodyForLossyString => None, + | Self::MissingBodyForLossyString + | Self::NonNullBodyForAbsent => None, Self::AttributeEncode { source, .. } => Some(source), Self::Arrow(e) => Some(e), } @@ -414,6 +427,9 @@ impl Builders { // is the retained line bytes on the §6.6 lossy path // (or `None` on the clean-attach path, reconstructed // from `template + params + separators` by the reader). + if r.body_kind == BodyKind::Absent && r.body.is_some() { + return Err(BatchError::NonNullBodyForAbsent); + } match r.body.as_deref() { Some(s) => self.body.append_value(s.as_bytes()), None => self.body.append_null(), @@ -844,6 +860,21 @@ mod tests { ); } + /// The §3.1 contract's other direction: an Absent record + /// carrying body bytes is a producer bug and must be rejected + /// loudly, never written. + #[test] + fn absent_body_kind_with_bytes_is_rejected() { + let mut rec = empty_record(); + rec.body_kind = BodyKind::Absent; + rec.body = Some("stray bytes".to_string()); + let err = mined_records_to_batch(&[rec]).expect_err("must reject"); + assert!( + matches!(err, BatchError::NonNullBodyForAbsent), + "expected NonNullBodyForAbsent, got {err:?}", + ); + } + /// Same canonical-encode path on the `resource_attributes` /// column: populated input round-trips through /// `encode_attributes` / `decode_attributes` and lands in diff --git a/crates/ourios-parquet/tests/rfc0025_absent_body.rs b/crates/ourios-parquet/tests/rfc0025_absent_body.rs index 0eaebbacf..b0bc3feab 100644 --- a/crates/ourios-parquet/tests/rfc0025_absent_body.rs +++ b/crates/ourios-parquet/tests/rfc0025_absent_body.rs @@ -91,7 +91,10 @@ fn pre_records() -> Vec { separators: Vec::new(), body: Some(r#"{"intValue":"42"}"#.to_string()), confidence: 1.0, - lossy_flag: true, + // Always false for structured rows (RFC 0001 §6.1) — + // the body carries canonical JSON, reconstruction is + // defined. + lossy_flag: false, }, ] } diff --git a/testdata/rfc0025/pre-amendment.parquet b/testdata/rfc0025/pre-amendment.parquet index 48f31ecd7d078e103a9e40d37fc38468b25554f1..39697a7936afb217bae682b76ced051b22fff5ed 100644 GIT binary patch delta 28 hcmbQ|FwbGb0TxDv%?DZTG6AXi?B95R)HIO}ZUC8b3Dy7r delta 28 jcmbQ|FwbGb0TxE4%?DZTGBGl4p3nY`2S`m5>EH$cnXL)h