From 8005a9589aa52c9fe29e4a3d34f004bcf61dfc2b Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Thu, 21 May 2026 10:35:26 +0200 Subject: [PATCH 1/3] feat(core): extend MinedRecord with the OTLP-envelope fields (PR-E1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap between MinedRecord (today's minimal shape) and RFC 0001 §6.1 / RFC 0005 §3.2 (the normative record schema). The miner now copies the full OTLP envelope from the input OtlpLogRecord through to the emitted MinedRecord; until the receiver (RFC 0003) lands, corpus / bench inputs leave these fields at their OtlpLogRecord::default() values, which surface as NULL or empty in the corresponding Parquet columns. Added MinedRecord fields (all populated from OtlpLogRecord by ourios-miner's record_envelope): - severity_text: Option - scope_version: Option - observed_time_unix_nano: Option - attributes: Vec - dropped_attributes_count: u32 - resource_attributes: Vec - trace_id: Option<[u8; 16]> - span_id: Option<[u8; 8]> - flags: u32 - event_name: Option Why now: the upcoming Parquet writer (PR-E2) needs every §3.2 row-level column to faithfully emit the schema. Landing the field set as its own PR keeps PR-E2 focused on the Arrow RecordBatch ↔ Parquet plumbing, and gives reviewers the contract change in one place rather than mixed with writer mechanics. Per CLAUDE.md §6.4: clean solution over papering over a half-implementation. Updated callers: - ourios-miner cluster.rs::record_envelope copies the 10 new fields from the incoming OtlpLogRecord. - Test fixtures in record.rs (sample_clean_record / sample_parse_failure_record) and reconstruct.rs's record_envelope helper get explicit zero / None / empty values for the new fields — keeping construction explicit (no Default impl) so the test fixtures remain a readable "expected shape" pin. Known follow-up: MinedRecord.body is Option while RFC 0001 §6.1 / RFC 0005 §3.2 specify raw bytes (potentially non-UTF-8). That gap is unchanged here; PR-E2 will paper over it on the write path (UTF-8 in → bytes out is a no-op) and the String-vs-Bytes type swap is a separate concern. Verified: - cargo build --all-features — clean - cargo test --all-features — 178 tests pass (no regressions) - cargo fmt --all --check — clean - cargo clippy --all-targets --all-features -- -D warnings — clean Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/ourios-core/src/record.rs | 74 +++++++++++++++++++++++--- crates/ourios-miner/src/cluster.rs | 10 ++++ crates/ourios-miner/src/reconstruct.rs | 10 ++++ 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/crates/ourios-core/src/record.rs b/crates/ourios-core/src/record.rs index 45b6625c1..902bd77a6 100644 --- a/crates/ourios-core/src/record.rs +++ b/crates/ourios-core/src/record.rs @@ -27,6 +27,7 @@ use std::sync::{Arc, Mutex}; use crate::audit::ParamType; +use crate::otlp::KeyValue; use crate::tenant::TenantId; /// RFC 0001 §6.1 *Body representation* discriminator. @@ -69,14 +70,13 @@ pub struct Param { /// One row the miner emits per ingested line — RFC 0001 §6.1 /// record schema. /// -/// Fields the §6.1 schema names but PR-A defers (everything in -/// `OtlpLogRecord` outside of the mining-output set — -/// `attributes`, `resource_attributes`, `trace_id`, `span_id`, -/// `flags`, `event_name`, etc.) land alongside the Parquet -/// writer, where they actually become observable. PR-A carries -/// only what the cluster has to populate today: the identity -/// triple, the template-key half of `(severity, scope)`, the -/// time field, and the mining outputs. +/// Carries the full §6.1 OTLP-derived envelope plus the mining +/// outputs. The receiver (RFC 0003, post-MVP) populates the +/// OTLP-derived fields from the wire; the miner copies them +/// through unchanged. Until the receiver lands, corpus / bench +/// inputs leave the OTLP envelope at its `OtlpLogRecord::default()` +/// values (zero / `None` / empty `Vec`), which surface as NULL +/// or empty in the corresponding RFC 0005 §3.2 Parquet columns. /// /// Records emitted on the parse-failure paths /// (`body::None`-with-record, empty input, over-cap, degenerate @@ -100,10 +100,48 @@ pub struct MinedRecord { /// so a reader can filter without joining back through the /// template store. pub severity_number: u8, + /// `LogRecord.severity_text` — the source's original severity + /// string, when set. Outside the §6.1 template key (the + /// numeric `severity_number` is canonical) but retained + /// per-record for query / display fidelity. + pub severity_text: Option, pub scope_name: Option, + /// `InstrumentationScope.version` — emitter library version, + /// retained per-record for drift / debugging. Outside the + /// template key. + pub scope_version: Option, /// Source event time per `OtlpLogRecord.time_unix_nano`. `0` /// = unknown. pub time_unix_nano: u64, + /// `LogRecord.observed_time_unix_nano` — collector observation + /// time, when set. + pub observed_time_unix_nano: Option, + /// `LogRecord.attributes` — per-occurrence structured context. + /// Mirrors RFC 0001 §6.1's `Vec`; the Parquet writer + /// (RFC 0005 §3.3) encodes this as canonical JSON in the + /// `attributes` `BYTE_ARRAY` column. Empty vec ↔ `[]` on disk + /// per RFC 0005 §3.2. + pub attributes: Vec, + /// `LogRecord.dropped_attributes_count` — truncation indicator + /// from the receiver. + pub dropped_attributes_count: u32, + /// `Resource.attributes` — source identity (`service.name`, + /// `host.*`, `k8s.*`, ...) copied onto every record under the + /// originating `ResourceLogs` group. Same on-disk encoding as + /// `attributes`. + pub resource_attributes: Vec, + /// `LogRecord.trace_id` — opaque 16 bytes (W3C Trace Context), + /// when set. RFC 0005 §3.2 stores this as + /// `FIXED_LEN_BYTE_ARRAY(16)` with no logical type — *not* + /// an RFC 4122 UUID. + pub trace_id: Option<[u8; 16]>, + /// `LogRecord.span_id` — opaque 8 bytes, when set. + pub span_id: Option<[u8; 8]>, + /// `LogRecord.flags` — lower 8 bits are W3C trace flags. + pub flags: u32, + /// `LogRecord.event_name` — identifier for structured-event + /// records. + pub event_name: Option, /// §6.1 *Body representation* fork. pub body_kind: BodyKind, /// Masked-parameter slots in template order. Empty for @@ -285,8 +323,18 @@ mod tests { template_id: 7, template_version: 1, severity_number: 9, + severity_text: None, scope_name: Some("lib.auth".to_string()), + scope_version: None, time_unix_nano: 1_700_000_000_000_000_000, + observed_time_unix_nano: None, + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0, + event_name: None, body_kind: BodyKind::String, params: vec![Param { type_tag: ParamType::Num, @@ -311,8 +359,18 @@ mod tests { template_id: 0, template_version: 0, severity_number: 0, + severity_text: None, scope_name: None, + scope_version: None, time_unix_nano: 0, + observed_time_unix_nano: None, + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0, + event_name: None, body_kind: BodyKind::String, params: Vec::new(), separators: Vec::new(), diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index d2b332c99..8b3134d23 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -514,8 +514,18 @@ impl MinerCluster { template_id: NO_TEMPLATE, template_version: 0, severity_number: record.severity_number, + severity_text: record.severity_text.clone(), scope_name: record.scope_name.clone(), + scope_version: record.scope_version.clone(), time_unix_nano: record.time_unix_nano, + observed_time_unix_nano: record.observed_time_unix_nano, + attributes: record.attributes.clone(), + dropped_attributes_count: record.dropped_attributes_count, + resource_attributes: record.resource_attributes.clone(), + trace_id: record.trace_id, + span_id: record.span_id, + flags: record.flags, + event_name: record.event_name.clone(), body_kind, params: Vec::new(), separators: Vec::new(), diff --git a/crates/ourios-miner/src/reconstruct.rs b/crates/ourios-miner/src/reconstruct.rs index 195c13d26..5bae3c0b0 100644 --- a/crates/ourios-miner/src/reconstruct.rs +++ b/crates/ourios-miner/src/reconstruct.rs @@ -168,8 +168,18 @@ mod tests { template_id: 0, template_version: 0, severity_number: 0, + severity_text: None, scope_name: None, + scope_version: None, time_unix_nano: 0, + observed_time_unix_nano: None, + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0, + event_name: None, body_kind, params: vec![], separators: vec![], From 4335d0832fd3f2a365fd3cb0375d188c295ab167 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Thu, 21 May 2026 10:42:21 +0200 Subject: [PATCH 2/3] fixup! feat(core): extend MinedRecord with the OTLP-envelope fields (PR-E1) --- crates/ourios-core/src/otlp.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/ourios-core/src/otlp.rs b/crates/ourios-core/src/otlp.rs index 8862eb7b7..9edcc2835 100644 --- a/crates/ourios-core/src/otlp.rs +++ b/crates/ourios-core/src/otlp.rs @@ -107,6 +107,20 @@ pub enum Body { /// the miner allocates or reuses the /// `(severity_number, scope_name, BodyKind::Structured)` /// sentinel template id per §6.1 *Template-key composition*. + /// + /// **Wire-export round-trip rule (RFC 0003 implementer note).** + /// `MinedRecord.body` for these rows is the OTLP-canonical + /// JSON encoding of the original `AnyValue` (RFC 0005 §3.3). + /// A future OTLP exporter MUST decode that JSON back into the + /// matching `AnyValue` variant (`Kvlist`, `Array`, `IntValue`, + /// `DoubleValue`, `BoolValue`, `BytesValue`) — *not* emit it + /// as `AnyValue::String` carrying the raw JSON text. The + /// latter shortcut is lossy: receivers (e.g. Grafana / Loki) + /// render `AnyValue::String` as text rather than walking the + /// structured tree, and "Body MUST support `AnyValue` to + /// preserve the semantics of structured logs" (OpenTelemetry + /// Logs Data Model §Body) is then violated end-to-end. + /// RFC 0003 will pin this as part of the exporter contract. Structured(AnyValue), } From 1a67de654362ac95dc18120ac2d889f347715ae4 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Thu, 21 May 2026 10:52:14 +0200 Subject: [PATCH 3/3] fixup! feat(core): extend MinedRecord with the OTLP-envelope fields (PR-E1) --- crates/ourios-core/src/otlp.rs | 32 +++++++++++++++++++----------- crates/ourios-miner/src/cluster.rs | 16 +++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/crates/ourios-core/src/otlp.rs b/crates/ourios-core/src/otlp.rs index 9edcc2835..1088a27cf 100644 --- a/crates/ourios-core/src/otlp.rs +++ b/crates/ourios-core/src/otlp.rs @@ -109,18 +109,26 @@ pub enum Body { /// sentinel template id per §6.1 *Template-key composition*. /// /// **Wire-export round-trip rule (RFC 0003 implementer note).** - /// `MinedRecord.body` for these rows is the OTLP-canonical - /// JSON encoding of the original `AnyValue` (RFC 0005 §3.3). - /// A future OTLP exporter MUST decode that JSON back into the - /// matching `AnyValue` variant (`Kvlist`, `Array`, `IntValue`, - /// `DoubleValue`, `BoolValue`, `BytesValue`) — *not* emit it - /// as `AnyValue::String` carrying the raw JSON text. The - /// latter shortcut is lossy: receivers (e.g. Grafana / Loki) - /// render `AnyValue::String` as text rather than walking the - /// structured tree, and "Body MUST support `AnyValue` to - /// preserve the semantics of structured logs" (OpenTelemetry - /// Logs Data Model §Body) is then violated end-to-end. - /// RFC 0003 will pin this as part of the exporter contract. + /// The *target* on-disk encoding for `MinedRecord.body` on + /// `Structured` rows is OTLP-canonical JSON per RFC 0005 §3.3. + /// The *current* miner implementation (in + /// `ourios-miner::cluster::ingest_structured`) writes the + /// `Debug` rendering of the decoded `AnyValue` (`format!( + /// "{any_value:?}")`) as an interim placeholder — the + /// canonicalisation PR replaces it before any wire-export + /// path lands. Either way, the future OTLP exporter MUST + /// decode the stored bytes back into the matching `AnyValue` + /// variant (the `opentelemetry_proto::tonic::common::v1:: + /// any_value::Value` enum — `KvlistValue`, `ArrayValue`, + /// `IntValue`, `DoubleValue`, `BoolValue`, `BytesValue`, + /// `StringValue`) — *not* emit the stored bytes as + /// `AnyValue::StringValue` carrying the raw text. The latter + /// shortcut is lossy: receivers (e.g. Grafana / Loki) render + /// `StringValue` as text rather than walking the structured + /// tree, and "Body MUST support `AnyValue` to preserve the + /// semantics of structured logs" (OpenTelemetry Logs Data + /// Model §Body) is then violated end-to-end. RFC 0003 will + /// pin this as part of the exporter contract. Structured(AnyValue), } diff --git a/crates/ourios-miner/src/cluster.rs b/crates/ourios-miner/src/cluster.rs index 8b3134d23..798c2d5f6 100644 --- a/crates/ourios-miner/src/cluster.rs +++ b/crates/ourios-miner/src/cluster.rs @@ -508,6 +508,22 @@ impl MinerCluster { /// `separators`, `body`, `confidence`, `lossy_flag`) are left /// at their zero / sentinel defaults; the calling site /// customises before calling [`Self::emit_record`]. + /// + /// **Per-record clone cost (deferred optimisation).** The + /// `attributes` and `resource_attributes` vectors are + /// `.clone()`-d once per emitted record. For corpus / bench + /// inputs today the vectors are empty (`Vec::clone` on an + /// empty `Vec` is essentially free), so there's no measured + /// cost. Once the RFC 0003 receiver populates them — and + /// especially `resource_attributes`, which is typically + /// identical across every record in a `ResourceLogs` group — + /// the deep clone becomes a hot-path concern worth measuring. + /// The shape options at that point (per-record-borrowed, + /// `Arc<[KeyValue]>` interning, take-ownership-from-receiver) + /// are RFC 0003 / `ourios-ingester` territory; pinning a + /// shape here would optimise without data. The + /// [`MinedRecord`] field type stays plain `Vec` for + /// now so it mirrors `OtlpLogRecord`'s shape exactly. fn record_envelope(record: &OtlpLogRecord, body_kind: BodyKind) -> MinedRecord { MinedRecord { tenant_id: record.tenant_id.clone(),