Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions crates/ourios-ingester/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ uuid = { version = "1", default-features = false, features = ["std", "v7"] }

[dev-dependencies]
tempfile = "=3.27.0"
# Read flushed files' Parquet schemas back in the RFC 0022 promoted-set
# threading test (version in lockstep with ourios-parquet's arrow stack).
parquet = { version = "58", default-features = false, features = ["arrow"] }
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
# Drive the axum `Router` in-process (`ServiceExt::oneshot`) for the
# HTTP-transport acceptance tests — no real socket.
tower = { version = "=0.5.3", default-features = false, features = ["util"] }
Expand Down
48 changes: 43 additions & 5 deletions crates/ourios-ingester/src/compactor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ use std::time::{Duration, Instant, SystemTime};
use ourios_core::audit::{AuditEvent, AuditPayload, AuditSink, NoOpAuditSink};
use ourios_core::tenant::TenantId;
use ourios_parquet::{
Committed, CompactionError, CompactionPolicy, PartitionKey, Store, compact_partition,
gc_orphans, percent_decode_tenant, plan_candidates,
Committed, CompactionError, CompactionPolicy, PartitionKey, PromotedAttributes, Store,
compact_partition_with_promoted, gc_orphans, percent_decode_tenant, plan_candidates,
};

use crate::metrics::CompactionMetrics;
Expand Down Expand Up @@ -140,7 +140,7 @@ pub struct CompactedFile {
/// Run one compaction sweep over `store`, as of wall-clock
/// `now_unix_nanos`: for each tenant, select its sealed candidate
/// partitions ([`plan_candidates`]) and consolidate each
/// ([`compact_partition`]), accumulating a [`SweepReport`].
/// ([`compact_partition_with_promoted`]), accumulating a [`SweepReport`].
///
/// Resilient: a tenant whose planning fails, or a partition whose
/// consolidation fails, is recorded in [`SweepReport::errors`] and
Expand All @@ -156,6 +156,27 @@ pub fn run_sweep(
store: &Store,
now_unix_nanos: u64,
policy: &CompactionPolicy,
) -> Result<SweepReport, IngestError> {
run_sweep_with_promoted(
store,
now_unix_nanos,
policy,
&PromotedAttributes::default(),
)
}

/// Like [`run_sweep`] but consolidating under an explicit RFC 0022 promoted
/// attribute set (§3.4: rewrites re-project with the *current* set). The bare
/// [`run_sweep`] delegates with the default (`service.name`-only) set.
///
/// # Errors
///
/// See [`run_sweep`].
pub fn run_sweep_with_promoted(
store: &Store,
now_unix_nanos: u64,
policy: &CompactionPolicy,
promoted: &PromotedAttributes,
) -> Result<SweepReport, IngestError> {
let mut report = SweepReport::default();
for tenant in tenants(store)? {
Expand All @@ -180,7 +201,7 @@ pub fn run_sweep(
partition.year, partition.month, partition.day, partition.hour,
)),
}
match compact_partition(store, &partition) {
match compact_partition_with_promoted(store, &partition, promoted) {
Ok(outcome) => {
if let Some(committed) = &outcome.committed {
report.partitions_compacted += 1;
Expand Down Expand Up @@ -251,6 +272,11 @@ pub struct Compactor {
store: Store,
policy: CompactionPolicy,
interval: Duration,
/// The RFC 0022 promoted attribute set consolidated files re-project
/// under (`storage.promoted_attributes`, §3.2/§3.4). Defaults to the
/// implicit `service.name`-only set; set via
/// [`Self::with_promoted_attributes`].
promoted: PromotedAttributes,
/// Where committed-compaction audit events go (RFC 0009 §3.6).
/// Defaults to [`NoOpAuditSink`]; set via [`Self::with_audit_sink`]
/// (the WAL-backed sink replaces it once `ourios-wal` lands).
Expand All @@ -264,6 +290,7 @@ impl std::fmt::Debug for Compactor {
.field("store", &self.store)
.field("policy", &self.policy)
.field("interval", &self.interval)
.field("promoted", &self.promoted)
.field("audit_sink", &"Box<dyn AuditSink>")
.finish()
}
Expand All @@ -281,10 +308,19 @@ impl Compactor {
store,
policy,
interval,
promoted: PromotedAttributes::default(),
audit_sink: Box::new(NoOpAuditSink::new()),
}
}

/// Set the RFC 0022 promoted attribute set consolidated files re-project
/// under (`storage.promoted_attributes`, §3.2/§3.4).
#[must_use]
pub fn with_promoted_attributes(mut self, promoted: PromotedAttributes) -> Self {
self.promoted = promoted;
self
}

/// Route committed-compaction audit events to `sink`.
#[must_use]
pub fn with_audit_sink(mut self, sink: Box<dyn AuditSink>) -> Self {
Expand Down Expand Up @@ -317,6 +353,7 @@ impl Compactor {
store,
policy,
interval,
promoted,
// Own the audit sink locally so it can move into each sweep's
// blocking task and back out. Its `emit` performs Parquet `put`s
// through the store — now S3 network I/O (RFC 0019 slice 2d) — so it
Expand All @@ -339,9 +376,10 @@ impl Compactor {
// (compaction is blocking I/O). `policy` is `Copy`, so the `move`
// closure copies it and the outer binding stays valid each loop.
let store = store.clone();
let promoted = promoted.clone();
let (result, elapsed, sink) = tokio::task::spawn_blocking(move || {
let start = Instant::now();
let result = run_sweep(&store, now_unix_nanos(), &policy);
let result = run_sweep_with_promoted(&store, now_unix_nanos(), &policy, &promoted);
// Emit the committed-compaction audit events here, on the
// blocking pool, since the sink's `put`s are blocking store I/O.
if let Ok(report) = &result {
Expand Down
2 changes: 1 addition & 1 deletion crates/ourios-ingester/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ pub mod record_sink;
pub mod recovery;
pub mod snapshot_store;

pub use compactor::{Compactor, IngestError, SweepReport, run_sweep};
pub use compactor::{Compactor, IngestError, SweepReport, run_sweep, run_sweep_with_promoted};
pub use metrics::CompactionMetrics;
pub use recovery::{RecoveryDriverError, RecoveryReport, TenantRecovery};
pub use snapshot_store::SnapshotStoreError;
40 changes: 30 additions & 10 deletions crates/ourios-ingester/src/record_sink.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ use std::time::{Duration, Instant};

use ourios_core::record::{MinedRecord, RecordSink};
use ourios_parquet::{
DEFAULT_ZSTD_LEVEL, PartitionKey, Store, StoreError, WriterError, encode_records_to_parquet,
DEFAULT_ZSTD_LEVEL, PartitionKey, PromotedAttributes, Store, StoreError, WriterError,
encode_records_to_parquet_with_promoted,
};
use uuid::Uuid;

Expand Down Expand Up @@ -109,6 +110,10 @@ pub struct ParquetRecordSink {
/// RFC 0014 §6.3 instruments (flush throughput/latency by trigger,
/// errors, buffer occupancy). No-op when no meter provider is installed.
metrics: SinkMetrics,
/// The RFC 0022 promoted attribute set every flushed file projects
/// (`storage.promoted_attributes`, §3.2). Defaults to the implicit
/// `service.name`-only set; set via [`Self::with_promoted_attributes`].
promoted: PromotedAttributes,
}

/// Cheap per-record footprint estimate driving the size trigger + ceiling — a
Expand Down Expand Up @@ -156,9 +161,18 @@ impl ParquetRecordSink {
derive_errors: 0,
audit_barrier: None,
metrics: SinkMetrics::new(),
promoted: PromotedAttributes::default(),
}
}

/// Set the RFC 0022 promoted attribute set flushed files project
/// (`storage.promoted_attributes`, §3.2).
#[must_use]
pub fn with_promoted_attributes(mut self, promoted: PromotedAttributes) -> Self {
self.promoted = promoted;
self
}

/// Install the audit-durability barrier (issue #302 fix #2): a closure run
/// before any inline size/ceiling publish that flushes the audit sink and
/// returns whether it fully drained. When it returns `false` the inline
Expand Down Expand Up @@ -245,10 +259,12 @@ impl ParquetRecordSink {
) -> Result<(), FlushError> {
let flush_start = Instant::now();
let bytes = match self.buffers.get(key) {
Some(buf) if !buf.records.is_empty() => {
encode_records_to_parquet(&buf.records, DEFAULT_ZSTD_LEVEL)
.map_err(FlushError::Encode)?
}
Some(buf) if !buf.records.is_empty() => encode_records_to_parquet_with_promoted(
&buf.records,
DEFAULT_ZSTD_LEVEL,
&self.promoted,
)
.map_err(FlushError::Encode)?,
_ => return Ok(()),
};
self.store
Expand Down Expand Up @@ -404,9 +420,10 @@ fn publish_partition(
store: &Store,
key: &PartitionKey,
records: &[MinedRecord],
promoted: &PromotedAttributes,
) -> Result<(), FlushError> {
let bytes =
encode_records_to_parquet(records, DEFAULT_ZSTD_LEVEL).map_err(FlushError::Encode)?;
let bytes = encode_records_to_parquet_with_promoted(records, DEFAULT_ZSTD_LEVEL, promoted)
.map_err(FlushError::Encode)?;
store
.put_blocking(&object_key(key), bytes)
.map_err(FlushError::Store)?;
Expand All @@ -425,7 +442,7 @@ fn publish_partition(
///
/// All access serializes on one mutex. `emit` is a short critical section, but
/// the flush triggers are **not**: `flush_all` / `flush_aged` hold the lock
/// across `encode_records_to_parquet` + `Store::put_blocking` (see
/// across `encode_records_to_parquet_with_promoted` + `Store::put_blocking` (see
/// [`ParquetRecordSink::flush_all`]), so a flush against a slow store blocks
/// every concurrent `emit` and trigger for the duration of the I/O. Callers
/// must treat them as blocking sections (the server runs them via
Expand Down Expand Up @@ -520,12 +537,15 @@ impl SharedParquetSink {
if batches.is_empty() {
return true;
}
let store = self.lock().store();
let (store, promoted) = {
let sink = self.lock();
(sink.store(), sink.promoted.clone())
};
let mut requeue = Vec::new();
let mut all_published = true;
for (key, records) in batches {
let start = Instant::now();
if publish_partition(&store, &key, &records).is_ok() {
if publish_partition(&store, &key, &records, &promoted).is_ok() {
self.lock()
.note_published(records.len(), start.elapsed(), trigger);
} else {
Expand Down
146 changes: 146 additions & 0 deletions crates/ourios-ingester/tests/rfc0022_promoted_threading.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
//! RFC 0022 §3.2 threading — a record sink configured with a promoted
//! attribute set projects it into every flushed file, on **both** write
//! paths: the locked flush (`flush_all`, the size/age/rotation triggers'
//! shared path) and the off-lock publish (`publish_owned`, the issue #302
//! coordinator path). The projection semantics themselves are the
//! `ourios-parquet` RFC0022.1/.2 suites' contract; here we pin that the
//! sink threads its configured set into them.

use std::path::Path;

use ourios_core::audit::ParamType;
use ourios_core::otlp::any_value::Value as AvValue;
use ourios_core::otlp::{AnyValue, KeyValue};
use ourios_core::record::{BodyKind, MinedRecord, Param, RecordSink};
use ourios_core::tenant::TenantId;
use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink};
use ourios_parquet::{PromotedAttributes, Store};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;

fn kv(key: &str, value: &str) -> KeyValue {
KeyValue {
key: key.to_string(),
value: Some(AnyValue {
value: Some(AvValue::StringValue(value.to_string())),
}),
..Default::default()
}
}

fn rec() -> MinedRecord {
MinedRecord {
tenant_id: TenantId::new("a"),
template_id: 1,
template_version: 1,
severity_number: 9,
severity_text: None,
scope_name: Some("lib.cart".to_string()),
scope_version: Some("1.0.0".to_string()),
scope_attributes: Vec::new(),
resource_schema_url: None,
scope_schema_url: None,
time_unix_nano: 1_775_127_480_000_000_000,
observed_time_unix_nano: None,
attributes: vec![kv("http.route", "/cart")],
dropped_attributes_count: 0,
resource_attributes: vec![kv("service.name", "api")],
trace_id: None,
span_id: None,
flags: 0x01,
event_name: None,
body_kind: BodyKind::String,
params: vec![Param {
type_tag: ParamType::Num,
value: "42".to_string(),
}],
separators: vec![String::new(), " ".to_string()],
body: None,
confidence: 1.0,
lossy_flag: false,
}
}

/// A "never trigger" flush config, so the test drives each path explicitly.
fn never_flush() -> FlushConfig {
FlushConfig {
target_bytes: usize::MAX,
max_buffer_age: std::time::Duration::from_secs(86_400),
ceiling_bytes: usize::MAX,
}
}

fn promoted_set() -> PromotedAttributes {
PromotedAttributes::new([], ["http.route".to_string()])
}

/// The column names of every flushed `*.parquet` file under `root`.
fn flushed_schemas(root: &Path) -> Vec<Vec<String>> {
let mut schemas = Vec::new();
let mut stack = vec![root.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else if path.extension().is_some_and(|x| x == "parquet") {
let file = std::fs::File::open(&path).expect("open flushed file");
let reader =
ParquetRecordBatchReaderBuilder::try_new(file).expect("read flushed footer");
schemas.push(
reader
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect(),
);
}
}
}
schemas
}

fn assert_promoted(schemas: &[Vec<String>]) {
assert_eq!(schemas.len(), 1, "exactly one file flushed");
let names = &schemas[0];
assert!(
names.iter().any(|n| n == "attr.http.route"),
"the configured key's column is projected: {names:?}"
);
assert!(
names.iter().any(|n| n == "resource.service.name"),
"the implicit promotion rides along: {names:?}"
);
}

/// The locked flush path (`flush_all` — size/age/rotation triggers).
#[test]
fn flush_path_projects_the_configured_set() {
let dir = tempfile::TempDir::new().expect("temp dir");
let mut sink = ParquetRecordSink::new(Store::local(dir.path()).expect("store"), never_flush())
.with_promoted_attributes(promoted_set());
sink.emit(rec());
sink.flush_all();
assert_promoted(&flushed_schemas(dir.path()));
}

/// The off-lock publish path (`publish_owned` — the issue #302 coordinator).
#[test]
fn publish_path_projects_the_configured_set() {
let dir = tempfile::TempDir::new().expect("temp dir");
let shared = SharedParquetSink::new(
ParquetRecordSink::new(Store::local(dir.path()).expect("store"), never_flush())
.with_promoted_attributes(promoted_set()),
);
let mut producer = shared.clone();
producer.emit(rec());
let batches = shared.drain_all();
assert!(
shared.publish_owned(batches, "test"),
"the drained partition publishes"
);
assert_promoted(&flushed_schemas(dir.path()));
}
Loading