Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 43 additions & 1 deletion crates/ourios-bench/src/harness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ where
run_streaming(
corpus.lines.iter().map(Ok),
capture_audit,
/* capture_snapshots */ true,
|input, record, snapshot| {
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
on_record(input, record, snapshot);
Ok(())
Expand All @@ -129,6 +130,7 @@ where
pub(crate) fn run_streaming<T, I, F>(
corpus: I,
capture_audit: bool,
capture_snapshots: bool,
mut on_record: F,
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
) -> Result<HarnessResult, BenchError>
where
Expand Down Expand Up @@ -185,7 +187,16 @@ where
// (decode the stored `AnyValue` bytes) rather than
// template + params, so the bench's C1 (which measures
// template-based reconstruction) doesn't apply.
let want_snapshot = !record.lossy_flag
// `capture_snapshots = false` skips the whole capture: every new
// `(template_id, template_version)` pair otherwise walks + clones
// the full template set via `templates_for`. On corpora with heavy
// widening churn (one version bump per few lines) that is
// quadratic — measured at ~3 KB/s on LogHub HDFS_v2, a ~57-day
// store build — and the query-store builds never read the
// snapshots at all (their callbacks ignore the argument). Only C1
// (reconstruction) needs them.
let want_snapshot = capture_snapshots
&& !record.lossy_flag
&& record.template_id != NO_TEMPLATE
&& matches!(record.body_kind, BodyKind::String);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if want_snapshot {
Expand Down Expand Up @@ -290,6 +301,37 @@ mod tests {
(tmp, load)
}

/// `capture_snapshots = false` (the query-store builds) skips the
/// per-`(id, version)` `templates_for` walk entirely: even a
/// non-lossy, real-template, string-body record — exactly the shape
/// that gets `Some(template)` under `run()` — receives `None`.
#[test]
fn store_build_mode_never_captures_snapshots() {
let (_tmp, load) = load_lines(&["user 42 logged in", "user 43 logged in"]);
let mut calls = 0usize;
run_streaming(
load.lines.iter().map(Ok),
/* capture_audit */ false,
/* capture_snapshots */ false,
|_input, record, snap| {
calls += 1;
assert!(!record.lossy_flag, "fixture lines mine cleanly");
assert_ne!(record.template_id, NO_TEMPLATE);
assert!(
matches!(record.body_kind, BodyKind::String),
"fixture lines carry string bodies",
);
assert!(
snap.is_none(),
"capture_snapshots = false must skip the templates_for walk",
);
Ok(())
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
},
)
.expect("harness runs");
assert_eq!(calls, load.lines.len());
}

/// One callback invocation per ingested line — the
/// RFC 0001 §6.1 emit contract the harness asserts.
#[test]
Expand Down
27 changes: 16 additions & 11 deletions crates/ourios-bench/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -386,17 +386,22 @@ fn build_store(
let mut min_ts = u64::MAX;
let mut max_ts = 0u64;

harness::run_streaming(stream, false, |input, emitted, _snap| {
let effective = effective_nanos(emitted)?;
append_record(&mut writers, bucket_root, emitted)?;
observe(input, emitted, effective)?;
rows += 1;
if effective != 0 {
min_ts = min_ts.min(effective);
max_ts = max_ts.max(effective);
}
Ok(())
})?;
harness::run_streaming(
stream,
/* capture_audit */ false,
/* capture_snapshots */ false,
|input, emitted, _snap| {
let effective = effective_nanos(emitted)?;
append_record(&mut writers, bucket_root, emitted)?;
observe(input, emitted, effective)?;
rows += 1;
if effective != 0 {
min_ts = min_ts.min(effective);
max_ts = max_ts.max(effective);
}
Ok(())
},
)?;
if rows == 0 {
// Parity with `corpus::load`'s empty-corpus rejection: a store
// with zero rows would make every query trivially instant.
Expand Down
Loading