Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 160 additions & 16 deletions crates/ourios-querier/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
//! `ourios-querier` — RFC 0007 querier (pillar #3, `DataFusion`).
//!
//! **Status: execution slice 1.** [`Querier::run`] executes a
//! **Status: execution slice 2.** [`Querier::run`] executes a
//! minimal query — tenant scope + optional time range + optional
//! template-exact id — against the RFC 0005 Parquet store via
//! `DataFusion`, returning a matching-row count. Tenant isolation
//! (RFC0007.5) is live + tested; the row-group pruning stats that
//! prove B1 (RFC0007.1) and the B2 latency bench come next.
//! `DataFusion`, returning a matching-row count **and the scan's
//! row-group pruning stats** ([`QueryStats`]). Tenant isolation
//! (RFC0007.5) and B1 pruning (RFC0007.1 — a selective query
//! provably skips row groups via statistics) are live + tested.
//! The B2 latency-vs-corpus-size bench (RFC0007.2) comes next.
//!
//! This crate is the **read path**: it runs the query against the
//! RFC 0005 store — scoped to the tenant's partition directory,
Expand All @@ -32,13 +34,18 @@
use std::path::PathBuf;
use std::sync::Arc;

use datafusion::arrow::array::{Array, Int64Array};
use datafusion::arrow::datatypes::DataType;
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::common::ScalarValue;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::{
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
};
use datafusion::error::DataFusionError;
use datafusion::functions_aggregate::expr_fn::count;
use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};
use datafusion::physical_plan::{ExecutionPlan, collect};
use datafusion::prelude::{SessionContext, col, lit};
use ourios_core::tenant::TenantId;
use ourios_parquet::columns;
Expand Down Expand Up @@ -174,6 +181,88 @@ fn has_published_parquet(dir: &std::path::Path) -> Result<bool, QueryError> {
Ok(false)
}

/// Pull the single aggregate count out of the result batches. A
/// `COUNT(*)` with no grouping always returns exactly one
/// `Int64` row; anything else means the plan/return-type changed
/// out from under us, so it's a surfaced error rather than a
/// silent (and wrong) zero.
fn count_value(batches: &[RecordBatch]) -> Result<u64, QueryError> {
let bad = |detail: String| QueryError::Storage {
detail: format!("count aggregate: {detail}"),
};
if batches.len() != 1 {
return Err(bad(format!(
"expected exactly 1 result batch, got {}",
batches.len(),
)));
}
let batch = &batches[0];
if batch.num_rows() != 1 || batch.num_columns() == 0 {
return Err(bad(format!(
"expected 1 row × ≥1 column, got {}×{}",
batch.num_rows(),
batch.num_columns(),
)));
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
}
let col = batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.ok_or_else(|| bad("count column is not Int64".to_string()))?;
if col.is_null(0) {
return Err(bad("count is null".to_string()));
}
u64::try_from(col.value(0)).map_err(|_| bad("negative count".to_string()))
}

/// Walk the executed physical plan and accumulate the scan
/// pruning / IO metrics into a [`QueryStats`]. Recursive — the
/// Parquet scan is a leaf under the aggregate.
fn scan_stats(plan: &dyn ExecutionPlan) -> QueryStats {
let mut stats = QueryStats::default();
accumulate_scan_stats(plan, &mut stats);
stats
}

fn accumulate_scan_stats(plan: &dyn ExecutionPlan, stats: &mut QueryStats) {
if let Some(metrics) = plan.metrics() {
// `aggregate_by_name` sums each metric across the scan's
// per-file / per-partition instances.
fold_metrics(&metrics.aggregate_by_name(), stats);
}
for child in plan.children() {
accumulate_scan_stats(child.as_ref(), stats);
}
}
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved

/// Fold the `DataFusion`-version-sensitive scan metrics — the
/// `row_groups_pruned_statistics` `PruningMetrics` and the
/// `bytes_scanned` `Count` — into `stats`. Pulled out of
/// [`accumulate_scan_stats`] so the metric-name / value-shape
/// matching is unit-testable without a live plan (the names are an
/// engine contract that can drift across `DataFusion` releases).
fn fold_metrics(metrics: &MetricsSet, stats: &mut QueryStats) {
for metric in metrics.iter() {
match metric.value() {
// `row_groups_pruned_statistics` is a PruningMetrics
// carrying both pruned (skipped via min/max stats) and
// matched (read) row-group counts — exactly the B1
// numerator + denominator.
MetricValue::PruningMetrics {
name,
pruning_metrics,
} if name == "row_groups_pruned_statistics" => {
stats.row_groups_pruned += pruning_metrics.pruned() as u64;
stats.row_groups_scanned += pruning_metrics.matched() as u64;
}
MetricValue::Count { name, count } if name == "bytes_scanned" => {
stats.bytes_read += count.value() as u64;
}
_ => {}
}
}
}

/// Map a `DataFusion` error to the Ourios-owned [`QueryError`] so
/// no `datafusion`/`arrow` type crosses the public boundary (§4.6).
// Takes the error by value so it drops in cleanly as
Expand Down Expand Up @@ -294,18 +383,21 @@ impl Querier {
.map_err(storage_err)?;
}

// `count()` aggregates without materialising the matched
// rows' columns — so the heavy `attributes` / `params` /
// `body` columns are never read for a count, and Parquet
// projection pushdown applies. (`collect()` would buffer
// every column of every match.)
let rows = df.count().await.map_err(storage_err)? as u64;
// QueryStats (row-group pruning / bytes) is slice 2 — it
// comes from the ParquetExec metrics, not the row count.
Ok(QueryResult {
rows,
stats: QueryStats::default(),
})
// Count via an aggregate so the heavy `attributes` /
// `params` / `body` columns are never materialised
// (projection pushdown). We build + execute the physical
// plan ourselves (rather than `df.count()`) so we can read
// the scan's pruning metrics off the retained plan.
let counted = df
.aggregate(vec![], vec![count(lit(1_i64)).alias("n")])
.map_err(storage_err)?;
let plan = counted.create_physical_plan().await.map_err(storage_err)?;
let batches = collect(Arc::clone(&plan), ctx.task_ctx())
.await
.map_err(storage_err)?;
let rows = count_value(&batches)?;
let stats = scan_stats(plan.as_ref());
Ok(QueryResult { rows, stats })
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
}
}

Expand Down Expand Up @@ -351,4 +443,56 @@ mod tests {
assert_eq!(r.stats.row_groups_pruned, 0);
assert_eq!(r.stats.bytes_read, 0);
}

/// Pin the metric-name / value-shape contract `fold_metrics`
/// depends on: a `row_groups_pruned_statistics` `PruningMetrics`
/// maps to pruned/matched row-group counts, `bytes_scanned`
/// `Count` maps to `bytes_read`, and any other metric is ignored.
/// If a `DataFusion` bump renames or reshapes these, this fails
/// locally rather than letting the live test silently report
/// always-zero stats.
#[test]
fn fold_metrics_extracts_pruning_and_bytes() {
use std::borrow::Cow;

use datafusion::physical_plan::metrics::{Count, Metric, PruningMetrics};

let pruning = PruningMetrics::new();
pruning.add_pruned(3);
pruning.add_matched(2);
let bytes = Count::new();
bytes.add(4096);
// A metric we don't track — must be left untouched.
let other = Count::new();
other.add(99);

let mut set = MetricsSet::new();
set.push(Arc::new(Metric::new(
MetricValue::PruningMetrics {
name: Cow::Borrowed("row_groups_pruned_statistics"),
pruning_metrics: pruning,
},
None,
)));
set.push(Arc::new(Metric::new(
MetricValue::Count {
name: Cow::Borrowed("bytes_scanned"),
count: bytes,
},
None,
)));
set.push(Arc::new(Metric::new(
MetricValue::Count {
name: Cow::Borrowed("output_rows"),
count: other,
},
None,
)));

let mut stats = QueryStats::default();
fold_metrics(&set, &mut stats);
assert_eq!(stats.row_groups_pruned, 3);
assert_eq!(stats.row_groups_scanned, 2);
assert_eq!(stats.bytes_read, 4096);
}
}
15 changes: 2 additions & 13 deletions crates/ourios-querier/tests/acceptance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,8 @@
//! the test ↔ criterion mapping is greppable (RFC 0007 §5 /
//! `docs/verification.md` §2).

#[ignore = "RFC 0007 red gate — execution pending (RFC0007.1)"]
#[test]
fn rfc0007_1_pushdown_prunes_row_groups() {
// B1: a template-exact query over a corpus where the target
// template lives in a minority of row groups prunes most of
// them — `row_groups_pruned / (row_groups_scanned +
// row_groups_pruned)` exceeds the floor, and `bytes_read` is
// sub-linear in corpus size for fixed result size.
unimplemented!(
"RFC0007.1 — assert QueryStats pruned-fraction ≥ floor and \
bytes_read sub-linear in corpus size"
);
}
// RFC0007.1 (B1 pushdown prunes row groups) is now a LIVE test —
// see `tests/execution.rs::rfc0007_1_pushdown_prunes_row_groups`.

#[ignore = "RFC 0007 red gate — execution pending (RFC0007.2)"]
#[test]
Expand Down
44 changes: 44 additions & 0 deletions crates/ourios-querier/tests/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,50 @@ async fn tenant_dir_without_committed_parquet_is_empty() {
);
}

/// RFC0007.1 (B1) — a selective `template_id` query prunes row
/// groups via Parquet statistics rather than scanning them.
/// Tenant "a" gets two files in different hours: one holds only
/// template 1, the other only template 2. A `template_id = 1`
/// query must skip the template-2 file's row group (its
/// `template_id` min/max can't satisfy `= 1`), so
/// `row_groups_pruned > 0` and the pruned fraction is positive.
#[tokio::test]
async fn rfc0007_1_pushdown_prunes_row_groups() {
let bucket = tempfile::TempDir::new().expect("temp");
// Different hours ⇒ different partition files; each file's
// single row group carries a distinct template_id min/max.
write_all(bucket.path(), &[rec("a", 1, TS0)]); // hour 10 → template 1
write_all(bucket.path(), &[rec("a", 2, TS0 + 3_600_000_000_000)]); // hour 11 → template 2

let q = Querier::new(bucket.path());
let r = q
.run(req("a", None, Some(1)))
.await
.expect("template-exact query");

assert_eq!(r.rows, 1, "only the one template-1 row matches");
let total = r.stats.row_groups_scanned + r.stats.row_groups_pruned;
// pruned fraction `pruned / total` is positive iff both are
// non-zero — assert that integer-wise (no float).
assert!(
r.stats.row_groups_pruned >= 1,
"the template-2 file's row group must be pruned by statistics; stats={:?}",
r.stats,
);
assert!(
total > r.stats.row_groups_pruned,
"at least one row group was also scanned (matched); stats={:?}",
r.stats,
);
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
// The `bytes_scanned` extraction must stay wired — a regression
// to always-0 would otherwise pass the row-group asserts above.
assert!(
r.stats.bytes_read > 0,
"the scanned row group reads bytes; stats={:?}",
r.stats,
);
}

/// A `bucket_root` whose path contains a space still resolves —
/// the URL is built from the canonical path (`DataFusion`
/// URI-encodes it), not a raw `file://{display}` string that would
Expand Down
Loading