From 7295b13bf38c43e9b9ddb530e0c1462f17acb0e6 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 20 Jul 2026 02:18:02 +0200 Subject: [PATCH 1/3] feat(bench): rfc 0009 d1/d2 sustained-ingest soak harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `ourios-bench soak`: an in-process soak that drives the real ingest path — OTLP export -> IngestPipeline -> group-commit WAL fsync (100 ms window) -> miner -> Parquet record sink on a local Store — at a paced target rate, and samples the compaction backlog while driving `run_sweep` manually. The core mechanism is a synthetic clock: compaction only acts on sealed partitions (hour end + grace), which is time-driven logic, so record timestamps advance on a compressed timeline (`time_compression` synthetic seconds per wall second; default 60 = one wall-minute of load per synthetic hour) and the same synthetic now feeds `run_sweep`/`plan_candidates`. The sealing logic runs unmodified — only the timestamps it compares are compressed — so the real seal -> sweep -> compact path is exercised without waiting real hours. D1 throughput and ack latency are measured in wall-clock time and are unaffected; ack latency is taken at the `IngestPipeline::ingest` boundary (commit wait + in-order miner hand-off), a conservative upper bound on the WAL-commit latency and exactly what an OTLP client sees. The report (JSON via --out + stdout summary) carries the achieved and per-core rates, ack-latency percentiles, the backlog timeseries, and D1/D2 verdicts with their exact bars (>= 100_000 lines/s/core with p99 <= 200 ms; backlog bounded, returning to zero and draining to a final zero). A workflow_dispatch-only soak-bench.yml runs it on the ci-runner (indicative, non-authoritative) with a verdict-table job summary and the JSON as artifact. Hazard #3 (WAL durability vs. latency): the harness changes no ingest code; it measures the existing batched-fsync path with the production 100 ms window through a shared-Wal Journal wrapper, on a multi-thread runtime so the coordinator's spawn_blocking fsync offload behaves as in the server. Hazard #4 (small files): the sink flush target is deliberately small so sustained ingest produces the multi-file partitions compaction exists to consolidate, and D2 asserts the sweep drains them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WQY9wfrfRggqSpMLH8Xj3Y --- .github/workflows/soak-bench.yml | 108 +++ crates/ourios-bench/Cargo.toml | 23 +- crates/ourios-bench/src/lib.rs | 8 +- crates/ourios-bench/src/main.rs | 185 ++++- crates/ourios-bench/src/soak.rs | 1204 ++++++++++++++++++++++++++++++ 5 files changed, 1514 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/soak-bench.yml create mode 100644 crates/ourios-bench/src/soak.rs diff --git a/.github/workflows/soak-bench.yml b/.github/workflows/soak-bench.yml new file mode 100644 index 000000000..7600256ba --- /dev/null +++ b/.github/workflows/soak-bench.yml @@ -0,0 +1,108 @@ +name: Soak bench (D1/D2) + +# RFC 0009 D1/D2 sustained-ingest soak (`ourios-bench soak`, +# `docs/benchmarks.md` §D1/§D2): the real ingest path in-process — OTLP → +# group-commit WAL fsync (100 ms window) → miner → Parquet sink — at a +# paced rate, with compaction sweeps driven on a synthetic (compressed) +# clock so the seal→sweep→compact cycle runs without waiting real hours. +# `workflow_dispatch`-only: an hour-long paced load is far too heavy for +# the per-PR loop, and the ci-runner wall-clock is indicative, +# non-authoritative — the §1 baseline (`baseline-8vcpu-32gib`) carries +# the authoritative D1/D2 numbers. + +on: + workflow_dispatch: + inputs: + duration_seconds: + description: "Load duration in wall seconds" + required: false + default: "3600" + rate_lines_per_sec: + description: "Target ingest rate (lines/s, whole process)" + required: false + default: "100000" + time_compression: + description: "Synthetic seconds per wall second (60 = one wall-minute per synthetic hour)" + required: false + default: "60" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + soak-bench: + name: D1/D2 sustained-ingest soak + runs-on: ubuntu-latest + # The load duration plus build time; a dispatch asking for the full + # hour still finishes with room to spare. + timeout-minutes: 120 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (pinned); channel via toolchain input + with: + toolchain: stable + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + + - name: Build ourios-bench (release) + run: cargo build -p ourios-bench --release --bin ourios-bench + + - name: Run D1/D2 soak + env: + DURATION_SECONDS: ${{ inputs.duration_seconds }} + RATE_LINES_PER_SEC: ${{ inputs.rate_lines_per_sec }} + TIME_COMPRESSION: ${{ inputs.time_compression }} + run: | + set -euo pipefail + target/release/ourios-bench soak \ + --duration-secs "$DURATION_SECONDS" \ + --target-lines-per-sec "$RATE_LINES_PER_SEC" \ + --time-compression "$TIME_COMPRESSION" \ + --out soak-report.json 2>&1 | tee soak-bench.log + + - name: Write job summary (D1/D2) + if: always() + run: | + { + echo '## ourios soak bench — D1/D2' + echo + echo '**Hardware: `ci-runner` (indicative — non-authoritative pending `baseline-8vcpu-32gib`).**' + echo + if [ -f soak-report.json ]; then + echo '| gate | measured | bar | verdict |' + echo '| --- | --- | --- | --- |' + jq -r ' + def verdict: if . then "PASS" else "FAIL" end; + "| D1 throughput | \(.per_core_lines_per_sec | round) lines/s/core (\(.achieved_lines_per_sec | round) total / \(.config.worker_threads) workers) | ≥ \(.d1.bar_lines_per_sec_per_core) lines/s/core | \(.d1.pass | verdict) |", + "| D1 ack latency | p99 \(.d1.p99_ack_ms) ms (p50 \(.latency.p50_ms) / max \(.latency.max_ms)) | p99 ≤ \(.d1.bar_p99_ack_ms) ms | \(.d1.pass | verdict) |", + "| D2 backlog | max \(.d2.max_backlog_partitions) partition(s), final \(.d2.final_backlog_partitions), returned to zero: \(.d2.returned_to_zero_after_max) | bounded, drains to 0 | \(.d2.pass | verdict) |" + ' soak-report.json + echo + echo 'Run shape:' + echo '```' + jq -r '"wall \(.wall_secs)s · \(.lines_acked) lines acked (\(.batches_acked) batches, \(.batches_failed) failed) · \(.total_partitions_compacted) partitions compacted over \(.samples | length) samples"' soak-report.json + echo '```' + else + echo '_No soak-report.json — the run failed before producing it._' + fi + echo + echo '> **Indicative only.** A hosted runner is not the `docs/benchmarks.md` §1' + echo '> baseline; the authoritative D1/D2 numbers come from `baseline-8vcpu-32gib`.' + echo '> The synthetic clock compresses record time so the seal→sweep→compact' + echo '> path runs for real; D1 throughput/latency are measured in wall-clock.' + echo '> Runs the maintainer selects are recorded by hand in docs/benchmarks.md §9.' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: soak-bench + path: | + soak-report.json + soak-bench.log + if-no-files-found: warn diff --git a/crates/ourios-bench/Cargo.toml b/crates/ourios-bench/Cargo.toml index e9080c248..58814fa66 100644 --- a/crates/ourios-bench/Cargo.toml +++ b/crates/ourios-bench/Cargo.toml @@ -74,7 +74,19 @@ tempfile = "3" # dev-only while the only in-process queries lived in the B2 criterion # bench, which still uses them via these regular deps.) ourios-querier = { path = "../ourios-querier" } -tokio = { version = "=1.52.3", default-features = false, features = ["rt"] } +# The RFC 0009 D1/D2 soak harness (`soak`) needs the multi-thread +# runtime: the group-commit coordinator offloads its fsync via +# `spawn_blocking`, which a current-thread runtime would serialize +# behind the load loop. `time` paces the load, `sync` bounds in-flight +# batches (Semaphore). +tokio = { version = "=1.52.3", default-features = false, features = ["rt", "rt-multi-thread", "time", "sync"] } +# The soak harness drives the real ingest path in-process: WAL → +# group-commit coordinator → `IngestPipeline` → record sink, with the +# compactor's `run_sweep` driven manually on the synthetic cadence. +# Regular (not dev) deps since the `soak` subcommand ships in the bin; +# the criterion benches use them through these same lines. +ourios-wal = { path = "../ourios-wal" } +ourios-ingester = { path = "../ourios-ingester" } # RFC 0031 L4 (frequency-aggregation) pair: decodes the querier's RFC 3339 # `bucket(width)` group-key rendering back to Unix nanoseconds for the # `AggKey` comparison against Loki. Same version `ourios-querier` pins. @@ -109,15 +121,6 @@ proptest = "=1.11.0" # which is what B2's "median latency" target wants; ourios-querier (a # regular dependency above) is the unit under test. criterion = "=0.8.2" -# RFC0008.3 recovery-non-amplification bench: times `Wal::replay` -# over N ∈ {1, 4, 16} segments. Supportive wall-clock evidence -# (the O(N) shape) mirroring b1/b2's role; the structural side is -# pinned by `ourios-wal`'s `rfc0008_3_*` integration tests. -ourios-wal = { path = "../ourios-wal" } -# Ingest write-path bench (CLAUDE.md §6.2 hot paths): drives the RFC 0014 -# `ParquetRecordSink` for the WAL → Parquet half; the WAL half uses -# `ourios-wal` directly. -ourios-ingester = { path = "../ourios-ingester" } [[bench]] name = "b1" diff --git a/crates/ourios-bench/src/lib.rs b/crates/ourios-bench/src/lib.rs index ac2c8ae36..fd35c9c74 100644 --- a/crates/ourios-bench/src/lib.rs +++ b/crates/ourios-bench/src/lib.rs @@ -27,7 +27,8 @@ //! `calibrate` + `reference` + `store` (RFC 0024 / the B1/B2 query //! stores), and the RFC 0031 comparative harness (`comparative` — the //! Loki equivalence check + measurement channel — and `lgates`, the -//! comparative must-win / floor gate math). +//! comparative must-win / floor gate math), plus the RFC 0009 D1/D2 +//! sustained-ingest soak harness (`soak`). #![deny(unsafe_code)] @@ -44,6 +45,7 @@ mod harness; mod lgates; mod reference; mod report; +mod soak; mod store; pub use calibrate::{CALIBRATION_DIR, extract_manifest, write_manifest}; @@ -61,6 +63,10 @@ pub use lgates::{ }; pub use reference::ReferenceCorpus; pub use report::{update_status_section, write_results_json}; +pub use soak::{ + BacklogSample, D1Verdict, D2Verdict, LatencySummary, SoakConfig, SoakError, SoakReport, + default_worker_threads, run_soak, +}; pub use store::{B1Store, BuiltStore, build_b1_store, build_comparative_store, build_query_store}; /// Configuration for one bench invocation. diff --git a/crates/ourios-bench/src/main.rs b/crates/ourios-bench/src/main.rs index 44046f410..e5e190aa0 100644 --- a/crates/ourios-bench/src/main.rs +++ b/crates/ourios-bench/src/main.rs @@ -15,9 +15,10 @@ use std::path::PathBuf; use std::process::ExitCode; -use clap::{Parser, ValueEnum}; +use clap::{Args, Parser, Subcommand, ValueEnum}; use ourios_bench::{ - BenchConfig, BenchError, GateSet, TxtSeverity, run, update_status_section, write_results_json, + BenchConfig, BenchError, GateSet, SoakConfig, SoakReport, TxtSeverity, run, run_soak, + update_status_section, write_results_json, }; /// The §9 Results doc the `--update-benchmarks-md` path @@ -34,12 +35,18 @@ const BASELINE_HARDWARE_TAG: &str = "baseline-8vcpu-32gib (8 vCPU / 32 GiB / gp3 #[derive(Parser, Debug)] #[command( name = "ourios-bench", - about = "RFC 0006 thesis-gate bench harness (A1 compression / C1 reconstruction / C2 convergence)" + about = "RFC 0006 thesis-gate bench harness (A1 compression / C1 reconstruction / C2 convergence)", + // Subcommands (`soak`) carry their own flag surface; without this, + // clap would still demand the gate run's --hardware-kind. + subcommand_negates_reqs = true, + args_conflicts_with_subcommands = true )] // Each bool is an independent operator flag; a state enum would fight // clap's derive, not clarify it. #[allow(clippy::struct_excessive_bools)] struct Cli { + #[command(subcommand)] + command: Option, /// Directory of corpus files to load (recursive). Walker /// dispatches on extension: `*.txt` (plain-text per RFC /// 0006 §3.3) and `*.jsonl` / `*.json` (OTLP/JSON Lines @@ -103,6 +110,67 @@ struct Cli { calibration_out: Option, } +#[derive(Subcommand, Debug)] +enum Command { + /// RFC 0009 D1/D2 sustained-ingest soak: drive the real ingest path + /// in-process at a paced rate and sample the compaction backlog + /// under a synthetic (compressed) clock. + Soak(SoakArgs), +} + +/// Flags mirroring [`SoakConfig`]; defaults match its `Default`. +#[derive(Args, Debug)] +struct SoakArgs { + /// Load duration in wall seconds. + #[arg(long, default_value_t = 3600)] + duration_secs: u64, + /// Target ingest rate for the whole process, in lines per second. + #[arg(long, default_value_t = 100_000)] + target_lines_per_sec: u64, + /// Log records per OTLP export batch. + #[arg(long, default_value_t = 1000)] + batch_size: usize, + /// Synthetic seconds per wall second (60 ⇒ one wall-minute of load + /// is one synthetic hour of record time — see the soak module doc). + #[arg(long, default_value_t = 60)] + time_compression: u64, + /// Wall seconds between backlog samples / compaction sweeps. + #[arg(long, default_value_t = 10)] + sample_every_secs: u64, + /// Record-sink flush threshold in bytes (small on purpose: the soak + /// must produce the multi-file partitions compaction consolidates). + #[arg(long, default_value_t = 4 * 1024 * 1024)] + sink_target_bytes: usize, + /// Seed for the deterministic batch generator. + #[arg(long, default_value_t = 0xD1D2_50AC)] + seed: u64, + /// Tokio worker threads (default: host parallelism) — also the + /// divisor for the D1 per-core rate. + #[arg(long)] + worker_threads: Option, + /// Where the JSON soak report lands. + #[arg(long, default_value = "soak-report.json")] + out: PathBuf, +} + +impl SoakArgs { + fn into_config(self) -> (SoakConfig, PathBuf) { + let config = SoakConfig { + duration_secs: self.duration_secs, + target_lines_per_sec: self.target_lines_per_sec, + batch_size: self.batch_size, + time_compression: self.time_compression, + sample_every_secs: self.sample_every_secs, + sink_target_bytes: self.sink_target_bytes, + seed: self.seed, + worker_threads: self + .worker_threads + .unwrap_or_else(ourios_bench::default_worker_threads), + }; + (config, self.out) + } +} + /// clap value parser for `--corpus-tag`: one normal path component /// (letters, digits, `-`, `_`, `.`; not `.`/`..`), because the tag /// becomes the manifest's file name under the calibration dir. @@ -167,6 +235,15 @@ impl Cli { fn main() -> ExitCode { let cli = Cli::parse(); + if let Some(Command::Soak(args)) = cli.command { + return match run_soak_command(args) { + Ok(code) => code, + Err(e) => { + eprintln!("ourios-bench: {e}"); + ExitCode::from(2) + } + }; + } match run_bench(cli) { Ok(code) => code, Err(e) => { @@ -176,6 +253,66 @@ fn main() -> ExitCode { } } +/// The `soak` subcommand: run the D1/D2 soak, write the JSON report, +/// print the human summary. Like A1/C2, the D1/D2 verdicts are +/// *reported*, not process failures — whether an indicative run's miss +/// pauses anything is the §7 escalation rule's human judgment. +fn run_soak_command(args: SoakArgs) -> Result { + let (config, out) = args.into_config(); + let report = run_soak(&config).map_err(|e| e.to_string())?; + let json = + serde_json::to_string_pretty(&report).map_err(|e| format!("serialize soak report: {e}"))?; + std::fs::write(&out, json).map_err(|e| format!("write {}: {e}", out.display()))?; + eprintln!("ourios-bench: soak report written to {}", out.display()); + print_soak_summary(&report); + Ok(ExitCode::SUCCESS) +} + +/// One-line-per-gate soak summary, mirroring [`print_summary`]'s shape. +fn print_soak_summary(report: &SoakReport) { + println!( + "soak — {:.1}s wall at target {} lines/s (batch {}, {} worker(s), ×{} synthetic time)", + report.wall_secs, + report.config.target_lines_per_sec, + report.config.batch_size, + report.config.worker_threads, + report.config.time_compression, + ); + println!( + " D1 ingest: {:.0} lines/s ({:.0}/core; bar ≥ {}/core) · ack p50 {:.2} p95 {:.2} \ + p99 {:.2} max {:.2} ms (bar p99 ≤ {} ms) — {}", + report.achieved_lines_per_sec, + report.d1.per_core_lines_per_sec, + report.d1.bar_lines_per_sec_per_core, + report.latency.p50_ms, + report.latency.p95_ms, + report.d1.p99_ack_ms, + report.latency.max_ms, + report.d1.bar_p99_ack_ms, + if report.d1.pass { "PASS" } else { "FAIL" }, + ); + println!( + " D2 backlog: max {} partition(s), final {} (returned to zero: {}) · {} compacted \ + over {} sample(s) — {}", + report.d2.max_backlog_partitions, + report.d2.final_backlog_partitions, + report.d2.returned_to_zero_after_max, + report.total_partitions_compacted, + report.samples.len(), + if report.d2.pass { "PASS" } else { "FAIL" }, + ); + println!( + " batches: {} acked, {} failed · {} line(s) acked", + report.batches_acked, report.batches_failed, report.lines_acked, + ); + if let Some(last) = report.samples.last() { + println!( + " WAL at last sample: {} segment(s), {} byte(s) on disk", + last.wal_segments, last.wal_disk_bytes, + ); + } +} + fn run_bench(cli: Cli) -> Result { if cli.calibrate { return run_calibrate(&cli); @@ -442,6 +579,48 @@ mod tests { ); } + /// The `soak` subcommand parses without the gate run's required + /// flags (`subcommand_negates_reqs`), and its flags land in + /// [`SoakConfig`] with the documented defaults. + #[test] + fn soak_subcommand_parses_without_gate_flags() { + let cli = Cli::try_parse_from(["ourios-bench", "soak"]) + .expect("bare soak parses without --hardware-kind"); + let Some(Command::Soak(args)) = cli.command else { + panic!("expected the soak subcommand"); + }; + let (config, out) = args.into_config(); + assert_eq!(config.duration_secs, 3600); + assert_eq!(config.target_lines_per_sec, 100_000); + assert_eq!(config.batch_size, 1000); + assert_eq!(config.time_compression, 60); + assert_eq!(config.sample_every_secs, 10); + assert!(config.worker_threads > 0); + assert_eq!(out, PathBuf::from("soak-report.json")); + + let cli = Cli::try_parse_from([ + "ourios-bench", + "soak", + "--duration-secs", + "5", + "--target-lines-per-sec", + "20000", + "--worker-threads", + "2", + "--out", + "x.json", + ]) + .expect("flagged soak parses"); + let Some(Command::Soak(args)) = cli.command else { + panic!("expected the soak subcommand"); + }; + let (config, out) = args.into_config(); + assert_eq!(config.duration_secs, 5); + assert_eq!(config.target_lines_per_sec, 20_000); + assert_eq!(config.worker_threads, 2); + assert_eq!(out, PathBuf::from("x.json")); + } + /// RFC0006.6 — `--gates` scopes the measurement; omitting /// it means all three. #[test] diff --git a/crates/ourios-bench/src/soak.rs b/crates/ourios-bench/src/soak.rs new file mode 100644 index 000000000..81660d637 --- /dev/null +++ b/crates/ourios-bench/src/soak.rs @@ -0,0 +1,1204 @@ +//! RFC 0009 D1/D2 sustained-ingest soak harness (`docs/benchmarks.md` +//! §D1 / §D2). +//! +//! Drives the **real** ingest path in-process — OTLP export → +//! [`IngestPipeline`] → group-commit WAL fsync (100 ms window, +//! `CLAUDE.md` §3.4) → miner → Parquet record sink on a local [`Store`] +//! over a tempdir — at a paced target rate, while a sampler interleaves +//! manual compaction sweeps ([`run_sweep`]) and records the WAL + +//! compaction backlog over time. +//! +//! # Synthetic clock +//! +//! Compaction only acts on **sealed** partitions: an hour partition is +//! sealed once its hour has ended plus a grace period +//! ([`plan_candidates`]' seal rule) — *time-driven* logic, so a +//! wall-clock soak would wait real hours to see one +//! seal → sweep → compact cycle. The harness therefore runs a +//! **synthetic clock**: record timestamps advance on a compressed +//! timeline ([`SoakConfig::time_compression`] synthetic seconds per wall +//! second; the default 60 makes one wall-minute of load one synthetic +//! hour), and the *same* synthetic now feeds [`run_sweep`] / +//! [`plan_candidates`]. The sealing logic runs unmodified — only the +//! timestamps it compares are compressed — so the real +//! seal → sweep → compact path is exercised without waiting real hours. +//! The D1 ack measurement is unaffected: throughput and ack latency are +//! measured in real wall-clock time; the synthetic clock only stamps +//! record timestamps and drives the compaction cadence. +//! +//! Ack latency is measured at the [`IngestPipeline::ingest`] boundary — +//! the elapsed time until the caller could ack, i.e. the group-commit +//! wait plus the in-order miner hand-off. That is a conservative upper +//! bound on the §D1 WAL-commit latency, and exactly what an OTLP client +//! observes. + +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; +use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use ourios_config::MinerConfig; +use ourios_ingester::compactor::run_sweep; +use ourios_ingester::receiver::pipeline::{Journal, ReceiveError}; +use ourios_ingester::receiver::{CommitCoordinator, IngestPipeline, TenantRule}; +use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; +use ourios_miner::cluster::MinerCluster; +use ourios_parquet::{CompactionPolicy, PartitionKey, Store, plan_candidates}; +use ourios_wal::{Wal, WalConfig, WalOffset}; +use serde::Serialize; + +/// §D1 throughput bar: sustained acked lines per second per core. +pub const D1_LINES_PER_SEC_PER_CORE: u64 = 100_000; +/// §D1 latency bar: p99 ingest-ack latency in milliseconds. +pub const D1_P99_ACK_MS: u64 = 200; + +/// The single soak tenant (derived from `service.name` by the production +/// [`TenantRule`], so the batches exercise the real fan-out). +const TENANT: &str = "soak"; +/// 2026-01-01T00:00:00Z — the synthetic timeline's fixed origin, so +/// partition keys are deterministic across runs. +const BASE_UNIX_NANOS: u64 = 1_767_225_600 * NANOS_PER_SEC; +const NANOS_PER_SEC: u64 = 1_000_000_000; +const HOUR_NANOS: u64 = 3_600 * NANOS_PER_SEC; +/// §D1's WAL knob: fsync batched at 100 ms. +const WAL_BATCH_WINDOW_MS: u64 = 100; +const WAL_SEGMENT_BYTES: u64 = 128 * 1024 * 1024; +/// Bounds in-flight ingest tasks. Well above the steady-state depth +/// (~rate × ack latency); reaching it means ingest is falling behind, +/// and the pacing loop then degrades — which the achieved rate records. +const MAX_IN_FLIGHT: usize = 512; + +/// Tunables for one soak run. Serialized into the report so a results +/// file is self-describing. +#[derive(Debug, Clone, Serialize)] +pub struct SoakConfig { + /// Load duration in wall seconds. + pub duration_secs: u64, + /// Paced target ingest rate (lines per second, whole process). + pub target_lines_per_sec: u64, + /// Log records per OTLP export batch. + pub batch_size: usize, + /// Synthetic seconds per wall second (see the module doc). The + /// default 60 makes one wall-minute of load one synthetic hour. + pub time_compression: u64, + /// Wall seconds between backlog samples (each sample also flushes + /// the record sink and runs one compaction sweep). + pub sample_every_secs: u64, + /// Record-sink flush threshold. Deliberately small so sustained + /// ingest produces the multi-file partitions compaction exists to + /// consolidate (hazard #4). + pub sink_target_bytes: usize, + /// Seed for the deterministic batch generator. + pub seed: u64, + /// Tokio worker threads for the load runtime — also the divisor for + /// the §D1 per-core rate. + pub worker_threads: usize, +} + +impl Default for SoakConfig { + fn default() -> Self { + Self { + duration_secs: 3_600, + target_lines_per_sec: 100_000, + batch_size: 1_000, + time_compression: 60, + sample_every_secs: 10, + sink_target_bytes: 4 * 1024 * 1024, + seed: 0xD1D2_50AC, + worker_threads: default_worker_threads(), + } + } +} + +/// Default [`SoakConfig::worker_threads`]: the host's parallelism. +#[must_use] +pub fn default_worker_threads() -> usize { + std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get) +} + +/// One backlog observation: candidate state *before* that sample's +/// sweep, what the sweep then compacted, and the WAL/sink gauges. +#[derive(Debug, Clone, Serialize)] +pub struct BacklogSample { + /// Wall seconds since load start. + pub wall_secs: f64, + /// The synthetic now the sweep ran at. + pub synthetic_unix_nanos: u64, + /// Sealed candidate partitions pending before the sweep. + pub backlog_partitions: usize, + /// Total bytes of the candidate partitions' Parquet files. + pub backlog_bytes: u64, + /// Partitions the sweep consolidated. + pub partitions_compacted: usize, + /// Planning/sweep/listing errors at this sample (0 on a clean pass). + pub errors: usize, + /// WAL segment count at sample time. + pub wal_segments: u32, + /// WAL on-disk bytes at sample time. + pub wal_disk_bytes: u64, + /// Records still buffered in the sink after the sample's flush. + pub sink_buffered_records: usize, +} + +/// Ack-latency percentiles over the whole run, in milliseconds. +#[derive(Debug, Clone, Serialize)] +pub struct LatencySummary { + pub p50_ms: f64, + pub p95_ms: f64, + pub p99_ms: f64, + pub max_ms: f64, +} + +/// §D1 verdict with its exact bars. +#[derive(Debug, Clone, Serialize)] +pub struct D1Verdict { + /// `per_core_lines_per_sec >= bar_lines_per_sec_per_core && + /// p99_ack_ms <= bar_p99_ack_ms`. + pub pass: bool, + pub per_core_lines_per_sec: f64, + pub bar_lines_per_sec_per_core: u64, + pub p99_ack_ms: f64, + pub bar_p99_ack_ms: u64, +} + +/// §D2 verdict: the backlog stayed bounded and drained. +#[derive(Debug, Clone, Serialize)] +pub struct D2Verdict { + /// `final_backlog_partitions == 0 && returned_to_zero_after_max`. + pub pass: bool, + pub max_backlog_partitions: usize, + /// Candidates still pending after the final drain sweep. + pub final_backlog_partitions: usize, + /// The backlog series hit zero at least once after its maximum + /// (trivially true when it never rose above zero). + pub returned_to_zero_after_max: bool, +} + +/// The soak run's full result — serialized as the `--out` JSON. +#[derive(Debug, Clone, Serialize)] +pub struct SoakReport { + pub config: SoakConfig, + /// Wall seconds from first batch to last ack (includes the drain of + /// in-flight batches). + pub wall_secs: f64, + pub lines_acked: u64, + pub batches_acked: u64, + pub batches_failed: u64, + pub achieved_lines_per_sec: f64, + /// `achieved_lines_per_sec / worker_threads` — the §D1 per-core rate. + pub per_core_lines_per_sec: f64, + pub latency: LatencySummary, + pub samples: Vec, + pub total_partitions_compacted: usize, + pub d1: D1Verdict, + pub d2: D2Verdict, +} + +/// Failure to set up or drive a soak run (load-side ingest errors are +/// *counted* in the report, not raised — a soak measures, it doesn't +/// abort on a failed batch). +#[derive(Debug)] +pub enum SoakError { + /// A config field is out of range. Names the constraint. + Config(&'static str), + /// Building the tokio runtime failed. + Runtime(std::io::Error), + /// Opening the WAL / store fixture failed. + Setup(String), +} + +impl std::fmt::Display for SoakError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Config(detail) => write!(f, "soak config: {detail}"), + Self::Runtime(e) => write!(f, "soak runtime: {e}"), + Self::Setup(detail) => write!(f, "soak setup: {detail}"), + } + } +} + +impl std::error::Error for SoakError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Runtime(e) => Some(e), + Self::Config(_) | Self::Setup(_) => None, + } + } +} + +/// Run one soak to completion and return the report. +/// +/// Builds a **multi-thread** tokio runtime: the group-commit +/// coordinator offloads its fsync via `spawn_blocking`, so a +/// current-thread runtime would serialize every fsync behind the load +/// loop and depress the D1 numbers. +/// +/// # Errors +/// +/// [`SoakError`] on an invalid config or a setup failure (runtime, WAL, +/// store). Ingest and sweep failures during the run are counted in the +/// report instead. +pub fn run_soak(config: &SoakConfig) -> Result { + validate(config)?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(config.worker_threads) + .enable_time() + .build() + .map_err(SoakError::Runtime)?; + runtime.block_on(soak(config)) +} + +fn validate(config: &SoakConfig) -> Result<(), SoakError> { + if config.duration_secs == 0 { + return Err(SoakError::Config("duration_secs must be > 0")); + } + if config.target_lines_per_sec == 0 { + return Err(SoakError::Config("target_lines_per_sec must be > 0")); + } + if config.batch_size == 0 { + return Err(SoakError::Config("batch_size must be > 0")); + } + if config.time_compression == 0 { + return Err(SoakError::Config("time_compression must be > 0")); + } + if config.sample_every_secs == 0 { + return Err(SoakError::Config("sample_every_secs must be > 0")); + } + if config.sink_target_bytes == 0 { + return Err(SoakError::Config("sink_target_bytes must be > 0")); + } + if config.worker_threads == 0 { + return Err(SoakError::Config("worker_threads must be > 0")); + } + Ok(()) +} + +async fn soak(config: &SoakConfig) -> Result { + let Some(pace) = batch_interval(config.target_lines_per_sec, config.batch_size) else { + return Err(SoakError::Config("target rate and batch size must be > 0")); + }; + let dir = + tempfile::tempdir().map_err(|e| SoakError::Setup(format!("create soak tempdir: {e}")))?; + let wal_root = dir.path().join("wal"); + let store_root = dir.path().join("store"); + for path in [&wal_root, &store_root] { + std::fs::create_dir_all(path) + .map_err(|e| SoakError::Setup(format!("create {}: {e}", path.display())))?; + } + + // Mirror the server's construction (`ourios-server` `serve`): WAL → + // group-commit coordinator → miner (record sink attached) → + // pipeline. `macos_full_fsync: false` matches the write-path bench. + // The WAL is shared behind a mutex so the sampler can read + // `Wal::metrics` while the coordinator owns the append/sync path. + let wal = Wal::open(WalConfig { + root: wal_root, + batch_window_ms: WAL_BATCH_WINDOW_MS, + segment_size_bytes: WAL_SEGMENT_BYTES, + segment_age_secs: 600, + housekeeping_secs: 60, + macos_full_fsync: false, + }) + .map_err(|e| SoakError::Setup(format!("open WAL: {e:?}")))?; + let wal = Arc::new(Mutex::new(wal)); + let coordinator = CommitCoordinator::new( + Box::new(SharedWal(Arc::clone(&wal))), + Duration::from_millis(WAL_BATCH_WINDOW_MS), + WAL_SEGMENT_BYTES, + ); + let store = + Store::local(&store_root).map_err(|e| SoakError::Setup(format!("open store: {e}")))?; + let sink = SharedParquetSink::new(ParquetRecordSink::new( + store.clone(), + FlushConfig { + target_bytes: config.sink_target_bytes, + max_buffer_age: Duration::from_secs(86_400), + ceiling_bytes: usize::MAX, + }, + )); + let miner = MinerCluster::new(MinerConfig::default()).with_record_sink(Box::new(sink.clone())); + let pipeline = Arc::new(IngestPipeline::new( + coordinator, + miner, + TenantRule::service_name(), + )); + + let clock = SyntheticClock { + started: Instant::now(), + compression: config.time_compression, + }; + let policy = CompactionPolicy::default(); + let stats = Arc::new(LoadStats::default()); + let stop = Arc::new(AtomicBool::new(false)); + let sampler_task = tokio::spawn(run_sampler(SamplerCtx { + store: store.clone(), + sink: sink.clone(), + wal: Arc::clone(&wal), + clock: clock.clone(), + policy, + every: Duration::from_secs(config.sample_every_secs), + stop: Arc::clone(&stop), + })); + + run_load(&pipeline, &clock, config, &stats, pace).await; + let wall_secs = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; + + stop.store(true, Ordering::Relaxed); + // A sampler panic loses the timeseries but not the run; report with + // no samples (d2 then fails loudly on the missing final backlog). + let mut samples: Vec = sampler_task.await.unwrap_or_default(); + + // Final drain: advance the synthetic clock past the last partition's + // hour end + grace so *every* partition seals, then sweep. This + // verifies compaction can fully clear the backlog the run produced; + // the mid-run samples carry the steady-state D2 evidence. + let drain_now = clock + .now_unix_nanos() + .saturating_add(HOUR_NANOS) + .saturating_add(policy.grace_nanos) + .saturating_add(1); + let drain_wall = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; + let (drain_sample, final_backlog) = { + let store = store.clone(); + let sink = sink.clone(); + let wal = Arc::clone(&wal); + match tokio::task::spawn_blocking(move || { + let sample = blocking_sample(&store, &sink, &wal, &policy, drain_wall, drain_now); + // An error listing here means the final backlog is unknown — + // fail the D2 verdict loudly rather than claim a drained run. + let final_backlog = + plan_candidates(&store, TENANT, drain_now, &policy).map_or(usize::MAX, |c| c.len()); + (sample, final_backlog) + }) + .await + { + Ok(result) => result, + Err(_) => (error_sample(drain_wall, drain_now), usize::MAX), + } + }; + samples.push(drain_sample); + + Ok(build_report( + config, + wall_secs, + &stats, + samples, + final_backlog, + )) +} + +fn build_report( + config: &SoakConfig, + wall_secs: f64, + stats: &LoadStats, + samples: Vec, + final_backlog: usize, +) -> SoakReport { + let mut latencies = stats + .latencies_us + .lock() + .unwrap_or_else(PoisonError::into_inner); + latencies.sort_unstable(); + let latency = LatencySummary { + p50_ms: us_to_ms(percentile_us(&latencies, 50, 100)), + p95_ms: us_to_ms(percentile_us(&latencies, 95, 100)), + p99_ms: us_to_ms(percentile_us(&latencies, 99, 100)), + max_ms: us_to_ms(latencies.last().copied().unwrap_or(0)), + }; + drop(latencies); + + let lines_acked = stats.lines_acked.load(Ordering::Relaxed); + let achieved = if wall_secs > 0.0 { + as_f64(lines_acked) / wall_secs + } else { + 0.0 + }; + let per_core = achieved / as_f64(to_u64(config.worker_threads)); + let backlog_series: Vec = samples.iter().map(|s| s.backlog_partitions).collect(); + let total_compacted = samples.iter().map(|s| s.partitions_compacted).sum(); + + let p99_ms = latency.p99_ms; + SoakReport { + config: config.clone(), + wall_secs, + lines_acked, + batches_acked: stats.batches_acked.load(Ordering::Relaxed), + batches_failed: stats.batches_failed.load(Ordering::Relaxed), + achieved_lines_per_sec: achieved, + per_core_lines_per_sec: per_core, + latency, + samples, + total_partitions_compacted: total_compacted, + d1: d1_verdict(per_core, p99_ms), + d2: d2_verdict(&backlog_series, final_backlog), + } +} + +/// The §D1 verdict over the measured per-core rate and p99 ack latency. +#[must_use] +pub fn d1_verdict(per_core_lines_per_sec: f64, p99_ack_ms: f64) -> D1Verdict { + D1Verdict { + pass: per_core_lines_per_sec >= as_f64(D1_LINES_PER_SEC_PER_CORE) + && p99_ack_ms <= as_f64(D1_P99_ACK_MS), + per_core_lines_per_sec, + bar_lines_per_sec_per_core: D1_LINES_PER_SEC_PER_CORE, + p99_ack_ms, + bar_p99_ack_ms: D1_P99_ACK_MS, + } +} + +/// The §D2 verdict over the per-sample backlog series and the +/// post-drain-sweep final backlog. +#[must_use] +pub fn d2_verdict(backlog_series: &[usize], final_backlog: usize) -> D2Verdict { + let max = backlog_series.iter().copied().max().unwrap_or(0); + let returned = if max == 0 { + // The backlog never accumulated; nothing to return from. + final_backlog == 0 + } else { + let after_max = backlog_series + .iter() + .position(|&b| b == max) + .map_or(&[][..], |i| &backlog_series[i + 1..]); + after_max.contains(&0) || final_backlog == 0 + }; + D2Verdict { + pass: final_backlog == 0 && returned, + max_backlog_partitions: max, + final_backlog_partitions: final_backlog, + returned_to_zero_after_max: returned, + } +} + +/// The pacing interval: one `batch_size`-line batch per tick sustains +/// `target_lines_per_sec`. `None` when either input is zero. Floors at +/// 1 ns (tokio's interval rejects zero). +#[must_use] +pub fn batch_interval(target_lines_per_sec: u64, batch_size: usize) -> Option { + if target_lines_per_sec == 0 || batch_size == 0 { + return None; + } + let nanos = u128::from(to_u64(batch_size)) * u128::from(NANOS_PER_SEC) + / u128::from(target_lines_per_sec); + Some(Duration::from_nanos(saturating_u64(nanos).max(1))) +} + +/// Nearest-rank percentile (`num`/`den`, e.g. 99/100 for p99) over an +/// ascending-sorted slice. `0` for an empty slice. +#[must_use] +pub fn percentile_us(sorted: &[u64], num: usize, den: usize) -> u64 { + if sorted.is_empty() || den == 0 { + return 0; + } + let rank = (sorted.len() * num).div_ceil(den).max(1); + sorted[rank.min(sorted.len()) - 1] +} + +// ---------------------------------------------------------------------- +// Load generation +// ---------------------------------------------------------------------- + +#[derive(Default)] +struct LoadStats { + latencies_us: Mutex>, + lines_acked: AtomicU64, + batches_acked: AtomicU64, + batches_failed: AtomicU64, +} + +async fn run_load( + pipeline: &Arc, + clock: &SyntheticClock, + config: &SoakConfig, + stats: &Arc, + pace: Duration, +) { + let semaphore = Arc::new(tokio::sync::Semaphore::new(MAX_IN_FLIGHT)); + let mut ticker = tokio::time::interval(pace); + let deadline = clock.started + Duration::from_secs(config.duration_secs); + let mut batch_idx = 0u64; + while Instant::now() < deadline { + ticker.tick().await; + let Ok(permit) = Arc::clone(&semaphore).acquire_owned().await else { + break; // Closed semaphore: cannot happen while we hold it. + }; + let pipeline = Arc::clone(pipeline); + let stats = Arc::clone(stats); + let batch = build_batch( + config.seed, + batch_idx, + config.batch_size, + clock.now_unix_nanos(), + ); + tokio::spawn(async move { + let started = Instant::now(); + match pipeline.ingest(batch).await { + Ok(lines) => { + let elapsed_us = saturating_u64(started.elapsed().as_micros()); + stats + .lines_acked + .fetch_add(to_u64(lines), Ordering::Relaxed); + stats.batches_acked.fetch_add(1, Ordering::Relaxed); + stats + .latencies_us + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(elapsed_us); + } + Err(_) => { + stats.batches_failed.fetch_add(1, Ordering::Relaxed); + } + } + drop(permit); + }); + batch_idx += 1; + } + // Drain: once every permit is reacquirable, no batch is in flight. + let permits = u32::try_from(MAX_IN_FLIGHT).unwrap_or(u32::MAX); + drop(semaphore.acquire_many(permits).await); +} + +/// A printf-style template: fixed text segments with one generated +/// numeric parameter between each adjacent pair. +struct TemplateSpec { + severity_number: i32, + severity_text: &'static str, + segments: &'static [&'static str], +} + +/// A small fixed template mix (bodies ~100–200 bytes) so the miner +/// converges to a stable template set while the params vary — the +/// workload shape the D-gates assume. +const TEMPLATES: &[TemplateSpec] = &[ + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "GET /api/v1/orders/", + " completed in ", + " ms with status ", + " for client session ", + " carrying a paged response body", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "POST /api/v1/checkout accepted cart ", + " with ", + " items totalling ", + " cents; reserved inventory across ", + " warehouse shards", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "connection pool lease granted for backend ", + " after ", + " us of queueing (", + " of ", + " connections currently busy)", + ], + }, + TemplateSpec { + severity_number: 5, + severity_text: "DEBUG", + segments: &[ + "cache lookup for key shard ", + " resolved in ", + " us: generation ", + " hit ratio holding near expectations for the window", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "user ", + " authenticated from device ", + " in region ", + "; session token minted with standard expiry policy applied", + ], + }, + TemplateSpec { + severity_number: 13, + severity_text: "WARN", + segments: &[ + "retrying upstream call to inventory service, attempt ", + " of ", + " after ", + " ms backoff; circuit breaker remains closed for now", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "scheduled reconciliation batch ", + " processed ", + " ledger entries in ", + " ms with zero divergent balances detected", + ], + }, + TemplateSpec { + severity_number: 17, + severity_text: "ERROR", + segments: &[ + "payment authorization ", + " declined by processor with code ", + " after ", + " ms; customer notified through the standard fallback flow", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "grpc stream ", + " closed cleanly after ", + " messages and ", + " bytes; peer acknowledged final frame without truncation", + ], + }, + TemplateSpec { + severity_number: 13, + severity_text: "WARN", + segments: &[ + "queue depth for topic partition ", + " reached ", + " messages (threshold ", + "); consumer group lag is being monitored closely", + ], + }, + TemplateSpec { + severity_number: 5, + severity_text: "DEBUG", + segments: &[ + "compacted segment ", + " merged ", + " entries into ", + " blocks; tombstone ratio stayed inside the configured budget", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "DELETE /api/v1/sessions/", + " completed in ", + " ms; ", + " downstream cache invalidations fanned out successfully", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "search query shard ", + " returned ", + " hits in ", + " ms with ranking pass applied under the default profile", + ], + }, + TemplateSpec { + severity_number: 17, + severity_text: "ERROR", + segments: &[ + "dns resolution for replica ", + " failed after ", + " ms and ", + " attempts; falling back to the cached endpoint set", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "tls handshake with peer ", + " completed in ", + " ms using the negotiated cipher suite for connection ", + " on the shared listener", + ], + }, + TemplateSpec { + severity_number: 13, + severity_text: "WARN", + segments: &[ + "slow query detected on statement ", + ": ", + " ms elapsed scanning ", + " rows; plan cache entry flagged for review", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "background export job ", + " uploaded ", + " objects (", + " bytes) to the archive bucket within the maintenance window", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "feature flag evaluation for cohort ", + " served variant ", + " in ", + " us; assignment recorded for the experimentation pipeline", + ], + }, + TemplateSpec { + severity_number: 5, + severity_text: "DEBUG", + segments: &[ + "heartbeat from worker ", + " received after ", + " ms; lease ", + " renewed and membership view left unchanged this round", + ], + }, + TemplateSpec { + severity_number: 9, + severity_text: "INFO", + segments: &[ + "PUT /api/v1/profiles/", + " updated ", + " fields in ", + " ms; audit trail entry appended with the request identifier", + ], + }, +]; + +/// Deterministic 64-bit generator (splitmix64) — no runtime `proptest` +/// and no external RNG dependency for the load loop. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// Build one deterministic OTLP export: `batch_size` records over the +/// fixed template mix, params drawn from a per-batch seed, timestamps +/// on the synthetic timeline (1 µs apart within the batch). +#[must_use] +pub fn build_batch( + seed: u64, + batch_idx: u64, + batch_size: usize, + ts_unix_nanos: u64, +) -> ExportLogsServiceRequest { + let mut rng_state = seed ^ batch_idx.wrapping_mul(0xA076_1D64_78BD_642F); + let log_records = (0..batch_size) + .map(|i| { + let pick = splitmix64(&mut rng_state); + let template = &TEMPLATES[saturating_usize(pick) % TEMPLATES.len()]; + let mut body = String::with_capacity(192); + for (slot, segment) in template.segments.iter().enumerate() { + if slot > 0 { + let param = splitmix64(&mut rng_state) % 1_000_000; + body.push_str(¶m.to_string()); + } + body.push_str(segment); + } + LogRecord { + time_unix_nano: ts_unix_nanos.saturating_add(to_u64(i).saturating_mul(1_000)), + severity_number: template.severity_number, + severity_text: template.severity_text.to_string(), + body: Some(AnyValue { + value: Some(any_value::Value::StringValue(body)), + }), + ..LogRecord::default() + } + }) + .collect(); + ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(TENANT.to_string())), + }), + ..KeyValue::default() + }], + ..Resource::default() + }), + scope_logs: vec![ScopeLogs { + log_records, + ..ScopeLogs::default() + }], + ..ResourceLogs::default() + }], + } +} + +// ---------------------------------------------------------------------- +// Synthetic clock + sampler +// ---------------------------------------------------------------------- + +#[derive(Clone)] +struct SyntheticClock { + started: Instant, + compression: u64, +} + +impl SyntheticClock { + fn now_unix_nanos(&self) -> u64 { + let wall = self.started.elapsed().as_nanos(); + let advanced = wall.saturating_mul(u128::from(self.compression)); + BASE_UNIX_NANOS.saturating_add(saturating_u64(advanced)) + } +} + +struct SamplerCtx { + store: Store, + sink: SharedParquetSink, + wal: Arc>, + clock: SyntheticClock, + policy: CompactionPolicy, + every: Duration, + stop: Arc, +} + +/// Sample + sweep on a fixed wall cadence until stopped. Sequential by +/// construction: each sweep completes before the next tick is awaited, +/// so sweeps never overlap. +async fn run_sampler(ctx: SamplerCtx) -> Vec { + let mut samples = Vec::new(); + loop { + tokio::time::sleep(ctx.every).await; + if ctx.stop.load(Ordering::Relaxed) { + break; + } + let store = ctx.store.clone(); + let sink = ctx.sink.clone(); + let wal = Arc::clone(&ctx.wal); + let policy = ctx.policy; + let wall_secs = as_f64(saturating_u64(ctx.clock.started.elapsed().as_millis())) / 1_000.0; + let synthetic_now = ctx.clock.now_unix_nanos(); + let sample = match tokio::task::spawn_blocking(move || { + blocking_sample(&store, &sink, &wal, &policy, wall_secs, synthetic_now) + }) + .await + { + Ok(sample) => sample, + Err(_) => error_sample(wall_secs, synthetic_now), + }; + samples.push(sample); + } + samples +} + +/// One blocking observation: flush the sink so buffered rows are +/// visible, record the pre-sweep candidate backlog, run one sweep at the +/// synthetic now, and read the WAL gauges. +fn blocking_sample( + store: &Store, + sink: &SharedParquetSink, + wal: &Arc>, + policy: &CompactionPolicy, + wall_secs: f64, + synthetic_now: u64, +) -> BacklogSample { + sink.flush_all(); + let (backlog_partitions, backlog_bytes, mut errors) = backlog(store, synthetic_now, policy); + let partitions_compacted = if let Ok(report) = run_sweep(store, synthetic_now, policy) { + errors += report.errors.len(); + report.partitions_compacted + } else { + errors += 1; + 0 + }; + let (wal_segments, wal_disk_bytes) = { + let metrics = lock_wal(wal).metrics(); + (metrics.segment_count, metrics.disk_bytes) + }; + BacklogSample { + wall_secs, + synthetic_unix_nanos: synthetic_now, + backlog_partitions, + backlog_bytes, + partitions_compacted, + errors, + wal_segments, + wal_disk_bytes, + sink_buffered_records: sink.buffered_records(), + } +} + +fn error_sample(wall_secs: f64, synthetic_now: u64) -> BacklogSample { + BacklogSample { + wall_secs, + synthetic_unix_nanos: synthetic_now, + backlog_partitions: 0, + backlog_bytes: 0, + partitions_compacted: 0, + errors: 1, + wal_segments: 0, + wal_disk_bytes: 0, + sink_buffered_records: 0, + } +} + +/// The candidate backlog as of `now`: sealed candidate partitions and +/// the summed size of their Parquet files (derived from the store +/// listing — the backlog has no stored byte counter). +fn backlog(store: &Store, now: u64, policy: &CompactionPolicy) -> (usize, u64, usize) { + match plan_candidates(store, TENANT, now, policy) { + Ok(candidates) => { + let mut bytes = 0u64; + let mut errors = 0usize; + for key in &candidates { + match store.list_with_sizes_blocking(Some(&partition_prefix(key))) { + Ok(entries) => { + bytes = bytes.saturating_add( + entries + .iter() + .filter(|(key, _)| key.ends_with(".parquet")) + .map(|(_, size)| *size) + .sum(), + ); + } + Err(_) => errors += 1, + } + } + (candidates.len(), bytes, errors) + } + Err(_) => (0, 0, 1), + } +} + +/// The partition's object-key prefix relative to the store root +/// (`data/tenant_id=…/year=…/…/hour=…`), matching the writer's layout. +fn partition_prefix(key: &PartitionKey) -> String { + key.data_path(Path::new("")) + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") +} + +// ---------------------------------------------------------------------- +// WAL sharing +// ---------------------------------------------------------------------- + +/// [`Journal`] over a shared WAL handle, so the sampler can read +/// [`Wal::metrics`] while the group-commit coordinator drives +/// append/sync. Delegates to the production `impl Journal for Wal`; the +/// coordinator already serializes journal access, so this mutex is +/// uncontended except for the sampler's brief metrics reads. +struct SharedWal(Arc>); + +fn lock_wal(wal: &Arc>) -> MutexGuard<'_, Wal> { + wal.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl Journal for SharedWal { + fn append_batch(&mut self, payload: &[u8]) -> Result<(), ReceiveError> { + Journal::append_batch(&mut *lock_wal(&self.0), payload) + } + + fn sync(&mut self) -> Result { + Journal::sync(&mut *lock_wal(&self.0)) + } + + fn unflushed_bytes(&self) -> u64 { + Journal::unflushed_bytes(&*lock_wal(&self.0)) + } +} + +// ---------------------------------------------------------------------- +// Numeric helpers +// ---------------------------------------------------------------------- + +fn to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn saturating_u64(value: u128) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn saturating_usize(value: u64) -> usize { + usize::try_from(value).unwrap_or(usize::MAX) +} + +// Report-scale values (lines, µs) are far below 2^52, so the f64 +// conversion is exact in practice; the report is ms/lines-per-sec scale. +#[allow(clippy::cast_precision_loss)] +fn as_f64(value: u64) -> f64 { + value as f64 +} + +fn us_to_ms(us: u64) -> f64 { + as_f64(us) / 1_000.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pacing_interval_matches_target_rate() { + assert_eq!( + batch_interval(100_000, 1_000), + Some(Duration::from_millis(10)), + "100k lines/s in 1000-line batches = one batch per 10 ms", + ); + assert_eq!(batch_interval(5_000, 100), Some(Duration::from_millis(20))); + assert_eq!(batch_interval(0, 100), None); + assert_eq!(batch_interval(100, 0), None); + assert_eq!( + batch_interval(u64::MAX, 1), + Some(Duration::from_nanos(1)), + "floors at 1 ns rather than a zero interval", + ); + } + + #[test] + fn percentile_is_nearest_rank() { + let v: Vec = (1..=100).collect(); + assert_eq!(percentile_us(&v, 50, 100), 50); + assert_eq!(percentile_us(&v, 95, 100), 95); + assert_eq!(percentile_us(&v, 99, 100), 99); + assert_eq!(percentile_us(&[7], 99, 100), 7); + assert_eq!(percentile_us(&[], 99, 100), 0); + assert_eq!(percentile_us(&[1, 2, 3], 99, 0), 0); + } + + #[test] + fn d1_verdict_applies_both_bars() { + assert!(d1_verdict(100_000.0, 200.0).pass, "exactly at both bars"); + assert!(!d1_verdict(99_999.0, 10.0).pass, "rate below the bar"); + assert!(!d1_verdict(500_000.0, 200.1).pass, "p99 above the bar"); + let v = d1_verdict(1.0, 1.0); + assert_eq!(v.bar_lines_per_sec_per_core, 100_000); + assert_eq!(v.bar_p99_ack_ms, 200); + } + + #[test] + fn d2_verdict_requires_drain_and_return_to_zero() { + assert!(d2_verdict(&[0, 1, 2, 0, 1], 0).pass, "returned after max"); + assert!( + d2_verdict(&[1, 2, 2], 0).pass, + "the final drained backlog counts as the return to zero", + ); + assert!(!d2_verdict(&[0, 2, 1], 1).pass, "final backlog nonzero"); + assert!(!d2_verdict(&[0, 3, 1], usize::MAX).pass, "unknown final"); + assert!(d2_verdict(&[], 0).pass, "no accumulation, drained"); + let v = d2_verdict(&[0, 3, 0, 1], 0); + assert_eq!(v.max_backlog_partitions, 3); + assert!(v.returned_to_zero_after_max); + } + + #[test] + fn batch_generation_is_deterministic_and_tenant_scoped() { + let a = build_batch(42, 7, 50, BASE_UNIX_NANOS); + let b = build_batch(42, 7, 50, BASE_UNIX_NANOS); + assert_eq!(a, b, "same seed + index ⇒ identical batch"); + let c = build_batch(42, 8, 50, BASE_UNIX_NANOS); + assert_ne!(a, c, "a different batch index draws different params"); + let records = &a.resource_logs[0].scope_logs[0].log_records; + assert_eq!(records.len(), 50); + let attrs = &a.resource_logs[0] + .resource + .as_ref() + .expect("resource") + .attributes; + assert_eq!(attrs[0].key, "service.name"); + } + + #[test] + fn config_validation_rejects_zeroes() { + let ok = SoakConfig::default(); + assert!(validate(&ok).is_ok()); + for broken in [ + SoakConfig { + duration_secs: 0, + ..ok.clone() + }, + SoakConfig { + target_lines_per_sec: 0, + ..ok.clone() + }, + SoakConfig { + batch_size: 0, + ..ok.clone() + }, + SoakConfig { + time_compression: 0, + ..ok.clone() + }, + SoakConfig { + sample_every_secs: 0, + ..ok.clone() + }, + SoakConfig { + worker_threads: 0, + ..ok.clone() + }, + ] { + assert!(matches!(validate(&broken), Err(SoakError::Config(_)))); + } + } + + /// End-to-end smoke soak: a few seconds of paced load under an + /// aggressive synthetic clock must drive the full + /// seal → sweep → compact cycle — at least one sample observes a + /// nonzero candidate backlog, at least one sweep compacts, and the + /// final drain leaves zero backlog. + #[test] + fn smoke_soak_runs_the_seal_sweep_compact_cycle() { + let config = SoakConfig { + duration_secs: 4, + target_lines_per_sec: 5_000, + batch_size: 100, + // 1 wall second = 2000 synthetic seconds: an hour partition + // seals (hour + 15 min grace) every ~2.25 wall seconds. + time_compression: 2_000, + sample_every_secs: 1, + // Small flush target so each synthetic hour lands as several + // Parquet files (a compaction candidate needs ≥ 2). + sink_target_bytes: 64 * 1024, + seed: 42, + worker_threads: 4, + }; + let report = run_soak(&config).expect("smoke soak runs"); + + assert!(report.lines_acked > 0, "load was ingested and acked"); + assert_eq!(report.batches_failed, 0, "no batch failed to commit"); + assert!(report.latency.p99_ms > 0.0, "latencies were recorded"); + assert!(report.achieved_lines_per_sec > 0.0); + assert!( + report.samples.iter().any(|s| s.backlog_partitions > 0), + "some sample observed a sealed candidate partition: {:?}", + report.samples, + ); + assert!( + report.total_partitions_compacted >= 1, + "at least one sweep compacted a partition: {:?}", + report.samples, + ); + assert_eq!( + report.samples.iter().map(|s| s.errors).sum::(), + 0, + "no sweep/listing errors: {:?}", + report.samples, + ); + assert_eq!(report.d2.final_backlog_partitions, 0, "drained"); + assert!(report.d2.pass, "the smoke run's backlog is bounded"); + // The report is the workflow artifact — it must serialize. + let json = serde_json::to_string(&report).expect("report serializes"); + assert!(json.contains("\"d1\"")); + } +} From cc02a7e9af273cf3f8c8d99d4fef8b5a2015b417 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 20 Jul 2026 05:57:36 +0200 Subject: [PATCH 2/3] =?UTF-8?q?fix(bench):=20soak=20review=20round=20?= =?UTF-8?q?=E2=80=94=20pacing,=20bounded=20latencies,=20wall=20split,=20si?= =?UTF-8?q?ngle-list=20backlog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #558 review fixes, all accepted findings: 1. Load loop checks the deadline AFTER awaiting the tick, so a tick landing past the deadline schedules no extra batch (the 5 s smoke now acks exactly duration/pace batches). Test: load_loop_schedules_no_batch_past_the_deadline. 2. Both tickers pace with MissedTickBehavior::Delay: the sampler runs on an aligned interval (interval_at) instead of sleep-then-work, so the period no longer stretches by each sample's blocking time, and the load ticker no longer bursts back-to-back catch-up batches after a stall (which broke the paced-load assumption and inflated in-flight depth). 3. Ack latencies are bounded: LatencyRecorder caps stored samples at 2^22 and decimates by two on overflow, keeping a systematic 1-in-2^k sample of the stream (percentiles stay valid at bounded memory); the report carries latency_samples_stored/_total. Test: latency_recorder_decimates_to_a_systematic_sample. 4. wall_secs split into load_wall_secs (the D1 rate denominator, measured at end of load drain) and total_wall_secs (covers the sampler join + drain sweep, so no sample timestamp exceeds it); JSON, stdout summary, and the workflow jq updated. Smoke test asserts the ordering invariants. 5. D2Verdict::returned_to_zero_after_max docstring now states plainly that the post-load drain sweep reaching zero is itself the return-to-zero evidence; logic unchanged, pinned by an extended d2_verdict test. 6. Backlog bytes come from ONE tenant-wide listing per sample matched against the candidates' partition prefixes, not one listing per candidate — bounded list ops per sample on non-local stores. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WQY9wfrfRggqSpMLH8Xj3Y --- .github/workflows/soak-bench.yml | 2 +- crates/ourios-bench/src/main.rs | 6 +- crates/ourios-bench/src/soak.rs | 276 ++++++++++++++++++++++++++----- 3 files changed, 238 insertions(+), 46 deletions(-) diff --git a/.github/workflows/soak-bench.yml b/.github/workflows/soak-bench.yml index 7600256ba..028a42659 100644 --- a/.github/workflows/soak-bench.yml +++ b/.github/workflows/soak-bench.yml @@ -85,7 +85,7 @@ jobs: echo echo 'Run shape:' echo '```' - jq -r '"wall \(.wall_secs)s · \(.lines_acked) lines acked (\(.batches_acked) batches, \(.batches_failed) failed) · \(.total_partitions_compacted) partitions compacted over \(.samples | length) samples"' soak-report.json + jq -r '"wall \(.load_wall_secs)s load / \(.total_wall_secs)s total · \(.lines_acked) lines acked (\(.batches_acked) batches, \(.batches_failed) failed) · \(.total_partitions_compacted) partitions compacted over \(.samples | length) samples · latency basis \(.latency_samples_stored)/\(.latency_samples_total) samples"' soak-report.json echo '```' else echo '_No soak-report.json — the run failed before producing it._' diff --git a/crates/ourios-bench/src/main.rs b/crates/ourios-bench/src/main.rs index e5e190aa0..9934eee3e 100644 --- a/crates/ourios-bench/src/main.rs +++ b/crates/ourios-bench/src/main.rs @@ -271,8 +271,10 @@ fn run_soak_command(args: SoakArgs) -> Result { /// One-line-per-gate soak summary, mirroring [`print_summary`]'s shape. fn print_soak_summary(report: &SoakReport) { println!( - "soak — {:.1}s wall at target {} lines/s (batch {}, {} worker(s), ×{} synthetic time)", - report.wall_secs, + "soak — {:.1}s load / {:.1}s total wall at target {} lines/s (batch {}, {} worker(s), \ + ×{} synthetic time)", + report.load_wall_secs, + report.total_wall_secs, report.config.target_lines_per_sec, report.config.batch_size, report.config.worker_threads, diff --git a/crates/ourios-bench/src/soak.rs b/crates/ourios-bench/src/soak.rs index 81660d637..b270e1640 100644 --- a/crates/ourios-bench/src/soak.rs +++ b/crates/ourios-bench/src/soak.rs @@ -47,7 +47,9 @@ use ourios_ingester::receiver::pipeline::{Journal, ReceiveError}; use ourios_ingester::receiver::{CommitCoordinator, IngestPipeline, TenantRule}; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_miner::cluster::MinerCluster; -use ourios_parquet::{CompactionPolicy, PartitionKey, Store, plan_candidates}; +use ourios_parquet::{ + CompactionPolicy, PartitionKey, Store, percent_encode_tenant, plan_candidates, +}; use ourios_wal::{Wal, WalConfig, WalOffset}; use serde::Serialize; @@ -173,8 +175,12 @@ pub struct D2Verdict { pub max_backlog_partitions: usize, /// Candidates still pending after the final drain sweep. pub final_backlog_partitions: usize, - /// The backlog series hit zero at least once after its maximum - /// (trivially true when it never rose above zero). + /// The backlog drained after its maximum: either the sample series + /// itself hit zero at least once after the max, **or** the post-load + /// drain sweep brought the final backlog to zero — the drain sweep + /// reaching zero is itself the return-to-zero evidence, observed at + /// the end of the run rather than mid-series. Trivially true when + /// the backlog never rose above zero. pub returned_to_zero_after_max: bool, } @@ -183,8 +189,11 @@ pub struct D2Verdict { pub struct SoakReport { pub config: SoakConfig, /// Wall seconds from first batch to last ack (includes the drain of - /// in-flight batches). - pub wall_secs: f64, + /// in-flight batches) — the rate denominator for the §D1 numbers. + pub load_wall_secs: f64, + /// Wall seconds for the whole run, including the sampler join and + /// the final drain sweep; every sample's `wall_secs` is ≤ this. + pub total_wall_secs: f64, pub lines_acked: u64, pub batches_acked: u64, pub batches_failed: u64, @@ -192,6 +201,12 @@ pub struct SoakReport { /// `achieved_lines_per_sec / worker_threads` — the §D1 per-core rate. pub per_core_lines_per_sec: f64, pub latency: LatencySummary, + /// Ack observations offered to the recorder (one per acked batch). + pub latency_samples_total: u64, + /// Observations retained as the percentile basis after the bounded + /// recorder's decimation (a systematic 1-in-2^k sample once the + /// stored cap is hit, so percentiles stay valid at bounded memory). + pub latency_samples_stored: usize, pub samples: Vec, pub total_partitions_compacted: usize, pub d1: D1Verdict, @@ -345,7 +360,9 @@ async fn soak(config: &SoakConfig) -> Result { })); run_load(&pipeline, &clock, config, &stats, pace).await; - let wall_secs = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; + // The D1 rate denominator stops here — the sampler join and the + // drain sweep below are measurement overhead, not load. + let load_wall_secs = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; stop.store(true, Ordering::Relaxed); // A sampler panic loses the timeseries but not the run; report with @@ -381,10 +398,12 @@ async fn soak(config: &SoakConfig) -> Result { } }; samples.push(drain_sample); + let total_wall_secs = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; Ok(build_report( config, - wall_secs, + load_wall_secs, + total_wall_secs, &stats, samples, final_backlog, @@ -393,27 +412,31 @@ async fn soak(config: &SoakConfig) -> Result { fn build_report( config: &SoakConfig, - wall_secs: f64, + load_wall_secs: f64, + total_wall_secs: f64, stats: &LoadStats, samples: Vec, final_backlog: usize, ) -> SoakReport { - let mut latencies = stats - .latencies_us - .lock() - .unwrap_or_else(PoisonError::into_inner); + let (mut latencies, latency_samples_total) = { + let mut recorder = stats + .latencies_us + .lock() + .unwrap_or_else(PoisonError::into_inner); + (std::mem::take(&mut recorder.samples), recorder.total) + }; latencies.sort_unstable(); + let latency_samples_stored = latencies.len(); let latency = LatencySummary { p50_ms: us_to_ms(percentile_us(&latencies, 50, 100)), p95_ms: us_to_ms(percentile_us(&latencies, 95, 100)), p99_ms: us_to_ms(percentile_us(&latencies, 99, 100)), max_ms: us_to_ms(latencies.last().copied().unwrap_or(0)), }; - drop(latencies); let lines_acked = stats.lines_acked.load(Ordering::Relaxed); - let achieved = if wall_secs > 0.0 { - as_f64(lines_acked) / wall_secs + let achieved = if load_wall_secs > 0.0 { + as_f64(lines_acked) / load_wall_secs } else { 0.0 }; @@ -424,13 +447,16 @@ fn build_report( let p99_ms = latency.p99_ms; SoakReport { config: config.clone(), - wall_secs, + load_wall_secs, + total_wall_secs, lines_acked, batches_acked: stats.batches_acked.load(Ordering::Relaxed), batches_failed: stats.batches_failed.load(Ordering::Relaxed), achieved_lines_per_sec: achieved, per_core_lines_per_sec: per_core, latency, + latency_samples_total, + latency_samples_stored, samples, total_partitions_compacted: total_compacted, d1: d1_verdict(per_core, p99_ms), @@ -452,7 +478,9 @@ pub fn d1_verdict(per_core_lines_per_sec: f64, p99_ack_ms: f64) -> D1Verdict { } /// The §D2 verdict over the per-sample backlog series and the -/// post-drain-sweep final backlog. +/// post-drain-sweep final backlog. A `final_backlog` of zero counts as +/// the return-to-zero evidence even when no mid-series sample caught +/// the backlog at zero (see [`D2Verdict::returned_to_zero_after_max`]). #[must_use] pub fn d2_verdict(backlog_series: &[usize], final_backlog: usize) -> D2Verdict { let max = backlog_series.iter().copied().max().unwrap_or(0); @@ -502,9 +530,67 @@ pub fn percentile_us(sorted: &[u64], num: usize, den: usize) -> u64 { // Load generation // ---------------------------------------------------------------------- +/// Cap on stored ack-latency samples (2^22 × 8 B = 32 MiB). Unbounded +/// storage would reach hundreds of millions of entries over an hour at +/// 100k lines/s with a small batch size. +const LATENCY_SAMPLE_CAP: usize = 1 << 22; + +/// Bounded ack-latency recorder: stores every observation until the cap, +/// then decimates by two. The retained set is always the *systematic* +/// 1-in-2^`shift` sample of the observation stream (an observation is +/// kept iff its index ≡ 0 mod 2^`shift`), a property halving preserves — +/// so percentiles over the retained set stay valid estimates of the full +/// stream at bounded memory. +struct LatencyRecorder { + samples: Vec, + /// log2 of the decimation stride: observation `i` is retained iff + /// `i % (1 << shift) == 0`. + shift: u32, + /// Observations offered, retained or not. + total: u64, + cap: usize, +} + +impl Default for LatencyRecorder { + fn default() -> Self { + Self::with_cap(LATENCY_SAMPLE_CAP) + } +} + +impl LatencyRecorder { + /// A cap below 2 could never halve; clamp so `record` always makes + /// progress. + fn with_cap(cap: usize) -> Self { + Self { + samples: Vec::new(), + shift: 0, + total: 0, + cap: cap.max(2), + } + } + + fn record(&mut self, value_us: u64) { + let index = self.total; + self.total += 1; + let stride = 1u64.checked_shl(self.shift).unwrap_or(u64::MAX); + if !index.is_multiple_of(stride) { + return; + } + self.samples.push(value_us); + if self.samples.len() >= self.cap { + // Keep every other retained sample: retained observation + // indices go from multiples of 2^shift to multiples of + // 2^(shift+1), matching the go-forward stride. + let halved: Vec = self.samples.iter().copied().step_by(2).collect(); + self.samples = halved; + self.shift += 1; + } + } +} + #[derive(Default)] struct LoadStats { - latencies_us: Mutex>, + latencies_us: Mutex, lines_acked: AtomicU64, batches_acked: AtomicU64, batches_failed: AtomicU64, @@ -519,10 +605,20 @@ async fn run_load( ) { let semaphore = Arc::new(tokio::sync::Semaphore::new(MAX_IN_FLIGHT)); let mut ticker = tokio::time::interval(pace); + // A stall must not burst back-to-back catch-up batches (default + // `Burst`) — that breaks the paced-load assumption and inflates + // in-flight depth; `Delay` keeps a full pace gap, as the sampler does. + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let deadline = clock.started + Duration::from_secs(config.duration_secs); let mut batch_idx = 0u64; - while Instant::now() < deadline { + loop { ticker.tick().await; + // Deadline check *after* the tick: a tick landing past the + // deadline must not schedule one extra batch, or the last batch + // would fall outside the wall/rate accounting window. + if Instant::now() >= deadline { + break; + } let Ok(permit) = Arc::clone(&semaphore).acquire_owned().await else { break; // Closed semaphore: cannot happen while we hold it. }; @@ -547,7 +643,7 @@ async fn run_load( .latencies_us .lock() .unwrap_or_else(PoisonError::into_inner) - .push(elapsed_us); + .record(elapsed_us); } Err(_) => { stats.batches_failed.fetch_add(1, Ordering::Relaxed); @@ -872,13 +968,19 @@ struct SamplerCtx { stop: Arc, } -/// Sample + sweep on a fixed wall cadence until stopped. Sequential by -/// construction: each sweep completes before the next tick is awaited, +/// Sample + sweep on an aligned wall cadence until stopped: ticks are +/// scheduled every `every` from the sampler's start (first at +/// `start + every`), so the period does not stretch by each sample's +/// blocking work. A sample that overruns its period delays the next +/// tick by a full `every` (`MissedTickBehavior::Delay`) rather than +/// bursting — and each sweep completes before the next tick is awaited, /// so sweeps never overlap. async fn run_sampler(ctx: SamplerCtx) -> Vec { let mut samples = Vec::new(); + let mut ticker = tokio::time::interval_at(tokio::time::Instant::now() + ctx.every, ctx.every); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { - tokio::time::sleep(ctx.every).await; + ticker.tick().await; if ctx.stop.load(Ordering::Relaxed) { break; } @@ -955,28 +1057,36 @@ fn error_sample(wall_secs: f64, synthetic_now: u64) -> BacklogSample { /// The candidate backlog as of `now`: sealed candidate partitions and /// the summed size of their Parquet files (derived from the store /// listing — the backlog has no stored byte counter). +/// +/// One tenant-wide listing per call, matched against the candidates' +/// partition prefixes — not one listing per candidate. On a non-local +/// store every listing is a remote call, so the per-sample list cost +/// must stay bounded regardless of backlog depth. fn backlog(store: &Store, now: u64, policy: &CompactionPolicy) -> (usize, u64, usize) { - match plan_candidates(store, TENANT, now, policy) { - Ok(candidates) => { - let mut bytes = 0u64; - let mut errors = 0usize; - for key in &candidates { - match store.list_with_sizes_blocking(Some(&partition_prefix(key))) { - Ok(entries) => { - bytes = bytes.saturating_add( - entries - .iter() - .filter(|(key, _)| key.ends_with(".parquet")) - .map(|(_, size)| *size) - .sum(), - ); - } - Err(_) => errors += 1, - } - } - (candidates.len(), bytes, errors) + let Ok(candidates) = plan_candidates(store, TENANT, now, policy) else { + return (0, 0, 1); + }; + if candidates.is_empty() { + return (0, 0, 0); + } + let prefixes: Vec = candidates + .iter() + .map(|key| format!("{}/", partition_prefix(key))) + .collect(); + let tenant_prefix = format!("data/tenant_id={}", percent_encode_tenant(TENANT)); + match store.list_with_sizes_blocking(Some(&tenant_prefix)) { + Ok(entries) => { + let bytes = entries + .iter() + .filter(|(key, _)| { + key.ends_with(".parquet") + && prefixes.iter().any(|p| key.starts_with(p.as_str())) + }) + .map(|(_, size)| *size) + .fold(0u64, u64::saturating_add); + (candidates.len(), bytes, 0) } - Err(_) => (0, 0, 1), + Err(_) => (candidates.len(), 0, 1), } } @@ -1099,6 +1209,61 @@ mod tests { let v = d2_verdict(&[0, 3, 0, 1], 0); assert_eq!(v.max_backlog_partitions, 3); assert!(v.returned_to_zero_after_max); + // The drain sweep reaching zero is itself the return-to-zero + // evidence: the mid-series never hits zero after its max here. + let drained_only = d2_verdict(&[1, 2, 2], 0); + assert!(drained_only.returned_to_zero_after_max); + assert!( + !d2_verdict(&[1, 2, 2], 1).returned_to_zero_after_max, + "without the drain reaching zero, this series never returned", + ); + } + + /// The bounded recorder keeps a systematic 1-in-2^k sample: with + /// cap 8 over observations 0..32, three halvings leave exactly the + /// multiples of 8, and `total` still counts every observation. + #[test] + fn latency_recorder_decimates_to_a_systematic_sample() { + let mut recorder = LatencyRecorder::with_cap(8); + for i in 0..32u64 { + recorder.record(i); + } + assert_eq!(recorder.total, 32); + assert_eq!(recorder.samples, vec![0, 8, 16, 24]); + + // Below the cap nothing is dropped. + let mut small = LatencyRecorder::with_cap(8); + for i in 0..5u64 { + small.record(i); + } + assert_eq!(small.total, 5); + assert_eq!(small.samples, vec![0, 1, 2, 3, 4]); + assert_eq!(small.shift, 0); + } + + /// A tick landing past the deadline must not schedule one extra + /// batch: with a 400 ms pace and a 1 s deadline only the ticks at + /// ~0/400/800 ms may send (3 batches); the tick at ~1200 ms breaks + /// first. Delayed ticks under load can only reduce the count. + #[test] + fn load_loop_schedules_no_batch_past_the_deadline() { + let config = SoakConfig { + duration_secs: 1, + target_lines_per_sec: 250, + batch_size: 100, // pace = 100 / 250 = 400 ms + time_compression: 1, + sample_every_secs: 1, + sink_target_bytes: 4 * 1024 * 1024, + seed: 7, + worker_threads: 2, + }; + let report = run_soak(&config).expect("deadline soak runs"); + let batches = report.batches_acked + report.batches_failed; + assert!( + (1..=3).contains(&batches), + "3 ticks fit before the 1 s deadline, got {batches}", + ); + assert!(report.lines_acked <= 300); } #[test] @@ -1179,11 +1344,36 @@ mod tests { assert_eq!(report.batches_failed, 0, "no batch failed to commit"); assert!(report.latency.p99_ms > 0.0, "latencies were recorded"); assert!(report.achieved_lines_per_sec > 0.0); + assert_eq!( + report.latency_samples_total, report.batches_acked, + "one latency observation per acked batch", + ); + assert!(report.latency_samples_stored <= saturating_usize(report.latency_samples_total)); + assert!( + report.total_wall_secs >= report.load_wall_secs, + "total wall covers the sampler join + drain sweep", + ); + for sample in &report.samples { + assert!( + sample.wall_secs <= report.total_wall_secs, + "no sample timestamp past the run's total wall: {sample:?}", + ); + } assert!( report.samples.iter().any(|s| s.backlog_partitions > 0), "some sample observed a sealed candidate partition: {:?}", report.samples, ); + assert!( + report + .samples + .iter() + .filter(|s| s.backlog_partitions > 0) + .all(|s| s.backlog_bytes > 0), + "candidate partitions carry bytes (the tenant-wide listing \ + matched their prefixes): {:?}", + report.samples, + ); assert!( report.total_partitions_compacted >= 1, "at least one sweep compacted a partition: {:?}", From 9b476ef2b2b61bc3277e327ebe56d4e22e147499 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 20 Jul 2026 06:20:23 +0200 Subject: [PATCH 3/3] =?UTF-8?q?fix(bench):=20soak=20polish=20=E2=80=94=20w?= =?UTF-8?q?rite=20params=20in=20place,=20prompt=20sampler=20shutdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more accepted PR #558 review findings: 1. The batch generator writes each param into the body via `write!` (std::fmt::Write) instead of allocating an intermediate String per record slot in the hot loop. 2. Sampler shutdown is prompt: the stop flag rides a tokio watch channel and races the tick in a `select!`, so the sampler exits as soon as stop is flagged instead of idling out up to one full `sample_every_secs` tick (which inflated total_wall_secs for nothing — the drain sample already exists). The aligned-cadence / no-overlap properties are unchanged: samples still run strictly between ticks, one at a time. tokio grows the `macros` feature for the `select!`. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WQY9wfrfRggqSpMLH8Xj3Y --- crates/ourios-bench/Cargo.toml | 5 +++-- crates/ourios-bench/src/soak.rs | 31 +++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crates/ourios-bench/Cargo.toml b/crates/ourios-bench/Cargo.toml index 58814fa66..fab836d21 100644 --- a/crates/ourios-bench/Cargo.toml +++ b/crates/ourios-bench/Cargo.toml @@ -78,8 +78,9 @@ ourios-querier = { path = "../ourios-querier" } # runtime: the group-commit coordinator offloads its fsync via # `spawn_blocking`, which a current-thread runtime would serialize # behind the load loop. `time` paces the load, `sync` bounds in-flight -# batches (Semaphore). -tokio = { version = "=1.52.3", default-features = false, features = ["rt", "rt-multi-thread", "time", "sync"] } +# batches (Semaphore) + carries the sampler stop signal (watch), and +# `macros` gives the sampler's tick-vs-stop `select!`. +tokio = { version = "=1.52.3", default-features = false, features = ["rt", "rt-multi-thread", "time", "sync", "macros"] } # The soak harness drives the real ingest path in-process: WAL → # group-commit coordinator → `IngestPipeline` → record sink, with the # compactor's `run_sweep` driven manually on the synthetic cadence. diff --git a/crates/ourios-bench/src/soak.rs b/crates/ourios-bench/src/soak.rs index b270e1640..8f75841ec 100644 --- a/crates/ourios-bench/src/soak.rs +++ b/crates/ourios-bench/src/soak.rs @@ -32,8 +32,9 @@ //! bound on the §D1 WAL-commit latency, and exactly what an OTLP client //! observes. +use std::fmt::Write as _; use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; @@ -348,7 +349,7 @@ async fn soak(config: &SoakConfig) -> Result { }; let policy = CompactionPolicy::default(); let stats = Arc::new(LoadStats::default()); - let stop = Arc::new(AtomicBool::new(false)); + let (stop_tx, stop_rx) = tokio::sync::watch::channel(false); let sampler_task = tokio::spawn(run_sampler(SamplerCtx { store: store.clone(), sink: sink.clone(), @@ -356,7 +357,7 @@ async fn soak(config: &SoakConfig) -> Result { clock: clock.clone(), policy, every: Duration::from_secs(config.sample_every_secs), - stop: Arc::clone(&stop), + stop: stop_rx, })); run_load(&pipeline, &clock, config, &stats, pace).await; @@ -364,7 +365,8 @@ async fn soak(config: &SoakConfig) -> Result { // drain sweep below are measurement overhead, not load. let load_wall_secs = as_f64(saturating_u64(clock.started.elapsed().as_millis())) / 1_000.0; - stop.store(true, Ordering::Relaxed); + // A send error just means the sampler already exited. + let _ = stop_tx.send(true); // A sampler panic loses the timeseries but not the run; report with // no samples (d2 then fails loudly on the missing final backlog). let mut samples: Vec = sampler_task.await.unwrap_or_default(); @@ -904,7 +906,9 @@ pub fn build_batch( for (slot, segment) in template.segments.iter().enumerate() { if slot > 0 { let param = splitmix64(&mut rng_state) % 1_000_000; - body.push_str(¶m.to_string()); + // Infallible for String, and no per-param allocation + // in this hot loop (one write per record slot). + let _ = write!(body, "{param}"); } body.push_str(segment); } @@ -965,7 +969,7 @@ struct SamplerCtx { clock: SyntheticClock, policy: CompactionPolicy, every: Duration, - stop: Arc, + stop: tokio::sync::watch::Receiver, } /// Sample + sweep on an aligned wall cadence until stopped: ticks are @@ -974,14 +978,21 @@ struct SamplerCtx { /// blocking work. A sample that overruns its period delays the next /// tick by a full `every` (`MissedTickBehavior::Delay`) rather than /// bursting — and each sweep completes before the next tick is awaited, -/// so sweeps never overlap. -async fn run_sampler(ctx: SamplerCtx) -> Vec { +/// so sweeps never overlap. Shutdown is prompt: the stop signal races +/// the tick, so the sampler never idles out a tick period (up to a full +/// `every`) just to notice it should exit. +async fn run_sampler(mut ctx: SamplerCtx) -> Vec { let mut samples = Vec::new(); let mut ticker = tokio::time::interval_at(tokio::time::Instant::now() + ctx.every, ctx.every); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { - ticker.tick().await; - if ctx.stop.load(Ordering::Relaxed) { + tokio::select! { + _ = ticker.tick() => {} + // The only value ever sent is `true`, and a dropped sender + // equally means the run is over — exit on either. + _ = ctx.stop.changed() => break, + } + if *ctx.stop.borrow() { break; } let store = ctx.store.clone();