diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 0c00c4063..527af01a2 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -49,11 +49,14 @@ reviews: # MAINTENANCE: these instructions name specific modules, invariants, and code # constructs. Re-read and update them whenever those modules are refactored or - # renamed — stale instructions silently misdirect the reviewer. (Both the - # `unified_pipeline` and `pipeline` paths are listed: issue #330 renames the - # former to the latter, so the same instructions apply to whichever is present.) + # renamed — stale instructions silently misdirect the reviewer. (All three + # pipeline paths are listed: issue #330 renames `src/lib/unified_pipeline` to + # `src/lib/pipeline` and extracts its engine into the `fgumi-pipeline-core` + # crate, so the same instructions apply to whichever is present. A glob that + # names only the `src/lib/` homes silently stops reviewing the engine the day + # it moves into the crate.) path_instructions: - - path: "src/lib/{unified_pipeline,pipeline}/**/*.rs" + - path: "{src/lib/unified_pipeline,src/lib/pipeline,crates/fgumi-pipeline-core/src}/**/*.rs" instructions: >- This is the hand-rolled concurrent step pipeline; its bugs are deadlocks, lost output, and unbounded memory, not style. Treat any change to the diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index a781ce7b0..829a03627 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -5,12 +5,19 @@ name: Miri (undefined-behavior check) # undefined behavior (out-of-bounds, invalid pointer use, aliasing violations), # turning those invariants into a machine-checked gate. # -# Scope: `fgumi-raw-bam`, which contains the raw-pointer queryname comparator -# (`natural_compare` / `natural_compare_nul`, via `get_unchecked` and `*const u8` -# walks) and has no FFI. `fgumi-sort` is intentionally not covered yet: its -# `memory_probe` module calls the mimalloc / mach2 FFI, which Miri cannot execute, -# so covering its radix-sort unsafe needs `#[cfg(not(miri))]` guards first -# (tracked as a follow-up). +# Scope: the two crates whose approved `unsafe` is pure Rust (no FFI), each +# narrowed to the module that carries it — +# - `fgumi-raw-bam::sort` — the raw-pointer queryname comparator +# (`natural_compare` / `natural_compare_nul`, via `get_unchecked` and +# `*const u8` walks). +# - `fgumi-pipeline-core::erased` — the typed-handle dispatch cache, four +# `mem::transmute`s that widen `&'a Handle` to `&'static` for storage and +# narrow it back on read. Aliasing/lifetime UB is exactly what Stacked +# Borrows checks, so this is the one place Miri adds signal the type system +# cannot. +# `fgumi-sort` is intentionally not covered yet: its `memory_probe` module calls +# the mimalloc / mach2 FFI, which Miri cannot execute, so covering its radix-sort +# unsafe needs `#[cfg(not(miri))]` guards first (tracked as a follow-up). # # Runs on a schedule (and on demand) rather than per-PR: Miri needs the nightly # toolchain (which can regress independently of this repo) and is much slower than @@ -61,4 +68,65 @@ jobs: # cannot interpret, and the BAM encode/decode roundtrip tests are slow and # exercise no `unsafe` of ours. The `sort` tests (comparator + coordinate/ # queryname raw-key tests) run clean under Miri in a few seconds. - run: cargo +nightly miri test -p fgumi-raw-bam sort + # + # `--list` first and fail on an empty selection: `cargo test ` + # exits 0 when the filter matches nothing, so renaming or moving the + # module would silently retire this gate while CI stayed green — the same + # zero-match hazard `compile_fail.rs`'s `EXPECTED_FIXTURES` guards. + run: | + set -euo pipefail + # The zero-match guard below proves the filter selects tests; it cannot + # prove the filter still COVERS the crate's `unsafe`. Pin that too: if an + # `#[allow(unsafe_code)]` site moves out of `sort.rs`, the filter keeps + # matching and Miri silently stops checking the moved site. + stray=$(grep -rl 'allow(unsafe_code)' crates/fgumi-raw-bam/src \ + | grep -v '^crates/fgumi-raw-bam/src/sort.rs$' || true) + if [ -n "${stray}" ]; then + echo "::error::unsafe_code outside the Miri-scoped 'sort' module: ${stray}" >&2 + exit 1 + fi + cargo +nightly miri test -p fgumi-raw-bam sort -- --list > "${RUNNER_TEMP}/miri-raw-bam-sort.list" + n=$(grep -c ': test$' "${RUNNER_TEMP}/miri-raw-bam-sort.list" || true) + echo "miri: the 'sort' filter matched ${n} test(s) in fgumi-raw-bam" + if [ "${n}" -eq 0 ]; then + echo "::error::the 'sort' filter matched no tests in fgumi-raw-bam — the Miri scope is stale" >&2 + exit 1 + fi + cargo +nightly miri test -p fgumi-raw-bam sort + - name: Miri — fgumi-pipeline-core typed-handle dispatch cache + # Scope to the `erased` module for the same reason: all four + # `#[allow(unsafe_code)]` sites in the crate live there + # (`TypedStep::resolve_input`/`resolve_outputs` and the `TypedStep2` + # pair). These 25 tests run clean under Miri in ~6s. The rest of the + # crate is deliberately excluded: the `builder` / `runtime` tests spawn + # worker threads and run a full pipeline, which takes Miri well over ten + # minutes, and three of them (`detached_two_sided_no_deadlock`, + # `driver_round_robins_all_live_before_parking`, + # `sticky_holding_source_yields_to_its_draining_consumer`) guard against + # a wedge with a WALL-CLOCK watchdog that `process::abort()`s — under + # Miri's slowdown that fires on a healthy run. Widening this scope means + # giving those watchdogs a `#[cfg(miri)]` budget first. + # + # Zero-match guard, as on the `fgumi-raw-bam` step above: this filter is + # the only thing pointing Miri at the crate's four `unsafe` sites, and an + # empty selection would pass silently. + run: | + set -euo pipefail + # Location guard, as on the `fgumi-raw-bam` step above: the step comment + # claims all four `#[allow(unsafe_code)]` sites live in `erased`, and + # nothing else enforces it. A site that moves elsewhere would leave the + # filter matching and the moved site unchecked. + stray=$(grep -rl 'allow(unsafe_code)' crates/fgumi-pipeline-core/src \ + | grep -v '^crates/fgumi-pipeline-core/src/erased.rs$' || true) + if [ -n "${stray}" ]; then + echo "::error::unsafe_code outside the Miri-scoped 'erased' module: ${stray}" >&2 + exit 1 + fi + cargo +nightly miri test -p fgumi-pipeline-core erased -- --list > "${RUNNER_TEMP}/miri-pipeline-core-erased.list" + n=$(grep -c ': test$' "${RUNNER_TEMP}/miri-pipeline-core-erased.list" || true) + echo "miri: the 'erased' filter matched ${n} test(s) in fgumi-pipeline-core" + if [ "${n}" -eq 0 ]; then + echo "::error::the 'erased' filter matched no tests in fgumi-pipeline-core — the Miri scope is stale" >&2 + exit 1 + fi + cargo +nightly miri test -p fgumi-pipeline-core erased diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a27e41571..2403e457e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -87,6 +87,7 @@ jobs: fgumi-fmt fgumi-cli-macros fgumi-cli-common + fgumi-pipeline-core fgumi-raw-bam fgumi-bam-io fgumi-umi diff --git a/CLAUDE.md b/CLAUDE.md index 1991ecfac..7fc81dc71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -210,6 +210,37 @@ regresses `samtools sort -n`–style throughput. SAFETY: the buffers are `to_vec()` + push, so the pointer is valid and null-terminated for the call's lifetime. +### Approved typed-handle cache (fgumi-pipeline-core) + +The pipeline runtime hands each step its input/output handles as +`&dyn Any`, so the `ErasedStep` adapter must `downcast_ref` them back to +their concrete types. That downcast sits on the per-item dispatch path: a +4-thread CODEC 8M run spends ≈1.6% of samples on `TypeId` compares alone. +The adapter resolves the handles once and caches them, which requires +storing a reference whose real lifetime the struct cannot name. + +- **`crates/fgumi-pipeline-core/src/erased.rs`** — four `#[allow(unsafe_code)]` + sites, two on `TypedStep` (`resolve_input`, `resolve_outputs`) and two on + `TypedStep2` (`resolve_inputs`, `resolve_outputs`). Each is a + `std::mem::transmute` that extends a `&'a Handle` to `&'static Handle` for + storage in the cache slot, and narrows it back to `&'a` on read. No pointer + is dereferenced through the `'static` form. SAFETY rests on three invariants + documented on `TypedStep` itself: the handle boxes are owned by + `ChainContexts` (alive for the whole `Pipeline::run`), every step instance is + dropped before those contexts are, and every dispatch passes the *same* box + for a given `step_idx`. The third is the one the compiler cannot check, so + each cache slot stores the address of the erased box it was resolved from and + every cached hit `assert!`s it — **unconditionally, release included**, so a + step reused across two *live* `ChainContexts` panics instead of reading + through the wrong one. The check is one load and one compare, not the + `downcast_ref` `TypeId` probe the cache exists to elide. A debug-only + `debug_assert!` keeps re-resolving the typed pointer as a second diagnostic. + Treat the assert as defence in depth, not as the safety argument: it compares + data addresses only, so an allocator that reuses a freed box's address defeats + it. Soundness rests on the second invariant — every step instance is dropped + before the contexts it cached from — which is what must be preserved by any + future change. + Any new `unsafe` site must extend this list and explain why the safe alternative is unacceptable. Do not introduce `unsafe` outside the crates listed in this section. diff --git a/Cargo.lock b/Cargo.lock index 144cea757..f461f7502 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,6 +856,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "fgumi-pipeline-core" +version = "0.5.0" +dependencies = [ + "ahash", + "anyhow", + "crossbeam-queue", + "log", + "noodles", + "parking_lot", + "proptest", + "rstest", + "trybuild", +] + [[package]] name = "fgumi-raw-bam" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 8dd539a8d..be1d7b01c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/fgumi-raw-bam", "crates/fgumi-dna", "crates/fgumi-bgzf", "crates/fgumi-fmt", "crates/fgumi-metrics", "crates/fgumi-sam", "crates/fgumi-simd-fastq", "crates/fgumi-tag", "crates/fgumi-umi", "crates/fgumi-consensus", "crates/fgumi-bam-io", "crates/fgumi-sort", "crates/fgumi-cli-common", "crates/fgumi-cli-macros", "crates/xtask"] +members = [".", "crates/fgumi-raw-bam", "crates/fgumi-dna", "crates/fgumi-bgzf", "crates/fgumi-fmt", "crates/fgumi-metrics", "crates/fgumi-sam", "crates/fgumi-simd-fastq", "crates/fgumi-tag", "crates/fgumi-umi", "crates/fgumi-consensus", "crates/fgumi-bam-io", "crates/fgumi-sort", "crates/fgumi-cli-common", "crates/fgumi-cli-macros", "crates/fgumi-pipeline-core", "crates/xtask"] resolver = "2" [workspace.package] @@ -28,6 +28,7 @@ fgumi-fmt = { version = "0.5.0", path = "crates/fgumi-fmt" } fgumi-consensus = { version = "0.5.0", path = "crates/fgumi-consensus", default-features = false } fgumi-dna = { version = "0.5.0", path = "crates/fgumi-dna" } fgumi-metrics = { version = "0.5.0", path = "crates/fgumi-metrics" } +fgumi-pipeline-core = { version = "0.5.0", path = "crates/fgumi-pipeline-core" } fgumi-raw-bam = { version = "0.5.0", path = "crates/fgumi-raw-bam" } fgumi-sam = { version = "0.5.0", path = "crates/fgumi-sam" } fgumi-simd-fastq = { version = "0.5.0", path = "crates/fgumi-simd-fastq" } @@ -82,6 +83,7 @@ serde = { version = "1.0.228", features = ["derive"] } sysinfo = { version = "0.38", default-features = false, features = ["system"] } tempfile = "3.3.0" thiserror = "2" +trybuild = "1.0" wide = "1.5" [package] diff --git a/crates/fgumi-cli-macros/Cargo.toml b/crates/fgumi-cli-macros/Cargo.toml index 0e8843f5e..aee7df32b 100644 --- a/crates/fgumi-cli-macros/Cargo.toml +++ b/crates/fgumi-cli-macros/Cargo.toml @@ -19,7 +19,7 @@ proc-macro2 = "1" clap = { workspace = true } anyhow = { workspace = true } rstest = { workspace = true } -trybuild = "1.0" +trybuild = { workspace = true } [lints.clippy] pedantic = { level = "deny", priority = -1 } diff --git a/crates/fgumi-pipeline-core/Cargo.toml b/crates/fgumi-pipeline-core/Cargo.toml new file mode 100644 index 000000000..594424540 --- /dev/null +++ b/crates/fgumi-pipeline-core/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "fgumi-pipeline-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Typed-step pipeline framework core (steps, queues, reorder stage, scheduler) for fgumi" +repository.workspace = true +license.workspace = true + +[dependencies] +ahash = { workspace = true } +anyhow = { workspace = true } +crossbeam-queue = { workspace = true } +log = { workspace = true } +noodles = { workspace = true, features = ["sam"] } +parking_lot = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +rstest = { workspace = true } +trybuild = { workspace = true } + +[lints.clippy] +pedantic = { level = "deny", priority = -1 } diff --git a/crates/fgumi-pipeline-core/src/builder.rs b/crates/fgumi-pipeline-core/src/builder.rs new file mode 100644 index 000000000..5b5f0c271 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/builder.rs @@ -0,0 +1,3746 @@ +//! `Pipeline`, `PipelineBuilder`, `Chain`, `MultiChain2/3/4`, `BuildError`. +//! +//! The builder uses `RefCell` for the steps + graph so multiple `Chain` +//! handles (one per branch of a multi-output step) can coexist as `&self`. +//! Move semantics on `Chain` enforce single-consumer per branch at the +//! type-system level; the `ChainGraph` performs the runtime all-wired check. + +use std::cell::RefCell; +use std::marker::PhantomData; +use std::sync::Arc; + +use super::erased::{ErasedStep, TypedStep, TypedStep2}; +use super::item::HeapSize; +use super::outputs::{Single, StepOutputs}; +use super::runtime::stats::PipelineStats; +use super::signal::{CancelHandle, PipelineSignal}; +use super::step::{Step, Step2}; +use super::topology::{BranchIdx, ChainGraph, StepIdx}; + +/// Errors from `PipelineBuilder::build()`. +#[derive(Debug)] +pub enum BuildError { + UnwiredOutput { + step: &'static str, + branch: &'static str, + }, + Empty, + /// A step whose `Input = ()` (a source) has an edge wired INTO it. Its + /// input is implicit, so chain-context construction hands it a dummy unit + /// input handle and nothing ever pops the wired edge — the producer's + /// output would be silently discarded. + WiredIntoSource { + step: &'static str, + producer: &'static str, + }, +} + +impl std::fmt::Display for BuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnwiredOutput { step, branch } => { + write!(f, "step {step:?} has unwired output branch {branch:?}") + } + Self::Empty => write!(f, "pipeline has no steps"), + Self::WiredIntoSource { step, producer } => write!( + f, + "step {step:?} is a source (Input = ()) but {producer:?} is wired into it; \ + a source's input is implicit, so those items would never be consumed" + ), + } + } +} + +impl std::error::Error for BuildError {} + +/// How much pipeline instrumentation to collect. Off by default (zero overhead: +/// queues stay non-instrumented and no sampler thread spawns). Each higher level +/// is a superset of the one below. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum InstrumentationLevel { + /// No edge instrumentation. The pool hot path takes its existing zero-cost + /// route; this is the production default. + #[default] + Off, + /// Per-edge throughput/occupancy/latency counters + the occupancy sampler + + /// the end-of-run edge table and bottleneck verdict. + Summary, + /// `Summary` plus a per-tick occupancy/throughput timeline TSV. + Timeline, + /// `Timeline` plus direct dwell/park-time latency (Detached edges). + Deep, +} + +impl InstrumentationLevel { + /// `true` for any level above `Off` (edge metrics are collected). + #[must_use] + pub fn is_on(self) -> bool { + !matches!(self, Self::Off) + } + + /// `true` if the background occupancy sampler should run. + #[must_use] + pub fn samples(self) -> bool { + self.is_on() + } + + /// `true` if the per-tick timeline TSV should be written. + #[must_use] + pub fn timeline(self) -> bool { + matches!(self, Self::Timeline | Self::Deep) + } + + /// `true` if direct dwell/park-time latency should be recorded. + #[must_use] + pub fn deep(self) -> bool { + matches!(self, Self::Deep) + } +} + +#[derive(Debug, Clone)] +pub struct PipelineConfig { + pub threads: usize, + /// Optional shared stats collector. Construct via `Pipeline::stats()`, + /// then pass into the config; the framework writes per-step counters + /// during the run, and the caller reads them after `run` returns. + /// `None` (the default) keeps the worker loop on its zero-cost path. + pub stats: Option>, + /// Deadlock-detection timeout in seconds. When > 0 and `stats` is set, + /// `Pipeline::run` spawns a background monitor that polls the stats + /// snapshot every `timeout / 4` seconds (clamped to ≥1s). If no step + /// has progressed (or finished) for `timeout` seconds, the monitor + /// logs the snapshot at warn level so the user has a starting point + /// for debugging the stall. `0` (default) disables the monitor. + /// + /// Mirrors the legacy framework's `--deadlock-timeout` semantics + /// (default 10s, 0 = disabled). Stats must be enabled for the + /// monitor to have anything to read; the helpers in + /// `commands/common.rs` auto-attach a stats handle when this is + /// non-zero so callers don't have to pair the two flags manually. + pub deadlock_timeout_secs: u64, + /// Total byte budget for byte-bounded queues across the chain. + /// `None` keeps each queue at the per-step limit set by its + /// `QueueSpec::ByteBounded { limit_bytes }`. `Some(total)` + /// enables the queue-memory rebalancer: an initial pass evenly + /// distributes `total` across all byte-bounded queues, and a + /// background thread periodically reads queue fullness and + /// shifts budget toward consistently-full queues (producer-bound + /// bottlenecks) at the expense of consistently-empty ones. + /// + /// Floor of 1 MiB per queue is enforced regardless of the + /// rebalancer's decisions to prevent pathological starvation. + pub queue_memory_total: Option, + /// How much per-edge instrumentation to collect (default `Off` = zero + /// overhead). Threaded into queue construction (instrumented transports) and + /// the occupancy sampler. See [`InstrumentationLevel`]. + pub instrumentation: InstrumentationLevel, + /// Where to write the per-tick timeline TSV when + /// `instrumentation.timeline()`. `None` → a default `pipeline-trace.tsv`. + pub trace_path: Option, + /// Per-worker dispatch-order policy. The default + /// [`ChainOrderScheduler`](crate::runtime::ChainOrderScheduler) walks live + /// steps upstream-first (historical behaviour). A chain may opt into + /// [`DrainFirstScheduler`](crate::runtime::DrainFirstScheduler) to walk + /// downstream-first (drain buffered work before producing more) — e.g. the + /// sort chain, to overlap its serial boundary/key scan with the parallel + /// inflate instead of starving it behind inflate on the shared pool. + pub scheduler: Arc, +} + +impl Default for PipelineConfig { + fn default() -> Self { + Self { + threads: std::thread::available_parallelism().map_or(1, std::num::NonZero::get), + stats: None, + deadlock_timeout_secs: 0, + queue_memory_total: None, + instrumentation: InstrumentationLevel::Off, + trace_path: None, + scheduler: Arc::new(super::runtime::ChainOrderScheduler), + } + } +} + +impl PipelineConfig { + /// Builder-style helper to attach a stats collector. + #[must_use] + pub fn with_stats(mut self, stats: Arc) -> Self { + self.stats = Some(stats); + self + } + + /// Builder-style helper to set the deadlock-detection timeout. + /// `0` disables; the monitor only spawns when this is non-zero + /// AND `stats` is set. + #[must_use] + pub fn with_deadlock_timeout(mut self, timeout_secs: u64) -> Self { + self.deadlock_timeout_secs = timeout_secs; + self + } + + /// Builder-style helper to set the per-worker dispatch-order policy. + /// Defaults to [`ChainOrderScheduler`](crate::runtime::ChainOrderScheduler) + /// (upstream-first); pass + /// [`DrainFirstScheduler`](crate::runtime::DrainFirstScheduler) to walk + /// downstream-first. + #[must_use] + pub fn with_scheduler(mut self, scheduler: Arc) -> Self { + self.scheduler = scheduler; + self + } + + /// Builder-style helper to set the total queue-memory budget. + /// `None` keeps per-step `ByteBounded` limits at their static + /// values; `Some(total)` enables the rebalancer. + #[must_use] + pub fn with_queue_memory_total(mut self, total: Option) -> Self { + self.queue_memory_total = total; + self + } + + /// Builder-style helper to set the instrumentation level (default `Off`). + #[must_use] + pub fn with_instrumentation(mut self, level: InstrumentationLevel) -> Self { + self.instrumentation = level; + self + } +} + +pub struct PipelineBuilder { + inner: RefCell, +} + +struct BuilderInner { + steps: Vec>, + graph: ChainGraph, +} + +impl Default for PipelineBuilder { + fn default() -> Self { + Self::new() + } +} + +impl PipelineBuilder { + #[must_use] + pub fn new() -> Self { + Self { inner: RefCell::new(BuilderInner { steps: Vec::new(), graph: ChainGraph::new() }) } + } + + /// Add the first chain link. Requires `step: Step` (i.e., a source). + #[must_use = "pipeline branches must be wired to a sink"] + pub fn chain(&self, step: S) -> Chain<'_, S::Outputs> + where + S: Step, + { + let mut inner = self.inner.borrow_mut(); + // Input arity 0, not `register_step`'s default of 1: a source's input is + // implicit, so the graph must reject any attempt to wire an edge INTO + // it. See `append_source` for the failure the default lets through. + let producer = + inner.graph.register_step_with_input_arity(step.profile().name, S::Outputs::arity(), 0); + inner.steps.push(Box::new(TypedStep::new(step))); + + Chain { builder: self, producer, branch: BranchIdx(0), _phantom: PhantomData } + } + + /// Add a source step (first link) without requiring the typed `Chain` + /// return value. Used by the `ChainBuilder` in the parent `fgumi` crate's + /// `pipeline::chains` to accumulate steps across `add_source` / + /// `add_` / `add_sink` method calls. + /// + /// Returns `(StepIdx, BranchIdx(0))` for the newly registered step. + /// The caller tracks this tail and passes it to + /// [`Self::append_step`] for all subsequent steps. + /// + /// This deliberately bypasses the typed `Chain<'_, S::Outputs>` API. + /// Type correctness is NOT validated at [`Self::build`] time — `build()` + /// only checks that the chain is non-empty (rejecting a zero-step chain with + /// `BuildError::Empty`) and that every output branch is wired. A mis-typed step + /// sequence (e.g. a step whose `Input=A` consumes an output of type `B`) + /// builds successfully and panics at the first dispatch in + /// `TypedStep::resolve_input` with "input handle downcast failed — + /// chain topology invariant". That panic is loud and immediate but it + /// is a runtime check, not a compile-time or build-time one. Callers + /// are responsible for maintaining type correctness. `pub` so the + /// `chains` layer in the parent `fgumi` crate can drive incremental + /// assembly across the crate boundary. + pub fn append_source(&self, step: S) -> (StepIdx, BranchIdx) + where + S: Step, + { + let mut inner = self.inner.borrow_mut(); + // Input arity 0 — see `chain`. Leaving this at `register_step`'s default + // of 1 makes a source look like it has a free input slot: because + // `HeapSize for ()` exists, a producer whose output shape is + // `Single<()>` type-checks against a `Step`, `wire_to_slot` + // accepts slot 0 against the bogus arity, `build()` reports the chain + // fully wired, and `build_chain_contexts_inner` still takes the + // `is_source()` branch and hands the step a dummy unit input handle — + // so the wired edge's items are never popped. Registering arity 0 makes + // the graph reject that edge at wire time instead. + let producer = + inner.graph.register_step_with_input_arity(step.profile().name, S::Outputs::arity(), 0); + inner.steps.push(Box::new(TypedStep::new(step))); + (producer, BranchIdx(0)) + } + + /// Append a step to the chain by wiring it to the current tail + /// `(prev_producer, prev_branch)`. Used by the parent crate's + /// `ChainBuilder` for the same reason as [`Self::append_source`] — + /// type-erased incremental chain assembly across method-boundary calls. + /// + /// Returns `(StepIdx, BranchIdx(0))` for the newly registered step. + pub fn append_step( + &self, + step: S, + prev: (StepIdx, BranchIdx), + ) -> (StepIdx, BranchIdx) { + let mut inner = self.inner.borrow_mut(); + let consumer = inner.graph.register_step(step.profile().name, S::Outputs::arity()); + inner.graph.wire(prev.0, prev.1, consumer); + inner.steps.push(Box::new(TypedStep::new(step))); + (consumer, BranchIdx(0)) + } + + /// Append a two-input [`Step2`] step, wiring `prev_a` into input slot 0 + /// and `prev_b` into input slot 1. The type-erased counterpart of + /// [`MultiChain2Ordered::join`] — used by the parent crate's + /// `ChainBuilder::add_zipper` to wire the unmapped and mapped source + /// chains into the zipper-merge step across method-boundary calls. + /// + /// Returns `(StepIdx, BranchIdx(0))` for the newly registered step. + pub fn append_step2( + &self, + step: S, + prev_a: (StepIdx, BranchIdx), + prev_b: (StepIdx, BranchIdx), + ) -> (StepIdx, BranchIdx) { + let mut inner = self.inner.borrow_mut(); + let consumer = + inner.graph.register_step_with_input_arity(step.profile().name, S::Outputs::arity(), 2); + inner.graph.wire_to_slot(prev_a.0, prev_a.1, consumer, 0); + inner.graph.wire_to_slot(prev_b.0, prev_b.1, consumer, 1); + inner.steps.push(Box::new(TypedStep2::new(step))); + (consumer, BranchIdx(0)) + } + + /// Finalize the chain. Returns `Err(UnwiredOutput)` if any output branch + /// is dangling, `Err(Empty)` if the chain has zero steps. + /// + /// # Errors + /// + /// See `BuildError`. + pub fn build(self) -> Result { + let inner = self.inner.into_inner(); + if inner.steps.is_empty() { + return Err(BuildError::Empty); + } + if let Some((producer, _branch, branch_name)) = inner.graph.first_unwired() { + return Err(BuildError::UnwiredOutput { + step: inner.graph.step_name(producer), + branch: branch_name, + }); + } + // Registering sources with `input_arity = 0` makes `wire_to_slot` reject + // an edge into a source built through `PipelineBuilder::{chain, + // append_source}`. It does NOT cover a source reached as a *consumer*: + // `Chain::chain` / `append_step` register any consumer with arity 1, so + // a `Step` appended there wires cleanly. `is_source()` is + // `Input == ()` regardless of how the step was registered, so check it + // here — this is the one place every wiring path converges. + for (idx, step) in inner.steps.iter().enumerate() { + if !step.is_source() { + continue; + } + let consumer = super::topology::StepIdx(idx); + if let Some(producer) = inner.graph.first_producer_into(consumer) { + return Err(BuildError::WiredIntoSource { + step: inner.graph.step_name(consumer), + producer: inner.graph.step_name(producer), + }); + } + } + Ok(Pipeline { steps: inner.steps, graph: inner.graph, signal: PipelineSignal::new() }) + } +} + +/// In-progress chain handle. Each `.chain()` call consumes self and returns +/// a fresh `Chain` rooted at the new tail (Rust move semantics enforce +/// single-consumer at the type-system level). +#[must_use = "pipeline branches must be wired to a sink"] +pub struct Chain<'b, O> { + builder: &'b PipelineBuilder, + producer: StepIdx, + branch: BranchIdx, + _phantom: PhantomData O>, +} + +impl<'b, T: Send + HeapSize + 'static> Chain<'b, Single> { + /// Extend the chain with a step accepting `T`. + #[must_use = "pipeline branches must be wired to a sink"] + pub fn chain(self, step: S) -> Chain<'b, S::Outputs> + where + S: Step, + { + let mut inner = self.builder.inner.borrow_mut(); + let consumer = inner.graph.register_step(step.profile().name, S::Outputs::arity()); + inner.graph.wire(self.producer, self.branch, consumer); + inner.steps.push(Box::new(TypedStep::new(step))); + + Chain { + builder: self.builder, + producer: consumer, + branch: BranchIdx(0), + _phantom: PhantomData, + } + } +} + +impl<'b, T: Send + super::item::HeapSize + super::item::Ordered + 'static> + Chain<'b, super::outputs::OrderedBytesSingle> +{ + /// Extend the chain with a step accepting `T`. Mirror of + /// `Chain>::chain` for the heap-aware ordered output shape + /// used by Phase 3 BAM steps. + #[must_use = "pipeline branches must be wired to a sink"] + pub fn chain(self, step: S) -> Chain<'b, S::Outputs> + where + S: Step, + { + let mut inner = self.builder.inner.borrow_mut(); + let consumer = inner.graph.register_step(step.profile().name, S::Outputs::arity()); + inner.graph.wire(self.producer, self.branch, consumer); + inner.steps.push(Box::new(TypedStep::new(step))); + + Chain { + builder: self.builder, + producer: consumer, + branch: BranchIdx(0), + _phantom: PhantomData, + } + } +} + +impl Chain<'_, ()> { + /// Convenience to drop a sink-tail chain handle without `let _ = ...`. + /// Sinks have `Outputs = ()` (zero branches) so the all-wired check + /// never flags them; this method exists only to suppress the + /// `#[must_use]` warning at the call site when a chain naturally ends + /// at a sink. + pub fn into_sink_marker(self) { + let _ = (self.builder, self.producer, self.branch); + } +} + +impl<'b, A: Send + HeapSize + 'static, B: Send + HeapSize + 'static> Chain<'b, (A, B)> { + /// Convert a 2-output chain into per-branch sub-chains. + #[must_use = "all chain branches must be wired to a sink"] + pub fn into_multi(self) -> MultiChain2<'b, A, B> { + MultiChain2 { + b0: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(0), + _phantom: PhantomData, + }, + b1: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(1), + _phantom: PhantomData, + }, + _phantom: PhantomData, + } + } +} + +impl<'b, A, B> Chain<'b, super::outputs::OrderedBytesTuple2> +where + A: Send + super::item::HeapSize + super::item::Ordered + 'static, + B: Send + super::item::HeapSize + super::item::Ordered + 'static, +{ + /// Convert a 2-output ordered + byte-bounded chain into per-branch + /// sub-chains. Each branch is exposed as + /// `Chain>` so downstream chained steps see + /// the byte-aware ordered shape (matching what + /// `Chain>::chain` accepts). + #[must_use = "all chain branches must be wired to a sink"] + pub fn into_multi(self) -> MultiChain2Ordered<'b, A, B> { + MultiChain2Ordered { + b0: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(0), + _phantom: PhantomData, + }, + b1: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(1), + _phantom: PhantomData, + }, + _phantom: PhantomData, + } + } +} + +impl<'b, A, B, C> Chain<'b, (A, B, C)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + #[must_use = "all chain branches must be wired to a sink"] + pub fn into_multi(self) -> MultiChain3<'b, A, B, C> { + MultiChain3 { + b0: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(0), + _phantom: PhantomData, + }, + b1: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(1), + _phantom: PhantomData, + }, + b2: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(2), + _phantom: PhantomData, + }, + _phantom: PhantomData, + } + } +} + +impl<'b, A, B, C, D> Chain<'b, (A, B, C, D)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + #[must_use = "all chain branches must be wired to a sink"] + pub fn into_multi(self) -> MultiChain4<'b, A, B, C, D> { + MultiChain4 { + b0: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(0), + _phantom: PhantomData, + }, + b1: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(1), + _phantom: PhantomData, + }, + b2: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(2), + _phantom: PhantomData, + }, + b3: Chain { + builder: self.builder, + producer: self.producer, + branch: BranchIdx(3), + _phantom: PhantomData, + }, + _phantom: PhantomData, + } + } +} + +#[must_use = "all chain branches must be wired to a sink"] +pub struct MultiChain2<'b, A: Send + HeapSize + 'static, B: Send + HeapSize + 'static> { + pub b0: Chain<'b, Single>, + pub b1: Chain<'b, Single>, + pub(crate) _phantom: PhantomData<&'b PipelineBuilder>, +} + +impl<'b, A: Send + HeapSize + 'static, B: Send + HeapSize + 'static> MultiChain2<'b, A, B> { + /// Construct a `MultiChain2` from two independent source-side + /// chains that each produce `Single` / `Single`. Used when + /// two parallel source subchains converge at a `Step2` consumer + /// (e.g. zipper's mapped + unmapped BAM source subchains, AAM's + /// aligner-output + original-record-buffer subchains). + /// + /// Mirrors [`Chain::into_multi`]'s output-side counterpart: that + /// method takes one producer with two output branches and splits + /// it into a `MultiChain2`; this method takes two distinct + /// producers (each with one output branch) and packages them. + /// + /// # Panics + /// + /// Panics if `a` and `b` are anchored at different + /// `PipelineBuilder` instances (i.e. the framework would have to + /// wire across pipelines, which is meaningless). + #[must_use = "pipeline branches must be wired to a sink"] + pub fn from_chains(a: Chain<'b, Single>, b: Chain<'b, Single>) -> Self { + assert!( + std::ptr::eq(a.builder, b.builder), + "MultiChain2::from_chains: both chains must be anchored at the same builder", + ); + Self { b0: a, b1: b, _phantom: PhantomData } + } + + /// Join two parallel sub-chains into a single [`Step2`] consumer. + /// + /// Wires `b0` into the consumer's input slot 0 + /// ([`StepCtx2`](crate::step::StepCtx2)'s `a`) and `b1` into input slot 1 + /// ([`StepCtx2`](crate::step::StepCtx2)'s `b`), registering the consumer with input arity + /// 2 in the chain graph. Each branch's typed producer-side + /// [`OutputQueueSet`](crate::handles::OutputQueueSet) is later (at chain-run time) drained into + /// the consumer's + /// [`crate::handles::TwoInputHandles`] + /// via [`TypedStep2::build_two_input_handles`]. + /// + /// Returns a single-branch downstream [`Chain`] typed by the + /// joined step's `S::Outputs`, ready to chain further single-input + /// steps onto. + /// + /// # Type bounds + /// + /// `S: Step2` — the joined step's input + /// types must match the two upstream branch types exactly. + /// + /// # Panics + /// + /// `wire_to_slot`'s defensive panic fires if either upstream + /// branch was already wired (the `Chain` move semantics prevent + /// this in well-formed code). + pub fn join(self, step: S) -> Chain<'b, S::Outputs> + where + S: Step2, + { + let builder = self.b0.builder; + let p0 = self.b0.producer; + let br0 = self.b0.branch; + let p1 = self.b1.producer; + let br1 = self.b1.branch; + + let mut inner = builder.inner.borrow_mut(); + let consumer = + inner.graph.register_step_with_input_arity(step.profile().name, S::Outputs::arity(), 2); + inner.graph.wire_to_slot(p0, br0, consumer, 0); + inner.graph.wire_to_slot(p1, br1, consumer, 1); + inner.steps.push(Box::new(TypedStep2::new(step))); + + Chain { builder, producer: consumer, branch: BranchIdx(0), _phantom: PhantomData } + } +} + +/// Ordered-bytes variant of `MultiChain2`. Each branch is typed as +/// `Chain>` so downstream chained steps see the +/// byte-aware ordered representation (matching the queue topology +/// `OrderedBytesTuple2` actually constructed). +#[must_use = "all chain branches must be wired to a sink"] +pub struct MultiChain2Ordered<'b, A, B> +where + A: Send + super::item::HeapSize + super::item::Ordered + 'static, + B: Send + super::item::HeapSize + super::item::Ordered + 'static, +{ + pub b0: Chain<'b, super::outputs::OrderedBytesSingle>, + pub b1: Chain<'b, super::outputs::OrderedBytesSingle>, + pub(crate) _phantom: PhantomData<&'b PipelineBuilder>, +} + +impl<'b, A, B> MultiChain2Ordered<'b, A, B> +where + A: Send + super::item::HeapSize + super::item::Ordered + 'static, + B: Send + super::item::HeapSize + super::item::Ordered + 'static, +{ + /// Construct a `MultiChain2Ordered` from two independent + /// source-side chains that each produce `OrderedBytesSingle` / + /// `OrderedBytesSingle`. The ordered/byte-bounded counterpart + /// of [`MultiChain2::from_chains`] — used when two parallel + /// source subchains coming out of ordered, byte-bounded BAM/FASTQ + /// step libraries (decompress → boundaries → decode → group) need + /// to converge at a [`Step2`] consumer. + /// + /// Anchored at the same `PipelineBuilder` as both input chains; + /// panics otherwise. + /// + /// # Panics + /// + /// Panics if `a` and `b` are anchored at different + /// `PipelineBuilder` instances. + #[must_use = "pipeline branches must be wired to a sink"] + pub fn from_chains( + a: Chain<'b, super::outputs::OrderedBytesSingle>, + b: Chain<'b, super::outputs::OrderedBytesSingle>, + ) -> Self { + assert!( + std::ptr::eq(a.builder, b.builder), + "MultiChain2Ordered::from_chains: both chains must be anchored at the same builder", + ); + Self { b0: a, b1: b, _phantom: PhantomData } + } + + /// Join two parallel ordered/byte-bounded sub-chains into a single + /// [`Step2`] consumer. Ordered counterpart of + /// [`MultiChain2::join`]: wires `b0` into the consumer's input + /// slot 0 ([`StepCtx2`](crate::step::StepCtx2)'s `a`) and `b1` into input slot 1 + /// ([`StepCtx2`](crate::step::StepCtx2)'s `b`), registering the consumer with input arity 2. + /// + /// Returns a single-branch downstream [`Chain`] typed by the + /// joined step's `S::Outputs`. + /// + /// # Type bounds + /// + /// `S: Step2` — the joined step's input + /// types must match the two upstream branch element types exactly. + /// The framework only requires `HeapSize` on `Step2::InputA` / + /// `Step2::InputB`; the `Ordered` bound carried by + /// `OrderedBytesSingle` is upstream-side typing and does not flow + /// into the consumer's input handle (steps see plain + /// `InputHandle` regardless of upstream queue topology). + /// + /// # Panics + /// + /// `wire_to_slot`'s defensive panic fires if either upstream + /// branch was already wired (the `Chain` move semantics prevent + /// this in well-formed code). + pub fn join(self, step: S) -> Chain<'b, S::Outputs> + where + S: Step2, + { + let builder = self.b0.builder; + let p0 = self.b0.producer; + let br0 = self.b0.branch; + let p1 = self.b1.producer; + let br1 = self.b1.branch; + + let mut inner = builder.inner.borrow_mut(); + let consumer = + inner.graph.register_step_with_input_arity(step.profile().name, S::Outputs::arity(), 2); + inner.graph.wire_to_slot(p0, br0, consumer, 0); + inner.graph.wire_to_slot(p1, br1, consumer, 1); + inner.steps.push(Box::new(TypedStep2::new(step))); + + Chain { builder, producer: consumer, branch: BranchIdx(0), _phantom: PhantomData } + } +} + +#[must_use = "all chain branches must be wired to a sink"] +pub struct MultiChain3<'b, A, B, C> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + pub b0: Chain<'b, Single>, + pub b1: Chain<'b, Single>, + pub b2: Chain<'b, Single>, + pub(crate) _phantom: PhantomData<&'b PipelineBuilder>, +} + +#[must_use = "all chain branches must be wired to a sink"] +pub struct MultiChain4<'b, A, B, C, D> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + pub b0: Chain<'b, Single>, + pub b1: Chain<'b, Single>, + pub b2: Chain<'b, Single>, + pub b3: Chain<'b, Single>, + pub(crate) _phantom: PhantomData<&'b PipelineBuilder>, +} + +/// A built pipeline ready to run. +/// +/// `steps` is consumed by `Pipeline::run`; `graph` is read by `run` and +/// `dag()`; `signal` backs both `cancel_handle` and `run`'s outcome +/// plumbing. +pub struct Pipeline { + pub(crate) steps: Vec>, + pub(crate) graph: ChainGraph, + pub(crate) signal: Arc, +} + +impl Pipeline { + #[must_use] + pub fn builder() -> PipelineBuilder { + PipelineBuilder::new() + } + + #[must_use] + pub fn cancel_handle(&self) -> CancelHandle { + CancelHandle::from_signal(Arc::clone(&self.signal)) + } + + /// Construct a fresh `PipelineStats` collector sized to this pipeline's + /// chain. Wrap the returned `Arc` and pass it into `PipelineConfig::stats` + /// (or via `PipelineConfig::with_stats`) before calling `run`. Counters + /// can be read at any time after the run completes. + #[must_use] + pub fn stats(&self) -> Arc { + let names: Vec<&'static str> = self.steps.iter().map(|s| s.profile().name).collect(); + Arc::new(PipelineStats::new(names)) + } + + /// Render the chain shape as a multi-line debug string. Lists each step + /// in chain order with its profile (kind, sticky, branch count) and + /// the consumer for each output branch. Used for diagnostics and for + /// runall-style `--explain` output. + /// + /// The output isn't a stable serialization format — it's a developer- + /// readable summary, intended to be `println!`'d during debugging or + /// embedded in error messages. + #[must_use] + pub fn dag(&self) -> String { + use std::fmt::Write as _; + + let mut s = String::new(); + let _ = writeln!( + s, + "Pipeline DAG ({} step{}):", + self.steps.len(), + if self.steps.len() == 1 { "" } else { "s" } + ); + for (idx, step) in self.steps.iter().enumerate() { + let profile = step.profile(); + let n_branches = self.graph.branch_count(super::topology::StepIdx(idx)); + // Render the *effective* (post-collapse) ordering so the diagnostic + // matches the transport actually built. Single-input Serial / + // Exclusive producers collapse declared `ByOrdinal` / `ByItemOrdinal` + // to `None` (no reorder stage); `Step2` producers (input_arity == 2) + // and `Parallel` producers keep their declared ordering verbatim. + // Using the shared `effective_branch_orderings` helper keeps `dag()` + // and `build_output_set` from drifting. + let effective_orderings = if step.input_arity() == 2 { + profile.branch_ordering.clone() + } else { + super::erased::effective_branch_orderings(profile.kind, &profile.branch_ordering) + }; + let _ = write!( + s, + " [{idx}] {name:<24} {kind:?} sticky={sticky} branches={n_branches}", + idx = idx, + name = profile.name, + kind = profile.kind, + sticky = profile.sticky, + n_branches = n_branches, + ); + if n_branches == 0 { + let _ = writeln!(s, " (sink)"); + } else { + let _ = writeln!(s); + for branch_usize in 0..n_branches { + let branch = super::topology::BranchIdx(branch_usize); + let consumer = self.graph.consumer(super::topology::StepIdx(idx), branch); + let consumer_name = match consumer { + Some(c) => self.graph.step_name(c), + None => "", + }; + let queue_spec = profile + .output_queues + .get(branch_usize) + .copied() + .unwrap_or(super::queues::QueueSpec::Unbounded); + let ordering = effective_orderings + .get(branch_usize) + .copied() + .unwrap_or(super::reorder::BranchOrdering::None); + let _ = writeln!( + s, + " .{branch_usize}: {queue_spec:?} {ordering:?} → {consumer_name}", + ); + } + } + } + s + } + + /// Run the pipeline to completion. + /// + /// Spawns `config.threads` worker threads, runs each step's `try_run` + /// until every step has reported `Finished`, joins on completion, and + /// returns `Ok(())` on clean exit or an `Err(PipelineError)` if any step + /// returned `Err` or the caller cancelled via the `CancelHandle`. + /// + /// # Errors + /// + /// Returns `PipelineError::NotEnoughThreads` if the chain has more + /// `Exclusive` steps than `config.threads`. Returns `PipelineError::Io` + /// if any step's `try_run` returned `Err`. + /// Returns `PipelineError::Cancelled` if `cancel_handle().cancel()` was + /// called during the run. + /// + /// # Panics + /// + /// Panics if a worker thread panics — the panic is propagated via + /// `JoinHandle::join`. Worker panics indicate a framework or step bug + /// (e.g., a contract violation that triggered a `debug_assert!`). + #[allow(clippy::too_many_lines)] + pub fn run(self, config: PipelineConfig) -> Result<(), super::signal::PipelineError> { + use std::thread; + + use super::runtime::{ + StepDrainCounter, WorkerCore, assign_exclusive_owners, assign_sticky_owners, + build_chain_contexts, build_worker_storage, extract_detached_steps, + run_detached_driver, run_fused_single_thread, run_worker_loop, + should_fuse_single_thread, + }; + use super::step::{DetachedGroup, StepKind}; + use super::topology::StepIdx; + + let Self { mut steps, graph, signal } = self; + let n_threads = config.threads; + let stats_arc = config.stats; + let deadlock_timeout_secs = config.deadlock_timeout_secs; + let scheduler = Arc::clone(&config.scheduler); + assert!(n_threads > 0, "PipelineConfig::threads must be > 0"); + if let Some(stats) = stats_arc.as_ref() { + assert_eq!( + stats.n_steps(), + steps.len(), + "PipelineConfig::stats was sized for {} steps but pipeline has {}; \ + obtain the stats handle from `Pipeline::stats()` after `build()`", + stats.n_steps(), + steps.len() + ); + } + if deadlock_timeout_secs > 0 && stats_arc.is_none() { + log::warn!( + "PipelineConfig::deadlock_timeout_secs is {deadlock_timeout_secs} but \ + PipelineConfig::stats is None; deadlock monitor cannot run without \ + stats. Disable one or pair them via the helpers in commands/common.rs." + ); + } + + // 0. Fused single-thread fast path (issue #330). A single-source + // source→sink chain at one worker is driven inline over direct buffers, + // skipping the scheduler's round-robin poll / contention / reorder + // overhead (~2/3 of `try_run` calls at t=1 are otherwise wasted). + // Fan-out is allowed (e.g. the `--rejects` kept/rejects split); only + // two-input `Step2` merges (zipper, align) and `--threads ≥ 2` fall + // through to the scheduled worker pool below. The deadlock monitor and + // queue rebalancer are not spawned: a single-worker inline drive cannot + // deadlock, and its edges are driven producer-then-consumer in one pass, + // so there is no cross-worker imbalance to rebalance. Those edges DO + // carry each step's profiled byte bound, so `queue_memory_total` is + // handed to `run_fused_single_thread` and applied to them there — the + // fused contexts are built inside that call, not here. + // Instrumentation forces the scheduled path (see `should_fuse_single_thread`): + // the fused path has no per-edge metrics / occupancy sampler / verdict. + if should_fuse_single_thread(n_threads, config.instrumentation, &steps, &graph) { + log::debug!( + "Using fused single-thread pipeline ({} steps, direct buffers)", + steps.len() + ); + let result = run_fused_single_thread( + steps, + &graph, + &signal, + stats_arc.as_ref(), + config.queue_memory_total, + deadlock_timeout_secs, + ); + // Same end-of-run stats snapshot the scheduled path emits, so + // `--pipeline-stats` shows the fused chain's per-step counters. + if let Some(stats) = stats_arc.as_ref() { + let snapshot = stats.snapshot(); + log::info!("Pipeline end-of-run stats:"); + for line in format!("{snapshot}").lines() { + log::info!("{line}"); + } + } + return result; + } + + // 1. Assign Exclusive owners; bail if too many. + let owners = assign_exclusive_owners(&steps, n_threads)?; + + // 1a. Compute per-worker sticky-driven step (Exclusive sticky union + // with Serial+sticky+Affinity targeting). Indexed by worker id. + let sticky_owners = assign_sticky_owners(&steps, &owners, n_threads); + + // 2. Build per-step contexts (input + output handles). + let contexts = Arc::new(build_chain_contexts(&steps, &graph, config.instrumentation)); + + // 2-pre-monitor invariant: if the deadlock monitor will be armed, every + // output transport must be ByteBounded so `in_flight_bytes` can see a + // wedge on it. A CountBounded/Unbounded edge is invisible to the probe + // and would silently disable fail-fast on that edge. Checked here in + // every build (not debug-only — it returns a real `PipelineError`) while + // `steps` is still alive, before it is consumed by + // `build_worker_storage` below. Test chains use CountBounded/Unbounded + // but do not arm the monitor, so this only fires for a fail-fast run. + if deadlock_timeout_secs > 0 && stats_arc.is_some() { + ensure_monitor_visible_transports(&steps, &graph)?; + } + + // 2a. If a total queue-memory budget was supplied, evenly + // redistribute it across all byte-bounded queues now (before + // workers start) so the initial state matches the user's + // budget instead of the per-step defaults baked into each + // step's `QueueSpec::ByteBounded { limit_bytes }`. Floor at + // 1 MiB per queue to prevent zero-budget queues that would + // always reject pushes. + if let Some(total) = config.queue_memory_total { + apply_initial_queue_budget(&contexts.bounded_queues, total); + } + + // 3. Per-step drain counter — init N for Parallel, 1 otherwise. + let drain_counters: Vec> = steps + .iter() + .map(|step| { + let initial = match step.profile().kind { + StepKind::Parallel => n_threads, + // `Detached` runs on a single dedicated thread, so (like + // `Serial`/`Exclusive`) exactly one finisher closes its + // output edge. + StepKind::Serial | StepKind::Exclusive | StepKind::Detached => 1, + }; + StepDrainCounter::new(initial) + }) + .collect(); + + // 3a. Extract `Detached` steps' real instances for their dedicated + // threads, leaving same-position placeholders so `build_worker_storage` + // (which Skips Detached on every worker) and the `step_idx`-aligned + // `ChainContexts` stay correct. The contexts were already built from + // `&steps` above, so each extracted step's input/output handles live in + // `contexts[step_idx]`. Done before `build_worker_storage` consumes + // `steps`. Empty for every non-sort chain (nothing declares Detached). + let detached_steps = extract_detached_steps(&mut steps); + + // 4. Build per-worker step storage (consumes `steps`). + let mut worker_entries = build_worker_storage(steps, &owners, n_threads); + + let signal_arc = Arc::clone(&signal); + + // 4a. Optional deadlock-detection monitor. Spawns a watcher + // thread that periodically samples the stats snapshot; if no + // step's `progress + finished` counter has advanced for + // `deadlock_timeout_secs` seconds, it logs the snapshot at + // `warn` level so the user has a starting point. Polls every + // `max(1, deadlock_timeout_secs / 4)` seconds. + let (monitor_stop, monitor_handle) = match (deadlock_timeout_secs, stats_arc.as_ref()) { + (n, Some(stats)) if n > 0 => { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + let stats_clone = Arc::clone(stats); + let contexts_clone = Arc::clone(&contexts); + let signal_clone = Arc::clone(&signal); + let warn_timeout = std::time::Duration::from_secs(deadlock_timeout_secs); + let fatal_timeout = std::time::Duration::from_secs( + deadlock_timeout_secs.saturating_mul(DEADLOCK_FATAL_MULTIPLE), + ); + let poll_interval = + std::time::Duration::from_secs(deadlock_timeout_secs.max(4) / 4); + let handle = thread::Builder::new() + .name("fgumi-deadlock-monitor".to_string()) + .spawn(move || { + run_deadlock_monitor( + &stop_clone, + &stats_clone, + &contexts_clone, + &signal_clone, + warn_timeout, + fatal_timeout, + poll_interval, + ); + }) + .expect("failed to spawn deadlock monitor thread"); + (Some(stop), Some(handle)) + } + _ => (None, None), + }; + + // 4b. Optional queue-memory rebalancer. Spawns a watcher + // thread that periodically samples each registered queue's + // `current_bytes / limit_bytes` ratio and shifts budget + // toward consistently-full queues at the expense of + // consistently-empty ones. Total budget is preserved. + let (rebalancer_stop, rebalancer_handle) = + if config.queue_memory_total.is_some() && !contexts.bounded_queues.is_empty() { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + // Capture handles into a contiguous Vec — the monitor + // doesn't need step indices/names beyond debug logging. + let handles: Vec> = + contexts.bounded_queues.iter().map(|rq| Arc::clone(&rq.handle)).collect(); + let names: Vec<&'static str> = + contexts.bounded_queues.iter().map(|rq| rq.producer_step_name).collect(); + let handle = thread::Builder::new() + .name("fgumi-queue-rebalancer".to_string()) + .spawn(move || { + run_queue_rebalancer(&stop_clone, &handles, &names); + }) + .expect("failed to spawn queue rebalancer thread"); + (Some(stop), Some(handle)) + } else { + (None, None) + }; + + // 4c. Optional occupancy sampler (`--pipeline-trace`). Polls each + // byte-bounded edge's depth into its `EdgeMetrics` histogram on a fixed + // cadence. Runs whenever instrumentation samples AND there are edges to + // sample (`contexts.edges` is empty at level `Off` and on the fused + // single-thread fast path, so this stays inert there). Read-only over + // the live queues — never perturbs the worker hot path. + let (sampler_stop, sampler_handle) = + if config.instrumentation.samples() && !contexts.edges.is_empty() { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + let contexts_clone = Arc::clone(&contexts); + // Timeline TSV path only when the level requests it. + let trace_path = if config.instrumentation.timeline() { + Some( + config + .trace_path + .clone() + .unwrap_or_else(|| std::path::PathBuf::from("pipeline-trace.tsv")), + ) + } else { + None + }; + let handle = thread::Builder::new() + .name("fgumi-occupancy-sampler".to_string()) + .spawn(move || { + crate::runtime::sampler::run_occupancy_sampler( + &stop_clone, + &contexts_clone.edges, + crate::runtime::sampler::DEFAULT_SAMPLE_INTERVAL, + trace_path, + ); + }) + .expect("failed to spawn occupancy sampler thread"); + (Some(stop), Some(handle)) + } else { + (None, None) + }; + + // 4d. Spawn one dedicated OS thread per `Detached` driver GROUP, in chain + // order, BEFORE the workers — same lifecycle slot as the monitor / + // rebalancer / sampler. Each thread drives its group with the SAME + // `run_worker_loop` the pool uses (via `run_detached_driver`), off the + // work-stealing pool, so a Detached step never consumes a pool worker + // slot (the "N + 2" threading). A group with one step is the legacy + // one-thread-per-detached-step case (`DetachedGroup::PerStep`); a + // `Shared` group co-locates several steps on one driver thread. Joined + // after the workers (step 6d). Empty for every non-sort chain, so this + // is a no-op there. + let detached_handles: Vec> = detached_steps + .into_iter() + .map(|group| { + let contexts_clone = Arc::clone(&contexts); + let signal_clone = Arc::clone(&signal_arc); + let drain_counters_clone: Vec> = + drain_counters.iter().map(Arc::clone).collect(); + let stats_clone = stats_arc.as_ref().map(Arc::clone); + let thread_name = match group.label() { + DetachedGroup::Shared(label) => format!("fgumi-driver-{label}"), + DetachedGroup::PerStep => { + format!("fgumi-detached-{}", group.primary_name()) + } + }; + thread::Builder::new() + .name(thread_name) + .spawn(move || { + // Catch a driver-thread panic so we can signal + // cancellation before unwinding — a wedged pool worker + // parked on this group's (now-dead) edge only exits on + // `is_done()`. Re-raise after signalling so the join in + // step 6d still collects the payload. + if let Err(panic) = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_detached_driver( + group, + &contexts_clone, + &drain_counters_clone, + &signal_clone, + stats_clone.as_ref(), + ); + })) + { + signal_clone.cancel(); + std::panic::resume_unwind(panic); + } + }) + .expect("failed to spawn detached driver thread") + }) + .collect(); + + // Holds the first worker panic payload; re-raised after helper threads + // are cleaned up so monitor/rebalancer shutdown always executes. + let mut worker_panic: Option> = None; + + if n_threads == 1 { + // Single-threaded fast path: run the worker loop directly on + // the caller's thread instead of spawning + joining a fresh + // OS thread. The framework machinery (`build_worker_storage`, + // drain counters, stats) is identical to the multi-threaded + // path — only the spawn/join is skipped. + // + // Savings: thread-spawn-and-join (a few ms one-time on Apple + // Silicon / Linux), and the caller's thread name / TLS is + // preserved (matters for log correlation in some tools). + // + // Per-call cost gap vs the legacy single-threaded path is + // dominated by the framework's per-record book-keeping + // (queue byte tracking, ordinal allocation, drain checks), + // not the spawn — see commit message benchmarks. + let entries = worker_entries + .pop() + .expect("build_worker_storage with n_threads=1 returns one entry vec"); + debug_assert!(worker_entries.is_empty()); + let exclusive_owner = owners + .iter() + .enumerate() + .find_map(|(idx, &own)| if own == Some(0) { Some(StepIdx(idx)) } else { None }); + let sticky_owner = sticky_owners[0]; + let mut worker = WorkerCore::new(0, exclusive_owner, sticky_owner); + let mut entries_local = entries; + // Defer a panic on the single-threaded fast path the same way the + // multi-worker join loop does: capture the payload, signal + // cancellation, and let the common monitor/rebalancer shutdown run + // before re-raising at step 7. Without this, a worker-loop panic + // unwinds straight through the caller and leaks the helper threads. + // `AssertUnwindSafe` is sound: after a panic we never touch `worker` + // or `entries_local` again — the run is shutting down. + if let Err(panic) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_worker_loop( + &mut worker, + &mut entries_local, + &contexts, + &drain_counters, + &signal_arc, + stats_arc.as_ref(), + scheduler.as_ref(), + ); + })) { + signal_arc.cancel(); + worker_panic = Some(panic); + } + } else { + // 5. Spawn worker threads. + let mut handles = Vec::with_capacity(n_threads); + for (worker_id, entries) in worker_entries.into_iter().enumerate() { + let exclusive_owner = owners.iter().enumerate().find_map(|(idx, &own)| { + if own == Some(worker_id) { Some(StepIdx(idx)) } else { None } + }); + let sticky_owner = sticky_owners[worker_id]; + + let contexts_clone = Arc::clone(&contexts); + let signal_clone = Arc::clone(&signal_arc); + let drain_counters_clone: Vec> = + drain_counters.iter().map(Arc::clone).collect(); + let stats_clone = stats_arc.as_ref().map(Arc::clone); + let scheduler_clone = Arc::clone(&scheduler); + + let handle = thread::Builder::new() + .name(format!("fgumi-worker-{worker_id}")) + .spawn(move || { + let mut worker = WorkerCore::new(worker_id, exclusive_owner, sticky_owner); + let mut entries_local = entries; + // Catch a worker-loop panic so we can signal cancellation + // *before* unwinding. A peer parked in its retry loop on a + // full/empty queue only exits when it observes + // `signal.is_done()`; without an early `cancel()` here, the + // join loop below could block forever on an earlier, + // now-wedged worker and never reach this thread's panic. + // We re-raise after signalling so the join still collects + // the payload (preserving the deferred re-raise at step 7). + // `AssertUnwindSafe` is sound: on panic the run is tearing + // down and neither local is used again. + if let Err(panic) = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_worker_loop( + &mut worker, + &mut entries_local, + &contexts_clone, + &drain_counters_clone, + &signal_clone, + stats_clone.as_ref(), + scheduler_clone.as_ref(), + ); + })) + { + signal_clone.cancel(); + std::panic::resume_unwind(panic); + } + }) + .expect("failed to spawn worker thread"); + handles.push(handle); + } + + // 6. Join workers. Capture the first worker panic payload so cleanup + // can proceed; re-raise after monitor/rebalancer threads are stopped. + for h in handles { + if let Err(panic) = h.join() + && worker_panic.is_none() + { + worker_panic = Some(panic); + } + } + } + + // 6d. Join the Detached step threads (after the workers). The sort + // writer (Detached, consuming the pool's Compress output) and merge + // (Detached, producing to the pool's Serialize input) finish their final + // flush only after their pool peers have drained, so joining them here — + // after the worker join, before the monitor stop — captures the full + // chain completion (and any Detached-thread panic) in the same deferred + // re-raise path the workers use. A no-op when no step is Detached. + for h in detached_handles { + if let Err(panic) = h.join() + && worker_panic.is_none() + { + worker_panic = Some(panic); + } + } + + // 6a. Stop and join the deadlock monitor (if spawned). We + // signal stop *after* workers join so the monitor sees the + // final stats state and doesn't fire spurious warnings during + // normal pipeline shutdown (where steps stop progressing + // because they're done, not stuck). + if let Some(stop) = monitor_stop { + stop.store(true, std::sync::atomic::Ordering::Relaxed); + } + if let Some(handle) = monitor_handle { + let _ = handle.join(); + } + + // 6b. Stop and join the queue rebalancer (if spawned). + if let Some(stop) = rebalancer_stop { + stop.store(true, std::sync::atomic::Ordering::Relaxed); + } + if let Some(handle) = rebalancer_handle { + let _ = handle.join(); + } + + if let Some(stop) = sampler_stop { + stop.store(true, std::sync::atomic::Ordering::Relaxed); + } + if let Some(handle) = sampler_handle { + let _ = handle.join(); + } + + // 6c. End-of-run stats snapshot — emitted at info-level so + // bench profiling can see which step accumulated the most + // wall-clock time inside `try_run`. Only logged when stats + // were enabled (`with_stats(...)` on the config); cheap + // either way. + if let Some(stats) = stats_arc.as_ref() { + // When instrumentation is on, fold the per-edge table + bottleneck + // verdict into the snapshot (the edges live in `contexts`, only + // reachable here inside `run`); otherwise the edge-less snapshot. + let snapshot = if config.instrumentation.is_on() { + stats.snapshot_with_edges(&contexts.edges, stats.elapsed_ns()) + } else { + stats.snapshot() + }; + log::info!("Pipeline end-of-run stats:"); + for line in format!("{snapshot}").lines() { + log::info!("{line}"); + } + } + + // 7. Re-raise worker panics after helper threads are cleaned up so + // monitor/rebalancer shutdown code always executes. + if let Some(panic) = worker_panic { + std::panic::resume_unwind(panic); + } + + // 8. Surface error or cancellation. PipelineError isn't Clone + // (io::Error isn't Clone); `to_result` reconstructs the recorded + // outcome and, for an external cancel whose payload isn't yet visible + // to this thread, synthesizes `Cancelled` from the terminal state. + let _ = graph; + signal.to_result() + } +} + +/// Default multiple of the warn window (`--deadlock-timeout`) after which a +/// persistent stall *with work still in flight* is treated as a fatal wedge. +/// Diverges from the legacy single-window kill: a single huge dispatch +/// (busy-locus group, large sort merge) can legitimately flatten progress for +/// many seconds, so we only fail after the stall persists well past the warn +/// window. A real deadlock hangs forever, so waiting longer to be sure is free. +const DEADLOCK_FATAL_MULTIPLE: u64 = 6; + +/// Total bytes currently held across the **byte-bounded** transport queues and +/// their reorder overflow stashes. Non-zero means work is stuck on a byte-bounded +/// edge; zero means those edges are idle (e.g. waiting on a slow upstream pipe). +/// This is the signal that distinguishes a real wedge from upstream starvation +/// on byte-bounded edges. +/// +/// LIMITATION: only `ByteBounded` branches register a queue handle (see +/// `build_chain_contexts_inner`), so a `CountBounded`/`Unbounded` transport — and +/// any reorder stash on such a branch — would not contribute here, making the +/// monitor blind to a wedge living entirely on such an edge. +/// +/// This is safe because **no production pipeline edge uses +/// `CountBounded`/`Unbounded`**: every production step profile declares +/// `QueueSpec::ByteBounded`, so the probe covers every production transport. +/// (The sort spill→merge path is fully byte-bounded too — its deadlock-free +/// backpressure is the internal `SortMergeSlot` slot table, not a pipeline +/// `CountBounded` edge.) [`ensure_monitor_visible_transports`] enforces this +/// invariant in **every** build (it returns +/// [`PipelineError::MonitorBlindTransport`], not a debug assertion) whenever the +/// monitor is armed, so a future step that declares a monitor-blind transport on +/// a fail-fast pipeline fails at startup rather than silently losing the wedge +/// verdict. Extending the probe to count-based queues for full defense-in-depth +/// is tracked separately. +/// +/// SECOND LIMITATION: byte accounting is `T::heap_size()`-only — +/// `ByteBoundedQueue` never counts `size_of::()` — so a `ByteBounded` edge +/// holding items whose `heap_size()` is 0 also reports zero here, however many +/// are queued. A wedge stranding only such items therefore classifies as +/// [`StallVerdict::Starving`], which resets the stall clock on every poll, so +/// `deadlock_timeout_secs` can never fail it. Production item types +/// (record batches, BGZF blocks) all carry real heap payloads, so this affects +/// zero-heap item types only. +fn in_flight_bytes(contexts: &crate::runtime::contexts::ChainContexts) -> u64 { + contexts + .bounded_queues + .iter() + .map(|rq| { + rq.handle.current_bytes() + + rq.reorder_cap.as_ref().map_or(0, |r| r.current_buffer_bytes()) + }) + .sum() +} + +/// Return the first `(step_name, QueueSpec)` whose output transport is invisible +/// to the deadlock monitor's [`in_flight_bytes`] probe — i.e. a `CountBounded` +/// or `Unbounded` branch, which registers no byte-probe handle. `None` means +/// every output transport is monitor-visible (`ByteBounded`). +/// +/// Used by [`ensure_monitor_visible_transports`] to pin the "no production edge +/// is monitor-blind" invariant the [`in_flight_bytes`] doc relies on. Iterates +/// every branch the graph declares for each step (`0..branch_count`) rather than +/// just the explicit `output_queues` entries: a step may declare fewer specs than +/// it has branches, and `dag()`/context-building resolve those missing branches +/// to `QueueSpec::Unbounded`. Using the same `unwrap_or(Unbounded)` fallback here +/// keeps the guard from overlooking an implicit (and therefore monitor-blind) +/// Unbounded branch. +fn first_monitor_blind_transport( + steps: &[Box], + graph: &super::topology::ChainGraph, +) -> Option<(&'static str, super::queues::QueueSpec)> { + use super::queues::QueueSpec; + for (step_idx, step) in steps.iter().enumerate() { + let profile = step.profile(); + for branch in 0..graph.branch_count(super::topology::StepIdx(step_idx)) { + let spec = profile.output_queues.get(branch).copied().unwrap_or(QueueSpec::Unbounded); + match spec { + QueueSpec::ByteBounded { .. } => {} + QueueSpec::CountBounded { .. } | QueueSpec::Unbounded => { + return Some((profile.name, spec)); + } + } + } + } + None +} + +/// Invariant check (run only when the deadlock monitor is armed): every +/// production output transport must be `ByteBounded` so the [`in_flight_bytes`] +/// probe can see a wedge on it. A `CountBounded`/`Unbounded` edge would be +/// invisible to the monitor, silently disabling fail-fast on that edge — exactly +/// the blind spot this guard exists to catch. The framework still permits +/// `CountBounded`/`Unbounded` for `#[cfg(test)]` chains (which do not arm the +/// monitor), so this only fires for a real fail-fast pipeline. +/// +/// Returns a [`PipelineError::MonitorBlindTransport`] (rather than panicking or +/// being a debug-only check) so the guard runs in release builds — where the +/// blind spot actually matters — yet a misconfigured chain fails gracefully at +/// startup, consistent with the other build/run-time validations (e.g. +/// [`PipelineError::NotEnoughThreads`]) rather than crashing the process. +fn ensure_monitor_visible_transports( + steps: &[Box], + graph: &super::topology::ChainGraph, +) -> Result<(), super::signal::PipelineError> { + if let Some((name, spec)) = first_monitor_blind_transport(steps, graph) { + return Err(super::signal::PipelineError::MonitorBlindTransport { + step: name, + spec: format!("{spec:?}"), + }); + } + Ok(()) +} + +/// Background deadlock monitor body. Polls `stats` every `poll_interval`, +/// tracking the cumulative `progress + finished` counter across all steps. +/// +/// On a stall (no advance), it consults [`in_flight_bytes`] to tell a real +/// wedge from upstream starvation (mirrors legacy `check_deadlock_and_restore`): +/// - idle with nothing in flight → starvation, reset the clock, keep watching; +/// - stuck work past `warn_timeout` → `warn` snapshot, once per warn window; +/// - stuck work past `fatal_timeout` → record [`PipelineError::TimedOut`] and +/// `cancel()`, so workers observe `is_done()` and the run fails fast instead +/// of hanging forever. +/// +/// Exits when `stop` is set (workers joined) or the pipeline is already done. +fn run_deadlock_monitor( + stop: &Arc, + stats: &Arc, + contexts: &Arc, + signal: &Arc, + warn_timeout: std::time::Duration, + fatal_timeout: std::time::Duration, + poll_interval: std::time::Duration, +) { + let mut mon_state = StallMonitorState { + last_total: total_progress(stats), + stall_start: std::time::Instant::now(), + last_warn: None, + }; + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + sleep_until_stop(stop, poll_interval); + if stop.load(std::sync::atomic::Ordering::Relaxed) || signal.is_done() { + break; + } + let now = std::time::Instant::now(); + let now_total = total_progress(stats); + let progressed = now_total != mon_state.last_total; + let stall_secs = now.duration_since(mon_state.stall_start).as_secs(); + let stuck = in_flight_bytes(contexts); + let verdict = classify_stall( + progressed, + stall_secs, + warn_timeout.as_secs(), + fatal_timeout.as_secs(), + stuck, + ); + if apply_stall_verdict( + verdict, + now, + now_total, + stall_secs, + stuck, + warn_timeout, + stats, + signal, + &mut mon_state, + ) { + break; + } + } +} + +/// Rolling state the deadlock monitor carries across poll iterations. +struct StallMonitorState { + last_total: u64, + stall_start: std::time::Instant, + last_warn: Option, +} + +/// React to one classified [`StallVerdict`], mutating the rolling monitor +/// `mon_state` and performing the verdict's side effect (a warn snapshot, or +/// recording a fatal [`PipelineError::TimedOut`] + `cancel`). Extracted from +/// [`run_deadlock_monitor`]'s poll loop so each per-verdict action is +/// unit-testable without spawning a thread or waiting on wall-clock time — +/// mirroring the pure `classify_stall` / `in_flight_bytes` split. Returns +/// `true` when a fatal wedge was recorded and the monitor should stop. +#[allow(clippy::too_many_arguments)] +fn apply_stall_verdict( + verdict: StallVerdict, + now: std::time::Instant, + now_total: u64, + stall_secs: u64, + stuck: u64, + warn_timeout: std::time::Duration, + stats: &PipelineStats, + signal: &PipelineSignal, + mon_state: &mut StallMonitorState, +) -> bool { + use super::signal::PipelineError; + match verdict { + StallVerdict::Progressing => { + mon_state.last_total = now_total; + mon_state.stall_start = now; + mon_state.last_warn = None; + } + StallVerdict::Starving => { + // Idle, waiting on a slow upstream — not a deadlock. Reset the + // stall clock so an idle gap never accumulates toward a fatal. + mon_state.stall_start = now; + mon_state.last_warn = None; + } + StallVerdict::Watching => {} + StallVerdict::Stalled => { + // Warn at most once per warn window: a long stall emits periodic + // diagnostics without spamming every poll. + if mon_state.last_warn.is_none_or(|w| now.duration_since(w) >= warn_timeout) { + log::warn!( + "Pipeline stall: no progress for {stall_secs}s with {stuck} bytes \ + still in flight. Snapshot follows." + ); + let snapshot = stats.snapshot(); + for line in format!("{snapshot}").lines() { + log::warn!("{line}"); + } + mon_state.last_warn = Some(now); + } + } + StallVerdict::Wedged => { + log::error!( + "Pipeline deadlock: no progress for {stall_secs}s with {stuck} bytes \ + stuck in flight; failing the pipeline. Snapshot follows." + ); + let snapshot = stats.snapshot(); + for line in format!("{snapshot}").lines() { + log::error!("{line}"); + } + signal.record_error(PipelineError::TimedOut { stalled_secs: stall_secs }); + signal.cancel(); + return true; + } + } + false +} + +/// Sum of `progress_count + finished_count` across all steps. The +/// monitor uses this as a single scalar progress watermark; a change +/// means *some* step is making forward progress. +fn total_progress(stats: &PipelineStats) -> u64 { + let snap = stats.snapshot(); + snap.steps.iter().map(|(_, s)| s.progress_count + s.finished_count).sum() +} + +/// Sleep up to `dur`, returning early as soon as `stop` is set. Polls the +/// flag in short slices so a background helper thread (deadlock monitor, +/// queue rebalancer) exits within tens of milliseconds at teardown instead +/// of blocking the main thread's `join()` for a full poll interval after +/// the pipeline has already finished. A plain `thread::sleep(poll_interval)` +/// here adds a fixed dead-time tail (up to `poll_interval`) to every run — +/// negligible on long jobs but a large *relative* regression on short ones +/// (e.g. FASTQ extract), since the worker pool is already idle and waiting. +fn sleep_until_stop(stop: &std::sync::atomic::AtomicBool, dur: std::time::Duration) { + const SLICE: std::time::Duration = std::time::Duration::from_millis(25); + let deadline = std::time::Instant::now() + dur; + loop { + if stop.load(std::sync::atomic::Ordering::Relaxed) { + return; + } + let now = std::time::Instant::now(); + if now >= deadline { + return; + } + std::thread::sleep(SLICE.min(deadline - now)); + } +} + +/// What the deadlock monitor should do after one poll. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StallVerdict { + /// Global progress advanced since the last poll — reset and keep watching. + Progressing, + /// No progress, but nothing is in flight: the pipeline is idle waiting on + /// a slow upstream (e.g. a stdin pipe), not deadlocked. Reset the stall + /// clock and keep watching. Mirrors legacy `check_deadlock_and_restore`'s + /// starvation guard (an empty pipeline is never a deadlock). + Starving, + /// No progress with work stuck, but below the warn threshold — no-op. + Watching, + /// No progress with work stuck past the warn (but below the fatal) + /// threshold — log a diagnostic snapshot and keep watching. + Stalled, + /// No progress with work stuck past the fatal threshold — a genuine wedge. + /// Fail the pipeline fast instead of hanging forever. + Wedged, +} + +/// Classify one deadlock-monitor poll. +/// +/// - `progressed`: did the global progress counter advance since the last poll? +/// - `stall_secs`: how long progress has been flat (0 if it just advanced). +/// - `warn_secs` / `fatal_secs`: the warn and fatal stall thresholds. +/// - `in_flight_bytes`: bytes currently held across transport queues **and** +/// reorder buffers. +/// +/// The starvation guard (no progress + nothing in flight ⇒ not a deadlock) and +/// fatal-on-stuck-work behavior mirror the legacy pipeline's +/// `check_deadlock_and_restore`. The split warn/fatal thresholds diverge from +/// legacy's single 10s kill: a single huge dispatch (busy-locus group, large +/// sort merge) can legitimately flatten progress for many seconds with work +/// stuck in queues, so we only fail after the stall persists well past the warn +/// window. +fn classify_stall( + progressed: bool, + stall_secs: u64, + warn_secs: u64, + fatal_secs: u64, + in_flight_bytes: u64, +) -> StallVerdict { + if progressed { + return StallVerdict::Progressing; + } + if in_flight_bytes == 0 { + return StallVerdict::Starving; + } + if stall_secs >= fatal_secs { + return StallVerdict::Wedged; + } + if stall_secs >= warn_secs { + return StallVerdict::Stalled; + } + StallVerdict::Watching +} + +/// Per-queue floor: never let the rebalancer take a queue below this +/// (`ByteBoundedQueue` panics if `limit_bytes == 0`, and very small +/// limits effectively wedge the producer). +const MIN_PER_QUEUE_BYTES: u64 = 1024 * 1024; + +/// Floor for the per-branch reorder overflow stash. Liveness needs no floor +/// (`next_serial` is always exempt — any cap ≥ 0 is deadlock-free), so this +/// is purely a throughput knob: keep enough lookahead headroom that a +/// reorder-heavy edge doesn't thrash one item per round-robin pass. +const MIN_REORDER_OVERFLOW_BYTES: u64 = 4 * 1024 * 1024; + +/// Initial-allocation pass for `queue_memory_total`. Distributes +/// `total` evenly across all byte-bounded queues. Floors each queue +/// at `MIN_PER_QUEUE_BYTES` even if the per-queue share would be +/// smaller — in that case the effective total exceeds the user's +/// budget, but starvation is the worse failure mode. +/// +/// Each ordered byte-bounded branch's reorder overflow stash is sized from +/// the SAME `per_queue` value (clamped to `[MIN_REORDER_OVERFLOW_BYTES, +/// DEFAULT_REORDER_OVERFLOW_BYTES]`), so the off-budget stash tracks the +/// transport budget instead of a fixed 256 MiB. The clamp ceiling is the +/// prior fixed value, so high thread counts (large `per_queue`) keep today's +/// reorder headroom — no `--threads N` regression — while low thread counts +/// (small `per_queue`) get a streaming-sized stash. +pub(crate) fn apply_initial_queue_budget( + queues: &[crate::runtime::contexts::RegisteredQueue], + total: u64, +) { + if queues.is_empty() { + return; + } + let per_queue = (total / (queues.len() as u64)).max(MIN_PER_QUEUE_BYTES); + let reorder_cap = reorder_cap_for(per_queue); + for rq in queues { + rq.handle.set_limit_bytes(per_queue); + if let Some(reorder) = &rq.reorder_cap { + reorder.set_max_overflow_bytes(reorder_cap); + } + } +} + +/// The reorder overflow cap for a branch whose transport budget is +/// `per_queue`: track the transport budget, clamped to +/// `[MIN_REORDER_OVERFLOW_BYTES, DEFAULT_REORDER_OVERFLOW_BYTES]`. The +/// ceiling is the prior fixed value, so a large `per_queue` (high thread +/// counts) keeps today's reorder headroom; a small `per_queue` (low thread +/// counts / lean budget) shrinks the off-budget stash to a streaming size. +fn reorder_cap_for(per_queue: u64) -> u64 { + per_queue.clamp(MIN_REORDER_OVERFLOW_BYTES, crate::reorder::DEFAULT_REORDER_OVERFLOW_BYTES) +} + +/// Background queue-memory rebalancer body. Polls each queue's +/// `current_bytes / limit_bytes` fullness ratio every 1 second. +/// Identifies the most-full producer (likely bottleneck) and the +/// least-full consumer (over-budget). Shifts a fraction of budget +/// from least to most full, preserving total budget. +/// +/// The algorithm is deliberately simple — incremental shifts (10% +/// of the source's limit per tick) converge gradually so transient +/// spikes don't overshoot. Floors each queue at `MIN_PER_QUEUE_BYTES`. +/// +/// Exits when `stop` is set (typically after workers join). +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn run_queue_rebalancer( + stop: &Arc, + handles: &[Arc], + names: &[&'static str], +) { + if handles.len() < 2 { + // Nothing to rebalance with one or zero queues. + return; + } + let poll_interval = std::time::Duration::from_secs(1); + let shift_fraction: f64 = 0.10; + + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + sleep_until_stop(stop, poll_interval); + if stop.load(std::sync::atomic::Ordering::Relaxed) { + break; + } + + // Snapshot fullness ratios. + let snapshot: Vec<(usize, u64, u64, f64)> = handles + .iter() + .enumerate() + .map(|(idx, h)| { + let cur = h.current_bytes(); + let lim = h.limit_bytes(); + let ratio = if lim == 0 { 0.0 } else { (cur as f64) / (lim as f64) }; + (idx, cur, lim, ratio) + }) + .collect(); + + // Find the most-full and least-full queues. + let max = snapshot + .iter() + .max_by(|a, b| a.3.partial_cmp(&b.3).unwrap_or(std::cmp::Ordering::Equal)) + .copied(); + let min = snapshot + .iter() + .min_by(|a, b| a.3.partial_cmp(&b.3).unwrap_or(std::cmp::Ordering::Equal)) + .copied(); + + let (Some((max_idx, _, max_lim, max_ratio)), Some((min_idx, _, min_lim, min_ratio))) = + (max, min) + else { + continue; + }; + if max_idx == min_idx { + continue; + } + // Only rebalance when the imbalance is meaningful: the + // fullest queue is ≥80% full AND the emptiest is ≤20% full. + // Otherwise the system is in steady state and we shouldn't + // perturb the limits. + if max_ratio < 0.80 || min_ratio > 0.20 { + continue; + } + + // Shift from min to max. + let to_shift = ((min_lim as f64) * shift_fraction) as u64; + if to_shift == 0 { + continue; + } + let new_min = min_lim.saturating_sub(to_shift).max(MIN_PER_QUEUE_BYTES); + if new_min == min_lim { + // Floor reached; can't shrink further. + continue; + } + let actual_shift = min_lim - new_min; + let new_max = max_lim.saturating_add(actual_shift); + handles[min_idx].set_limit_bytes(new_min); + handles[max_idx].set_limit_bytes(new_max); + log::debug!( + "queue rebalance: shift {} bytes {} ({} -> {}) -> {} ({} -> {})", + actual_shift, + names[min_idx], + min_lim, + new_min, + names[max_idx], + max_lim, + new_max + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + + use rstest::rstest; + + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{Step, StepCtx, StepKind, StepOutcome, StepProfile}; + + // Each case pins one level's (is_on/samples, timeline, deep) predicates, so a + // failure identifies the specific level that regressed. `samples()` tracks + // `is_on()`, so the two share the `is_on` column. + #[rstest] + #[case(InstrumentationLevel::Off, false, false, false)] + #[case(InstrumentationLevel::Summary, true, false, false)] + #[case(InstrumentationLevel::Timeline, true, true, false)] + #[case(InstrumentationLevel::Deep, true, true, true)] + fn instrumentation_level_predicates( + #[case] level: InstrumentationLevel, + #[case] is_on: bool, + #[case] timeline: bool, + #[case] deep: bool, + ) { + assert_eq!(level.is_on(), is_on); + assert_eq!(level.samples(), is_on); + assert_eq!(level.timeline(), timeline); + assert_eq!(level.deep(), deep); + } + + #[test] + fn instrumentation_defaults_are_off() { + assert_eq!(InstrumentationLevel::default(), InstrumentationLevel::Off); + assert_eq!(PipelineConfig::default().instrumentation, InstrumentationLevel::Off); + assert!(PipelineConfig::default().trace_path.is_none()); + } + + #[test] + fn apply_initial_queue_budget_sets_registered_reorder_cap() { + // End-to-end wiring: a registered ordered byte-bounded branch's reorder + // cap is re-sized by the budget pass (not left at its construction + // default). Guards the Pass-1.5 registration + the `set` call together. + use crate::queues::{BoundedQueueHandle, ByteBoundedQueue, ItemQueue}; + use crate::reorder::{ + DEFAULT_REORDER_OVERFLOW_BYTES, ReorderCapHandle, ReorderStage, Sequenced, + }; + use crate::runtime::contexts::RegisteredQueue; + use crate::topology::{BranchIdx, StepIdx}; + + // A reorder stage constructed at the 256 MiB fallback; keep a concrete + // handle so we can read the cap back after the budget pass. + let transport: Arc>> = + Arc::new(ByteBoundedQueue::>::new(1024 * 1024)); + let stage = Arc::new(ReorderStage::::with_max_overflow_bytes( + transport, + DEFAULT_REORDER_OVERFLOW_BYTES, + )); + assert_eq!(stage.current_max_overflow_bytes(), DEFAULT_REORDER_OVERFLOW_BYTES); + let reorder_dyn: Arc = stage.clone(); + + // A transport-limit handle for the `RegisteredQueue.handle` slot. + let transport_q = Arc::new(ByteBoundedQueue::::new(1024 * 1024)); + let transport_handle: Arc = transport_q; + + let registered = vec![RegisteredQueue { + producer_step_name: "TestStep", + producer_step: StepIdx(0), + branch: BranchIdx(0), + handle: transport_handle, + reorder_cap: Some(reorder_dyn), + }]; + + // Lean total → per_queue = 8 MiB (1 queue) → reorder clamp = 8 MiB. + apply_initial_queue_budget(®istered, 8 * 1024 * 1024); + assert_eq!( + stage.current_max_overflow_bytes(), + 8 * 1024 * 1024, + "budget pass must re-size the registered reorder cap to the clamped per_queue" + ); + + // Huge total → per_queue huge → reorder clamped back to the ceiling. + apply_initial_queue_budget(®istered, 100 * 1024 * 1024 * 1024); + assert_eq!( + stage.current_max_overflow_bytes(), + DEFAULT_REORDER_OVERFLOW_BYTES, + "high budget clamps the reorder cap to the 256 MiB ceiling (no t>1 regression)" + ); + } + + #[test] + fn apply_initial_queue_budget_floors_each_queue_at_min_when_budget_tiny() { + // When `total / n_queues < MIN_PER_QUEUE_BYTES`, the budget pass floors + // each transport at `MIN_PER_QUEUE_BYTES` even though the effective total + // then exceeds the user's budget — starvation (a zero/tiny-budget queue + // that always rejects pushes, wedging the producer) is the worse failure + // mode. A regression dropping the `.max(MIN_PER_QUEUE_BYTES)` would not be + // caught by the lean/huge cases the sibling test covers. + use crate::queues::{BoundedQueueHandle, ByteBoundedQueue}; + use crate::runtime::contexts::RegisteredQueue; + use crate::topology::{BranchIdx, StepIdx}; + + // Four registered queues, each constructed at the 1 MiB floor. + let handles: Vec>> = + (0..4).map(|_| Arc::new(ByteBoundedQueue::::new(MIN_PER_QUEUE_BYTES))).collect(); + let registered: Vec = handles + .iter() + .enumerate() + .map(|(i, h)| RegisteredQueue { + producer_step_name: "TestStep", + producer_step: StepIdx(i), + branch: BranchIdx(0), + handle: Arc::clone(h) as Arc, + reorder_cap: None, + }) + .collect(); + + // total = 1 byte over 4 queues → per_queue would be 0, floored to 1 MiB. + apply_initial_queue_budget(®istered, 1); + for h in &handles { + assert_eq!( + h.limit_bytes(), + MIN_PER_QUEUE_BYTES, + "each queue's transport limit must be floored to MIN_PER_QUEUE_BYTES \ + when the per-queue share underflows the floor" + ); + } + } + + #[test] + fn reorder_cap_tracks_per_queue_clamped_to_floor_and_ceiling() { + let ceiling = crate::reorder::DEFAULT_REORDER_OVERFLOW_BYTES; + // Mid-range per_queue passes through unchanged. + assert_eq!(reorder_cap_for(32 * 1024 * 1024), 32 * 1024 * 1024); + // Tiny per_queue (lean / low-thread) is floored — but stays small. + assert_eq!(reorder_cap_for(1024), MIN_REORDER_OVERFLOW_BYTES); + assert_eq!(reorder_cap_for(MIN_REORDER_OVERFLOW_BYTES - 1), MIN_REORDER_OVERFLOW_BYTES); + // Huge per_queue (high thread counts) is capped at today's ceiling → + // no `--threads N` regression. + assert_eq!(reorder_cap_for(4 * ceiling), ceiling); + assert_eq!(reorder_cap_for(ceiling), ceiling); + } + + // ───── Test stubs ───── + + #[derive(Clone)] + struct StubSource; + impl Step for StubSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Source", + kind: StepKind::Exclusive, + sticky: true, + output_queues: vec![QueueSpec::CountBounded { capacity: 64 }], + branch_ordering: vec![BranchOrdering::ByOrdinal], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + } + + #[derive(Clone)] + struct StubTransform; + impl Step for StubTransform { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Transform", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 64 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct StubFanOut2; + impl Step for StubFanOut2 { + type Input = u64; + type Outputs = (u32, String); + fn profile(&self) -> StepProfile { + StepProfile { + name: "FanOut2", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![ + QueueSpec::CountBounded { capacity: 32 }, + QueueSpec::CountBounded { capacity: 32 }, + ], + branch_ordering: vec![BranchOrdering::None, BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct StubSinkU32; + impl Step for StubSinkU32 { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "SinkU32", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[derive(Clone)] + struct StubSinkString; + impl Step for StubSinkString { + type Input = String; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "SinkString", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + // ───── Tests ───── + + #[test] + fn empty_builder_returns_empty_error() { + let builder = PipelineBuilder::new(); + assert!(matches!(builder.build(), Err(BuildError::Empty))); + } + + #[test] + fn unwired_source_returns_unwired_output() { + let builder = PipelineBuilder::new(); + let _chain = builder.chain(StubSource); + let result = builder.build(); + assert!(matches!(result, Err(BuildError::UnwiredOutput { step: "Source", branch: "0" }))); + } + + /// Sources register with `input_arity = 0`, matching the contract + /// `ChainGraph::register_step_with_input_arity` documents. A source's input + /// is implicit, so the graph must have no slot to wire an edge into. + #[rstest] + #[case::chain(false)] + #[case::append_source(true)] + fn sources_register_with_zero_input_arity(#[case] via_append_source: bool) { + let builder = PipelineBuilder::new(); + let source = if via_append_source { + builder.append_source(StubSource).0 + } else { + let chain = builder.chain(StubSource); + // `Chain` does not expose its `StepIdx`; the source is step 0. + drop(chain); + StepIdx(0) + }; + assert_eq!( + builder.inner.borrow().graph.input_arity(source), + 0, + "a source's input is implicit — it must have no input slot" + ); + } + + /// Wiring a producer into a source is rejected at build time. + /// + /// Arity 0 makes `wire_to_slot` panic for a source registered through + /// `PipelineBuilder::{chain, append_source}`, but a source appended as a + /// *consumer* goes through `Chain::chain`, which registers every consumer + /// with arity 1 — so the wire succeeds and only `build()` can catch it. + /// Without the check the chain builds clean, `build_chain_contexts_inner` + /// hands the second source a dummy unit input handle because `is_source()` + /// is true, and `StubSource`'s output is silently discarded. + #[test] + fn build_rejects_a_producer_wired_into_a_source() { + /// Emits `()`, so its `Chain>` type-checks against any + /// `Step` — i.e. against another source. + #[derive(Clone)] + struct UnitSource; + impl Step for UnitSource { + type Input = (); + type Outputs = Single<()>; + fn profile(&self) -> StepProfile { + StepProfile { + name: "UnitSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + let builder = PipelineBuilder::new(); + // `StubSource: Step` — accepted here purely because + // `HeapSize for ()` exists, not because the wiring is meaningful. + builder.chain(UnitSource).chain(StubSource).chain(StubSinkU32).into_sink_marker(); + + assert!( + matches!( + builder.build(), + Err(BuildError::WiredIntoSource { step: "Source", producer: "UnitSource" }) + ), + "a producer wired into a source must be a build error, not silent data loss" + ); + } + + #[test] + fn source_to_transform_unwired_at_transform_returns_unwired() { + let builder = PipelineBuilder::new(); + let _chain = builder.chain(StubSource).chain(StubTransform); + let result = builder.build(); + assert!(matches!( + result, + Err(BuildError::UnwiredOutput { step: "Transform", branch: "0" }) + )); + } + + #[test] + fn fully_wired_source_to_sink_succeeds() { + let builder = PipelineBuilder::new(); + builder.chain(StubSource).chain(StubSinkU32).into_sink_marker(); + let result = builder.build(); + assert!(result.is_ok()); + let pipeline = result.unwrap(); + assert_eq!(pipeline.graph.n_steps(), 2); + } + + #[test] + fn unwired_fanout_branch_is_detected() { + let builder = PipelineBuilder::new(); + let after_fanout = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut2); + let multi = after_fanout.into_multi(); + // Wire branch 0 to a sink, drop branch 1 (unwired). + multi.b0.chain(StubSinkU32).into_sink_marker(); + drop(multi.b1); + + let result = builder.build(); + assert!(matches!(result, Err(BuildError::UnwiredOutput { step: "FanOut2", branch: "1" }))); + } + + #[test] + fn fanout_with_both_branches_wired_succeeds() { + let builder = PipelineBuilder::new(); + let after_fanout = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut2); + let multi = after_fanout.into_multi(); + multi.b0.chain(StubSinkU32).into_sink_marker(); + multi.b1.chain(StubSinkString).into_sink_marker(); + + let result = builder.build(); + assert!(result.is_ok()); + let pipeline = result.unwrap(); + assert_eq!(pipeline.graph.n_steps(), 5); + } + + #[test] + fn pipeline_config_default_uses_available_parallelism() { + let cfg = PipelineConfig::default(); + assert!(cfg.threads >= 1); + } + + #[derive(Clone)] + struct StubFanOut3; + impl Step for StubFanOut3 { + type Input = u64; + type Outputs = (u32, String, u64); + fn profile(&self) -> StepProfile { + StepProfile { + name: "FanOut3", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }; 3], + branch_ordering: vec![BranchOrdering::None; 3], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct StubFanOut4; + impl Step for StubFanOut4 { + type Input = u64; + type Outputs = (u32, String, u64, u32); + fn profile(&self) -> StepProfile { + StepProfile { + name: "FanOut4", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }; 4], + branch_ordering: vec![BranchOrdering::None; 4], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct StubSinkU64; + impl Step for StubSinkU64 { + type Input = u64; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "SinkU64", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// `into_multi` must hand back one `Chain` per declared branch, each + /// pinned to its own `BranchIdx`. If two sub-chains shared a branch index + /// the builder would report a phantom unwired branch even though the + /// caller wired every one — so wiring all of them must build cleanly. + #[test] + fn into_multi_3_exposes_one_chain_per_branch() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut3).into_multi(); + multi.b0.chain(StubSinkU32).into_sink_marker(); + multi.b1.chain(StubSinkString).into_sink_marker(); + multi.b2.chain(StubSinkU64).into_sink_marker(); + + let pipeline = builder.build().expect("all three branches wired"); + assert_eq!(pipeline.graph.n_steps(), 6, "source + transform + fanout + 3 sinks"); + } + + /// The 3-branch counterpart of `unwired_fanout_branch_is_detected`: the + /// dropped branch must be named by index, proving `into_multi` gave branch + /// 2 its own identity rather than aliasing an earlier one. + #[test] + fn into_multi_3_reports_the_branch_left_unwired() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut3).into_multi(); + multi.b0.chain(StubSinkU32).into_sink_marker(); + multi.b1.chain(StubSinkString).into_sink_marker(); + drop(multi.b2); + + assert!(matches!( + builder.build(), + Err(BuildError::UnwiredOutput { step: "FanOut3", branch: "2" }) + )); + } + + #[test] + fn into_multi_4_exposes_one_chain_per_branch() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut4).into_multi(); + multi.b0.chain(StubSinkU32).into_sink_marker(); + multi.b1.chain(StubSinkString).into_sink_marker(); + multi.b2.chain(StubSinkU64).into_sink_marker(); + multi.b3.chain(StubSinkU32).into_sink_marker(); + + let pipeline = builder.build().expect("all four branches wired"); + assert_eq!(pipeline.graph.n_steps(), 7, "source + transform + fanout + 4 sinks"); + } + + #[test] + fn into_multi_4_reports_the_branch_left_unwired() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(StubSource).chain(StubTransform).chain(StubFanOut4).into_multi(); + multi.b0.chain(StubSinkU32).into_sink_marker(); + multi.b1.chain(StubSinkString).into_sink_marker(); + multi.b2.chain(StubSinkU64).into_sink_marker(); + drop(multi.b3); + + assert!(matches!( + builder.build(), + Err(BuildError::UnwiredOutput { step: "FanOut4", branch: "3" }) + )); + } + + #[derive(Clone, Copy)] + struct Ord32 { + ordinal: u64, + } + impl crate::item::HeapSize for Ord32 {} + impl crate::item::Ordered for Ord32 { + fn ordinal(&self) -> u64 { + self.ordinal + } + } + + #[derive(Clone)] + struct OrderedSource; + impl Step for OrderedSource { + type Input = (); + type Outputs = crate::outputs::OrderedBytesTuple2; + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrderedSource", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { limit_bytes: 1024 }; 2], + branch_ordering: vec![BranchOrdering::ByItemOrdinal; 2], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct OrderedSink; + impl Step for OrderedSink { + type Input = Ord32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrderedSink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// The ordered + byte-bounded 2-way fan-out — the shape real BAM source + /// chains produce — is the one `into_multi` variant with no coverage. Its + /// sub-chains are `Chain>`, not `Chain>`, + /// so a branch mix-up would surface as a type error at the call site; what + /// this pins is that both branches are exposed and separately wireable. + #[test] + fn into_multi_ordered_bytes_2_exposes_one_chain_per_branch() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(OrderedSource).into_multi(); + let _: &Chain<'_, crate::outputs::OrderedBytesSingle> = &multi.b0; + multi.b0.chain(OrderedSink).into_sink_marker(); + multi.b1.chain(OrderedSink).into_sink_marker(); + + let pipeline = builder.build().expect("both ordered branches wired"); + assert_eq!(pipeline.graph.n_steps(), 3, "source + 2 sinks"); + } + + /// The unwired branch is reported BY INDEX, so the two sub-chains are + /// distinct edges rather than two views of branch 0. + #[test] + fn into_multi_ordered_bytes_2_reports_the_branch_left_unwired() { + let builder = PipelineBuilder::new(); + let multi = builder.chain(OrderedSource).into_multi(); + multi.b0.chain(OrderedSink).into_sink_marker(); + drop(multi.b1); + + assert!(matches!( + builder.build(), + Err(BuildError::UnwiredOutput { step: "OrderedSource", branch: "1" }) + )); + } + + // Each case pins one `BuildError` variant's rendering, so a failure names + // the variant whose message drifted rather than a combined assert. + #[rstest] + #[case::empty(BuildError::Empty, "pipeline has no steps")] + #[case::unwired( + BuildError::UnwiredOutput { step: "FanOut2", branch: "1" }, + "step \"FanOut2\" has unwired output branch \"1\"" + )] + fn build_error_displays_actionably(#[case] err: BuildError, #[case] expected: &str) { + assert_eq!(err.to_string(), expected); + } + + /// A read of one `PipelineConfig` field, so each setter case can name the + /// field it targets and assert on a comparable value. + #[derive(Debug, PartialEq, Eq)] + enum ConfigProbe { + DeadlockTimeout(u64), + StatsPresent(bool), + QueueMemoryTotal(Option), + Instrumentation(InstrumentationLevel), + SchedulerName(&'static str), + } + + impl ConfigProbe { + /// Read the same field this probe names out of `cfg`. + fn read_from(&self, cfg: &PipelineConfig) -> Self { + match self { + Self::DeadlockTimeout(_) => Self::DeadlockTimeout(cfg.deadlock_timeout_secs), + Self::StatsPresent(_) => Self::StatsPresent(cfg.stats.is_some()), + Self::QueueMemoryTotal(_) => Self::QueueMemoryTotal(cfg.queue_memory_total), + Self::Instrumentation(_) => Self::Instrumentation(cfg.instrumentation), + Self::SchedulerName(_) => Self::SchedulerName(cfg.scheduler.name()), + } + } + } + + /// The builder-style setters are the only way a caller configures a + /// `PipelineConfig`. Each case applies one setter and asserts two things: + /// the field it targets took the new value, and a field it does NOT target + /// is still at its default — a copy-paste slip that assigned the wrong field + /// would silently disable instrumentation or the deadlock monitor. + /// + /// `scheduler` is a trait object, so its case asserts the observable + /// `name()` rather than allocation identity: pointer inequality would pass + /// even if the setter stored the wrong scheduler. + #[rstest] + #[case::with_deadlock_timeout( + &|c: PipelineConfig| c.with_deadlock_timeout(30), + ConfigProbe::DeadlockTimeout(30), + ConfigProbe::StatsPresent(false) + )] + #[case::with_stats( + &|c: PipelineConfig| c.with_stats(Arc::new(PipelineStats::new(vec!["Source"]))), + ConfigProbe::StatsPresent(true), + ConfigProbe::DeadlockTimeout(0) + )] + #[case::with_queue_memory_total_some( + &|c: PipelineConfig| c.with_queue_memory_total(Some(64 * 1024 * 1024)), + ConfigProbe::QueueMemoryTotal(Some(64 * 1024 * 1024)), + ConfigProbe::StatsPresent(false) + )] + // `None` is the documented "keep per-step static limits, rebalancer off" + // value, so it must round-trip as None rather than be treated as unset. + #[case::with_queue_memory_total_none( + &|c: PipelineConfig| c.with_queue_memory_total(None), + ConfigProbe::QueueMemoryTotal(None), + ConfigProbe::StatsPresent(false) + )] + #[case::with_instrumentation( + &|c: PipelineConfig| c.with_instrumentation(InstrumentationLevel::Deep), + ConfigProbe::Instrumentation(InstrumentationLevel::Deep), + ConfigProbe::DeadlockTimeout(0) + )] + #[case::with_scheduler( + &|c: PipelineConfig| c.with_scheduler(Arc::new(crate::runtime::DrainFirstScheduler)), + ConfigProbe::SchedulerName("drain-first"), + ConfigProbe::Instrumentation(InstrumentationLevel::Off) + )] + fn pipeline_config_setter_sets_only_its_own_field( + #[case] apply: &dyn Fn(PipelineConfig) -> PipelineConfig, + #[case] expected: ConfigProbe, + #[case] untouched: ConfigProbe, + ) { + let defaults = PipelineConfig::default(); + assert_eq!( + untouched.read_from(&defaults), + untouched, + "the case's `untouched` probe must state the actual default" + ); + + let cfg = apply(PipelineConfig::default()); + assert_eq!(expected.read_from(&cfg), expected, "setter set its own field"); + assert_eq!(untouched.read_from(&cfg), untouched, "setter left the other field alone"); + } + + /// The default scheduler must stay upstream-first; `with_scheduler`'s case + /// above is only meaningful against a known baseline. + #[test] + fn default_scheduler_walks_chain_order() { + let cfg = PipelineConfig::default(); + assert_eq!(cfg.scheduler.name(), "chain-order"); + assert_eq!(cfg.scheduler.walk(), crate::runtime::WalkDirection::Forward); + } + + /// `append_source` / `append_step` are the type-erased assembly API the + /// parent crate's `ChainBuilder` drives across method-boundary calls, where + /// the fluent `chain` API cannot be used because the chain type changes at + /// every step. Wiring by returned `(StepIdx, BranchIdx)` must produce the + /// same graph the fluent API does. + #[test] + fn append_source_and_append_step_build_the_same_graph_as_chaining() { + let builder = PipelineBuilder::new(); + let src = builder.append_source(StubSource); + let mid = builder.append_step(StubTransform, src); + let _sink = builder.append_step(StubSinkU64, mid); + + let pipeline = builder.build().expect("every branch wired by index"); + assert_eq!(pipeline.graph.n_steps(), 3); + } + + #[test] + fn append_step_leaves_an_unwired_branch_detectable() { + // Appending a fan-out and wiring only one of its branches must still be + // caught by `build`, exactly as with the fluent API. + let builder = PipelineBuilder::new(); + let src = builder.append_source(StubSource); + let mid = builder.append_step(StubTransform, src); + let fanout = builder.append_step(StubFanOut2, mid); + let _sink = builder.append_step(StubSinkU32, fanout); + + assert!(matches!( + builder.build(), + Err(BuildError::UnwiredOutput { step: "FanOut2", branch: "1" }) + )); + } + + #[derive(Clone)] + struct StubJoin; + impl crate::step::Step2 for StubJoin { + type InputA = u32; + type InputB = String; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "Join", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run( + &mut self, + _ctx: &mut crate::step::StepCtx2<'_, Self>, + ) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// `append_step2` must consume BOTH upstream branches — one into input slot + /// 0 and one into slot 1. If it wired only one, `build` would report the + /// other as unwired. + #[test] + fn append_step2_consumes_both_upstream_branches() { + let builder = PipelineBuilder::new(); + let src = builder.append_source(StubSource); + let mid = builder.append_step(StubTransform, src); + let fanout = builder.append_step(StubFanOut2, mid); + let branch_b = (fanout.0, BranchIdx(1)); + let _join = builder.append_step2(StubJoin, fanout, branch_b); + + let pipeline = builder.build().expect("both fan-out branches consumed by the join"); + assert_eq!(pipeline.graph.n_steps(), 4, "source + transform + fanout + join"); + } + + #[test] + fn dag_renders_chain_shape() { + let builder = PipelineBuilder::new(); + builder.chain(StubSource).chain(StubSinkU32).into_sink_marker(); + let pipeline = builder.build().unwrap(); + let dag = pipeline.dag(); + // Verify shape rendering — names and (sink) marker. + assert!(dag.contains("Source"), "DAG missing source name: {dag}"); + assert!(dag.contains("SinkU32"), "DAG missing sink name: {dag}"); + assert!(dag.contains("(sink)"), "DAG missing sink marker: {dag}"); + assert!(dag.contains("→ SinkU32"), "DAG missing source→sink wiring: {dag}"); + } + + #[test] + fn dag_renders_effective_collapsed_ordering_for_serial_exclusive() { + // `StubSource` is `Exclusive` and declares `BranchOrdering::ByOrdinal`, + // but `build_output_set` collapses that to `None` (no reorder stage) for + // single-input Serial/Exclusive producers. `dag()` must render the + // *effective* ordering so the diagnostic matches the transport actually + // built — it must not print the un-collapsed declared `ByOrdinal`. + let builder = PipelineBuilder::new(); + builder.chain(StubSource).chain(StubSinkU32).into_sink_marker(); + let pipeline = builder.build().unwrap(); + let dag = pipeline.dag(); + // The source's output branch line is `.0: → SinkU32`. + let source_branch_line = dag + .lines() + .find(|l| l.contains("→ SinkU32")) + .expect("DAG must have the source→sink branch line"); + assert!( + source_branch_line.contains("None"), + "DAG must render the collapsed (effective) ordering `None`: {source_branch_line}" + ); + assert!( + !source_branch_line.contains("ByOrdinal"), + "DAG must NOT render the un-collapsed declared `ByOrdinal`: {source_branch_line}" + ); + } + + // ───────────────────────────────────────────────────────────────────── + // Pipeline::run smoke tests + // ───────────────────────────────────────────────────────────────────── + + use std::sync::atomic::{AtomicU32, Ordering as AtomicOrd}; + + use crate::signal::PipelineError; + + /// Source emitting `remaining` items via a shared atomic counter; safe + /// for both single-worker and multi-worker `Parallel` execution. + /// + /// Uses an `Unbounded` output queue so the test never hits backpressure + /// (which would require the source to use the `HeldSlot>` + /// retry pattern — exercised in the bigger end-to-end smoke test below). + #[derive(Clone)] + struct SharedCountingSource { + remaining: Arc, + } + impl Step for SharedCountingSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SharedSource", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::Unbounded], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + let n = self.remaining.load(AtomicOrd::Acquire); + if n == 0 { + return Ok(StepOutcome::Finished); + } + // CAS down to claim this item; only push on successful claim. + if self + .remaining + .compare_exchange(n, n - 1, AtomicOrd::AcqRel, AtomicOrd::Acquire) + .is_ok() + { + ctx.outputs.push(n).map_err(|_| { + std::io::Error::other("Unbounded queue rejected push (impossible)") + })?; + Ok(StepOutcome::Progress) + } else { + Ok(StepOutcome::NoProgress) + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// Byte-bounded variant of [`SharedCountingSource`]. Identical claim/push + /// logic but declares a `ByteBounded` output transport so it is + /// monitor-visible — required by any test that arms the deadlock monitor + /// (`deadlock_timeout_secs > 0` + stats), which requires every output edge + /// to be `ByteBounded` (see `ensure_monitor_visible_transports`) before + /// workers spawn. Using the `Unbounded` source there would fail the run with + /// `PipelineError::MonitorBlindTransport` before the worker-panic path is + /// ever reached. + #[derive(Clone)] + struct SharedCountingSourceByteBounded { + remaining: Arc, + /// A value claimed from `remaining` but not yet accepted by the output + /// (the byte-bounded push hit backpressure). Held per-worker and retried + /// on a later tick so the exact ordinal survives — rolling `remaining` + /// back instead would let a peer re-claim the count and drop/duplicate an + /// ordinal under contention. + pending: Option, + } + impl Step for SharedCountingSourceByteBounded { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SharedSourceByteBounded", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { limit_bytes: 1 << 20 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + // First flush any value claimed on a prior tick whose push was + // rejected by backpressure. Retrying the exact held ordinal (rather + // than rolling `remaining` back) keeps the emitted set a clean + // permutation of `1..=N` even under contention. + if let Some(n) = self.pending { + return if ctx.outputs.push(n).is_ok() { + self.pending = None; + Ok(StepOutcome::Progress) + } else { + Ok(StepOutcome::NoProgress) + }; + } + let n = self.remaining.load(AtomicOrd::Acquire); + if n == 0 { + return Ok(StepOutcome::Finished); + } + if self + .remaining + .compare_exchange(n, n - 1, AtomicOrd::AcqRel, AtomicOrd::Acquire) + .is_ok() + { + // Byte-bounded push can hit backpressure; hold the claimed + // ordinal and retry it on a later tick. Holding a claimed item + // counts as progress per the `StepOutcome` contract ("pushed or + // held an item" — see `step.rs`) and matches the production + // held-slot source (`sort::merge`), so the scheduler's deadlock + // accounting sees the claim as forward motion rather than a stall. + if ctx.outputs.push(n).is_ok() { + Ok(StepOutcome::Progress) + } else { + self.pending = Some(n); + Ok(StepOutcome::Progress) + } + } else { + Ok(StepOutcome::NoProgress) + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// Sink that pops and counts. + #[derive(Clone)] + struct ParallelCountingSink { + received: Arc, + /// Every value popped, so a test can assert record *identity* and not + /// just the count — a framework bug that duplicates one item and drops + /// another keeps the count at N. Shared across worker copies. + seen: Arc>>, + } + impl ParallelCountingSink { + fn new(received: &Arc) -> Self { + Self { + received: Arc::clone(received), + seen: Arc::new(parking_lot::Mutex::new(Vec::new())), + } + } + } + impl Step for ParallelCountingSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "ParallelSink", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + match ctx.input.pop() { + Some(v) => { + self.seen.lock().push(v); + self.received.fetch_add(1, AtomicOrd::Relaxed); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// `SharedCountingSource` emits the distinct ordinals `n..=1`, so a drain + /// test can assert the exact multiset the sink received rather than only its + /// size. Sorted because worker interleaving makes arrival order a valid + /// scheduling detail; identity is the contract, order is not. + fn assert_received_every_ordinal_once(sink: &ParallelCountingSink, n: u32) { + let mut got = sink.seen.lock().clone(); + got.sort_unstable(); + let expected: Vec = (1..=n).collect(); + assert_eq!(got, expected, "every emitted ordinal 1..={n} must arrive exactly once"); + } + + #[test] + fn pipeline_run_with_threads_1_drains_chain() { + let remaining = Arc::new(AtomicU32::new(10)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 1, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 10); + assert_received_every_ordinal_once(&sink, 10); + } + + #[test] + fn pipeline_run_with_threads_4_drains_chain() { + let remaining = Arc::new(AtomicU32::new(50)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 4, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 50); + assert_received_every_ordinal_once(&sink, 50); + } + + /// A `u32 -> u32` pass-through step that runs on a dedicated Detached + /// thread. Pops one item per `try_run`, pushes it on (holding on + /// output-full backpressure), finishes once its input drains. + #[derive(Clone)] + struct DetachedPassThrough { + held: Option, + } + impl Step for DetachedPassThrough { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "DetachedPassThrough", + kind: StepKind::Detached, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + // A rejected push reports `NoProgress`, not `Contention`: the driver + // treats them identically, but `Contention` means "a Serial step's + // mutex was held by another worker" and feeds `contention_count`, + // which the bottleneck verdict turns into its SPIN ratio. Using it + // for ordinary output backpressure invents contention that never + // happened. + if let Some(v) = self.held.take() { + if ctx.outputs.push(v).is_err() { + self.held = Some(v); + return Ok(StepOutcome::NoProgress); + } + return Ok(StepOutcome::Progress); + } + match ctx.input.pop() { + Some(v) => match ctx.outputs.push(v) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + }, + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// L2.3: a `source(Parallel) -> passthrough(Detached) -> sink(Parallel)` + /// chain runs to completion at `--threads 4` with the Detached step on its + /// own dedicated thread (off the pool). All N items flow through. + #[test] + fn detached_step_runs_and_drains_multithreaded() { + let remaining = Arc::new(AtomicU32::new(200)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(DetachedPassThrough { held: None }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 4, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!( + received.load(AtomicOrd::Relaxed), + 200, + "all items flowed through Detached step" + ); + assert_received_every_ordinal_once(&sink, 200); + } + + /// L2.3: a chain with a Detached step and ZERO items (source finishes + /// immediately) cleanly drains the Detached thread and the run completes. + #[test] + fn detached_step_zero_items_run_completes() { + let remaining = Arc::new(AtomicU32::new(0)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(DetachedPassThrough { held: None }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 4, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 0); + // Empty expectation: nothing was emitted, so nothing may arrive — this + // also catches a spurious item the bare count check would miss if the + // counter and the sink ever disagreed. + assert_received_every_ordinal_once(&sink, 0); + } + + /// L2.3 step 7: at `--threads 1` the fusible linear chain runs the Detached + /// step **inline** in the fused single-thread driver — no dedicated thread + /// is spawned (the fused path returns before `extract_detached_steps`), yet + /// the Detached step still drives to completion. + #[test] + fn detached_collapses_inline_at_t1() { + let remaining = Arc::new(AtomicU32::new(30)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(DetachedPassThrough { held: None }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 1, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 30); + assert_received_every_ordinal_once(&sink, 30); + } + + /// Sink whose worker loop panics the moment it pops an item — used to drive + /// the worker-panic deferral paths in `Pipeline::run`. + #[derive(Clone)] + struct PanickingSink; + impl Step for PanickingSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "PanickingSink", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + match ctx.input.pop() { + Some(_) => panic!("intentional worker panic for test"), + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// Run `f` with the panic hook silenced so an *expected* worker panic does + /// not spew a backtrace into the test log. Safe under `nextest`, which runs + /// each test in its own process. + fn with_silenced_panic_hook(f: impl FnOnce() -> R) -> R { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = f(); + std::panic::set_hook(prev); + result + } + + /// Assert a caught panic carries `PanickingSink`'s payload. + /// + /// `with_silenced_panic_hook` suppresses the payload, so a bare + /// `is_err()` is satisfied by *any* panic — a `thread::Builder::spawn` + /// `expect`, a framework `debug_assert!`, or the monitor-visibility guard + /// tripping would all pass while the worker-panic deferral path never ran. + fn assert_is_the_intentional_worker_panic(payload: &(dyn std::any::Any + Send), ctx: &str) { + let msg = payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .expect("panic payload is a string"); + assert!( + msg.contains("intentional worker panic for test"), + "{ctx}: run() must re-raise the SINK's panic, got: {msg}" + ); + } + + #[test] + fn pipeline_run_reraises_worker_panic_single_threaded() { + // A worker-loop panic on the single-threaded fast path must propagate + // out of `run` (after the common monitor/rebalancer shutdown), not be + // swallowed. The test completing at all proves the run did not hang. + let remaining = Arc::new(AtomicU32::new(10)); + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(PanickingSink) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = with_silenced_panic_hook(|| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + pipeline.run(PipelineConfig { threads: 1, ..Default::default() }) + })) + }); + let payload = result.expect_err("single-threaded worker panic must propagate out of run()"); + assert_is_the_intentional_worker_panic(payload.as_ref(), "single-threaded"); + } + + #[test] + fn pipeline_run_reraises_worker_panic_with_monitor_enabled() { + // With the deadlock monitor enabled (stats + non-zero timeout), a + // multi-worker panic must still re-raise — after the monitor is stopped + // and joined — rather than deadlocking the join loop or leaking the + // helper thread. The panicking worker signals cancellation so any wedged + // peer observes `is_done()` and exits, letting every join complete. + // + // The source MUST be byte-bounded: arming the monitor (stats + + // non-zero timeout) runs `ensure_monitor_visible_transports`, which + // requires every output edge to be `ByteBounded` in every build. An + // `Unbounded` source would fail the run with `MonitorBlindTransport` + // before any worker is spawned, so the test would "pass" on the wrong + // error and never exercise the worker-panic deferral path it covers. + let remaining = Arc::new(AtomicU32::new(1_000)); + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSourceByteBounded { + remaining: Arc::clone(&remaining), + pending: None, + }) + .chain(PanickingSink) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + let stats = pipeline.stats(); + + let result = with_silenced_panic_hook(|| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + pipeline.run(PipelineConfig { + threads: 4, + stats: Some(Arc::clone(&stats)), + deadlock_timeout_secs: 5, + ..Default::default() + }) + })) + }); + let payload = result.expect_err("multi-worker panic must propagate out of run()"); + assert_is_the_intentional_worker_panic(payload.as_ref(), "multi-worker with monitor"); + } + + #[test] + fn pipeline_stats_handle_matches_chain_size() { + let builder = PipelineBuilder::new(); + builder.chain(StubSource).chain(StubSinkU32).into_sink_marker(); + let pipeline = builder.build().unwrap(); + let stats = pipeline.stats(); + assert_eq!(stats.n_steps(), 2); + assert_eq!(stats.step_name(StepIdx(0)), "Source"); + assert_eq!(stats.step_name(StepIdx(1)), "SinkU32"); + } + + #[test] + fn pipeline_run_populates_stats_when_enabled() { + let remaining = Arc::new(AtomicU32::new(20)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + let stats = pipeline.stats(); + + let cfg = + PipelineConfig { threads: 2, stats: Some(Arc::clone(&stats)), ..Default::default() }; + let result = pipeline.run(cfg); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 20); + + let snap = stats.snapshot(); + assert_eq!(snap.steps.len(), 2); + assert_eq!(snap.steps[0].0, "SharedSource"); + assert_eq!(snap.steps[1].0, "ParallelSink"); + + // Sink saw exactly 20 items (one Progress per pop with item). + assert_eq!(snap.steps[1].1.progress_count, 20); + // Source must have made progress at least 20 times to push the items. + assert!(snap.steps[0].1.progress_count >= 20); + // Both steps accumulated wall time. + assert!(snap.steps[0].1.total_run_ns > 0); + assert!(snap.steps[1].1.total_run_ns > 0); + // No errors recorded. + assert_eq!(snap.steps[0].1.error_count, 0); + assert_eq!(snap.steps[1].1.error_count, 0); + // The source returned Finished at least once across the workers. + assert!(snap.steps[0].1.finished_count >= 1); + } + + #[test] + fn pipeline_run_without_stats_succeeds_unchanged() { + let remaining = Arc::new(AtomicU32::new(5)); + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + + let builder = PipelineBuilder::new(); + builder + .chain(SharedCountingSource { remaining: Arc::clone(&remaining) }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 2, stats: None, ..Default::default() }); + assert!(result.is_ok(), "run failed: {:?}", result.err()); + assert_eq!(received.load(AtomicOrd::Relaxed), 5); + } + + #[test] + fn pipeline_run_responds_to_cancellation() { + /// Source that emits forever (never returns `Finished`) until cancelled. + #[derive(Clone)] + struct InfiniteSource; + impl Step for InfiniteSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "InfiniteSource", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + let _ = ctx.outputs.push(0); + Ok(StepOutcome::Progress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct DiscardSink; + impl Step for DiscardSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "DiscardSink", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + match ctx.input.pop() { + Some(_) => Ok(StepOutcome::Progress), + None => Ok(StepOutcome::NoProgress), + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + let builder = PipelineBuilder::new(); + builder.chain(InfiniteSource).chain(DiscardSink).into_sink_marker(); + let pipeline = builder.build().unwrap(); + let cancel = pipeline.cancel_handle(); + + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(100)); + cancel.cancel(); + }); + + let result = pipeline.run(PipelineConfig { threads: 4, ..Default::default() }); + assert!(matches!(result, Err(PipelineError::Cancelled))); + } + + #[test] + fn pipeline_run_propagates_step_error() { + /// Source that emits `n` items, then returns `Err`. + #[derive(Clone)] + struct FailingSource { + remaining: Arc, + } + impl Step for FailingSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "FailingSource", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + let n = self.remaining.load(AtomicOrd::Acquire); + if n == 0 { + return Err(std::io::Error::other("source failed")); + } + if self + .remaining + .compare_exchange(n, n - 1, AtomicOrd::AcqRel, AtomicOrd::Acquire) + .is_ok() + { + let _ = ctx.outputs.push(n); + Ok(StepOutcome::Progress) + } else { + Ok(StepOutcome::NoProgress) + } + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + let received = Arc::new(AtomicU32::new(0)); + let sink = ParallelCountingSink::new(&received); + let remaining = Arc::new(AtomicU32::new(5)); + let builder = PipelineBuilder::new(); + builder + .chain(FailingSource { remaining: Arc::clone(&remaining) }) + .chain(sink.clone()) + .into_sink_marker(); + let pipeline = builder.build().unwrap(); + + let result = pipeline.run(PipelineConfig { threads: 4, ..Default::default() }); + match result { + Err(PipelineError::Io { step, source }) => { + assert_eq!(step, "FailingSource"); + assert_eq!(source.kind(), std::io::ErrorKind::Other); + } + other => panic!("expected Io error, got {other:?}"), + } + } + + // ───── sleep_until_stop ───── + + /// When `stop` is already set, the helper must return effectively + /// immediately, never sleeping the full duration. This is the teardown + /// fast-path: the main thread sets `stop` then `join()`s, and the helper + /// must not block for a poll interval afterward. + #[test] + fn sleep_until_stop_returns_immediately_when_already_stopped() { + let stop = std::sync::atomic::AtomicBool::new(true); + let start = std::time::Instant::now(); + sleep_until_stop(&stop, std::time::Duration::from_secs(10)); + assert!( + start.elapsed() < std::time::Duration::from_millis(100), + "expected near-immediate return, took {:?}", + start.elapsed() + ); + } + + /// When `stop` is set partway through the sleep, the helper must wake and + /// return well before the full duration elapses (within a couple of poll + /// slices), proving it interrupts a long sleep rather than waiting it out. + #[test] + fn sleep_until_stop_wakes_when_stopped_midway() { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + let setter = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(50)); + stop_clone.store(true, std::sync::atomic::Ordering::Relaxed); + }); + let start = std::time::Instant::now(); + sleep_until_stop(&stop, std::time::Duration::from_secs(10)); + let elapsed = start.elapsed(); + setter.join().unwrap(); + // Woke shortly after the 50ms flag flip — far below the 10s budget. + assert!(elapsed >= std::time::Duration::from_millis(40), "woke too early: {elapsed:?}"); + assert!(elapsed < std::time::Duration::from_millis(500), "woke too late: {elapsed:?}"); + } + + /// When `stop` is never set, the helper sleeps for approximately the full + /// duration (it does not return early). Generous upper bound keeps it + /// non-flaky on a loaded CI host. + #[test] + fn sleep_until_stop_sleeps_full_duration_when_never_stopped() { + let stop = std::sync::atomic::AtomicBool::new(false); + let start = std::time::Instant::now(); + sleep_until_stop(&stop, std::time::Duration::from_millis(100)); + let elapsed = start.elapsed(); + assert!(elapsed >= std::time::Duration::from_millis(95), "returned too early: {elapsed:?}"); + assert!(elapsed < std::time::Duration::from_secs(2), "ran far too long: {elapsed:?}"); + } + + #[test] + fn classify_stall_progress_is_healthy() { + // Any advance in the global progress counter clears the stall, + // regardless of how long the prior stall was or what is in flight. + assert_eq!(classify_stall(true, 999, 10, 60, 1_000_000), StallVerdict::Progressing); + } + + #[test] + fn classify_stall_empty_pipeline_is_starvation_not_deadlock() { + // No progress, but nothing is stuck anywhere: the pipeline is idle + // waiting on a slow upstream (e.g. a stdin pipe). Never fatal — mirrors + // legacy `check_deadlock_and_restore`'s starvation guard. + assert_eq!(classify_stall(false, 120, 10, 60, 0), StallVerdict::Starving); + } + + /// The `in_flight_bytes` probe is blind to queued items whose `heap_size()` + /// is 0, because `ByteBoundedQueue` accounts `T::heap_size()` only and never + /// `size_of::()`. A `ByteBounded` edge can therefore hold items and still + /// report zero bytes — which `classify_stall` reads as `Starving`, resetting + /// the stall clock on every poll so `deadlock_timeout_secs` never fires. + /// + /// This pins the accounting limitation documented on `in_flight_bytes`, so a + /// future change that starts counting `size_of::()` (closing the blind + /// spot) fails here and prompts the doc to be updated with it. + #[test] + fn byte_bounded_queue_of_zero_heap_items_reports_no_bytes_in_flight() { + use crate::queues::{ByteBoundedQueue, ItemQueue}; + + // `u32: HeapSize` reports 0 heap bytes (see `item.rs`). + let q = ByteBoundedQueue::::new(64 * 1024); + for i in 0..100u32 { + q.try_push(i).expect("64 KiB budget accepts zero-heap items"); + } + assert!(!q.is_empty(), "the items really are queued"); + assert_eq!( + q.current_bytes(), + 0, + "heap_size()-only accounting reports zero for zero-heap items" + ); + // ...and a wedge stranding exactly those items is unclassifiable. + assert_eq!( + classify_stall(false, 999, 10, 60, q.current_bytes()), + StallVerdict::Starving, + "a wedge holding only zero-heap items cannot reach Wedged" + ); + } + + #[test] + fn classify_stall_stuck_work_below_fatal_only_warns() { + // No progress with work stuck, but the stall has not persisted long + // enough to be sure it is a wedge rather than one slow dispatch. + assert_eq!(classify_stall(false, 15, 10, 60, 4096), StallVerdict::Stalled); + } + + #[test] + fn classify_stall_below_warn_threshold_keeps_watching() { + // Stalled with stuck work but not yet past the warn threshold: no-op. + assert_eq!(classify_stall(false, 5, 10, 60, 4096), StallVerdict::Watching); + } + + #[test] + fn classify_stall_stuck_work_past_fatal_is_wedged() { + // No progress with work stuck for >= the fatal threshold: a genuine + // wedge — fail fast instead of hanging forever. + assert_eq!(classify_stall(false, 60, 10, 60, 4096), StallVerdict::Wedged); + } + + /// L2.4: a Detached step legitimately blocked on EMPTY input (slow upstream) + /// must NOT trip the monitor. The Detached merge's data flows through the + /// internal `SortMergeSlot` table — invisible to `in_flight_bytes` — and its + /// framework input edge (setup events) is drained early, so while it waits + /// for decompress the byte-bounded edges feeding it are EMPTY: + /// `in_flight_bytes == 0` → `Starving` (progress-shaped), never `Wedged`, + /// no matter how long the wait. The starvation guard already encodes this; + /// this test pins the Detached interpretation against a regression. + #[test] + fn monitor_no_false_positive_on_idle_detached() { + // Long stall, well past the fatal threshold, but nothing is stuck on a + // visible byte-bounded edge (the Detached step is parked on its empty + // input). Must be Starving, not Wedged. + assert_eq!(classify_stall(false, 600, 10, 60, 0), StallVerdict::Starving); + // And the instant the slow producer feeds it, global progress advances. + assert_eq!(classify_stall(true, 600, 10, 60, 0), StallVerdict::Progressing); + } + + /// L2.4 (revision item 10): a *genuine* wedge of the Detached merge still + /// trips fatal. If the merge is truly stuck, its pool upstream (decompress) + /// can't push to the full slots either, so the byte-bounded spill→decompress + /// edge stays FULL with no progress — `in_flight_bytes > 0` past the fatal + /// window → `Wedged`. The slot-table blindness does not hide a real wedge, + /// because the upstream byte edge always reflects it. + #[test] + fn monitor_still_trips_on_genuine_detached_wedge() { + assert_eq!(classify_stall(false, 60, 10, 60, 1_048_576), StallVerdict::Wedged); + } + + /// A step with a single output branch of the given `QueueSpec`, used to + /// exercise the monitor-visibility transport check. + fn step_with_output_spec( + name: &'static str, + spec: QueueSpec, + ) -> Box { + #[derive(Clone)] + struct SpecStep { + name: &'static str, + spec: QueueSpec, + } + impl Step for SpecStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: self.name, + kind: StepKind::Serial, + sticky: false, + output_queues: vec![self.spec], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + Box::new(crate::erased::TypedStep::new(SpecStep { name, spec })) + } + + /// A `ChainGraph` whose per-step branch count matches each step's declared + /// `output_queues` length — the common shape where every branch has a spec. + fn graph_matching_specs( + steps: &[Box], + ) -> crate::topology::ChainGraph { + let mut g = crate::topology::ChainGraph::new(); + for step in steps { + g.register_step(step.profile().name, step.profile().output_queues.len()); + } + g + } + + #[test] + fn first_monitor_blind_transport_finds_count_and_unbounded() { + // All-ByteBounded: every transport is monitor-visible. + let visible = vec![ + step_with_output_spec("A", QueueSpec::ByteBounded { limit_bytes: 1 << 20 }), + step_with_output_spec("B", QueueSpec::ByteBounded { limit_bytes: 1 << 20 }), + ]; + assert!(first_monitor_blind_transport(&visible, &graph_matching_specs(&visible)).is_none()); + + // A CountBounded branch is flagged (it registers no byte probe). + let with_count = vec![ + step_with_output_spec("A", QueueSpec::ByteBounded { limit_bytes: 1 << 20 }), + step_with_output_spec("Blind", QueueSpec::CountBounded { capacity: 8 }), + ]; + let (name, spec) = + first_monitor_blind_transport(&with_count, &graph_matching_specs(&with_count)).unwrap(); + assert_eq!(name, "Blind"); + assert!(matches!(spec, QueueSpec::CountBounded { .. })); + + // Unbounded is flagged too. + let with_unbounded = vec![step_with_output_spec("U", QueueSpec::Unbounded)]; + assert!( + first_monitor_blind_transport(&with_unbounded, &graph_matching_specs(&with_unbounded)) + .is_some() + ); + } + + #[test] + fn first_monitor_blind_transport_flags_implicit_unbounded_branch() { + // A step whose graph branch count exceeds its declared `output_queues`: + // the extra branch resolves to `QueueSpec::Unbounded` (matching `dag()` + // and context-building), which is monitor-blind and must be flagged even + // though the step declared only one explicit, ByteBounded spec. + let steps = vec![step_with_output_spec( + "HasImplicitBranch", + QueueSpec::ByteBounded { limit_bytes: 1 << 20 }, + )]; + let mut graph = crate::topology::ChainGraph::new(); + graph.register_step("HasImplicitBranch", 2); // 2 branches, 1 explicit spec + let (name, spec) = first_monitor_blind_transport(&steps, &graph).unwrap(); + assert_eq!(name, "HasImplicitBranch"); + assert!(matches!(spec, QueueSpec::Unbounded), "implicit 2nd branch is Unbounded"); + } + + #[test] + fn ensure_monitor_visible_transports_ok_for_all_byte_bounded() { + // The armed-monitor invariant holds (returns Ok) when every output + // transport is ByteBounded. + let steps = vec![ + step_with_output_spec("A", QueueSpec::ByteBounded { limit_bytes: 1 << 20 }), + step_with_output_spec("B", QueueSpec::ByteBounded { limit_bytes: 1 << 20 }), + ]; + assert!(ensure_monitor_visible_transports(&steps, &graph_matching_specs(&steps)).is_ok()); + } + + #[test] + fn ensure_monitor_visible_transports_errs_on_count_bounded() { + // A monitor-blind transport on an armed pipeline is rejected with a + // graceful error (in every build, release included) instead of silently + // losing the wedge verdict or crashing the process. + let steps = vec![step_with_output_spec("Blind", QueueSpec::CountBounded { capacity: 8 })]; + let err = ensure_monitor_visible_transports(&steps, &graph_matching_specs(&steps)) + .expect_err("a CountBounded transport on an armed pipeline must be rejected"); + assert!( + matches!(err, PipelineError::MonitorBlindTransport { step: "Blind", .. }), + "expected MonitorBlindTransport for the blind step, got {err:?}" + ); + } + + #[test] + fn apply_stall_verdict_wedged_records_timeout_and_cancels() { + use crate::signal::PipelineError; + let stats = PipelineStats::new(vec![]); + let signal = PipelineSignal::new(); + let now = std::time::Instant::now(); + let mut mon_state = StallMonitorState { last_total: 0, stall_start: now, last_warn: None }; + + let stop = apply_stall_verdict( + StallVerdict::Wedged, + now, + 0, + 60, + 4096, + std::time::Duration::from_secs(10), + &stats, + &signal, + &mut mon_state, + ); + + assert!(stop, "a wedge must stop the monitor"); + assert!(signal.is_done(), "a wedge must make workers observe is_done()"); + match signal.outcome() { + Some(PipelineError::TimedOut { stalled_secs }) => assert_eq!(*stalled_secs, 60), + other => panic!("expected TimedOut, got {other:?}"), + } + // Exercises the `TimedOut` Display arm. + assert!(signal.outcome().unwrap().to_string().contains("no progress")); + } + + #[test] + fn apply_stall_verdict_stalled_warns_once_per_window() { + let stats = PipelineStats::new(vec![]); + let signal = PipelineSignal::new(); + let now = std::time::Instant::now(); + let mut mon_state = StallMonitorState { last_total: 0, stall_start: now, last_warn: None }; + let warn = std::time::Duration::from_secs(10); + + // First stall in the window arms the throttle without failing the run. + let stop = apply_stall_verdict( + StallVerdict::Stalled, + now, + 0, + 15, + 4096, + warn, + &stats, + &signal, + &mut mon_state, + ); + assert!(!stop); + assert!(mon_state.last_warn.is_some(), "a stall must arm the warn throttle"); + assert!(!signal.is_done(), "a stall must not fail the run"); + + // A second stall inside the same window must not re-arm (no re-warn). + let armed = mon_state.last_warn; + let stop2 = apply_stall_verdict( + StallVerdict::Stalled, + now, + 0, + 16, + 4096, + warn, + &stats, + &signal, + &mut mon_state, + ); + assert!(!stop2); + assert_eq!(mon_state.last_warn, armed, "must not re-warn within the same window"); + } + + #[test] + fn apply_stall_verdict_progressing_and_starving_reset_the_clock() { + let stats = PipelineStats::new(vec![]); + let signal = PipelineSignal::new(); + let t0 = std::time::Instant::now(); + let warn = std::time::Duration::from_secs(10); + let mut mon_state = + StallMonitorState { last_total: 0, stall_start: t0, last_warn: Some(t0) }; + + // Progress advances the watermark and clears the warn throttle. + let later = t0 + std::time::Duration::from_secs(5); + let stop = apply_stall_verdict( + StallVerdict::Progressing, + later, + 42, + 0, + 0, + warn, + &stats, + &signal, + &mut mon_state, + ); + assert!(!stop); + assert_eq!(mon_state.last_total, 42); + assert_eq!(mon_state.stall_start, later); + assert!(mon_state.last_warn.is_none()); + assert!(!signal.is_done()); + + // Starvation (nothing in flight) resets the clock without failing. + mon_state.last_warn = Some(t0); + let even_later = later + std::time::Duration::from_secs(5); + let stop2 = apply_stall_verdict( + StallVerdict::Starving, + even_later, + 99, + 0, + 0, + warn, + &stats, + &signal, + &mut mon_state, + ); + assert!(!stop2); + assert_eq!(mon_state.stall_start, even_later); + assert!(mon_state.last_warn.is_none()); + assert!(!signal.is_done()); + } + + #[test] + fn in_flight_bytes_counts_transport_and_reorder_stash() { + // The wedge-vs-starvation signal must include the reorder overflow + // stash, not just transport queues — items can sit in a reorder buffer + // (waiting for a missing serial) while every transport reads empty. + use crate::item::HeapSize; + use crate::queues::{BoundedQueueHandle, ByteBoundedQueue, CountBoundedQueue, ItemQueue}; + use crate::reorder::{ReorderCapHandle, ReorderStage, Sequenced}; + use crate::runtime::contexts::{ChainContexts, RegisteredQueue}; + use crate::topology::{BranchIdx, StepIdx}; + + #[derive(Debug)] + struct Heavy(Vec); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + self.0.len() + } + } + + // Transport queue holding 200 bytes. + let transport = Arc::new(ByteBoundedQueue::::new(10_000)); + transport.try_push(Heavy(vec![0u8; 200])).unwrap(); + assert_eq!(transport.current_bytes(), 200); + + // Reorder stage with 300 bytes stuck in its overflow stash: with + // next_serial (0) absent and the inner transport (cap 1) full, the + // second push overflows into the buffer. + let inner: Arc>> = + Arc::new(CountBoundedQueue::>::new(1)); + let reorder = Arc::new(ReorderStage::with_max_overflow_bytes(inner, 100_000)); + reorder.try_push(5, Heavy(vec![0u8; 100])).unwrap(); // -> inner transport + reorder.try_push(6, Heavy(vec![0u8; 300])).unwrap(); // -> overflow stash + assert_eq!(reorder.current_buffer_bytes(), 300); + + let handle: Arc = transport.clone(); + let cap: Arc = reorder.clone(); + let rq = RegisteredQueue { + producer_step_name: "test", + producer_step: StepIdx(0), + branch: BranchIdx(0), + handle, + reorder_cap: Some(cap), + }; + let contexts = ChainContexts { + inputs: vec![], + outputs: vec![], + bounded_queues: vec![rq], + edges: vec![], + }; + // 200 (transport) + 300 (reorder stash). + assert_eq!(in_flight_bytes(&contexts), 500); + + // An empty registry reads zero — the starvation (idle) signal. + let empty = ChainContexts { + inputs: vec![], + outputs: vec![], + bounded_queues: vec![], + edges: vec![], + }; + assert_eq!(in_flight_bytes(&empty), 0); + } +} diff --git a/crates/fgumi-pipeline-core/src/erased.rs b/crates/fgumi-pipeline-core/src/erased.rs new file mode 100644 index 000000000..541ea3a3a --- /dev/null +++ b/crates/fgumi-pipeline-core/src/erased.rs @@ -0,0 +1,1487 @@ +//! Type erasure: `ErasedStep` trait + `TypedStep` adapter. +//! +//! The runtime holds a heterogeneous chain in `Vec>`. +//! `TypedStep` is the adapter that bridges between the type-erased +//! dispatch in the worker loop and the concrete `S::try_run` body. +//! +//! Each `ErasedStep` exposes the methods the runtime needs: +//! - `clone_boxed` — make per-worker copies for `Parallel` steps +//! - `build_output_set` — construct the producer's queue set + view from +//! `StepProfile::output_queues` + `branch_ordering` +//! - `build_input_handle` — pull the consumer's typed input handle out of +//! the producer's `OutputQueueSet` (mutable: takes ownership of the +//! branch's slot) +//! - `wrap_outputs_view` — wrap the type-erased view into a typed +//! `OutputHandles` for the worker to pass into `ctx.outputs` +//! - `mark_outputs_drained` — close all output branches (called by the +//! driver when a step returns `StepOutcome::Finished`, counter-gated for +//! `Parallel` so only the last clone closes the shared output) +//! - `is_source` — true iff `S::Input == ()` (used by chain-context +//! construction to pick the source's unit-input path) + +use std::any::{Any, TypeId}; +use std::io; +use std::marker::PhantomData; +use std::sync::Arc; + +use super::handles::{BranchInputHandle, OutputQueueSet}; +use super::outputs::StepOutputs; +use super::reorder::BranchOrdering; +use super::signal::PipelineSignal; +use super::step::{ + Affinity, DetachedGroup, OutputHandles, OutputsViewAny, Step, StepCtx, StepKind, StepOutcome, + StepProfile, +}; + +/// The branch orderings the framework actually *builds* for a single-input +/// producer of the given [`StepKind`], given its declared profile orderings. +/// +/// `Serial` / `Exclusive` producers emit items in arrival order by construction +/// (the framework's mutex serializes pushes; an Exclusive-owned step has a +/// single dispatcher), so inserting a `ReorderStage` on their output edges is +/// pure overhead. The framework collapses any declared +/// `ByOrdinal` / `ByItemOrdinal` to [`BranchOrdering::None`] for those kinds — +/// `build_queues` then constructs the direct transport with no reorder stage. +/// `Parallel` producers keep their declared orderings verbatim. +/// +/// # Precondition for `ByItemOrdinal` +/// +/// The collapse preserves *arrival* order, which equals *ordinal* order only if +/// the producer already receives its input in ordinal order. A single-input +/// `Serial` / `Exclusive` / `Detached` step fed by a +/// [`BranchOrdering::None`] edge can be handed items out of ordinal order +/// (multiple `Parallel` upstream workers push in nondeterministic order), and +/// collapsing its declared `ByItemOrdinal` output to `None` then propagates that +/// disorder to a consumer that asked for ordinal order. +/// +/// So a chain declaring `ByItemOrdinal` must keep every edge upstream of that +/// producer ordered — which the canonical BAM pattern does: the source declares +/// an ordering and every intermediate transform propagates the input's serial +/// onto its outputs. Declaring `ByItemOrdinal` downstream of an unordered +/// (`None`) edge is a chain-construction error, not something this collapse can +/// detect: the rule is a function of `(kind, declared)` only and has no view of +/// the graph. +/// +/// This is the single source of truth for the collapse rule, shared by +/// [`TypedStep::build_output_set`] (which constructs the transport) and +/// `Pipeline::dag` (which must render the *effective* — post-collapse — +/// ordering so the diagnostic matches the transport actually built). Note +/// `Step2` producers (`TypedStep2`) do **not** collapse, so this helper is only +/// for single-input steps. +#[must_use] +pub(crate) fn effective_branch_orderings( + kind: StepKind, + declared: &[BranchOrdering], +) -> Vec { + match kind { + StepKind::Parallel => declared.to_vec(), + // Single-producer kinds emit in-order from one thread, so a downstream + // reorder stage is redundant. `Detached` is single-producer too (one + // dedicated thread), so it collapses like `Serial`/`Exclusive`. + StepKind::Serial | StepKind::Exclusive | StepKind::Detached => { + declared.iter().map(|_| BranchOrdering::None).collect() + } + } +} + +/// Type-erased step interface used by the worker loop. +pub trait ErasedStep: Send + 'static { + fn profile(&self) -> StepProfile; + + /// The step's static name, returned WITHOUT building a `StepProfile`. + /// + /// `dispatch_one_step` reads the name on every dispatch (for stats / + /// error reporting), so going through `profile()` there would heap- + /// allocate the profile's two `Vec`s per dispatch (a virtual call, so + /// the optimizer cannot elide them). Adapters cache the name at + /// construction and return it here for free. + fn name(&self) -> &'static str; + + /// The step's [`StepKind`], returned WITHOUT building a `StepProfile`. + /// + /// The setup passes (`pool::assign_exclusive_owners`, + /// `pool::assign_sticky_owners`, `storage::build_worker_storage`) read the + /// kind per step; going through `profile()` there heap-allocates the + /// profile's two `Vec`s per read (a virtual call, so the optimizer cannot + /// elide them). Adapters cache the kind at construction and return it free. + fn kind(&self) -> StepKind; + + /// Whether the step is `sticky`, returned WITHOUT building a `StepProfile`. + /// Same rationale as [`Self::kind`]: the sticky-owner assignment passes read + /// it per step. Adapters cache it at construction. + fn sticky(&self) -> bool; + + /// Forward `Step::affinity` for worker-eligibility gating. Only + /// consulted for `Serial` steps; the runtime calls this once during + /// `build_worker_storage` to decide which workers get a `Shared` + /// entry vs a `Skip` placeholder. + fn affinity(&self) -> Affinity; + + /// Forward `Step::detached_group` — which dedicated driver thread a + /// `Detached` step runs on. Read once by `extract_detached_steps` to group + /// detached steps onto shared driver threads (the N+2 model). Only + /// meaningful for `Detached` kinds. + fn detached_group(&self) -> DetachedGroup; + + /// Dispatch `S::try_run` after downcasting queue handles. + /// + /// # Errors + /// + /// Forwards any I/O error from the step body. + fn try_run_erased(&mut self, ctx: &mut ErasedStepCtx<'_>) -> io::Result; + + /// Construct a fresh per-worker copy of this step. Used for Parallel + /// steps. Cheap — implementing types call `S::clone()`, where typical + /// state is unit-struct or `Arc` (one atomic increment). + fn clone_boxed(&self) -> Box; + + /// Take ownership of the consumer's input handle from the producer's + /// output queue set. Used by the framework when constructing chain + /// topology; each branch is taken exactly once. + fn build_input_handle( + &self, + producer_set: &mut OutputQueueSet, + branch_idx: usize, + ) -> Box; + + /// Input arity. Default `1` for single-input + /// [`crate::step::Step`] impls; multi-input + /// adapters (`TypedStep2`, future `StepN`) override to return their + /// arity. Used by [`crate::runtime::contexts`] + /// to decide which input-construction path to take. + fn input_arity(&self) -> usize { + 1 + } + + /// Take ownership of TWO consumer input handles from two + /// upstream output queue sets, paired into a typed + /// [`crate::handles::TwoInputHandles`] + /// per the consumer's [`crate::step::Step2`] + /// associated types. + /// + /// Default impl panics — single-input steps never have arity 2. + /// `TypedStep2` overrides to call `take_typed_input` twice + /// (once per input slot) and wrap the pair. + /// + /// # Panics + /// + /// Default impl always panics; multi-input adapters override. + fn build_two_input_handles( + &self, + _producer_sets: &mut [OutputQueueSet], + _p0_idx: usize, + _p0_branch: usize, + _p1_idx: usize, + _p1_branch: usize, + ) -> Box { + let p = self.profile(); + panic!( + "build_two_input_handles called on '{}' (kind = {:?}); \ + only Step2 adapters (`TypedStep2`) support arity-2 input \ + construction. This is a framework bug — chain-build code \ + should only dispatch arity-2 input construction to steps \ + that override `input_arity` to return 2.", + p.name, p.kind + ); + } + + /// Build this step's output queue set + outputs view from the profile's + /// per-branch queue specs and ordering directives. + fn build_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny); + + /// Like [`Self::build_output_set`] but forces every output branch to a + /// **direct** transport (no reorder stage), keeping the profile's configured + /// queue bound. + /// + /// Used only by the single-thread *fused* driver + /// ([`crate::runtime::run_fused_single_thread`]). At one worker, FIFO push + /// order is already the correct order, so the reorder stage is dead weight. + /// The count/byte bound stays: the fused driver runs a producer before its + /// consumer in each pass, so a step emitting more items per `try_run` than + /// its consumer removes would grow the edge every pass. The bound turns that + /// into ordinary backpressure — the producer holds and retries, as it must + /// under the scheduled driver anyway — instead of unbounded growth. + fn build_fused_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny); + + /// Wrap a typed `OutputsViewAny` into `Box>` + /// (type-erased as `Any`). The runtime stores this box per step so the + /// worker loop's `try_run_erased` can downcast it to the typed + /// `OutputHandles` the step expects in its `ctx.outputs`. + fn wrap_outputs_view(&self, view: OutputsViewAny) -> Box; + + /// Mark all of this step's output branches drained. Downcasts the + /// type-erased outputs handle to `OutputHandles` and calls + /// the typed `mark_all_drained`. Called by the driver when a step returns + /// `StepOutcome::Finished` (counter-gated for `Parallel`). + fn mark_outputs_drained(&self, outputs: &(dyn Any + Send + Sync)); + + /// Returns `true` iff `S::Input = ()` — i.e., this is a source step. + /// Used by chain-context construction to pick the source's unit-input + /// path (a source has no upstream queue to take an input handle from). + fn is_source(&self) -> bool; +} + +/// Identity of a type-erased handle box: the address of the value the +/// `&dyn Any` points at. +/// +/// The typed-handle caches below record this alongside each cached pointer and +/// compare it on every cached hit, so a step handed a *different* +/// `ChainContexts` is rejected instead of reading through a stale pointer. A +/// `usize` rather than a raw pointer keeps `TypedStep` / `TypedStep2` auto-`Send` +/// (raw pointers are `!Send`); the value is only ever compared, never +/// dereferenced. +fn erased_addr(handle: &(dyn Any + Send + Sync)) -> usize { + std::ptr::from_ref(handle).cast::<()>().addr() +} + +/// One slot of a step adapter's typed-handle cache: the resolved handle plus the +/// [`erased_addr`] of the `&dyn Any` box it came from. +/// +/// The `'static` lifetime is a lie — the real bound is `ChainContexts` — held up +/// by the invariants documented on [`TypedStep`], the third of which is what the +/// stored address is checked against on every cached hit. +type CachedHandle = Option<(usize, &'static H)>; + +/// Context the worker loop hands to `ErasedStep` methods. +pub struct ErasedStepCtx<'a> { + /// Boxed `BranchInputHandle`. Adapter downcasts. + pub input: &'a (dyn Any + Send + Sync), + /// `OutputHandles`. Adapter downcasts. + pub outputs: &'a (dyn Any + Send + Sync), + /// Shared signal (error/cancel). Workers consult; steps don't directly. + pub signal: &'a Arc, +} + +/// Adapter that wraps a concrete `Step` impl as an `ErasedStep`. +/// +/// ## Cached typed handles +/// +/// `try_run_erased` needs typed +/// `&BranchInputHandle` / `&OutputHandles` views +/// of the type-erased context boxes. The first call resolves them via +/// `Any::downcast_ref` (a `TypeId` compare + transmute); subsequent calls +/// reuse the cached references through the `cached_input` / +/// `cached_outputs` fields. Without the cache, a 4-thread CODEC 8M +/// run pays ≈230 samples (~1.6%) on `downcast_ref` `TypeId` compares +/// across the dispatch hot path; the cache eliminates them entirely. +/// +/// The cache is sound because: +/// +/// 1. The boxes are owned by `ChainContexts` (an `Arc` held alive +/// for the entire `Pipeline::run` call by every worker thread). +/// 2. Every `TypedStep` instance — owned (`Parallel`), shared +/// (`Serial`, behind `Mutex`), or pinned (`Exclusive`) — is +/// destroyed before `ChainContexts` goes out of scope (workers +/// exit, then the runtime drops the contexts). +/// 3. Every dispatch passes the **same** box reference for a given +/// `step_idx` (see `run_worker_loop`'s +/// `contexts.inputs[step_idx.0].as_ref()`). The cached pointer +/// always refers to that same box. +/// +/// Only (3) is invisible to the compiler, so each cache slot also records the +/// `erased_addr` of the box it was resolved from and every cached hit asserts +/// it — unconditionally, in release too. A step handed a different, +/// **still-live** `ChainContexts` panics on the spot instead of reading through +/// a pointer into the other one. +/// +/// That check is a best-effort guard, not a proof. It compares data addresses +/// only, so it cannot detect a *freed* box whose address the allocator handed +/// back out: a new `ChainContexts` whose input box lands on a dead one's address +/// passes the assert and the `transmute` reads through a dangling pointer. +/// Soundness therefore still rests entirely on invariant (2) — every +/// `TypedStep` is dropped before the `ChainContexts` it cached from. Relaxing +/// (2) is not made safe by this guard. +/// +/// `clone_boxed` calls `TypedStep::new(...)` which initializes the +/// cache to `None`, so per-worker clones (`Parallel` steps) start +/// with a fresh cache. +pub struct TypedStep { + inner: S, + /// Static step name, cached from `inner.profile().name` at + /// construction so `ErasedStep::name()` (read per dispatch) never + /// rebuilds the profile's `Vec`s. See the `ErasedStep::name` doc. + name: &'static str, + /// `StepKind`, cached at construction so the setup passes read it without + /// rebuilding the profile's `Vec`s. See the `ErasedStep::kind` doc. + kind: StepKind, + /// `sticky` flag, cached at construction for the same reason as `kind`. + sticky: bool, + /// Cached downcast of `ctx.input` after the first dispatch, paired with the + /// [`erased_addr`] of the box it was resolved from (checked on every cached + /// hit). `'static` is a lie — the actual lifetime is bounded by + /// `ChainContexts` — enforced via the `// SAFETY:` comment below. + cached_input: CachedHandle>, + /// Cached downcast of `ctx.outputs`, with its box address. Same lifetime story. + cached_outputs: CachedHandle>, + _phantom: PhantomData S>, +} + +impl TypedStep { + pub fn new(step: S) -> Self { + let profile = step.profile(); + let name = profile.name; + let kind = profile.kind; + let sticky = profile.sticky; + Self { + inner: step, + name, + kind, + sticky, + cached_input: None, + cached_outputs: None, + _phantom: PhantomData, + } + } + + /// Resolve the typed input handle from the dispatch context, caching + /// the result for subsequent dispatches. + /// + /// # Panics + /// + /// Panics on the first dispatch if the `ctx.input` box doesn't + /// downcast to `BranchInputHandle` (a chain topology + /// invariant violation; the builder should have caught this). + #[allow(unsafe_code)] + fn resolve_input<'a>(&mut self, ctx: &ErasedStepCtx<'a>) -> &'a BranchInputHandle { + if let Some((cached_addr, cached)) = self.cached_input { + // Point 3 -- every dispatch passes the same box for a given + // `step_idx` -- is the invariant the cache rests on, and the one the + // type system cannot enforce. Violating it makes `cached` point at a + // box this dispatch does not own, so reject a mismatch in EVERY + // build: a `debug_assert!` alone would leave release builds reading + // through a stale pointer. The check is one load and one compare — + // not the `downcast_ref` `TypeId` probe this cache exists to elide. + assert_eq!( + erased_addr(ctx.input), + cached_addr, + "cached input handle does not match the dispatch context — a \ + TypedStep was reused across two ChainContexts" + ); + // Same box implies the same typed pointer, so this adds nothing in + // release; keep it in debug as a check that the cached pointer was + // derived from that box correctly in the first place. + debug_assert!( + ctx.input + .downcast_ref::>() + .is_some_and(|live| std::ptr::eq(live, cached)), + "cached input handle disagrees with a fresh downcast of the same box" + ); + // SAFETY: Lifetime extension from `'static` (cache slot) back + // to `'a` (the dispatch context's lifetime). The cached + // pointer was originally a `&'a BranchInputHandle` + // pulled out of `ChainContexts.inputs[step_idx]`; that box + // outlives every `TypedStep` (point 2 in the type-level + // doc). Dispatches always pass the same box for the same + // `step_idx` (point 3), so the pointer is still pointing + // at the live box on every subsequent dispatch. + return unsafe { + std::mem::transmute::<&BranchInputHandle, &'a BranchInputHandle>( + cached, + ) + }; + } + let r: &'a BranchInputHandle = ctx + .input + .downcast_ref::>() + .expect("input handle downcast failed — chain topology invariant"); + // SAFETY: Lifetime extension from `'a` to `'static` for storage. + // The same point-2/point-3 invariants from `// SAFETY:` above + // apply: the box outlives `self`, and we only ever read the + // cache through `resolve_input`, which immediately re-extends + // back to a bounded `'a` before handing it to user code. + let cached: &'static BranchInputHandle = unsafe { + std::mem::transmute::< + &'a BranchInputHandle, + &'static BranchInputHandle, + >(r) + }; + self.cached_input = Some((erased_addr(ctx.input), cached)); + r + } + + /// Resolve the typed outputs handle from the dispatch context, + /// caching the result for subsequent dispatches. See `resolve_input` + /// for the safety argument. + #[allow(unsafe_code)] + fn resolve_outputs<'a>(&mut self, ctx: &ErasedStepCtx<'a>) -> &'a OutputHandles { + if let Some((cached_addr, cached)) = self.cached_outputs { + // See `resolve_input` for why the address check is unconditional and + // the downcast re-check is debug-only. + assert_eq!( + erased_addr(ctx.outputs), + cached_addr, + "cached outputs handle does not match the dispatch context — a \ + TypedStep was reused across two ChainContexts" + ); + debug_assert!( + ctx.outputs + .downcast_ref::>() + .is_some_and(|live| std::ptr::eq(live, cached)), + "cached outputs handle disagrees with a fresh downcast of the same box" + ); + // SAFETY: see `resolve_input`; same boxes/lifetimes story. + return unsafe { + std::mem::transmute::<&OutputHandles, &'a OutputHandles>( + cached, + ) + }; + } + let r: &'a OutputHandles = ctx + .outputs + .downcast_ref::>() + .expect("outputs handle downcast failed — chain topology invariant"); + // SAFETY: see `resolve_input`; same boxes/lifetimes story. + let cached: &'static OutputHandles = unsafe { + std::mem::transmute::<&'a OutputHandles, &'static OutputHandles>( + r, + ) + }; + self.cached_outputs = Some((erased_addr(ctx.outputs), cached)); + r + } +} + +impl ErasedStep for TypedStep +where + S: Step, +{ + fn profile(&self) -> StepProfile { + self.inner.profile() + } + + fn name(&self) -> &'static str { + self.name + } + + fn kind(&self) -> StepKind { + self.kind + } + + fn sticky(&self) -> bool { + self.sticky + } + + fn affinity(&self) -> Affinity { + self.inner.affinity() + } + + fn detached_group(&self) -> DetachedGroup { + self.inner.detached_group() + } + + fn try_run_erased(&mut self, ctx: &mut ErasedStepCtx<'_>) -> io::Result { + let input = self.resolve_input(ctx); + let outputs = self.resolve_outputs(ctx); + let mut step_ctx = StepCtx { input, outputs }; + self.inner.try_run(&mut step_ctx) + } + + fn clone_boxed(&self) -> Box { + // `TypedStep::new` initializes `cached_input` / `cached_outputs` + // to `None` so the new clone resolves them fresh on its first + // dispatch (the box references in *this* `TypedStep`'s cache + // are still valid for the new clone too — they refer to the + // same `ChainContexts` boxes — but resolving fresh is simpler + // and keeps the cache lifetime story local to each clone). + Box::new(TypedStep::new(self.inner.new_worker_copy())) + } + + fn build_input_handle( + &self, + producer_set: &mut OutputQueueSet, + branch_idx: usize, + ) -> Box { + let handle: BranchInputHandle = + producer_set.take_typed_input::(branch_idx); + Box::new(handle) + } + + fn build_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + let profile = self.inner.profile(); + // Producers with `StepKind::Serial` or `StepKind::Exclusive` emit + // items in arrival order by construction (the framework's mutex + // serializes pushes; an Exclusive-owned step has a single + // dispatcher). Inserting a `ReorderStage` on those output edges + // is pure overhead — every push pays an ordinal allocation + + // `Sequenced` wrap + reorder mutex hop for items that are + // already in order. The framework collapses any + // `BranchOrdering::ByOrdinal` / `ByItemOrdinal` declaration to + // `None` here, so `build_queues` constructs the direct transport + // path with no reorder stage. Step authors keep declaring + // `ByItemOrdinal` (preserves the intent in the profile; + // documents what the consumer needs); the framework is just + // smart enough to skip the wrap when the producer can already + // satisfy that need. + let effective_orderings = + effective_branch_orderings(profile.kind, &profile.branch_ordering); + ::build_queues( + &profile.output_queues, + &effective_orderings, + level, + ) + } + + fn build_fused_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + // Force a DIRECT transport on every branch — at one worker FIFO push + // order is already the correct order, so the reorder stage is pure + // overhead. The profile's queue specs are kept as-is: the fused driver + // dispatches a producer before its consumer in each pass, so a step that + // emits k > 1 items per `try_run` while its consumer takes one would + // grow the edge by k - 1 every pass. Dropping the bound turns that into + // unbounded memory growth; keeping it makes the producer hold and retry, + // exactly as it already must under the scheduled driver. + let profile = self.inner.profile(); + let orderings = vec![BranchOrdering::None; profile.output_queues.len()]; + ::build_queues(&profile.output_queues, &orderings, level) + } + + fn wrap_outputs_view(&self, view: OutputsViewAny) -> Box { + let outputs: OutputHandles = OutputHandles::new(view); + Box::new(outputs) + } + + fn mark_outputs_drained(&self, outputs: &(dyn Any + Send + Sync)) { + let typed = outputs + .downcast_ref::>() + .expect("outputs handle downcast failed in mark_outputs_drained"); + ::mark_all_drained(typed); + } + + fn is_source(&self) -> bool { + TypeId::of::() == TypeId::of::<()>() + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// TypedStep2 — adapter for `Step2` impls (two-input merge steps). +// +// Same shape as TypedStep for outputs: one `OutputHandles` +// cached after the first dispatch. Inputs differ — instead of a single +// `BranchInputHandle`, the adapter caches a single +// `&TwoInputHandles` and lends per-branch refs +// (`ctx.a`, `ctx.b`) into the wrapped `Step2::try_run`. +// +// Drain detection: a `Step2` consumer is "drained" when **both** input +// branches report drained. The step itself checks this in `try_run` +// (`ctx.a.is_drained() && ctx.b.is_drained()`) to decide when to report +// `Finished`; the typed-erased input box is a `TwoInputHandles`. +// ───────────────────────────────────────────────────────────────────────────── + +use super::handles::TwoInputHandles; +use super::step::{Step2, StepCtx2}; + +/// Adapter wrapping a [`Step2`] impl as an [`ErasedStep`]. +/// +/// Cached typed handles follow the same pattern as [`TypedStep`]: +/// the first dispatch resolves `&TwoInputHandles` +/// / `&OutputHandles` via `Any::downcast_ref`, subsequent +/// dispatches reuse the cached references through unsafe lifetime +/// extension. The safety argument is identical (the boxes are owned +/// by `ChainContexts` which outlives every adapter instance, and +/// every dispatch passes the same box reference for a given +/// `step_idx`). +pub struct TypedStep2 { + inner: S, + /// Static step name, cached at construction. See `ErasedStep::name`. + name: &'static str, + /// `StepKind`, cached at construction. See `ErasedStep::kind`. + kind: StepKind, + /// `sticky` flag, cached at construction. See `ErasedStep::sticky`. + sticky: bool, + /// Cached downcasts, each paired with the [`erased_addr`] of the box it was + /// resolved from — see [`CachedHandle`]. + cached_inputs: CachedHandle>, + cached_outputs: CachedHandle>, + _phantom: PhantomData S>, +} + +impl TypedStep2 { + pub fn new(step: S) -> Self { + let profile = step.profile(); + let name = profile.name; + let kind = profile.kind; + let sticky = profile.sticky; + Self { + inner: step, + name, + kind, + sticky, + cached_inputs: None, + cached_outputs: None, + _phantom: PhantomData, + } + } + + #[allow(unsafe_code)] + fn resolve_inputs<'a>( + &mut self, + ctx: &ErasedStepCtx<'a>, + ) -> &'a TwoInputHandles { + if let Some((cached_addr, cached)) = self.cached_inputs { + // See `TypedStep::resolve_input` for why the address check is + // unconditional and the downcast re-check is debug-only. + assert_eq!( + erased_addr(ctx.input), + cached_addr, + "cached input handles do not match the dispatch context — a \ + TypedStep2 was reused across two ChainContexts" + ); + debug_assert!( + ctx.input + .downcast_ref::>() + .is_some_and(|live| std::ptr::eq(live, cached)), + "cached input handles disagree with a fresh downcast of the same box" + ); + // SAFETY: lifetime extension from `'static` (cache slot) + // back to `'a` (the dispatch context's lifetime). The + // cached pointer was originally a + // `&'a TwoInputHandles` pulled out + // of `ChainContexts.inputs[step_idx]`; that box outlives + // every `TypedStep2` instance (same point-2/point-3 + // invariants as `TypedStep::resolve_input`). + return unsafe { + std::mem::transmute::< + &TwoInputHandles, + &'a TwoInputHandles, + >(cached) + }; + } + let r: &'a TwoInputHandles = ctx + .input + .downcast_ref::>() + .expect("input handle downcast failed — Step2 chain topology invariant"); + // SAFETY: lifetime extension from `'a` to `'static` for + // storage. The box outlives `self`; we only ever read the + // cache through `resolve_inputs`, which immediately re-extends + // back to a bounded `'a` before handing it to user code. + let cached: &'static TwoInputHandles = unsafe { + std::mem::transmute::< + &'a TwoInputHandles, + &'static TwoInputHandles, + >(r) + }; + self.cached_inputs = Some((erased_addr(ctx.input), cached)); + r + } + + #[allow(unsafe_code)] + fn resolve_outputs<'a>(&mut self, ctx: &ErasedStepCtx<'a>) -> &'a OutputHandles { + if let Some((cached_addr, cached)) = self.cached_outputs { + // See `TypedStep::resolve_input` for why the address check is + // unconditional and the downcast re-check is debug-only. + assert_eq!( + erased_addr(ctx.outputs), + cached_addr, + "cached outputs handle does not match the dispatch context — a \ + TypedStep2 was reused across two ChainContexts" + ); + debug_assert!( + ctx.outputs + .downcast_ref::>() + .is_some_and(|live| std::ptr::eq(live, cached)), + "cached outputs handle disagrees with a fresh downcast of the same box" + ); + // SAFETY: lifetime extension from `'static` (cache slot) back to + // `'a`; same boxes/lifetimes story as `resolve_inputs`. + return unsafe { + std::mem::transmute::<&OutputHandles, &'a OutputHandles>( + cached, + ) + }; + } + let r: &'a OutputHandles = ctx + .outputs + .downcast_ref::>() + .expect("outputs handle downcast failed — Step2 chain topology invariant"); + // SAFETY: lifetime extension from `'a` to `'static` for storage; same + // boxes/lifetimes story as `resolve_inputs`. + let cached: &'static OutputHandles = unsafe { + std::mem::transmute::<&'a OutputHandles, &'static OutputHandles>( + r, + ) + }; + self.cached_outputs = Some((erased_addr(ctx.outputs), cached)); + r + } +} + +impl ErasedStep for TypedStep2 { + fn profile(&self) -> StepProfile { + self.inner.profile() + } + + fn name(&self) -> &'static str { + self.name + } + + fn kind(&self) -> StepKind { + self.kind + } + + fn sticky(&self) -> bool { + self.sticky + } + + fn affinity(&self) -> Affinity { + self.inner.affinity() + } + + fn detached_group(&self) -> DetachedGroup { + self.inner.detached_group() + } + + fn try_run_erased(&mut self, ctx: &mut ErasedStepCtx<'_>) -> io::Result { + let inputs = self.resolve_inputs(ctx); + let outputs = self.resolve_outputs(ctx); + let mut typed_ctx = StepCtx2:: { a: &inputs.a, b: &inputs.b, outputs }; + self.inner.try_run(&mut typed_ctx) + } + + fn clone_boxed(&self) -> Box { + Box::new(TypedStep2::new(self.inner.new_worker_copy())) + } + + fn build_input_handle( + &self, + _producer_set: &mut OutputQueueSet, + _branch_idx: usize, + ) -> Box { + let p = self.profile(); + panic!( + "build_input_handle called on Step2 adapter '{}' (kind = {:?}); \ + multi-input steps build their inputs via build_two_input_handles. \ + This is a framework bug — chain-build code should dispatch on \ + input_arity().", + p.name, p.kind + ); + } + + fn input_arity(&self) -> usize { + 2 + } + + fn build_two_input_handles( + &self, + producer_sets: &mut [OutputQueueSet], + p0_idx: usize, + p0_branch: usize, + p1_idx: usize, + p1_branch: usize, + ) -> Box { + if p0_idx == p1_idx { + assert_ne!( + p0_branch, + p1_branch, + "Step2 inputs must consume distinct branches when they share \ + producer step {p0_idx} for step '{}'.", + self.profile().name + ); + let set = &mut producer_sets[p0_idx]; + let a: BranchInputHandle = set.take_typed_input::(p0_branch); + let b: BranchInputHandle = set.take_typed_input::(p1_branch); + return Box::new(TwoInputHandles::::new(a, b)); + } + // Borrow two disjoint elements of `producer_sets` simultaneously. + // `split_at_mut(lo+1)` puts producer_sets[lo] in the first half; + // we index into the second half for the hi side. + let (lo_idx, hi_idx, swap) = + if p0_idx < p1_idx { (p0_idx, p1_idx, false) } else { (p1_idx, p0_idx, true) }; + let (lo_half, hi_half) = producer_sets.split_at_mut(lo_idx + 1); + let lo_set: &mut OutputQueueSet = &mut lo_half[lo_idx]; + let hi_set: &mut OutputQueueSet = &mut hi_half[hi_idx - (lo_idx + 1)]; + let (a_set, b_set) = if swap { + (hi_set, lo_set) // p0 = high, p1 = low + } else { + (lo_set, hi_set) // p0 = low, p1 = high + }; + let a: BranchInputHandle = a_set.take_typed_input::(p0_branch); + let b: BranchInputHandle = b_set.take_typed_input::(p1_branch); + Box::new(TwoInputHandles::::new(a, b)) + } + + fn build_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + // Unlike `TypedStep::build_output_set`, a `Step2` does NOT collapse a + // `ByOrdinal` / `ByItemOrdinal` output to `None` for `Serial` / + // `Exclusive` kinds — the ordering is passed through verbatim and the + // reorder stage is kept. + // + // The single-input collapse rests on a universal property: a Serial + // single-input step consumes one already-ordered stream, so any input + // ordinal it propagates onto an output is emitted in push order, making + // the reorder stage redundant. A two-input MERGE has no such guarantee. + // It interleaves two branches, so a `ByItemOrdinal` output that + // propagates an *input's* ordinal can be pushed out of ordinal order + // even under serial (mutex-serialized, single-dispatcher) execution — + // e.g. when branch B's next-needed ordinal hasn't arrived yet but + // branch A's later one has. For such a step the reorder stage is + // load-bearing: dropping it would deliver records out of order. + // + // Today's production `Step2`s happen not to need it (`PairRawFastq` + // declares `BranchOrdering::None`; `ZipperMergeStep` assigns fresh + // *sequential* output ordinals, so its push order already equals its + // ordinal order). But the framework cannot assume that for an arbitrary + // merge, so it conservatively keeps the reorder for every ordered Step2 + // output. Do not add the single-input collapse here: a `Step2` that + // propagates an input ordinal can emit out of ordinal order even under + // serial execution, so dropping the reorder would deliver records out of + // order. + // Bind the profile once — each `profile()` call rebuilds its two `Vec`s. + // Matches the single-input adapter and this adapter's own + // `build_fused_output_set`. + let profile = self.inner.profile(); + ::build_queues( + &profile.output_queues, + &profile.branch_ordering, + level, + ) + } + + fn build_fused_output_set( + &self, + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + // `Step2` never appears in a linear (single-input) fused chain, so this + // is unreachable in practice; provided for trait completeness with the + // same direct-transport / profile-bounds semantics as the single-input + // adapter. + let profile = self.inner.profile(); + let orderings = vec![BranchOrdering::None; profile.output_queues.len()]; + ::build_queues(&profile.output_queues, &orderings, level) + } + + fn wrap_outputs_view(&self, view: OutputsViewAny) -> Box { + let typed: OutputHandles = OutputHandles::::new(view); + Box::new(typed) + } + + fn mark_outputs_drained(&self, outputs: &(dyn Any + Send + Sync)) { + let typed = outputs + .downcast_ref::>() + .expect("mark_outputs_drained downcast failed — Step2 chain topology invariant"); + ::mark_all_drained(typed); + } + + fn is_source(&self) -> bool { + // Step2 consumers are never sources by definition (they have + // two non-unit input branches). + false + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::handles::BranchInputHandle; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{ + InputHandle, OutputHandles, Step, StepCtx, StepCtx2, StepKind, StepOutcome, StepProfile, + }; + + /// Trivial step: u32 → u32+1 (single output). + #[derive(Clone)] + struct AddOne; + + impl Step for AddOne { + type Input = u32; + type Outputs = Single; + + fn profile(&self) -> StepProfile { + StepProfile { + name: "AddOne", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(n) => { + let _ = ctx.outputs.push(n + 1); + Ok(StepOutcome::Progress) + } + None => Ok(StepOutcome::NoProgress), + } + } + + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// Build a `(producer_set, producer_outputs)` pair so we can simulate a + /// chain link by manually constructing the upstream side. + fn build_addone_outputs() -> (OutputQueueSet, OutputHandles>) { + let producer: Box = Box::new(TypedStep::new(AddOne)); + let (queue_set, outputs_view) = + producer.build_output_set(crate::builder::InstrumentationLevel::Off); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + (queue_set, outputs) + } + + #[rstest] + #[case(StepKind::Serial, false)] + #[case(StepKind::Serial, true)] + #[case(StepKind::Parallel, false)] + #[case(StepKind::Parallel, true)] + #[case(StepKind::Exclusive, false)] + #[case(StepKind::Exclusive, true)] + #[case(StepKind::Detached, false)] + #[case(StepKind::Detached, true)] + fn cached_kind_and_sticky_match_profile(#[case] kind: StepKind, #[case] sticky: bool) { + // The `kind()` / `sticky()` accessors return values cached at + // construction (so the setup passes don't rebuild the profile's `Vec`s). + // They must agree with `profile()` for every kind/sticky combination, + // or a stale cache could mis-route worker assignment. Each combination is + // an independent `#[case]` so a failure isolates the offending pair. + #[derive(Clone)] + struct Tagged(StepKind, bool); + impl Step for Tagged { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Tagged", + kind: self.0, + sticky: self.1, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + let erased: Box = Box::new(TypedStep::new(Tagged(kind, sticky))); + let profile = erased.profile(); + assert_eq!(erased.kind(), profile.kind, "kind() must match profile for {kind:?}"); + assert_eq!( + erased.sticky(), + profile.sticky, + "sticky() must match profile for {kind:?}/{sticky}" + ); + } + + #[rstest] + #[case(StepKind::Serial, false)] + #[case(StepKind::Serial, true)] + #[case(StepKind::Parallel, false)] + #[case(StepKind::Parallel, true)] + #[case(StepKind::Exclusive, false)] + #[case(StepKind::Exclusive, true)] + #[case(StepKind::Detached, false)] + #[case(StepKind::Detached, true)] + fn cached_kind_and_sticky_match_profile_step2(#[case] kind: StepKind, #[case] sticky: bool) { + // Mirror `cached_kind_and_sticky_match_profile` for the `Step2` adapter. + // `TypedStep2::{new, kind, sticky}` caches the same metadata at + // construction, and runtime setup reads `ErasedStep::kind()`/`sticky()` + // for ALL erased steps — `Step` and `Step2` alike (e.g. zipper's merge + // step). A stale `TypedStep2` cache could mis-route owner assignment for + // a merge step just as a `TypedStep` cache could, so pin both paths. + #[derive(Clone)] + struct Tagged2(StepKind, bool); + impl Step2 for Tagged2 { + type InputA = u32; + type InputB = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Tagged2", + kind: self.0, + sticky: self.1, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx2<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + let erased: Box = Box::new(TypedStep2::new(Tagged2(kind, sticky))); + let profile = erased.profile(); + assert_eq!(erased.kind(), profile.kind, "kind() must match profile for {kind:?}"); + assert_eq!( + erased.sticky(), + profile.sticky, + "sticky() must match profile for {kind:?}/{sticky}" + ); + } + + #[test] + fn typed_step_round_trips_through_erased_dispatch() { + // Producer's output set carries the input handle that the consumer + // (also an AddOne) will pull from. The consumer's own output set is + // separate. + let (mut producer_set, producer_outputs) = build_addone_outputs(); + + // Push a u32 onto the producer's output (which is the consumer's input). + producer_outputs.push(41).unwrap(); + + // Consumer takes the typed input handle out of the producer's set. + let mut consumer: Box = Box::new(TypedStep::new(AddOne)); + let input_any = consumer.build_input_handle(&mut producer_set, 0); + + // Consumer needs its own output set + view to run. + let (consumer_set, consumer_view) = + consumer.build_output_set(crate::builder::InstrumentationLevel::Off); + let consumer_outputs: OutputHandles> = OutputHandles::new(consumer_view); + + let signal = PipelineSignal::new(); + let mut ctx = ErasedStepCtx { + input: input_any.as_ref(), + outputs: &consumer_outputs as &(dyn Any + Send + Sync), + signal: &signal, + }; + let outcome = consumer.try_run_erased(&mut ctx).unwrap(); + assert_eq!(outcome, StepOutcome::Progress); + + // Consumer pushed (41 + 1) = 42 onto its own output. Pull it. + let mut consumer_set = consumer_set; + let consumer_input = consumer_set.take_typed_input::(0); + assert_eq!(consumer_input.pop(), Some(42)); + } + + /// Every other dispatch test calls `try_run_erased` exactly once, so + /// `resolve_input` / `resolve_outputs` always take the first-resolve path + /// and the cached-hit arm — which holds both `transmute`s and the + /// box-address assert guarding the dispatch-identity invariant — is never + /// executed. Dispatch twice against the same `ErasedStepCtx` so the second + /// call resolves from cache. + #[test] + fn second_dispatch_resolves_handles_from_the_cache() { + let (mut producer_set, producer_outputs) = build_addone_outputs(); + producer_outputs.push(1).unwrap(); + producer_outputs.push(2).unwrap(); + + let mut consumer: Box = Box::new(TypedStep::new(AddOne)); + let input_any = consumer.build_input_handle(&mut producer_set, 0); + let (mut consumer_set, consumer_view) = + consumer.build_output_set(crate::builder::InstrumentationLevel::Off); + let consumer_outputs: OutputHandles> = OutputHandles::new(consumer_view); + + let signal = PipelineSignal::new(); + let mut ctx = ErasedStepCtx { + input: input_any.as_ref(), + outputs: &consumer_outputs as &(dyn Any + Send + Sync), + signal: &signal, + }; + // First dispatch populates the cache; the second must hit it — and both + // its unconditional box-address assert and the debug-only downcast + // re-check, which is live in this test build. + assert_eq!(consumer.try_run_erased(&mut ctx).unwrap(), StepOutcome::Progress); + assert_eq!(consumer.try_run_erased(&mut ctx).unwrap(), StepOutcome::Progress); + + let out = consumer_set.take_typed_input::(0); + assert_eq!( + (out.pop(), out.pop()), + (Some(2), Some(3)), + "both dispatches must produce output through the cached handles" + ); + } + + /// Point 3 of the `TypedStep` safety argument — every dispatch passes the + /// same box for a given `step_idx` — is the one the compiler cannot enforce. + /// A step whose cache is populated and is then handed a *different* input box + /// must panic rather than resolve from the stale entry. The check is an + /// unconditional `assert!`, so this holds in release builds too; when it was + /// a `debug_assert!` a release build read through the pointer into the + /// previous `ChainContexts` instead. + #[test] + #[should_panic(expected = "cached input handle does not match the dispatch context")] + fn dispatch_against_a_different_context_box_panics() { + let (mut first_producer_set, first_producer_outputs) = build_addone_outputs(); + first_producer_outputs.push(1).unwrap(); + let (mut second_producer_set, second_producer_outputs) = build_addone_outputs(); + second_producer_outputs.push(2).unwrap(); + + let mut consumer: Box = Box::new(TypedStep::new(AddOne)); + let first_input = consumer.build_input_handle(&mut first_producer_set, 0); + let second_input = consumer.build_input_handle(&mut second_producer_set, 0); + let (_consumer_set, consumer_view) = + consumer.build_output_set(crate::builder::InstrumentationLevel::Off); + let consumer_outputs: OutputHandles> = OutputHandles::new(consumer_view); + let outputs_any = &consumer_outputs as &(dyn Any + Send + Sync); + + let signal = PipelineSignal::new(); + // First dispatch populates the cache from `first_input`. + let mut ctx = + ErasedStepCtx { input: first_input.as_ref(), outputs: outputs_any, signal: &signal }; + assert_eq!(consumer.try_run_erased(&mut ctx).unwrap(), StepOutcome::Progress); + + // Same step, different input box — the invariant violation. + let mut wrong_ctx = + ErasedStepCtx { input: second_input.as_ref(), outputs: outputs_any, signal: &signal }; + let _ = consumer.try_run_erased(&mut wrong_ctx); + } + + #[test] + fn typed_step_returns_noprogress_on_empty_input() { + let (mut producer_set, _producer_outputs) = build_addone_outputs(); + let mut consumer: Box = Box::new(TypedStep::new(AddOne)); + let input_any = consumer.build_input_handle(&mut producer_set, 0); + + let (_consumer_set, consumer_view) = + consumer.build_output_set(crate::builder::InstrumentationLevel::Off); + let consumer_outputs: OutputHandles> = OutputHandles::new(consumer_view); + + let signal = PipelineSignal::new(); + let mut ctx = ErasedStepCtx { + input: input_any.as_ref(), + outputs: &consumer_outputs as &(dyn Any + Send + Sync), + signal: &signal, + }; + let outcome = consumer.try_run_erased(&mut ctx).unwrap(); + assert_eq!(outcome, StepOutcome::NoProgress); + } + + #[test] + fn clone_boxed_yields_independent_step() { + let original: Box = Box::new(TypedStep::new(AddOne)); + let cloned = original.clone_boxed(); + assert_eq!(original.profile().name, "AddOne"); + assert_eq!(cloned.profile().name, "AddOne"); + } + + /// `ErasedStep::name()` (the per-dispatch, allocation-free accessor) + /// must agree with `profile().name`, including across `clone_boxed` + /// (the per-worker clone path that re-caches the name in `new`). + #[test] + fn erased_name_matches_profile_name() { + let step: Box = Box::new(TypedStep::new(AddOne)); + assert_eq!(step.name(), step.profile().name); + assert_eq!(step.name(), "AddOne"); + let cloned = step.clone_boxed(); + assert_eq!(cloned.name(), cloned.profile().name); + assert_eq!(cloned.name(), "AddOne"); + } + + #[test] + fn build_output_set_uses_profile_queues_and_ordering() { + let typed: Box = Box::new(TypedStep::new(AddOne)); + let (mut queue_set, _outputs_view) = + typed.build_output_set(crate::builder::InstrumentationLevel::Off); + assert_eq!(queue_set.n_branches(), 1); + // Verify the queue is u32-typed by taking the input handle. + let input = queue_set.take_typed_input::(0); + // Empty initially. + assert_eq!(input.pop(), None); + } + + /// The fused transport keeps the profile's queue bound; only the ordering is + /// dropped. It previously forced `QueueSpec::Unbounded` on every branch, so a + /// step emitting more items per `try_run` than its consumer removed grew the + /// edge by the difference on every pass of the fused driver — the driver runs + /// a producer before its consumer, so nothing capped it. `AddOne` declares + /// `CountBounded { capacity: 4 }`, so the fifth push must be rejected. + #[test] + fn build_fused_output_set_keeps_the_profile_queue_bound() { + let typed: Box = Box::new(TypedStep::new(AddOne)); + let (mut queue_set, outputs_view) = + typed.build_fused_output_set(crate::builder::InstrumentationLevel::Off); + assert_eq!(queue_set.n_branches(), 1); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + + for n in 0..4u32 { + outputs.push(n).expect("the capacity-4 fused transport admits four items"); + } + assert!( + outputs.push(4).is_err(), + "the fused transport must apply the profile's capacity-4 backpressure \ + rather than growing without limit" + ); + + // Backpressure, not a wedge: draining one item frees exactly one slot. + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(0), "direct FIFO transport, no reorder stage"); + outputs.push(4).expect("one pop frees one slot"); + } + + #[test] + fn build_input_handle_downcasts_to_typed_branch_handle() { + // Producer pushes a value onto its output. + let (mut producer_set, producer_outputs) = build_addone_outputs(); + producer_outputs.push(7).unwrap(); + + // Consumer grabs the typed input handle. + let consumer: Box = Box::new(TypedStep::new(AddOne)); + let input_any = consumer.build_input_handle(&mut producer_set, 0); + let input = + input_any.downcast_ref::>().expect("input handle downcast"); + assert_eq!(input.pop(), Some(7)); + } + + #[test] + fn wrap_outputs_view_yields_typed_outputs_handles() { + let typed: Box = Box::new(TypedStep::new(AddOne)); + let (mut queue_set, view) = + typed.build_output_set(crate::builder::InstrumentationLevel::Off); + let outputs_any = typed.wrap_outputs_view(view); + + // Downcast back to OutputHandles> and push. + let outputs = outputs_any + .downcast_ref::>>() + .expect("OutputHandles downcast"); + outputs.push(99).unwrap(); + + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(99)); + } + + #[test] + fn mark_outputs_drained_propagates_to_consumer_input() { + // Producer is an AddOne; its outputs feed a consumer's input. + let producer: Box = Box::new(TypedStep::new(AddOne)); + let (mut producer_set, producer_view) = + producer.build_output_set(crate::builder::InstrumentationLevel::Off); + let producer_outputs_any = producer.wrap_outputs_view(producer_view); + + let consumer: Box = Box::new(TypedStep::new(AddOne)); + let input_any = consumer.build_input_handle(&mut producer_set, 0); + let consumer_input = input_any + .downcast_ref::>() + .expect("consumer input handle is BranchInputHandle"); + + // Initially not drained. + assert!(!InputHandle::is_drained(consumer_input)); + + // Producer marks its outputs drained via the type-erased path. + producer.mark_outputs_drained(producer_outputs_any.as_ref()); + + // Drained signal reaches consumer. + assert!(InputHandle::is_drained(consumer_input)); + } + + /// Multi-output step: u32 → (u32, u32) split into low + high bytes. + #[derive(Clone)] + struct SplitBytes; + + impl Step for SplitBytes { + type Input = u32; + type Outputs = (u32, u32); + + fn profile(&self) -> StepProfile { + StepProfile { + name: "SplitBytes", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![ + QueueSpec::CountBounded { capacity: 2 }, + QueueSpec::CountBounded { capacity: 2 }, + ], + branch_ordering: vec![BranchOrdering::None, BranchOrdering::None], + } + } + + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(n) => { + let v = ctx.outputs.view(); + let _ = v.a.push(n & 0xFF); + let _ = v.b.push((n >> 8) & 0xFF); + Ok(StepOutcome::Progress) + } + None => Ok(StepOutcome::NoProgress), + } + } + } + + #[test] + fn split_bytes_routes_to_two_branches() { + // Producer feeding `SplitBytes`: another AddOne (Single → Single). + // We just need a way to feed u32 into SplitBytes, so reuse AddOne's outputs. + let (mut producer_set, producer_outputs) = build_addone_outputs(); + producer_outputs.push(0xABCD).unwrap(); + + // SplitBytes is the consumer; takes the typed input from the producer. + let mut splitter: Box = Box::new(TypedStep::new(SplitBytes)); + let input_any = splitter.build_input_handle(&mut producer_set, 0); + + // Splitter's own output set has two branches. + let (mut splitter_set, splitter_view) = + splitter.build_output_set(crate::builder::InstrumentationLevel::Off); + let splitter_outputs: OutputHandles<(u32, u32)> = OutputHandles::new(splitter_view); + + let signal = PipelineSignal::new(); + let mut ctx = ErasedStepCtx { + input: input_any.as_ref(), + outputs: &splitter_outputs as &(dyn Any + Send + Sync), + signal: &signal, + }; + + let outcome = splitter.try_run_erased(&mut ctx).unwrap(); + assert_eq!(outcome, StepOutcome::Progress); + + let in_a = splitter_set.take_typed_input::(0); + let in_b = splitter_set.take_typed_input::(1); + assert_eq!(in_a.pop(), Some(0xCD)); + assert_eq!(in_b.pop(), Some(0xAB)); + } + + #[test] + fn is_source_true_for_unit_input_steps() { + #[derive(Clone)] + struct UnitInputStep; + impl Step for UnitInputStep { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "UnitInput", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::ByOrdinal], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + } + + let source: Box = Box::new(TypedStep::new(UnitInputStep)); + let mid: Box = Box::new(TypedStep::new(AddOne)); + assert!(source.is_source()); + assert!(!mid.is_source()); + } + + /// Pins the deliberate asymmetry between `TypedStep::build_output_set` + /// (collapses ordered output → `None` for Serial/Exclusive) and + /// `TypedStep2::build_output_set` (keeps the reorder stage). A two-input + /// merge can push a `ByItemOrdinal` output out of ordinal order even under + /// serial execution, so the reorder stage is load-bearing. This test fails + /// if a future change adds the single-input collapse to the Step2 path. + #[test] + fn step2_serial_byitemordinal_output_is_reordered_not_collapsed() { + use crate::item::{HeapSize, Ordered}; + use crate::outputs::OrderedBytesSingle; + use crate::step::{Step2, StepCtx2}; + + #[derive(Debug)] + struct Ord32 { + ordinal: u64, + } + impl HeapSize for Ord32 { + fn heap_size(&self) -> usize { + 0 + } + } + impl Ordered for Ord32 { + fn ordinal(&self) -> u64 { + self.ordinal + } + } + + // A Serial two-input merge with a `ByItemOrdinal` output. We only need + // its output edge (via `build_output_set`), never run it, so `try_run` + // is trivial. + struct OutOfOrderMerge; + impl Step2 for OutOfOrderMerge { + type InputA = u32; + type InputB = u32; + type Outputs = OrderedBytesSingle; + fn profile(&self) -> StepProfile { + StepProfile { + name: "OutOfOrderMerge", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { limit_bytes: 64 * 1024 }], + branch_ordering: vec![BranchOrdering::ByItemOrdinal], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx2<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + let step: Box = Box::new(TypedStep2::new(OutOfOrderMerge)); + let (mut queue_set, view) = + step.build_output_set(crate::builder::InstrumentationLevel::Off); + let outputs_any = step.wrap_outputs_view(view); + let outputs = outputs_any + .downcast_ref::>>() + .expect("OutputHandles downcast"); + + // Push OUT of ordinal order: 2, then 0, then 1. + outputs.push(Ord32 { ordinal: 2 }).unwrap(); + outputs.push(Ord32 { ordinal: 0 }).unwrap(); + outputs.push(Ord32 { ordinal: 1 }).unwrap(); + + // The reorder stage must deliver them in ORDINAL order (0, 1, 2). With + // the single-input collapse the consumer would see push order (2, 0, 1). + let input = queue_set.take_typed_input::(0); + let got: Vec = std::iter::from_fn(|| input.pop().map(|o| o.ordinal)).collect(); + assert_eq!(got, vec![0, 1, 2], "ordered Step2 output must be reordered by ordinal"); + } + + /// Input construction is dispatched on `input_arity()`, so a single-input + /// adapter must never be asked for two-input handles and a `Step2` adapter + /// must never be asked for one. Both defaults panic rather than silently + /// mis-wiring a chain, and both messages must name the step so a real hit + /// is debuggable. These pin the guard, not the happy path. + #[test] + #[should_panic(expected = "AddOne")] + fn build_two_input_handles_on_a_single_input_adapter_panics() { + let step: Box = Box::new(TypedStep::new(AddOne)); + let (mut producer_set, _view) = + step.build_output_set(crate::builder::InstrumentationLevel::Off); + let _ = step.build_two_input_handles(std::slice::from_mut(&mut producer_set), 0, 0, 0, 0); + } + + #[test] + #[should_panic(expected = "build_input_handle called on Step2 adapter")] + fn build_input_handle_on_a_step2_adapter_panics() { + use crate::step::{Step2, StepCtx2}; + + #[derive(Clone)] + struct JoinStub; + impl Step2 for JoinStub { + type InputA = u32; + type InputB = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "JoinStub", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx2<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + let (mut producer_set, _outputs) = build_addone_outputs(); + let step: Box = Box::new(TypedStep2::new(JoinStub)); + let _ = step.build_input_handle(&mut producer_set, 0); + } +} diff --git a/crates/fgumi-pipeline-core/src/finalize.rs b/crates/fgumi-pipeline-core/src/finalize.rs new file mode 100644 index 000000000..4170d9333 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/finalize.rs @@ -0,0 +1,39 @@ +//! Post-pipeline finalize hook contract. +//! +//! [`FinalizeHook`] is the bare trait that lets a chain register +//! heterogeneous post-pipeline cleanup actions — metrics drains, summary +//! logging, gate checks, rejects-file finalization, BAI indexing — in a single +//! `Vec>` that the caller drains after `Pipeline::run` +//! returns. +//! +//! It lives in `fgumi-pipeline-core` (rather than the umbrella `fgumi` crate) +//! so that any CLI crate built directly on the pipeline core — e.g. +//! `fgumi-sort-cli` — can implement the same contract without redefining a +//! parallel trait. Keeping the trait single-sourced here is what lets the sort +//! finalize hooks be defined once and re-exported by the umbrella, instead of +//! duplicated across both crates (X1-005). +//! +//! The richer machinery built on top of this trait — `BuiltPipeline`, +//! `drain_finalize`, and the built-in stats/timing hooks — stays in the +//! umbrella crate, since it references umbrella-only types. + +use anyhow::Result; + +/// Post-pipeline cleanup. Each stage with metrics, summary logging, +/// `--min-corrected`-style gates, or rejects-file finalization registers one or +/// more hooks during chain build. The caller iterates the registered hooks +/// (typically via `try_for_each`) after `Pipeline::run` returns. +/// +/// Trait object so heterogeneous hooks (correct's metrics drain, consensus's +/// summary log, AAM's records-aligned counter, sort's BAI indexer, etc.) can +/// share a `Vec>`. +pub trait FinalizeHook: Send { + /// Run the post-pipeline action. Called exactly once, after + /// `Pipeline::run` returns and before the caller exits. + /// + /// # Errors + /// + /// Returns the underlying I/O / aggregation / gate-check error. + /// Callers typically chain via `try_for_each`. + fn finalize(self: Box) -> Result<()>; +} diff --git a/crates/fgumi-pipeline-core/src/handles.rs b/crates/fgumi-pipeline-core/src/handles.rs new file mode 100644 index 000000000..0ad223372 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/handles.rs @@ -0,0 +1,2260 @@ +//! Framework-side input/output handles wired to the layered queue stack. +//! +//! Per-branch architecture: +//! +//! ```text +//! producer step +//! │ outputs.push(item) +//! ▼ +//! BranchOutputHandle +//! │ - allocates ordinal (if BranchOrdering::ByOrdinal) +//! │ - heap-size accounting happens inside ByteBoundedQueue +//! │ - forwards to either ReorderStage or transport directly +//! ▼ +//! ┌────────────────────────────────┐ +//! │ ReorderStage (optional) │ ← only if BranchOrdering::ByOrdinal +//! └────────────────────────────────┘ +//! ▼ +//! ┌──────────────────────────────────────┐ +//! │ Arc> │ ← Wrapped = Sequenced if ordered, else T +//! └──────────────────────────────────────┘ +//! ▼ +//! BranchInputHandle +//! │ pop() -> Option +//! ▼ +//! consumer step +//! ``` +//! +//! No code in this module surfaces serials or heap sizes to step authors — +//! those are framework-managed. Step authors see `outputs.push(item)` and +//! `input.pop() -> Option`. +//! +//! `QueueSpec::ByteBounded` requires `T: HeapSize`. Every build path — `Single`, +//! the tuple fan-outs, and the ordered-bytes shapes — bounds `T: HeapSize` and +//! routes through `build_branch_byte_aware` (or `build_branch_ordered_bytes`), so +//! `ByteBounded` is honored on every branch of every shape. Those byte-aware +//! entry points delegate every non-byte spec straight back to `build_branch`, so +//! count/unbounded branches are unaffected. +//! +//! `build_branch` itself keeps a `ByteBounded` panic as the guard on that +//! delegation: reaching it with a byte spec means a caller bypassed a byte-aware +//! entry point. The other two panics are for an item-carried ordinal without the +//! trait to read it — `ByItemOrdinal` declared without `Ordered`, and +//! `ByteBounded` + `ByItemOrdinal` on a path lacking the `Ordered` bound (which +//! must use `build_branch_ordered_bytes`). + +use std::any::Any; +use std::marker::PhantomData; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; + +use super::item::{HeapSize, Ordered}; +use super::outputs::Single; +use super::queues::{ByteBoundedQueue, CountBoundedQueue, ItemQueue, QueueSpec, UnboundedQueue}; +use super::reorder::{ + BranchOrdering, DEFAULT_REORDER_OVERFLOW_BYTES, ReorderCapHandle, ReorderStage, Sequenced, +}; +use super::step::{InputHandle, OutputHandles, OutputsViewAny}; + +// ───────────────────────────────────────────────────────────────────────────── +// BranchOutputHandle — one per output branch on the producer side. +// ───────────────────────────────────────────────────────────────────────────── + +/// Producer-side handle for one output branch. Step authors call `push(item)`. +/// The framework manages ordinal assignment and routes to the right destination +/// (transport or reorder stage). +pub struct BranchOutputHandle { + inner: BranchOutputInner, +} + +enum BranchOutputInner { + /// `BranchOrdering::None`: push directly to transport. + Direct(Arc>), + /// `BranchOrdering::ByOrdinal` or `ByItemOrdinal`: push through + /// `ReorderStage`. The stage internally wraps in `Sequenced` and + /// forwards to its transport. `OrdinalSource` decides where the + /// ordinal comes from on each push. + Ordered { stage: Arc>, ordinal_source: OrdinalSource }, +} + +/// Where each push's ordinal comes from on an `Ordered` branch. +enum OrdinalSource { + /// `BranchOrdering::ByOrdinal`: producer allocates a fresh ordinal via + /// this counter on every push. + Allocated(Arc), + /// `BranchOrdering::ByItemOrdinal`: items carry their own ordinal. The + /// function pointer is `|item: &T| item.ordinal()`, monomorphized at + /// branch construction time (requires `T: Ordered`). + ItemSerial(fn(&T) -> u64), +} + +impl OrdinalSource { + #[inline] + fn next(&self, item: &T) -> u64 { + match self { + Self::Allocated(c) => c.fetch_add(1, AtomicOrdering::AcqRel), + Self::ItemSerial(f) => f(item), + } + } +} + +/// An item the underlying queue rejected due to backpressure. +/// +/// `Unpushed` carries the rejected item back to the producer for retry, +/// **plus an opaque token** that — for ordered (`ByOrdinal`) branches — +/// preserves the ordinal allocated on the original push. Retrying via +/// [`BranchOutputHandle::retry`] reuses that ordinal so the consumer's +/// `ReorderStage` doesn't stall waiting for a sequence number that was +/// burned on a rejected attempt. +/// +/// Step authors store `HeldSlot>` (not `HeldSlot`) when their +/// step pushes to an ordered branch. Direct branches don't carry an ordinal, +/// so `Unpushed` is just a typed wrapper around the item there. +pub struct Unpushed { + item: T, + /// Pre-allocated ordinal from a rejected `Ordered` push. `None` for + /// `Direct` (FIFO) branches. + ordinal: Option, +} + +/// Outcome of [`OutputHandles::retry_held`] — a step's drain-time held-slot +/// retry. Lets the caller distinguish "nothing was held" from "flushed +/// successfully" (the two differ for a single-final-batch flusher: the former +/// must still build its batch, the latter is done). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HeldRetry { + /// The held slot was empty — nothing to retry. + WasEmpty, + /// A held item was retried and accepted; the slot is now empty. + Flushed, + /// A held item was retried, still rejected by backpressure, and put back + /// in the slot. The caller must yield (return `NoProgress`/`Contention` + /// and retry on the next dispatch). + StillHeld, +} + +impl std::fmt::Debug for Unpushed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Unpushed").field("ordinal", &self.ordinal).field("item", &"<...>").finish() + } +} + +impl Unpushed { + /// Borrow the rejected item (read-only). + pub fn item(&self) -> &T { + &self.item + } + + /// Discard the framework token and recover the item, abandoning the + /// pre-allocated ordinal (if any). After calling this, **do not** push + /// the item back into an ordered branch — the missing ordinal will + /// stall the consumer's `ReorderStage`. Use only when abandoning the + /// item entirely (e.g., during cancellation). + pub fn into_item(self) -> T { + self.item + } +} + +impl BranchOutputHandle { + /// Try to push a fresh item. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` when the underlying queue (or reorder + /// stage, for `ByOrdinal` branches) applied backpressure. The producer + /// must hand the `Unpushed` to a subsequent [`Self::retry`] call — + /// **not** call `push` with the recovered item — so any pre-allocated + /// ordinal is preserved. + pub fn push(&self, item: T) -> Result<(), Unpushed> { + match &self.inner { + BranchOutputInner::Direct(q) => { + q.try_push(item).map_err(|item| Unpushed { item, ordinal: None }) + } + BranchOutputInner::Ordered { stage, ordinal_source } => { + let ord = ordinal_source.next(&item); + stage + .try_push(ord, item) + .map_err(|(ord, item)| Unpushed { item, ordinal: Some(ord) }) + } + } + } + + /// Retry a previously-rejected push. Reuses the original ordinal for + /// ordered branches so the consumer's `ReorderStage` sees a contiguous + /// sequence. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` if the queue still applied backpressure; + /// the caller should re-hold and retry on the next worker iteration. + /// + /// # Panics + /// + /// Panics if the `Unpushed` came from a different branch kind than this + /// handle (a framework invariant violation). The `Ordered` + `ordinal = + /// None` case panics in **every** build: there is no recovery that + /// preserves the sequence (see the arm's comment). The `Direct` + + /// `ordinal = Some` case is recoverable — a `Direct` branch has no reorder + /// stage to desynchronize — so it panics only in debug builds and drops + /// the stray ordinal in release. + pub fn retry(&self, unpushed: Unpushed) -> Result<(), Unpushed> { + let Unpushed { item, ordinal } = unpushed; + match (&self.inner, ordinal) { + (BranchOutputInner::Direct(q), None) => { + q.try_push(item).map_err(|item| Unpushed { item, ordinal: None }) + } + (BranchOutputInner::Ordered { stage, .. }, Some(ord)) => stage + .try_push(ord, item) + .map_err(|(ord, item)| Unpushed { item, ordinal: Some(ord) }), + (BranchOutputInner::Direct(q), Some(_)) => { + debug_assert!(false, "Unpushed::ordinal=Some on a Direct branch"); + q.try_push(item).map_err(|item| Unpushed { item, ordinal: None }) + } + (BranchOutputInner::Ordered { .. }, None) => { + // Unrecoverable, so fail loudly in every build. `self.push` + // would allocate a FRESH ordinal and abandon the one this + // `Unpushed` was created with, leaving a permanent hole in the + // sequence — the consumer's `ReorderStage` then waits on the + // missing ordinal forever and only the deadlock monitor notices. + // A panic here names the framework bug instead. This is the + // hazard `Unpushed::into_item` documents. + panic!( + "Unpushed::ordinal=None on an Ordered branch — re-pushing would \ + abandon the allocated ordinal and stall the consumer's reorder stage" + ); + } + } + } + + /// Mark this branch drained (producer-side). Idempotent. + pub fn mark_drained(&self) { + match &self.inner { + BranchOutputInner::Direct(q) => q.mark_drained(), + BranchOutputInner::Ordered { stage, .. } => stage.mark_drained(), + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// BranchInputHandle — implements InputHandle. +// ───────────────────────────────────────────────────────────────────────────── + +/// Consumer-side input handle for one branch. Implements `InputHandle`. +pub struct BranchInputHandle { + inner: BranchInputInner, + /// `Some` only on an instrumented edge. The **consumer-pop** side of the + /// edge's `EdgeMetrics` is recorded here (not at the transport queue), + /// because an ordered edge pops through a `ReorderStage` that drains the + /// transport in bulk — `pop` here is the real consumer pop. Shares the same + /// `Arc` as the producer transport (push side). + metrics: Option>, + /// Whether to report each popped item's `heap_size()` to `record_pop`. + /// + /// True only on a byte-bounded edge, mirroring the push side: a + /// `CountBoundedQueue` / `UnboundedQueue` records `record_push(0)` because it + /// keeps no byte accounting, and `ordered_branch` gates its push-side byte + /// reporting on the same signal (`transport_handle.is_some()`). Recording + /// real bytes on the pop side of such an edge made `popped_bytes` nonzero + /// while `pushed_bytes` stayed 0, so `compute_edge_stats` reported a + /// `mibytes_per_s` throughput for an edge that measures no bytes at all. + record_item_bytes: bool, +} + +enum BranchInputInner { + Direct(Arc>), + Ordered(Arc>), + /// A permanently-empty, permanently-drained handle that owns no transport. + /// Used for source steps' dummy input (their input is implicitly drained + /// from t=0; the worker loop never pops from it), avoiding a per-source + /// `SegQueue` allocation just to report `is_drained() == true`. + AlwaysDrained(PhantomData T>), +} + +impl BranchInputHandle { + /// Construct a zero-state input handle that is always empty and always + /// drained, owning no backing queue. Used for source steps, whose input is + /// implicitly drained from the start and never popped. + #[must_use] + pub fn always_drained() -> Self { + Self { + inner: BranchInputInner::AlwaysDrained(PhantomData), + metrics: None, + record_item_bytes: false, + } + } +} + +impl InputHandle for BranchInputHandle { + fn pop(&self) -> Option { + // A `None` from an `Ordered` edge is *reorder-blocked* (not starved) while + // the reorder stage still holds out-of-order items waiting for an earlier + // ordinal — counting that as an empty pop inflates `pop_empties` and can + // misclassify a backlogged edge as starved. The `reorder_blocked` flag is + // reported by the SAME locked pop, so shared (`Parallel`) consumers can't + // observe a torn `(item, blocked)` pair. Direct / always-drained branches + // have no reorder buffer, so a `None` there is always a true empty pop. + let (item, reorder_blocked) = match &self.inner { + BranchInputInner::Direct(q) => (q.try_pop(), false), + BranchInputInner::Ordered(stage) => stage.try_pop_in_order_reporting_blocked(), + BranchInputInner::AlwaysDrained(_) => (None, false), + }; + if let Some(m) = &self.metrics { + if let Some(it) = &item { + // Bytes only on a byte-bounded edge — see `record_item_bytes`. + // usize→u64 is lossless on every target we build for. + #[allow(clippy::cast_possible_truncation)] + let bytes = if self.record_item_bytes { it.heap_size() as u64 } else { 0 }; + m.record_pop(bytes); + } else if !reorder_blocked { + m.record_empty(); + } + } + item + } + + fn is_drained(&self) -> bool { + match &self.inner { + BranchInputInner::Direct(q) => q.is_drained() && q.is_empty(), + BranchInputInner::Ordered(stage) => stage.is_drained(), + BranchInputInner::AlwaysDrained(_) => true, + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Branch construction (one branch at a time, dispatching on QueueSpec + Ordering). +// ───────────────────────────────────────────────────────────────────────────── + +/// The budget-resize handles for one byte-bounded branch: the transport +/// queue's limit setter, plus (for an ordered branch) the reorder stage's +/// overflow-cap setter. The framework's budget pass +/// (`apply_initial_queue_budget`) sets both — the transport limit and the +/// reorder cap — from the same per-edge budget, so they stay in lockstep +/// (single source of truth for `per_queue`). +pub(crate) struct BranchBudgetHandles { + pub(crate) transport: Arc, + /// `Some` for an ordered (`ByOrdinal` / `ByItemOrdinal`) byte-bounded + /// branch — its reorder overflow stash is sized from the same budget. + /// `None` for a direct (unordered) byte-bounded branch (no reorder stage). + pub(crate) reorder_cap: Option>, +} + +/// One end-to-end branch: an output handle and an input handle wired +/// to the same underlying transport (and optional reorder stage). +/// +/// `bounded_queue_handle` is `Some` iff the branch's transport is a +/// `ByteBoundedQueue`; it bundles the transport-limit and reorder-cap +/// setters (see [`BranchBudgetHandles`]). The pipeline-builder collects +/// these (across every branch in the chain) into a registry that the +/// budget pass + optional queue-memory rebalancer use to set/redistribute +/// budget. `None` for `CountBounded` / `Unbounded` branches. +pub(crate) struct Branch { + pub(crate) output: BranchOutputHandle, + pub(crate) input: BranchInputHandle, + pub(crate) bounded_queue_handle: Option, + /// `Some` on an instrumented edge — the shared `EdgeMetrics` (also held by + /// the transport queue for push counts and the input handle for pop counts). + /// Collected into the edge registry by `contexts.rs` Pass 1.5. + pub(crate) metrics: Option>, +} + +/// Mint a per-edge [`EdgeMetrics`] when instrumentation is on, else `None`. +/// Called once per branch in the `build_branch*` constructors; the same handle +/// is shared by the transport (push counts) and the input handle (pop counts). +fn edge_metrics( + level: crate::builder::InstrumentationLevel, +) -> Option> { + level.is_on().then(crate::runtime::metrics::EdgeMetrics::new) +} + +/// Build one branch from a queue spec + ordering directive, where `T` does +/// not need to impl `HeapSize` or `Ordered`. +/// +/// # Panics +/// +/// Panics on `QueueSpec::ByteBounded` (requires `T: HeapSize`; use the +/// byte-aware build path) or on `BranchOrdering::ByItemOrdinal` (requires +/// `T: Ordered`; use `build_branch_ordered`). +pub(crate) fn build_branch( + spec: QueueSpec, + ordering: BranchOrdering, + level: crate::builder::InstrumentationLevel, +) -> Branch { + match (spec, ordering) { + (QueueSpec::CountBounded { capacity }, BranchOrdering::None) => { + let m = edge_metrics(level); + let q: Arc> = + Arc::new(CountBoundedQueue::::maybe_instrumented(capacity, m.clone())); + direct_branch(q, None, m) + } + (QueueSpec::CountBounded { capacity }, BranchOrdering::ByOrdinal) => { + let m = edge_metrics(level); + let transport: Arc>> = + Arc::new(CountBoundedQueue::>::maybe_instrumented( + // Ordered transport is NOT instrumented: push/reject are recorded + // at the ReorderStage boundary (a stash turns a full-transport + // `Err` into an accepted `Ok`). Pop side is on the input handle. + capacity, None, + )); + ordered_branch( + transport, + OrdinalSource::Allocated(Arc::new(AtomicU64::new(0))), + DEFAULT_REORDER_OVERFLOW_BYTES, + None, + m, + ) + } + (QueueSpec::Unbounded, BranchOrdering::None) => { + let m = edge_metrics(level); + let q: Arc> = + Arc::new(UnboundedQueue::::maybe_instrumented(m.clone())); + direct_branch(q, None, m) + } + (QueueSpec::Unbounded, BranchOrdering::ByOrdinal) => { + let m = edge_metrics(level); + let transport: Arc>> = + // Ordered transport is NOT instrumented — push/reject recorded at + // the ReorderStage boundary (see the count-bounded note above). + Arc::new(UnboundedQueue::>::maybe_instrumented(None)); + ordered_branch( + transport, + OrdinalSource::Allocated(Arc::new(AtomicU64::new(0))), + DEFAULT_REORDER_OVERFLOW_BYTES, + None, + m, + ) + } + (_, BranchOrdering::ByItemOrdinal) => { + panic!( + "BranchOrdering::ByItemOrdinal requires `T: Ordered` — \ + use `build_branch_ordered::` instead. The plain \ + `build_branch::` path doesn't have the trait bound to \ + read `item.ordinal()` at push time." + ); + } + (QueueSpec::ByteBounded { .. }, _) => { + panic!( + "QueueSpec::ByteBounded requires `T: HeapSize` — \ + use the byte-aware build path (`build_branch_byte_aware` or \ + `build_branch_ordered_bytes`)." + ); + } + } +} + +/// Build one branch where `T: Ordered` (no `HeapSize` requirement). +/// Supports `BranchOrdering::ByItemOrdinal` (uses `item.ordinal()` for the +/// reorder stage's serial). Falls through to `build_branch::` for non- +/// `ByItemOrdinal` orderings. +/// +/// # Panics +/// +/// Panics on `QueueSpec::ByteBounded` (requires `T: HeapSize`; use the +/// `_ordered_bytes` build path). +pub(crate) fn build_branch_ordered( + spec: QueueSpec, + ordering: BranchOrdering, + level: crate::builder::InstrumentationLevel, +) -> Branch { + match (spec, ordering) { + (QueueSpec::CountBounded { capacity }, BranchOrdering::ByItemOrdinal) => { + let m = edge_metrics(level); + let transport: Arc>> = + Arc::new(CountBoundedQueue::>::maybe_instrumented( + // Ordered transport is NOT instrumented: push/reject are recorded + // at the ReorderStage boundary (a stash turns a full-transport + // `Err` into an accepted `Ok`). Pop side is on the input handle. + capacity, None, + )); + ordered_branch( + transport, + OrdinalSource::ItemSerial(|item: &T| item.ordinal()), + DEFAULT_REORDER_OVERFLOW_BYTES, + None, + m, + ) + } + (QueueSpec::Unbounded, BranchOrdering::ByItemOrdinal) => { + let m = edge_metrics(level); + let transport: Arc>> = + // Ordered transport is NOT instrumented — push/reject recorded at + // the ReorderStage boundary (see the count-bounded note above). + Arc::new(UnboundedQueue::>::maybe_instrumented(None)); + ordered_branch( + transport, + OrdinalSource::ItemSerial(|item: &T| item.ordinal()), + DEFAULT_REORDER_OVERFLOW_BYTES, + None, + m, + ) + } + (QueueSpec::ByteBounded { .. }, _) => { + panic!( + "QueueSpec::ByteBounded requires `T: HeapSize` — \ + `build_branch_ordered` only bounds `T: Ordered`. Use \ + `build_branch_ordered_bytes::` for the \ + combined case." + ); + } + (other, ord) => build_branch::(other, ord, level), + } +} + +/// Build one branch where `T: HeapSize + Ordered` (the canonical BAM step +/// case). Supports all `QueueSpec` × `BranchOrdering` combinations. +pub(crate) fn build_branch_ordered_bytes( + spec: QueueSpec, + ordering: BranchOrdering, + level: crate::builder::InstrumentationLevel, +) -> Branch { + use crate::queues::BoundedQueueHandle; + + match (spec, ordering) { + (QueueSpec::ByteBounded { limit_bytes }, BranchOrdering::None) => { + let m = edge_metrics(level); + let q = Arc::new(ByteBoundedQueue::::maybe_instrumented(limit_bytes, m.clone())); + let handle: Arc = Arc::clone(&q) as Arc; + let q_dyn: Arc> = q; + direct_branch(q_dyn, Some(handle), m) + } + (QueueSpec::ByteBounded { limit_bytes }, BranchOrdering::ByOrdinal) => { + let m = edge_metrics(level); + let transport_concrete = Arc::new( + // Ordered transport is NOT instrumented — see the note on the + // count-bounded ordered transport above; push/reject are recorded + // at the ReorderStage boundary. The byte `handle` (occupancy / + // budget resize) is derived from this same queue and is unaffected. + ByteBoundedQueue::>::maybe_instrumented(limit_bytes, None), + ); + let handle: Arc = + Arc::clone(&transport_concrete) as Arc; + let transport: Arc>> = transport_concrete; + // Cap the must-accept overflow buffer so one worker grinding on a + // large ordinal can't let every later ordinal overflow unbounded + // (#29). This `DEFAULT_REORDER_OVERFLOW_BYTES` is the no-budget + // FALLBACK: when `queue_memory_total` is set (production chains), + // `apply_initial_queue_budget` re-sizes this cap thread-awarely + // from the per-edge transport budget via the registered + // `ReorderCapHandle` — so at low thread counts the stash is small + // and at high thread counts it keeps this 256 MiB ceiling. + ordered_branch( + transport, + OrdinalSource::Allocated(Arc::new(AtomicU64::new(0))), + DEFAULT_REORDER_OVERFLOW_BYTES, + Some(handle), + m, + ) + } + (QueueSpec::ByteBounded { limit_bytes }, BranchOrdering::ByItemOrdinal) => { + let m = edge_metrics(level); + let transport_concrete = Arc::new( + // Ordered transport is NOT instrumented — see the note on the + // count-bounded ordered transport above; push/reject are recorded + // at the ReorderStage boundary. The byte `handle` (occupancy / + // budget resize) is derived from this same queue and is unaffected. + ByteBoundedQueue::>::maybe_instrumented(limit_bytes, None), + ); + let handle: Arc = + Arc::clone(&transport_concrete) as Arc; + let transport: Arc>> = transport_concrete; + ordered_branch( + transport, + OrdinalSource::ItemSerial(|item: &T| item.ordinal()), + DEFAULT_REORDER_OVERFLOW_BYTES, + Some(handle), + m, + ) + } + (other, ord) => build_branch_ordered::(other, ord, level), + } +} + +/// Build one branch where `T: HeapSize`. Supports all three queue specs +/// for non-`ByItemOrdinal` orderings. `ByItemOrdinal` requires `T: Ordered` +/// — use `build_branch_ordered_bytes` for the combined case. +/// +/// The build path `build_single_queues` uses for every `Single` output +/// (`T: HeapSize`, no `Ordered` bound): it honors `ByteBounded` for +/// `BranchOrdering::None` and `ByOrdinal`, and delegates every other spec +/// straight back to `build_branch::`. Steps needing both `HeapSize` and +/// `Ordered` go through `build_branch_ordered_bytes` instead. +/// +/// # Panics +/// +/// Panics on `ByteBounded` + `ByItemOrdinal`: reading an item-carried serial +/// needs `T: Ordered`, which this signature does not bound — so the ordinal is +/// unreachable here even when the item type happens to implement `Ordered`. A +/// step needing both declares `OrderedBytesSingle`, which routes to +/// `build_branch_ordered_bytes`; reaching this arm means a step asked for +/// `ByItemOrdinal` through `Single`, whose build path cannot read one. The +/// panic is that guard rather than a supported configuration. +pub(crate) fn build_branch_byte_aware( + spec: QueueSpec, + ordering: BranchOrdering, + level: crate::builder::InstrumentationLevel, +) -> Branch { + use crate::queues::BoundedQueueHandle; + + match (spec, ordering) { + (QueueSpec::ByteBounded { limit_bytes }, BranchOrdering::None) => { + let m = edge_metrics(level); + let q = Arc::new(ByteBoundedQueue::::maybe_instrumented(limit_bytes, m.clone())); + let handle: Arc = Arc::clone(&q) as Arc; + let q_dyn: Arc> = q; + direct_branch(q_dyn, Some(handle), m) + } + (QueueSpec::ByteBounded { limit_bytes }, BranchOrdering::ByOrdinal) => { + let m = edge_metrics(level); + let transport_concrete = Arc::new( + // Ordered transport is NOT instrumented — see the note on the + // count-bounded ordered transport above; push/reject are recorded + // at the ReorderStage boundary. The byte `handle` (occupancy / + // budget resize) is derived from this same queue and is unaffected. + ByteBoundedQueue::>::maybe_instrumented(limit_bytes, None), + ); + let handle: Arc = + Arc::clone(&transport_concrete) as Arc; + let transport: Arc>> = transport_concrete; + ordered_branch( + transport, + OrdinalSource::Allocated(Arc::new(AtomicU64::new(0))), + DEFAULT_REORDER_OVERFLOW_BYTES, + Some(handle), + m, + ) + } + (QueueSpec::ByteBounded { .. }, BranchOrdering::ByItemOrdinal) => { + panic!( + "ByteBounded + ByItemOrdinal requires `T: HeapSize + Ordered` — \ + use `build_branch_ordered_bytes` instead." + ); + } + (other, ord) => build_branch::(other, ord, level), + } +} + +fn direct_branch( + q: Arc>, + transport_handle: Option>, + metrics: Option>, +) -> Branch { + // A byte-bounded direct edge carries a `transport_handle`; count/unbounded + // ones don't. Same signal `ordered_branch` uses to gate its push-side byte + // reporting — the pop side must agree, or the edge reports popped bytes it + // never counted as pushed. + let record_item_bytes = transport_handle.is_some(); + // A direct (unordered) branch has no reorder stage, so no reorder cap. + let bounded_queue_handle = + transport_handle.map(|transport| BranchBudgetHandles { transport, reorder_cap: None }); + Branch { + output: BranchOutputHandle { inner: BranchOutputInner::Direct(Arc::clone(&q)) }, + input: BranchInputHandle { + inner: BranchInputInner::Direct(q), + metrics: metrics.clone(), + record_item_bytes, + }, + bounded_queue_handle, + metrics, + } +} + +/// Build one ordered branch: a `ReorderStage` over `transport`, plus the input / +/// output handles that share it. +/// +/// `max_overflow_bytes` is mandatory rather than optional. The stage's +/// must-accept overflow stash bypasses the transport's own bound by design, so an +/// ordered branch without a cap would let memory grow with *input* size instead +/// of with config. Taking it by value also keeps `ReorderStage::new`'s unbounded +/// default — which exists for tests — unreachable from any branch builder, so +/// there is no path here that produces an uncapped stash. +fn ordered_branch( + transport: Arc>>, + ordinal_source: OrdinalSource, + max_overflow_bytes: u64, + transport_handle: Option>, + metrics: Option>, +) -> Branch { + // A byte-bounded ordered edge carries a `transport_handle`; count/unbounded + // ones don't. Record the push side's byte size only on the byte-bounded ones + // (matching the queue's own byte accounting). Push/reject are recorded at the + // ReorderStage boundary — NOT on the transport — because a must-accept stash + // turns a full-transport `Err` into an accepted `Ok` (see `push_metrics`). + let record_item_bytes = transport_handle.is_some(); + let stage = Arc::new( + ReorderStage::::with_max_overflow_bytes(transport, max_overflow_bytes) + .with_push_metrics(metrics.clone(), record_item_bytes), + ); + // Pair the transport-limit setter with this stage's overflow-cap setter so + // the budget pass sizes both from one per-edge budget. Cloned (and coerced + // to the trait object) before the stage is moved into the input handle. + let reorder_cap: Arc = stage.clone(); + let bounded_queue_handle = transport_handle + .map(|transport| BranchBudgetHandles { transport, reorder_cap: Some(reorder_cap) }); + Branch { + output: BranchOutputHandle { + inner: BranchOutputInner::Ordered { stage: Arc::clone(&stage), ordinal_source }, + }, + input: BranchInputHandle { + inner: BranchInputInner::Ordered(stage), + metrics: metrics.clone(), + record_item_bytes, + }, + bounded_queue_handle, + metrics, + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// OutputQueueSet — per-step collection of typed input handles +// ───────────────────────────────────────────────────────────────────────────── + +/// Type-erased per-branch input handles, owned by the producer step's chain +/// entry until the consumer claims them. The framework moves one entry from +/// here into each consumer's `TypedStep` at chain-build time via +/// `take_typed_input`. +pub struct OutputQueueSet { + pub(crate) branches: Vec, +} + +pub(crate) struct BranchEntry { + /// `Box` carrying a `BranchInputHandle` for the branch's `T`. + /// Set to `Box::new(())` after `take_typed_input`. + pub(crate) input_handle: Box, + /// `Some` iff this branch's transport is a `ByteBoundedQueue`; bundles + /// the transport-limit setter and (for an ordered branch) the reorder + /// overflow-cap setter (see [`BranchBudgetHandles`]). The pipeline + /// builder collects these into a registry so the budget pass + + /// optional queue-memory rebalancer can set/reallocate budget. + pub(crate) bounded_queue_handle: Option, + /// `Some` iff this edge is instrumented (`--pipeline-trace`); the shared + /// `EdgeMetrics` (also held by the transport for push counts and the input + /// handle for pop counts). Collected into the edge registry by `contexts.rs` + /// Pass 1.5. Cleared (`None`) by `take_typed_input`'s placeholder. + pub(crate) metrics: Option>, +} + +impl OutputQueueSet { + pub(crate) fn new(branches: Vec) -> Self { + Self { branches } + } + + #[must_use] + pub fn n_branches(&self) -> usize { + self.branches.len() + } + + /// Take ownership of branch `i`'s typed input handle. Called by the + /// consumer step's chain-build code once. + /// + /// # Panics + /// + /// Panics if `branch_idx >= n_branches()`, if the branch was already + /// taken, or if the requested type `T` doesn't match the producer's + /// declared branch type (build-time invariant enforced by `Chain::chain`). + pub fn take_typed_input( + &mut self, + branch_idx: usize, + ) -> BranchInputHandle { + let entry = std::mem::replace( + &mut self.branches[branch_idx], + BranchEntry { input_handle: Box::new(()), bounded_queue_handle: None, metrics: None }, + ); + let handle: Box> = + entry.input_handle.downcast::>().unwrap_or_else(|_| { + panic!("OutputQueueSet branch type mismatch at index {branch_idx}") + }); + *handle + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Two-input handle wrapper — held inside ChainContexts.inputs[step_idx] for +// every `Step2` consumer. The `TypedStep2` adapter downcasts the +// type-erased box back to `&TwoInputHandles` per +// dispatch and exposes the per-branch refs as `ctx.a` / `ctx.b`. +// +// The wrapper itself is plain owned data; `ChainContexts` builds it once +// at chain-build time by pulling each branch's `BranchInputHandle` out +// of the matching upstream's `OutputQueueSet`. +// ───────────────────────────────────────────────────────────────────────────── + +/// Two per-branch input handles for a [`crate::step::Step2`] +/// consumer, paired by branch slot (`a` = consumer's input slot 0, +/// `b` = slot 1). Boxed type-erased into `ChainContexts.inputs[step_idx]` +/// at chain-build time; the +/// [`crate::erased::TypedStep2`] adapter +/// downcasts and lends per-branch references into [`crate::step::StepCtx2`]. +pub struct TwoInputHandles +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, +{ + pub(crate) a: BranchInputHandle, + pub(crate) b: BranchInputHandle, +} + +impl TwoInputHandles +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, +{ + /// Construct from two per-branch handles. Called by the + /// chain-context builder. + pub(crate) fn new(a: BranchInputHandle, b: BranchInputHandle) -> Self { + Self { a, b } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Per-arity typed views — held inside OutputsViewAny.inner +// ───────────────────────────────────────────────────────────────────────────── + +/// View for `Single` outputs. Internal: stored inside `OutputsViewAny` +/// and accessed via `OutputHandles>::push` / `retry`. +pub(crate) struct SingleOutputsView { + pub(crate) primary: BranchOutputHandle, +} + +impl SingleOutputsView { + pub fn mark_all_drained(&self) { + self.primary.mark_drained(); + } +} + +/// View for `OrderedBytesSingle` outputs. Same shape as `SingleOutputsView` +/// but the contained `BranchOutputHandle` is constructed via the +/// `_ordered_bytes` build path (which supports `T: HeapSize + Ordered` and +/// dispatches `BranchOrdering::ByItemOrdinal` correctly). +pub(crate) struct OrderedBytesSingleOutputsView { + pub(crate) primary: BranchOutputHandle, +} + +impl OrderedBytesSingleOutputsView { + pub fn mark_all_drained(&self) { + self.primary.mark_drained(); + } +} + +/// View for `(A, B)` tuple outputs. Internal: see `SingleOutputsView`. +pub(crate) struct Tuple2OutputsView { + pub(crate) a: BranchOutputHandle, + pub(crate) b: BranchOutputHandle, +} + +impl Tuple2OutputsView { + pub fn mark_all_drained(&self) { + self.a.mark_drained(); + self.b.mark_drained(); + } +} + +/// View for `(A, B, C)` tuple outputs. Internal: see `SingleOutputsView`. +pub(crate) struct Tuple3OutputsView +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + pub(crate) a: BranchOutputHandle, + pub(crate) b: BranchOutputHandle, + pub(crate) c: BranchOutputHandle, +} + +impl Tuple3OutputsView +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + pub fn mark_all_drained(&self) { + self.a.mark_drained(); + self.b.mark_drained(); + self.c.mark_drained(); + } +} + +/// View for `(A, B, C, D)` tuple outputs. Internal: see `SingleOutputsView`. +pub(crate) struct Tuple4OutputsView +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + pub(crate) a: BranchOutputHandle, + pub(crate) b: BranchOutputHandle, + pub(crate) c: BranchOutputHandle, + pub(crate) d: BranchOutputHandle, +} + +impl Tuple4OutputsView +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + pub fn mark_all_drained(&self) { + self.a.mark_drained(); + self.b.mark_drained(); + self.c.mark_drained(); + self.d.mark_drained(); + } +} + +/// View for `()` (sink) outputs. Internal: see `SingleOutputsView`. +/// No fields and no methods — sinks have no outputs to track or drain. +pub(crate) struct UnitOutputsView; + +// ───────────────────────────────────────────────────────────────────────────── +// Typed accessors on OutputHandles +// ───────────────────────────────────────────────────────────────────────────── + +/// The single canonical definition of the held-slot re-hold invariant, shared by +/// every single-output shape's `retry_held` (the `Single` and +/// `OrderedBytesSingle` variants differ only in which `retry` they hand in). +/// +/// If the slot holds an item, `retry` it; on rejection put it **back** in the slot +/// (so it is never dropped) and report [`HeldRetry::StillHeld`]. Used by step +/// flush-first preambles so each step doesn't re-implement the take/retry/put-back +/// dance (a copy that forgot the put-back would silently drop a final batch). +/// **Never spins** — the caller maps `StillHeld` to a yield (`NoProgress`/`Contention`) +/// and retries on the next dispatch. +#[inline] +fn retry_held_impl( + held: &mut crate::held::HeldSlot>, + retry: impl FnOnce(Unpushed) -> Result<(), Unpushed>, +) -> HeldRetry { + match held.take() { + None => HeldRetry::WasEmpty, + Some(unpushed) => match retry(unpushed) { + Ok(()) => HeldRetry::Flushed, + Err(again) => { + held.put(again); + HeldRetry::StillHeld + } + }, + } +} + +impl OutputHandles> { + /// Push a fresh item to the (single) output. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` when backpressure rejected the push. Hand + /// the rejected item to [`Self::retry`] on the next iteration so any + /// pre-allocated ordinal is preserved. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `SingleOutputsView` (a framework invariant violation). + #[inline] + pub fn push(&self, item: T) -> Result<(), Unpushed> { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Single outputs view downcast failed"); + view.primary.push(item) + } + + /// Retry a previously-rejected push. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` if backpressure still rejected the push. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `SingleOutputsView`. + #[inline] + pub fn retry(&self, unpushed: Unpushed) -> Result<(), Unpushed> { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Single outputs view downcast failed"); + view.primary.retry(unpushed) + } + + /// Retry a step's held output slot, re-holding on backpressure. + /// + /// Retry the held output slot for the `Single` shape, re-holding on + /// backpressure. Delegates to `retry_held_impl`, the shared canonical + /// re-hold invariant. + #[inline] + pub fn retry_held(&self, held: &mut crate::held::HeldSlot>) -> HeldRetry { + retry_held_impl(held, |unpushed| self.retry(unpushed)) + } +} + +impl OutputHandles> { + /// Push a fresh item to the heap-aware ordered output. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` when backpressure (count or byte) rejected + /// the push. Hand the rejected item to [`Self::retry`] on the next + /// iteration so the pre-allocated ordinal (if any) is preserved. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `OrderedBytesSingleOutputsView` (a framework invariant violation). + #[inline] + pub fn push(&self, item: T) -> Result<(), Unpushed> { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesSingle outputs view downcast failed"); + view.primary.push(item) + } + + /// Retry a previously-rejected push on the heap-aware ordered output. + /// + /// # Errors + /// + /// Returns `Err(Unpushed)` if backpressure still rejected the push. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `OrderedBytesSingleOutputsView`. + #[inline] + pub fn retry(&self, unpushed: Unpushed) -> Result<(), Unpushed> { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesSingle outputs view downcast failed"); + view.primary.retry(unpushed) + } + + /// Retry the held output slot for the `OrderedBytesSingle` shape, + /// re-holding on backpressure. Delegates to `retry_held_impl`, the shared + /// canonical re-hold invariant. + #[inline] + pub fn retry_held(&self, held: &mut crate::held::HeldSlot>) -> HeldRetry { + retry_held_impl(held, |unpushed| self.retry(unpushed)) + } +} + +impl OutputHandles<(A, B)> { + /// Borrow the typed per-branch view for a 2-tuple output. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple2OutputsView` (a framework invariant violation). + #[must_use] + #[inline] + pub fn view(&self) -> Tuple2View<'_, A, B> { + let v = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-2 outputs view downcast failed"); + Tuple2View { a: &v.a, b: &v.b } + } +} + +impl OutputHandles> +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, +{ + /// Borrow the typed per-branch view for an ordered + byte-bounded + /// 2-tuple output. Same view type as plain `(A, B)` (the view + /// only carries `BranchOutputHandle`s; the ordering is encoded in + /// the queue's transport, not the handle). + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple2OutputsView` (a framework invariant violation). + #[must_use] + #[inline] + pub fn view(&self) -> Tuple2View<'_, A, B> { + let v = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesTuple2 outputs view downcast failed"); + Tuple2View { a: &v.a, b: &v.b } + } +} + +impl OutputHandles> +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, + C: Send + HeapSize + Ordered + 'static, +{ + /// Borrow the typed per-branch view for an ordered + byte-bounded 3-tuple + /// output. Same view type as plain `(A, B, C)` — the view carries only + /// `BranchOutputHandle`s; the ordering is encoded in the queue transport. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple3OutputsView` (a framework invariant violation). + #[must_use] + #[inline] + pub fn view(&self) -> Tuple3View<'_, A, B, C> { + let v = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesTuple3 outputs view downcast failed"); + Tuple3View { a: &v.a, b: &v.b, c: &v.c } + } +} + +pub struct Tuple2View<'a, A: Send + HeapSize + 'static, B: Send + HeapSize + 'static> { + pub a: &'a BranchOutputHandle, + pub b: &'a BranchOutputHandle, +} + +impl OutputHandles<(A, B, C)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + /// Borrow the typed per-branch view for a 3-tuple output. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple3OutputsView` (a framework invariant violation). + #[must_use] + #[inline] + pub fn view(&self) -> Tuple3View<'_, A, B, C> { + let v = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-3 outputs view downcast failed"); + Tuple3View { a: &v.a, b: &v.b, c: &v.c } + } +} + +pub struct Tuple3View<'a, A, B, C> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + pub a: &'a BranchOutputHandle, + pub b: &'a BranchOutputHandle, + pub c: &'a BranchOutputHandle, +} + +impl OutputHandles<(A, B, C, D)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + /// Borrow the typed per-branch view for a 4-tuple output. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple4OutputsView` (a framework invariant violation). + #[must_use] + #[inline] + pub fn view(&self) -> Tuple4View<'_, A, B, C, D> { + let v = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-4 outputs view downcast failed"); + Tuple4View { a: &v.a, b: &v.b, c: &v.c, d: &v.d } + } +} + +pub struct Tuple4View<'a, A, B, C, D> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + pub a: &'a BranchOutputHandle, + pub b: &'a BranchOutputHandle, + pub c: &'a BranchOutputHandle, + pub d: &'a BranchOutputHandle, +} + +impl OutputHandles<()> { + /// Sinks have no output. Method exists for API symmetry. + pub fn noop(&self) {} +} + +// ───────────────────────────────────────────────────────────────────────────── +// Typed mark_all_drained on OutputHandles +// +// The framework's driver calls this through `TypedStep` when a step returns +// `StepOutcome::Finished` (counter-gated for `Parallel` so only the last clone +// closes the shared output). Each impl downcasts to the right per-arity view +// and forwards. +// ───────────────────────────────────────────────────────────────────────────── + +impl OutputHandles> { + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `SingleOutputsView` (a framework invariant violation). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Single outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles> { + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `OrderedBytesSingleOutputsView` (a framework invariant violation). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesSingle outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles<(A, B)> { + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple2OutputsView` (a framework invariant violation). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-2 outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles> +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, +{ + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple2OutputsView` (the view shape is shared with plain + /// `(A, B)`). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesTuple2 outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles> +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, + C: Send + HeapSize + Ordered + 'static, +{ + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple3OutputsView` (the view shape is shared with plain + /// `(A, B, C)`). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("OrderedBytesTuple3 outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles<(A, B, C)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple3OutputsView` (a framework invariant violation). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-3 outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles<(A, B, C, D)> +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + /// Mark all output branches drained. + /// + /// # Panics + /// + /// Panics if the type-erased outputs view doesn't downcast to + /// `Tuple4OutputsView` (a framework invariant violation). + pub fn mark_all_drained(&self) { + let view = self + .inner + .inner + .downcast_ref::>() + .expect("Tuple-4 outputs view downcast failed"); + view.mark_all_drained(); + } +} + +impl OutputHandles<()> { + /// Mark all output branches drained. Sinks have no outputs; no-op. + pub fn mark_all_drained(&self) { + // No-op for unit outputs. + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// build_*_queues per arity — entry points called from outputs.rs +// ───────────────────────────────────────────────────────────────────────────── + +pub(crate) fn build_single_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) { + assert_eq!(specs.len(), 1, "Single::build_queues requires 1 spec"); + assert_eq!(ordering.len(), 1, "Single::build_queues requires 1 ordering"); + + // `Single` bounds `T: HeapSize`, so use the byte-aware build path: it + // honors `QueueSpec::ByteBounded` for `BranchOrdering::None` and + // `ByOrdinal` — the two orderings a `Single` output can declare, since it + // carries no item serial — and delegates every non-byte spec straight back + // to `build_branch::`, so count/unbounded paths are unchanged. A step + // needing `ByItemOrdinal` declares `OrderedBytesSingle` instead; asking + // for it here panics (see `build_branch_byte_aware`). + let branch = build_branch_byte_aware::(specs[0], ordering[0], level); + let view = SingleOutputsView { primary: branch.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![BranchEntry { + input_handle: Box::new(branch.input), + bounded_queue_handle: branch.bounded_queue_handle, + metrics: branch.metrics, + }]); + (queue_set, outputs_view) +} + +/// Build queues for `OrderedBytesSingle` outputs (`T: HeapSize + Ordered`). +/// Supports every `QueueSpec` × `BranchOrdering` combination. +pub(crate) fn build_single_queues_ordered_bytes( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) { + assert_eq!(specs.len(), 1, "OrderedBytesSingle::build_queues requires 1 spec"); + assert_eq!(ordering.len(), 1, "OrderedBytesSingle::build_queues requires 1 ordering"); + + let branch = build_branch_ordered_bytes::(specs[0], ordering[0], level); + let view = OrderedBytesSingleOutputsView { primary: branch.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![BranchEntry { + input_handle: Box::new(branch.input), + bounded_queue_handle: branch.bounded_queue_handle, + metrics: branch.metrics, + }]); + (queue_set, outputs_view) +} + +/// Fan-out build path for an `(A, B)` tuple output. +/// +/// Every branch goes through `build_branch_byte_aware`, matching +/// `build_single_queues`: each type parameter is bounded `HeapSize`, so a branch +/// may declare `QueueSpec::ByteBounded`, and the plain `build_branch` panics on +/// that spec. Non-byte specs are delegated straight back to `build_branch`, so +/// count/unbounded branches are unchanged. This matters beyond fan-out ergonomics: +/// an armed deadlock monitor REQUIRES `ByteBounded` on every output transport +/// (`ensure_monitor_visible_transports`), so a fan-out chain could not be built +/// at all while these branches rejected the spec. +pub(crate) fn build_tuple2_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, +{ + assert_eq!(specs.len(), 2, "(A, B)::build_queues requires 2 specs"); + assert_eq!(ordering.len(), 2, "(A, B)::build_queues requires 2 orderings"); + + let ba = build_branch_byte_aware::(specs[0], ordering[0], level); + let bb = build_branch_byte_aware::(specs[1], ordering[1], level); + let view = Tuple2OutputsView { a: ba.output, b: bb.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![ + BranchEntry { + input_handle: Box::new(ba.input), + bounded_queue_handle: ba.bounded_queue_handle, + metrics: ba.metrics, + }, + BranchEntry { + input_handle: Box::new(bb.input), + bounded_queue_handle: bb.bounded_queue_handle, + metrics: bb.metrics, + }, + ]); + (queue_set, outputs_view) +} + +/// Build queues for `OrderedBytesTuple2` outputs. Both branches +/// support the full `QueueSpec × BranchOrdering` cross-product because +/// `A: Ordered + HeapSize` and `B: Ordered + HeapSize`. The +/// `Tuple2OutputsView` carries only `BranchOutputHandle`s (no ordering +/// metadata), so the view type is the same as for plain `(A, B)`. +pub(crate) fn build_tuple2_queues_ordered_bytes( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, +{ + assert_eq!(specs.len(), 2, "OrderedBytesTuple2::build_queues requires 2 specs"); + assert_eq!(ordering.len(), 2, "OrderedBytesTuple2::build_queues requires 2 orderings"); + + let ba = build_branch_ordered_bytes::(specs[0], ordering[0], level); + let bb = build_branch_ordered_bytes::(specs[1], ordering[1], level); + let view = Tuple2OutputsView { a: ba.output, b: bb.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![ + BranchEntry { + input_handle: Box::new(ba.input), + bounded_queue_handle: ba.bounded_queue_handle, + metrics: ba.metrics, + }, + BranchEntry { + input_handle: Box::new(bb.input), + bounded_queue_handle: bb.bounded_queue_handle, + metrics: bb.metrics, + }, + ]); + (queue_set, outputs_view) +} + +/// Fan-out build path for an `(A, B, C)` tuple output. +/// +/// Every branch goes through `build_branch_byte_aware`, matching +/// `build_single_queues`: each type parameter is bounded `HeapSize`, so a branch +/// may declare `QueueSpec::ByteBounded`, and the plain `build_branch` panics on +/// that spec. Non-byte specs are delegated straight back to `build_branch`, so +/// count/unbounded branches are unchanged. This matters beyond fan-out ergonomics: +/// an armed deadlock monitor REQUIRES `ByteBounded` on every output transport +/// (`ensure_monitor_visible_transports`), so a fan-out chain could not be built +/// at all while these branches rejected the spec. +pub(crate) fn build_tuple3_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + assert_eq!(specs.len(), 3, "(A, B, C)::build_queues requires 3 specs"); + assert_eq!(ordering.len(), 3, "(A, B, C)::build_queues requires 3 orderings"); + + let ba = build_branch_byte_aware::(specs[0], ordering[0], level); + let bb = build_branch_byte_aware::(specs[1], ordering[1], level); + let bc = build_branch_byte_aware::(specs[2], ordering[2], level); + let view = Tuple3OutputsView { a: ba.output, b: bb.output, c: bc.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![ + BranchEntry { + input_handle: Box::new(ba.input), + bounded_queue_handle: ba.bounded_queue_handle, + metrics: ba.metrics, + }, + BranchEntry { + input_handle: Box::new(bb.input), + bounded_queue_handle: bb.bounded_queue_handle, + metrics: bb.metrics, + }, + BranchEntry { + input_handle: Box::new(bc.input), + bounded_queue_handle: bc.bounded_queue_handle, + metrics: bc.metrics, + }, + ]); + (queue_set, outputs_view) +} + +/// Build queues for `OrderedBytesTuple3` outputs. All three branches +/// support the full `QueueSpec × BranchOrdering` cross-product because each is +/// `Ordered + HeapSize`. The `Tuple3OutputsView` carries only +/// `BranchOutputHandle`s (no ordering metadata), so the view type is the same +/// as for plain `(A, B, C)`. +pub(crate) fn build_tuple3_queues_ordered_bytes( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, + C: Send + HeapSize + Ordered + 'static, +{ + assert_eq!(specs.len(), 3, "OrderedBytesTuple3::build_queues requires 3 specs"); + assert_eq!(ordering.len(), 3, "OrderedBytesTuple3::build_queues requires 3 orderings"); + + let ba = build_branch_ordered_bytes::(specs[0], ordering[0], level); + let bb = build_branch_ordered_bytes::(specs[1], ordering[1], level); + let bc = build_branch_ordered_bytes::(specs[2], ordering[2], level); + let view = Tuple3OutputsView { a: ba.output, b: bb.output, c: bc.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![ + BranchEntry { + input_handle: Box::new(ba.input), + bounded_queue_handle: ba.bounded_queue_handle, + metrics: ba.metrics, + }, + BranchEntry { + input_handle: Box::new(bb.input), + bounded_queue_handle: bb.bounded_queue_handle, + metrics: bb.metrics, + }, + BranchEntry { + input_handle: Box::new(bc.input), + bounded_queue_handle: bc.bounded_queue_handle, + metrics: bc.metrics, + }, + ]); + (queue_set, outputs_view) +} + +/// Fan-out build path for an `(A, B, C, D)` tuple output. +/// +/// Every branch goes through `build_branch_byte_aware`, matching +/// `build_single_queues`: each type parameter is bounded `HeapSize`, so a branch +/// may declare `QueueSpec::ByteBounded`, and the plain `build_branch` panics on +/// that spec. Non-byte specs are delegated straight back to `build_branch`, so +/// count/unbounded branches are unchanged. This matters beyond fan-out ergonomics: +/// an armed deadlock monitor REQUIRES `ByteBounded` on every output transport +/// (`ensure_monitor_visible_transports`), so a fan-out chain could not be built +/// at all while these branches rejected the spec. +pub(crate) fn build_tuple4_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + assert_eq!(specs.len(), 4, "(A, B, C, D)::build_queues requires 4 specs"); + assert_eq!(ordering.len(), 4, "(A, B, C, D)::build_queues requires 4 orderings"); + + let ba = build_branch_byte_aware::(specs[0], ordering[0], level); + let bb = build_branch_byte_aware::(specs[1], ordering[1], level); + let bc = build_branch_byte_aware::(specs[2], ordering[2], level); + let bd = build_branch_byte_aware::(specs[3], ordering[3], level); + let view = Tuple4OutputsView { a: ba.output, b: bb.output, c: bc.output, d: bd.output }; + let outputs_view = OutputsViewAny { inner: Box::new(view) }; + let queue_set = OutputQueueSet::new(vec![ + BranchEntry { + input_handle: Box::new(ba.input), + bounded_queue_handle: ba.bounded_queue_handle, + metrics: ba.metrics, + }, + BranchEntry { + input_handle: Box::new(bb.input), + bounded_queue_handle: bb.bounded_queue_handle, + metrics: bb.metrics, + }, + BranchEntry { + input_handle: Box::new(bc.input), + bounded_queue_handle: bc.bounded_queue_handle, + metrics: bc.metrics, + }, + BranchEntry { + input_handle: Box::new(bd.input), + bounded_queue_handle: bd.bounded_queue_handle, + metrics: bd.metrics, + }, + ]); + (queue_set, outputs_view) +} + +pub(crate) fn build_unit_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + _level: crate::builder::InstrumentationLevel, +) -> (OutputQueueSet, OutputsViewAny) { + assert_eq!(specs.len(), 0, "()::build_queues requires 0 specs"); + assert_eq!(ordering.len(), 0, "()::build_queues requires 0 orderings"); + let outputs_view = OutputsViewAny { inner: Box::new(UnitOutputsView) }; + let queue_set = OutputQueueSet::new(Vec::new()); + (queue_set, outputs_view) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod handle_tests { + use super::*; + use rstest::rstest; + + #[test] + fn always_drained_handle_is_empty_and_drained() { + // The zero-state source input handle owns no transport: it pops nothing + // and reports drained from the start, so a source step sees its + // (implicit) input as immediately end-of-stream. + let h = BranchInputHandle::<()>::always_drained(); + assert_eq!(h.pop(), None, "always-drained handle yields no items"); + assert!(h.is_drained(), "always-drained handle reports drained"); + // Idempotent: still drained, still empty after repeated reads. + assert_eq!(h.pop(), None); + assert!(h.is_drained()); + } + + #[test] + fn build_branch_mints_metrics_iff_on() { + use crate::builder::InstrumentationLevel as L; + let on = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::None, + L::Summary, + ); + assert!(on.metrics.is_some(), "level on → edge metrics minted"); + let off = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::None, + L::Off, + ); + assert!(off.metrics.is_none(), "level off → no metrics (hot path metric-free)"); + } + + #[test] + fn metrics_shared_between_transport_push_and_input_handle_pop() { + // Producer-push (transport) and consumer-pop (input handle) share one + // EdgeMetrics: push lands via the queue, pop/empty via the input handle. + use crate::builder::InstrumentationLevel as L; + let b = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::None, + L::Summary, + ); + let m = b.metrics.clone().expect("metrics present"); + b.output.push(7).unwrap(); + assert_eq!(b.input.pop(), Some(7)); + assert_eq!(b.input.pop(), None); // empty + let s = m.snapshot(); + assert_eq!(s.pushed_items, 1, "producer push counted at the transport"); + assert_eq!(s.popped_items, 1, "consumer pop counted at the input handle"); + assert_eq!(s.pop_empties, 1, "empty pop counted at the input handle"); + } + + #[test] + fn ordered_reorder_blocked_pop_is_not_counted_empty() { + // Regression: an ordered edge's `try_pop_in_order` returns `None` both + // when the edge is genuinely starved AND when it is only reorder-blocked + // (later ordinals are buffered while it waits for the next in-order + // ordinal). Counting the reorder-blocked case as an empty pop inflates + // `pop_empties` and can misclassify a backlogged edge as starved, so + // the pop path must skip `record_empty()` while the reorder buffer holds + // out-of-order items. + use crate::runtime::metrics::EdgeMetrics; + + let transport: Arc>> = + Arc::new(CountBoundedQueue::>::new(8)); + let stage = Arc::new(ReorderStage::new(transport)); + // Buffer ordinal 1 while ordinal 0 is still absent: the next in-order + // pop is blocked, not starved. + stage.try_push(1, 100).unwrap(); + + let m = EdgeMetrics::new(); + let handle = BranchInputHandle { + inner: BranchInputInner::Ordered(stage.clone()), + metrics: Some(m.clone()), + // Count-bounded transport, so no byte accounting on either side. + record_item_bytes: false, + }; + + // Reorder-blocked: yields nothing yet, but there is buffered work. + assert_eq!(handle.pop(), None, "next ordinal (0) absent → no in-order item"); + assert_eq!( + m.snapshot().pop_empties, + 0, + "a reorder-blocked pop is backlog, not starvation — must not count as empty" + ); + + // Now the missing ordinal arrives and both items drain in order; still + // no empty pops recorded. + stage.try_push(0, 50).unwrap(); + assert_eq!(handle.pop(), Some(50)); + assert_eq!(handle.pop(), Some(100)); + assert_eq!(m.snapshot().pop_empties, 0, "in-order drains record no empties"); + + // Genuinely drained now: this pop IS a starved/empty pop. + assert_eq!(handle.pop(), None); + assert_eq!( + m.snapshot().pop_empties, + 1, + "an empty pop with no buffered work is a true empty pop" + ); + } + + #[test] + fn count_bounded_fifo_round_trip() { + let b = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Off, + ); + b.output.push(1).unwrap(); + b.output.push(2).unwrap(); + assert_eq!(b.input.pop(), Some(1)); + assert_eq!(b.input.pop(), Some(2)); + assert_eq!(b.input.pop(), None); + } + + #[test] + fn count_bounded_ordered_emits_in_order() { + let b = build_branch::( + QueueSpec::CountBounded { capacity: 8 }, + BranchOrdering::ByOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + // Single producer pushes ordinals 0,1,2 in arrival order. + b.output.push(100).unwrap(); + b.output.push(200).unwrap(); + b.output.push(300).unwrap(); + assert_eq!(b.input.pop(), Some(100)); + assert_eq!(b.input.pop(), Some(200)); + assert_eq!(b.input.pop(), Some(300)); + } + + #[test] + fn drained_signal_propagates() { + let b = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Off, + ); + b.output.push(1).unwrap(); + b.output.mark_drained(); + // Marker set but item still buffered: not drained. + assert!(!b.input.is_drained()); + assert_eq!(b.input.pop(), Some(1)); + assert!(b.input.is_drained()); + } + + #[test] + fn unbounded_branch_works() { + let b = build_branch::( + QueueSpec::Unbounded, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Off, + ); + for i in 0..1024 { + b.output.push(i).unwrap(); + } + for i in 0..1024 { + assert_eq!(b.input.pop(), Some(i)); + } + } + + #[derive(Debug)] + struct Bytes(Vec); + impl HeapSize for Bytes { + fn heap_size(&self) -> usize { + self.0.len() + } + } + + #[test] + fn byte_bounded_branch_works() { + let b = build_branch_byte_aware::( + QueueSpec::ByteBounded { limit_bytes: 1000 }, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Off, + ); + b.output.push(Bytes(vec![0; 500])).unwrap(); + let popped = b.input.pop().unwrap(); + assert_eq!(popped.0.len(), 500); + } + + /// Every fan-out shape must accept `QueueSpec::ByteBounded` on every branch. + /// The tuple builders used the non-byte-aware `build_branch`, which panics on + /// that spec, so a fan-out step declaring it could not be built — and an armed + /// deadlock monitor *requires* `ByteBounded` on every output transport + /// (`ensure_monitor_visible_transports`), which made a monitored fan-out chain + /// unbuildable. Asserting the registered byte handle (not just "it didn't + /// panic") also pins that the branch is genuinely byte-bounded and therefore + /// visible to the monitor and the queue-memory budget. + #[rstest] + #[case::tuple2(2)] + #[case::tuple3(3)] + #[case::tuple4(4)] + fn byte_bounded_accepted_on_every_fan_out_branch(#[case] n_branches: usize) { + let specs = vec![QueueSpec::ByteBounded { limit_bytes: 1000 }; n_branches]; + let orderings = vec![BranchOrdering::None; n_branches]; + let level = crate::builder::InstrumentationLevel::Off; + let (queue_set, _view) = match n_branches { + 2 => build_tuple2_queues::(&specs, &orderings, level), + 3 => build_tuple3_queues::(&specs, &orderings, level), + 4 => build_tuple4_queues::(&specs, &orderings, level), + other => panic!("unhandled branch count {other}"), + }; + assert_eq!(queue_set.n_branches(), n_branches); + for branch in 0..n_branches { + let handles = queue_set.branches[branch] + .bounded_queue_handle + .as_ref() + .unwrap_or_else(|| panic!("branch {branch} must register a byte-bounded handle")); + assert_eq!( + handles.transport.limit_bytes(), + 1000, + "branch {branch} must carry the declared byte bound" + ); + } + } + + /// `popped_bytes` must agree with `pushed_bytes` about whether an edge counts + /// bytes at all. A count/unbounded transport records `record_push(0)` by + /// design (no byte accounting on its hot path), so the pop side must report 0 + /// too — otherwise `compute_edge_stats` divides real popped bytes by wall time + /// and reports a `mibytes_per_s` throughput for an edge whose pushed bytes are + /// always 0. A byte-bounded edge reports real sizes on both sides. + #[rstest] + #[case::count_bounded(QueueSpec::CountBounded { capacity: 4 }, 0)] + #[case::unbounded(QueueSpec::Unbounded, 0)] + #[case::byte_bounded(QueueSpec::ByteBounded { limit_bytes: 4096 }, 700)] + fn popped_bytes_recorded_only_on_byte_bounded_edges( + #[case] spec: QueueSpec, + #[case] expected_popped_bytes: u64, + ) { + // `Summary` mints the edge metrics; `Off` would leave `metrics: None`. + let b = build_branch_byte_aware::( + spec, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Summary, + ); + b.output.push(Bytes(vec![0; 700])).expect("first push fits every spec here"); + assert!(b.input.pop().is_some(), "the pushed item comes back"); + + let ms = b.metrics.as_ref().expect("Summary mints edge metrics").snapshot(); + assert_eq!( + ms.popped_bytes, expected_popped_bytes, + "popped_bytes must mirror the push side's byte accounting" + ); + assert_eq!( + ms.pushed_bytes, expected_popped_bytes, + "pushed and popped byte accounting must agree for the same edge" + ); + assert_eq!(ms.popped_items, 1, "the item itself is always counted"); + } + + #[test] + #[should_panic(expected = "ByteBounded requires `T: HeapSize`")] + fn byte_bounded_panics_in_non_heap_aware_builder() { + let _ = build_branch::( + QueueSpec::ByteBounded { limit_bytes: 1000 }, + BranchOrdering::None, + crate::builder::InstrumentationLevel::Off, + ); + } + + #[test] + fn byte_bounded_single_builds_via_user_facing_build_queues() { + // Regression: a user step declaring `QueueSpec::ByteBounded` in its + // `StepProfile::output_queues` with a `Single` output reaches + // `StepOutputs::build_queues` → `build_single_queues`. `Single` + // bounds `T: HeapSize`, so this builds a byte-bounded queue (the + // documented "byte-bounded without item-carried serials" shape) + // rather than panicking. + use crate::outputs::{Single, StepOutputs}; + use crate::step::OutputHandles; + let (mut queue_set, outputs_view) = as StepOutputs>::build_queues( + &[QueueSpec::ByteBounded { limit_bytes: 1000 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + outputs.push(7).unwrap(); + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(7)); + } + + #[test] + fn single_retry_held_drains_then_reholds() { + // Exercises `OutputHandles>::retry_held`: empty → WasEmpty, + // a rejected push re-held → StillHeld while the queue is full → Flushed + // once a slot frees, with the held item never dropped. + use crate::held::HeldSlot; + use crate::outputs::{Single, StepOutputs}; + use crate::step::OutputHandles; + let (mut queue_set, outputs_view) = as StepOutputs>::build_queues( + &[QueueSpec::CountBounded { capacity: 1 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + let mut held: HeldSlot> = HeldSlot::new(); + + // Empty slot → WasEmpty (queue untouched). + assert!(matches!(outputs.retry_held(&mut held), HeldRetry::WasEmpty)); + + // Fill the capacity-1 queue; the next push is rejected and re-held. + outputs.push(1).unwrap(); + let rejected = outputs.push(2).expect_err("capacity-1 queue must reject the 2nd push"); + held.put(rejected); + + // Queue still full → StillHeld; the item is put back, not dropped. + assert!(matches!(outputs.retry_held(&mut held), HeldRetry::StillHeld)); + assert!(held.is_held()); + + // Drain one item; the held push now flushes. + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(1)); + assert!(matches!(outputs.retry_held(&mut held), HeldRetry::Flushed)); + assert!(!held.is_held()); + assert_eq!(input.pop(), Some(2)); + } + + #[test] + fn byte_bounded_plus_byordinal_works_after_phase3_amendment() { + // Phase 3 amendment 2 lifted the deferred ByteBounded + ByOrdinal + // path: `Sequenced: HeapSize` is now impl'd, so a byte-bounded + // transport can wrap `Sequenced` and the `Allocated` ordinal + // source provides per-branch ordinals. + let b = build_branch_byte_aware::( + QueueSpec::ByteBounded { limit_bytes: 1000 }, + BranchOrdering::ByOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + b.output.push(Bytes(vec![0; 100])).unwrap(); + let popped = b.input.pop().unwrap(); + assert_eq!(popped.0.len(), 100); + } + + #[test] + #[should_panic(expected = "BranchOrdering::ByItemOrdinal requires `T: Ordered`")] + fn by_item_ordinal_panics_in_non_ordered_builder() { + let _ = build_branch::( + QueueSpec::CountBounded { capacity: 4 }, + BranchOrdering::ByItemOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + } + + /// Trivial `Ordered`-impl test fixture for the by-item-ordinal builders. + #[derive(Debug, PartialEq, Eq)] + struct OrdItem { + ord: u64, + v: u32, + } + impl crate::item::Ordered for OrdItem { + fn ordinal(&self) -> u64 { + self.ord + } + } + + #[test] + fn by_item_ordinal_uses_item_serial() { + let b = build_branch_ordered::( + QueueSpec::CountBounded { capacity: 8 }, + BranchOrdering::ByItemOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + // Push out of order: ordinals 2, 0, 1. + b.output.push(OrdItem { ord: 2, v: 200 }).unwrap(); + b.output.push(OrdItem { ord: 0, v: 0 }).unwrap(); + b.output.push(OrdItem { ord: 1, v: 100 }).unwrap(); + // Consumer sees them in item-ordinal order. + assert_eq!(b.input.pop().unwrap().v, 0); + assert_eq!(b.input.pop().unwrap().v, 100); + assert_eq!(b.input.pop().unwrap().v, 200); + } + + impl HeapSize for OrdItem { + fn heap_size(&self) -> usize { + std::mem::size_of::() + } + } + + #[test] + fn ordered_bytes_supports_byte_bounded_with_item_serial() { + let b = build_branch_ordered_bytes::( + QueueSpec::ByteBounded { limit_bytes: 4096 }, + BranchOrdering::ByItemOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + b.output.push(OrdItem { ord: 1, v: 1 }).unwrap(); + b.output.push(OrdItem { ord: 0, v: 0 }).unwrap(); + assert_eq!(b.input.pop().unwrap().v, 0); + assert_eq!(b.input.pop().unwrap().v, 1); + } + + #[test] + fn ordered_branch_preserves_ordinal_across_retry() { + // Regression for C1 (ordinal-burn): with capacity 2, push three items + // through an ordered branch, draining one at a time. Without the + // Unpushed retry, the third push would burn an ordinal on + // backpressure and stall the reorder stage. + let b = build_branch::( + QueueSpec::CountBounded { capacity: 2 }, + BranchOrdering::ByOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + + // Pump 5 items through the branch, draining sequentially. Capacity 2 + // means every push after the second hits backpressure once. + let mut received = Vec::new(); + for n in 0..5_u32 { + // Try to push. Retry on backpressure until accepted, draining the + // input handle in between to make progress. + let mut held: Option> = None; + let mut fresh: Option = Some(n); + + loop { + if let Some(unpushed) = held.take() { + match b.output.retry(unpushed) { + Ok(()) => {} + Err(again) => { + held = Some(again); + } + } + } + if held.is_none() + && let Some(item) = fresh.take() + { + match b.output.push(item) { + Ok(()) => {} + Err(unpushed) => { + held = Some(unpushed); + } + } + } + if held.is_none() && fresh.is_none() { + break; + } + // Drain any items the consumer has ready, freeing transport space. + while let Some(v) = b.input.pop() { + received.push(v); + } + } + } + // Final drain — drain everything still buffered. + while let Some(v) = b.input.pop() { + received.push(v); + } + assert_eq!(received, vec![0, 1, 2, 3, 4], "ordinals preserved across retries"); + } + + /// An `Unpushed` carrying no ordinal handed to an `Ordered` branch's + /// `retry` must panic in every build, not silently re-push. Re-pushing + /// allocates a fresh ordinal and abandons the original, so the consumer's + /// `ReorderStage` waits on the missing ordinal forever — a stall the + /// deadlock monitor reports with no hint at the cause. + #[test] + #[should_panic(expected = "Unpushed::ordinal=None on an Ordered branch")] + fn ordered_retry_without_ordinal_panics() { + let b = build_branch::( + QueueSpec::CountBounded { capacity: 2 }, + BranchOrdering::ByOrdinal, + crate::builder::InstrumentationLevel::Off, + ); + let _ = b.output.retry(Unpushed { item: 7, ordinal: None }); + } + + #[test] + fn output_queue_set_take_typed_input() { + let (mut set, _view) = build_single_queues::( + &[QueueSpec::CountBounded { capacity: 4 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let _input: BranchInputHandle = set.take_typed_input::(0); + } +} + +#[cfg(test)] +mod build_queues_tests { + use super::*; + use crate::outputs::{Single, StepOutputs}; + use crate::step::OutputHandles; + + fn count_specs(arity: usize, capacity: usize) -> (Vec, Vec) { + (vec![QueueSpec::CountBounded { capacity }; arity], vec![BranchOrdering::None; arity]) + } + + #[test] + fn single_round_trip() { + let (specs, ordering) = count_specs(1, 4); + let (mut queue_set, outputs_view) = as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + outputs.push(7).unwrap(); + + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(7)); + } + + #[test] + fn tuple_2_round_trip() { + let (specs, ordering) = count_specs(2, 2); + let (mut queue_set, outputs_view) = <(u32, String) as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles<(u32, String)> = OutputHandles::new(outputs_view); + + let v = outputs.view(); + v.a.push(10).unwrap(); + v.b.push("hello".to_string()).unwrap(); + + let in_a = queue_set.take_typed_input::(0); + let in_b = queue_set.take_typed_input::(1); + assert_eq!(in_a.pop(), Some(10)); + assert_eq!(in_b.pop(), Some("hello".to_string())); + } + + #[test] + fn tuple_3_round_trip() { + let (specs, ordering) = count_specs(3, 2); + let (mut queue_set, outputs_view) = <(u32, u64, String) as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles<(u32, u64, String)> = OutputHandles::new(outputs_view); + + let v = outputs.view(); + v.a.push(1).unwrap(); + v.b.push(2).unwrap(); + v.c.push("three".to_string()).unwrap(); + + assert_eq!(queue_set.take_typed_input::(0).pop(), Some(1)); + assert_eq!(queue_set.take_typed_input::(1).pop(), Some(2)); + assert_eq!(queue_set.take_typed_input::(2).pop(), Some("three".to_string())); + } + + #[test] + fn tuple_4_round_trip() { + let (specs, ordering) = count_specs(4, 2); + let (mut queue_set, outputs_view) = + <(u32, u64, String, Vec) as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles<(u32, u64, String, Vec)> = OutputHandles::new(outputs_view); + + let v = outputs.view(); + v.a.push(1).unwrap(); + v.b.push(2).unwrap(); + v.c.push("three".to_string()).unwrap(); + v.d.push(vec![4u8, 5, 6]).unwrap(); + + assert_eq!(queue_set.take_typed_input::(0).pop(), Some(1)); + assert_eq!(queue_set.take_typed_input::(1).pop(), Some(2)); + assert_eq!(queue_set.take_typed_input::(2).pop(), Some("three".to_string())); + assert_eq!(queue_set.take_typed_input::>(3).pop(), Some(vec![4u8, 5, 6])); + } + + #[test] + fn unit_build_queues_yields_empty_set() { + let (queue_set, outputs_view) = + <() as StepOutputs>::build_queues(&[], &[], crate::builder::InstrumentationLevel::Off); + let outputs: OutputHandles<()> = OutputHandles::new(outputs_view); + outputs.noop(); + assert_eq!(queue_set.n_branches(), 0); + } + + #[test] + fn mark_all_drained_propagates_to_input_handle() { + let (specs, ordering) = count_specs(1, 4); + let (mut queue_set, outputs_view) = as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + + let input = queue_set.take_typed_input::(0); + assert!(!input.is_drained()); + outputs.mark_all_drained(); + assert!(input.is_drained()); + } + + #[test] + fn ordered_branch_preserves_emission_order_through_typed_path() { + let specs = vec![QueueSpec::CountBounded { capacity: 8 }]; + let ordering = vec![BranchOrdering::ByOrdinal]; + let (mut queue_set, outputs_view) = as StepOutputs>::build_queues( + &specs, + &ordering, + crate::builder::InstrumentationLevel::Off, + ); + let outputs: OutputHandles> = OutputHandles::new(outputs_view); + + outputs.push(10).unwrap(); + outputs.push(20).unwrap(); + outputs.push(30).unwrap(); + + let input = queue_set.take_typed_input::(0); + assert_eq!(input.pop(), Some(10)); + assert_eq!(input.pop(), Some(20)); + assert_eq!(input.pop(), Some(30)); + } +} diff --git a/crates/fgumi-pipeline-core/src/header.rs b/crates/fgumi-pipeline-core/src/header.rs new file mode 100644 index 000000000..7109d33b7 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/header.rs @@ -0,0 +1,495 @@ +//! One-shot lazy SAM/BAM header handle. +//! +//! `HeaderHandle` lets a downstream sink open its output file before the +//! full output header is known, and consume the header at first record +//! time once an upstream step has produced it. The primary motivating +//! consumer is the writer downstream of `AlignAndMergeStep`: the +//! aligner's `@PG` (and any `@RG`/`@CO` lines it adds) are runtime +//! contributions that aren't available at `Pipeline::build` time. +//! +//! ### Contract +//! +//! - `set` and `poison` are one-shot — calling either after the handle +//! has been resolved returns `AlreadySetError` without altering state. +//! - `try_get` is non-blocking and dispatcher-friendly: a step that +//! needs the header but observes `None` returns `StepOutcome::NoProgress` +//! and is rescheduled. +//! - Cloning shares state (via `Arc>`), so multiple +//! readers see the same set-or-poison outcome. The cross-thread +//! happens-before guarantee comes from `OnceLock`'s internal +//! synchronization (`set` is Release, `get` is Acquire); the +//! `Arc>` wrapper exists only for shareability. +//! +//! ### Producer-side invariant +//! +//! Every consumer that polls `try_get` assumes the handle will +//! eventually resolve via `set` or `poison`. Producer steps owning a +//! `HeaderHandle` must poison their handle in their `Drop` impl as a +//! backstop against panics (or any exit path that bypasses normal +//! completion). The framework does not enforce this; it's a step-level +//! convention. A future revision may split the handle into typed +//! setter / reader halves so an orphaned setter is statically detectable. +//! +//! An orphaned setter — a producer that exits without resolving — does +//! **not** hang the run, provided the consumer follows the shape below. +//! Two framework backstops catch it, both pinned by tests in this module +//! (`orphaned_setter_*`): +//! +//! 1. **Producer left no items.** When a step reports `Finished` the +//! framework closes its output edges, so the consumer's +//! `is_input_drained()` arm fires and it can fail with a diagnostic. +//! 2. **Producer left items queued.** A header-blocked consumer pops +//! nothing, so `is_drained()` (which is `drained && empty`) stays +//! false and backstop 1 is unreachable — but those stranded items are +//! in flight, so the deadlock monitor sees non-zero `in_flight_bytes`, +//! classifies the run `Wedged` past the fatal timeout, and fails it +//! with `PipelineError::TimedOut`. +//! +//! Backstop 2 depends on the stranded items carrying real heap bytes: +//! queue accounting is `HeapSize::heap_size()`-only, so a zero-heap item +//! type leaves `in_flight_bytes` at 0, which classifies as `Starving` and +//! resets the stall clock forever. That gap is general to the monitor +//! rather than specific to headers; see the `in_flight_bytes` docs in +//! `builder.rs`. +//! +//! ### Consumer-side contract +//! +//! A consumer must never poll `try_get` in isolation. Pair the probe with +//! `is_input_drained()` and treat "input drained before the header +//! resolved" as an error — that arm is backstop 1, and without it a +//! consumer whose producer left no items polls `None` forever. + +use std::fmt; +use std::io; +use std::sync::{Arc, OnceLock}; + +use noodles::sam::Header; + +/// Shared, one-shot header slot. +/// +/// See module docs for the full contract. +#[derive(Clone, Default, Debug)] +pub struct HeaderHandle { + inner: Arc>>>, +} + +impl HeaderHandle { + /// Construct an empty handle. `try_get` returns `None` until `set` + /// or `poison` is called. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Construct a handle that already carries `header`. `try_get` + /// returns `Some(Ok(&header))` immediately. Convenience for call + /// sites that have an eager header today and want to opt into the + /// handle-typed sink API without changing behavior. + /// + /// # Panics + /// The internal `set` call is on a freshly-constructed cell and + /// is therefore unreachable as a failure — the `expect` is a + /// belt-and-braces assertion documenting the invariant. + #[must_use] + #[allow(clippy::missing_panics_doc)] // documented above; clippy + // doesn't see the # Panics section across this constructor's + // delegation to `set`. + pub fn from_header(header: Header) -> Self { + let handle = Self::new(); + handle.set(header).expect("fresh HeaderHandle accepts its first set"); + handle + } + + /// Resolve the handle to `header`. Returns `AlreadySetError` if the + /// handle was previously set or poisoned. + /// + /// # Errors + /// Returns `AlreadySetError` on the second and subsequent calls. + pub fn set(&self, header: Header) -> Result<(), AlreadySetError> { + self.inner.set(Ok(header)).map_err(|_| AlreadySetError) + } + + /// Resolve the handle to a failure. Subsequent `try_get` calls + /// surface the error. + /// + /// # Errors + /// Returns `AlreadySetError` on the second and subsequent calls. + pub fn poison(&self, error: io::Error) -> Result<(), AlreadySetError> { + self.inner.set(Err(Arc::new(error))).map_err(|_| AlreadySetError) + } + + /// Non-blocking handle probe. + /// + /// Returns: + /// - `None` if neither `set` nor `poison` has been called yet — the + /// caller should yield (e.g. return `StepOutcome::NoProgress`). + /// - `Some(Ok(&header))` if `set(header)` was called. + /// - `Some(Err(e))` if `poison(e)` was called. A fresh `io::Error` + /// is constructed on every call from the stored kind + message + /// (because `io::Error` is not `Clone`). **Note:** the + /// original error's `source()` chain and any structured + /// payload are not preserved — only kind + display string + /// round-trip. Producers that need to surface structured + /// diagnostic context should encode it into the display + /// string before poisoning the handle. + #[must_use] + pub fn try_get(&self) -> Option> { + self.inner.get().map(|stored| match stored { + Ok(header) => Ok(header), + Err(err) => Err(io::Error::new(err.kind(), err.to_string())), + }) + } + + /// `true` once `set` or `poison` has resolved this handle. + #[must_use] + pub fn is_set(&self) -> bool { + self.inner.get().is_some() + } +} + +/// Returned by `set` / `poison` when the handle was previously +/// resolved. +#[derive(Debug, Clone, Copy)] +pub struct AlreadySetError; + +impl fmt::Display for AlreadySetError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "HeaderHandle has already been set or poisoned") + } +} + +impl std::error::Error for AlreadySetError {} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + + #[test] + fn new_handle_has_no_value() { + let h = HeaderHandle::new(); + assert!(!h.is_set()); + assert!(h.try_get().is_none()); + } + + #[test] + fn from_header_resolves_immediately() { + let h = HeaderHandle::from_header(Header::default()); + assert!(h.is_set()); + let got = h.try_get().expect("set").expect("ok"); + assert_eq!(got, &Header::default()); + } + + #[test] + fn set_then_try_get_returns_ok() { + let h = HeaderHandle::new(); + h.set(Header::default()).expect("first set"); + assert!(h.is_set()); + let got = h.try_get().expect("set").expect("ok"); + assert_eq!(got, &Header::default()); + } + + #[test] + fn poison_then_try_get_returns_err() { + let h = HeaderHandle::new(); + h.poison(io::Error::other("aligner crashed")).expect("first poison"); + assert!(h.is_set()); + let err = h.try_get().expect("set").expect_err("poisoned"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert_eq!(err.to_string(), "aligner crashed"); + } + + #[test] + fn second_set_returns_already_set() { + let h = HeaderHandle::new(); + h.set(Header::default()).expect("first"); + let err = h.set(Header::default()).expect_err("second set"); + let _ = err; // type asserted by the binding + } + + #[test] + fn set_then_poison_returns_already_set() { + let h = HeaderHandle::new(); + h.set(Header::default()).expect("first"); + let _err: AlreadySetError = + h.poison(io::Error::other("late")).expect_err("poison after set"); + } + + #[test] + fn poison_then_set_returns_already_set() { + let h = HeaderHandle::new(); + h.poison(io::Error::other("first")).expect("first"); + let _err: AlreadySetError = h.set(Header::default()).expect_err("set after poison"); + } + + #[test] + fn clones_share_state() { + let a = HeaderHandle::new(); + let b = a.clone(); + assert!(!a.is_set() && !b.is_set()); + a.set(Header::default()).expect("first"); + assert!(a.is_set() && b.is_set()); + assert!(b.try_get().expect("set").is_ok()); + } + + #[test] + fn poison_preserves_error_kind_and_message_across_calls() { + let h = HeaderHandle::new(); + h.poison(io::Error::new(io::ErrorKind::BrokenPipe, "stderr ring: foo")).unwrap(); + for _ in 0..3 { + let e = h.try_get().unwrap().unwrap_err(); + assert_eq!(e.kind(), io::ErrorKind::BrokenPipe); + assert_eq!(e.to_string(), "stderr ring: foo"); + } + } + + #[test] + fn set_from_another_thread_is_observable() { + let h = HeaderHandle::new(); + let h2 = h.clone(); + let join = thread::spawn(move || { + h2.set(Header::default()).expect("first set"); + }); + join.join().expect("thread join"); + assert!(h.is_set()); + assert!(h.try_get().unwrap().is_ok()); + } + + #[test] + fn from_header_then_set_returns_already_set() { + let h = HeaderHandle::from_header(Header::default()); + let _err: AlreadySetError = + h.set(Header::default()).expect_err("from_header consumes the slot"); + } + + #[test] + fn from_header_then_poison_returns_already_set() { + let h = HeaderHandle::from_header(Header::default()); + let _err: AlreadySetError = + h.poison(io::Error::other("late")).expect_err("from_header consumes the slot"); + } + + // ───────────────────────────────────────────────────────────────────── + // Orphaned-setter backstops. + // + // These pin the two escapes described in the module's "Producer-side + // invariant" section, so the claim there is tested rather than asserted. + // ───────────────────────────────────────────────────────────────────── + + /// Shared fixtures for the orphaned-setter tests: a producer that finishes + /// without ever resolving its handle, and a consumer shaped like the real + /// BGZF writer (holds records until the header resolves; treats "input + /// drained before the header arrived" as a hard error). + mod orphan { + use super::HeaderHandle; + use std::io; + + use crate::item::HeapSize; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{Step, StepCtx, StepKind, StepOutcome, StepProfile}; + + /// Item with a real heap payload, so a `ByteBounded` transport holding + /// one reports non-zero `current_bytes` (the queue's accounting is + /// `heap_size()`-only — `size_of::()` is not counted). + pub struct Block { + pub payload: Vec, + } + impl HeapSize for Block { + fn heap_size(&self) -> usize { + self.payload.capacity() + } + } + + /// Producer that emits `remaining` blocks and then finishes, never + /// calling `set`/`poison` on the handle it owns. + #[derive(Clone)] + pub struct OrphanedSetter { + pub remaining: u32, + pub block_bytes: usize, + pub spec: QueueSpec, + pub _handle: HeaderHandle, + } + impl Step for OrphanedSetter { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrphanedSetter", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![self.spec], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.remaining == 0 { + // Finishes WITHOUT resolving `_handle` — the orphan case. + return Ok(StepOutcome::Finished); + } + self.remaining -= 1; + // Asserted, not discarded: backstop 2's whole premise is that + // these items are stranded IN the queue. A silently dropped push + // would leave `in_flight_bytes` at 0 and the test would pass for + // the wrong reason. Budgets below always admit every block. + ctx.outputs + .push(Block { payload: vec![0u8; self.block_bytes] }) + .map_err(|_| io::Error::other("orphan test budget must admit every block"))?; + Ok(StepOutcome::Progress) + } + } + + /// Consumer mirroring the real `WriteBgzfFile` sink. + #[derive(Clone)] + pub struct HeaderBlockedSink { + pub handle: HeaderHandle, + } + impl Step for HeaderBlockedSink { + type Input = Block; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "HeaderBlockedSink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + // The header gates consumption: nothing may be written before + // it resolves, so a blocked sink pops nothing. + let header_ready = self.handle.try_get().transpose()?.is_some(); + if header_ready && ctx.input.pop().is_some() { + return Ok(StepOutcome::Progress); + } + if ctx.input.is_drained() { + if !header_ready { + return Err(io::Error::other( + "HeaderBlockedSink: input drained before HeaderHandle was resolved", + )); + } + return Ok(StepOutcome::Finished); + } + Ok(StepOutcome::NoProgress) + } + } + } + + /// Backstop 1 — orphaned setter that emitted **nothing**: the consumer's + /// `is_drained()` arm fires and the run fails fast with a diagnostic. + /// + /// `is_drained()` is `queue.is_drained() && queue.is_empty()`, so this arm + /// is only reachable when the orphaned producer left no items behind. That + /// precondition is what backstop 2 covers. + #[test] + fn orphaned_setter_with_empty_output_fails_the_run_fast() { + use std::sync::mpsc; + use std::time::Duration; + + use crate::queues::QueueSpec; + use crate::{Pipeline, PipelineConfig}; + + let handle = HeaderHandle::new(); + let for_run = handle.clone(); + let (tx, rx) = mpsc::channel(); + let join = thread::spawn(move || { + let builder = Pipeline::builder(); + builder + .chain(orphan::OrphanedSetter { + remaining: 0, + block_bytes: 0, + spec: QueueSpec::CountBounded { capacity: 8 }, + _handle: for_run.clone(), + }) + .chain(orphan::HeaderBlockedSink { handle: for_run }) + .into_sink_marker(); + let pipeline = builder.build().expect("pipeline build"); + let _ = tx.send(pipeline.run(PipelineConfig { threads: 2, ..Default::default() })); + }); + + let result = rx + .recv_timeout(Duration::from_secs(10)) + .expect("an orphaned setter with an empty output must not hang the run"); + join.join().expect("run thread panicked"); + + let err = result.expect_err("an unresolved HeaderHandle must fail the run"); + assert!( + format!("{err}").contains("input drained before HeaderHandle was resolved"), + "run must fail with the sink's drained-without-header error, got: {err}" + ); + assert!(!handle.is_set(), "nobody ever resolved the handle"); + } + + /// Backstop 2 — orphaned setter that emitted items: those items sit unpopped + /// (the blocked sink never consumes), so `is_drained()` stays false forever + /// and the drained arm above can never fire. The deadlock monitor is what + /// catches this: the stranded items are in flight on a `ByteBounded` edge, + /// so `in_flight_bytes` is non-zero, `classify_stall` returns `Wedged` past + /// the fatal timeout, and the run fails with `PipelineError::TimedOut`. + /// + /// The item must carry real heap bytes for that to hold — queue accounting + /// is `heap_size()`-only, so a zero-heap item type would leave + /// `in_flight_bytes` at 0, yielding `Starving`, which resets the stall clock + /// on every poll and leaves the wedge uncatchable. + #[test] + fn orphaned_setter_with_inflight_bytes_is_caught_by_the_deadlock_monitor() { + use std::sync::mpsc; + use std::time::Duration; + + use crate::queues::QueueSpec; + use crate::signal::PipelineError; + use crate::{Pipeline, PipelineConfig}; + + let handle = HeaderHandle::new(); + let for_run = handle.clone(); + let (tx, rx) = mpsc::channel(); + let join = thread::spawn(move || { + let builder = Pipeline::builder(); + builder + .chain(orphan::OrphanedSetter { + remaining: 4, + block_bytes: 4096, + spec: QueueSpec::ByteBounded { limit_bytes: 1 << 20 }, + _handle: for_run.clone(), + }) + .chain(orphan::HeaderBlockedSink { handle: for_run }) + .into_sink_marker(); + let pipeline = builder.build().expect("pipeline build"); + let stats = pipeline.stats(); + let _ = tx.send(pipeline.run(PipelineConfig { + threads: 2, + stats: Some(stats), + // warn at 1s, fatal at 1s * DEADLOCK_FATAL_MULTIPLE. + deadlock_timeout_secs: 1, + ..Default::default() + })); + }); + + let result = rx + .recv_timeout(Duration::from_secs(60)) + .expect("the deadlock monitor must fail a header-wedged run, not let it hang"); + join.join().expect("run thread panicked"); + + let err = result.expect_err("a header-wedged run must fail"); + assert!( + matches!(err, PipelineError::TimedOut { .. }), + "the wedge must surface as TimedOut, got: {err:?}" + ); + assert!(!handle.is_set(), "nobody ever resolved the handle"); + } + + #[test] + fn poison_visible_to_all_clones() { + let a = HeaderHandle::new(); + let b = a.clone(); + let c = a.clone(); + a.poison(io::Error::new(io::ErrorKind::BrokenPipe, "boom")).unwrap(); + for clone in [&a, &b, &c] { + let err = clone.try_get().expect("set").expect_err("poisoned"); + assert_eq!(err.kind(), io::ErrorKind::BrokenPipe); + assert_eq!(err.to_string(), "boom"); + } + } +} diff --git a/crates/fgumi-pipeline-core/src/held.rs b/crates/fgumi-pipeline-core/src/held.rs new file mode 100644 index 000000000..f4a2d3af0 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/held.rs @@ -0,0 +1,89 @@ +//! Generic held-item slot for non-blocking back-pressure. +//! +//! When a step can't push to a downstream queue (it's full), it stashes the +//! item in a `HeldSlot` and returns `StepOutcome::Progress`. The next +//! `try_run` call drains the held item before doing new work. +//! +//! Draining the held item before new work is a *step-author convention*, not +//! something `HeldSlot` enforces: the type only provides single-slot put/take +//! (with a double-put panic). The step is responsible for checking and draining +//! the slot first on each `try_run`. + +pub struct HeldSlot { + inner: Option, +} + +impl Default for HeldSlot { + fn default() -> Self { + Self { inner: None } + } +} + +impl HeldSlot { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn is_held(&self) -> bool { + self.inner.is_some() + } + + /// Stash an item. + /// + /// # Panics + /// + /// Panics if the slot is already occupied (a contract violation: callers + /// must drain via `take` before calling `put` again). This is a hard + /// `assert!` rather than `debug_assert!` because silently overwriting would + /// **drop the previously held item** — a record lost from the pipeline — + /// which must not pass unnoticed in release builds. The check is a single + /// predictable branch on the back-pressure path, so the cost is negligible. + pub fn put(&mut self, item: T) { + assert!(self.inner.is_none(), "HeldSlot already occupied"); + self.inner = Some(item); + } + + pub fn take(&mut self) -> Option { + self.inner.take() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_slot_is_empty() { + let s: HeldSlot = HeldSlot::new(); + assert!(!s.is_held()); + } + + #[test] + fn put_then_take_round_trips() { + let mut s = HeldSlot::new(); + s.put(42_u32); + assert!(s.is_held()); + assert_eq!(s.take(), Some(42)); + assert!(!s.is_held()); + } + + #[test] + fn take_from_empty_returns_none() { + let mut s: HeldSlot = HeldSlot::new(); + assert_eq!(s.take(), None); + } + + #[test] + #[should_panic(expected = "HeldSlot already occupied")] + fn double_put_panics() { + // Pin the message, not just the unwind: `catch_unwind` accepts ANY panic, + // so a `put` that started failing for an unrelated reason would still pass + // while the double-put guard itself had silently gone. Must fire in debug + // AND release — overwriting a held item is silent record loss. + let mut s = HeldSlot::new(); + s.put(1_u32); + s.put(2_u32); + } +} diff --git a/crates/fgumi-pipeline-core/src/item.rs b/crates/fgumi-pipeline-core/src/item.rs new file mode 100644 index 000000000..438361c2d --- /dev/null +++ b/crates/fgumi-pipeline-core/src/item.rs @@ -0,0 +1,162 @@ +//! Item-level orthogonal traits used by the queue layer. +//! +//! These traits are independent of each other: +//! - [`HeapSize`] is required only for items in a `ByteBoundedQueue`. +//! - [`Ordered`] is required only for items routed through a `ReorderStage`. +//! +//! An item type may implement both, neither, or just one, depending on which +//! queues and operators it flows through. The framework checks bounds at +//! queue construction time (see [`crate::queues`] and [`crate::reorder`]). + +/// Approximate heap footprint, in bytes, of an item. +/// +/// Used by `ByteBoundedQueue` to enforce a memory budget rather than an +/// item-count budget. Items that hold variable-size buffers (BAM batches, +/// FASTQ batches, decompressed BGZF blocks) implement this manually. +/// +/// **Default impl.** The default method body returns `0`, so an empty +/// `impl HeapSize for MyType {}` opts in but reports zero bytes — every +/// item then fits in any byte-bounded queue regardless of the limit. Step +/// authors must override `heap_size` for any type whose actual footprint +/// matters. +/// +/// **No blanket impl.** We do *not* provide `impl HeapSize for T` so that +/// pushing a non-impl type into a `ByteBoundedQueue` is a compile error +/// rather than a silent zero-count. +pub trait HeapSize { + fn heap_size(&self) -> usize { + 0 + } +} + +/// Source-step input type. Carries no heap allocation. +impl HeapSize for () {} + +// Primitive impls for the integer/string types that flow through tests +// and pipeline plumbing. Production BAM/FASTQ types impl `HeapSize` +// explicitly with their actual heap footprint; primitives report zero +// (their `Vec` capacity, when applicable, is reported by the wrapping +// type's own impl). +impl HeapSize for u8 {} +impl HeapSize for u16 {} +impl HeapSize for u32 {} +impl HeapSize for u64 {} +impl HeapSize for usize {} +impl HeapSize for i8 {} +impl HeapSize for i16 {} +impl HeapSize for i32 {} +impl HeapSize for i64 {} +impl HeapSize for isize {} +impl HeapSize for bool {} +impl HeapSize for f32 {} +impl HeapSize for f64 {} +impl HeapSize for char {} + +impl HeapSize for String { + fn heap_size(&self) -> usize { + self.capacity() + } +} + +impl HeapSize for Vec { + fn heap_size(&self) -> usize { + self.capacity() * std::mem::size_of::() + + self.iter().map(HeapSize::heap_size).sum::() + } +} + +impl HeapSize for Option { + fn heap_size(&self) -> usize { + self.as_ref().map_or(0, HeapSize::heap_size) + } +} + +/// Producer-emitted serial ordinal. +/// +/// Items routed through a `ReorderStage` carry a monotonically-increasing +/// ordinal assigned by the producer step at push time (the framework +/// allocates from a per-branch `AtomicU64`). The reorder stage uses this +/// ordinal to deliver items to the consumer in producer-emitted order +/// regardless of inter-thread arrival skew. +/// +/// Steps don't implement this directly. The framework wraps the user's +/// pushed item in an internal `Sequenced` newtype that carries the +/// ordinal alongside the item; `Sequenced` impls `Ordered`. See the +/// reorder stage for the wrapper type. +pub trait Ordered { + fn ordinal(&self) -> u64; +} + +#[cfg(test)] +mod tests { + use super::*; + + struct WithHeap(Vec); + impl HeapSize for WithHeap { + fn heap_size(&self) -> usize { + self.0.len() + } + } + + struct WithoutHeap; + impl HeapSize for WithoutHeap {} // explicit empty impl: opts in, default returns 0 + + struct WithOrdinal { + o: u64, + } + impl Ordered for WithOrdinal { + fn ordinal(&self) -> u64 { + self.o + } + } + + #[test] + fn heap_size_default_is_zero() { + assert_eq!(WithoutHeap.heap_size(), 0); + } + + #[test] + fn heap_size_override_works() { + assert_eq!(WithHeap(vec![0; 1024]).heap_size(), 1024); + } + + #[test] + fn ordered_returns_ordinal() { + assert_eq!(WithOrdinal { o: 42 }.ordinal(), 42); + } + + /// A `ByteBoundedQueue` charges each item its `heap_size()`, so the blanket + /// impls decide how much budget a queue of owned data actually accounts + /// for. They report *capacity*, not length — an over-allocated buffer costs + /// what it reserved, because that is what the process is holding. + #[test] + fn string_heap_size_is_its_capacity() { + let mut s = String::with_capacity(64); + s.push_str("abc"); + assert_eq!(s.heap_size(), 64, "capacity, not the 3 bytes in use"); + assert_eq!(String::new().heap_size(), 0, "an unallocated String costs nothing"); + } + + #[test] + fn vec_heap_size_counts_its_buffer_and_its_elements() { + // Flat elements: just the buffer. + let mut flat: Vec = Vec::with_capacity(8); + flat.push(1); + assert_eq!(flat.heap_size(), 8 * std::mem::size_of::()); + + // Nested elements add their own heap. Without the per-element sum a + // queue of `Vec>` would be charged only for the outer spine and + // could blow well past its byte budget. + let nested: Vec = vec![WithHeap(vec![0; 100]), WithHeap(vec![0; 200])]; + let spine = nested.capacity() * std::mem::size_of::(); + assert_eq!(nested.heap_size(), spine + 300); + + assert_eq!(Vec::::new().heap_size(), 0, "an unallocated Vec costs nothing"); + } + + #[test] + fn option_heap_size_delegates_to_the_payload_and_none_is_free() { + assert_eq!(Some(WithHeap(vec![0; 77])).heap_size(), 77); + assert_eq!(None::.heap_size(), 0); + } +} diff --git a/crates/fgumi-pipeline-core/src/lib.rs b/crates/fgumi-pipeline-core/src/lib.rs new file mode 100644 index 000000000..fa463736b --- /dev/null +++ b/crates/fgumi-pipeline-core/src/lib.rs @@ -0,0 +1,60 @@ +//! Core types and traits for the typed-step pipeline framework. +//! +//! A pipeline is a graph of typed [`Step`]s joined by bounded [`queues`] and run +//! by a work-stealing worker pool. This crate carries only the framework +//! primitives — the step traits, the queues, the [`reorder`] stage that restores +//! input order across parallel branches, and the [`runtime`] that schedules and +//! drives them. Nothing here reads or writes sequencing data, so the dependency +//! graph stays light (`ahash` / `crossbeam-queue` / `parking_lot` / `log`, plus +//! `anyhow` for the one [`FinalizeHook::finalize`] return type and one +//! `noodles::sam` type for the shared header handle) and the crate compiles +//! fast in isolation. +//! +//! The concrete steps that do the I/O and the computation live elsewhere and +//! plug in by implementing [`Step`] (one input) or [`Step2`] (two inputs); +//! [`PipelineBuilder`] wires them into a [`Pipeline`]. + +#![deny(unsafe_code)] + +pub mod builder; +pub mod erased; +pub mod finalize; +pub mod handles; +pub mod header; +pub mod held; +pub mod item; +pub mod outputs; +pub mod queues; +pub mod reorder; +pub mod runtime; +pub mod signal; +pub mod step; +pub mod topology; + +#[cfg(test)] +mod tests; + +pub use builder::{ + BuildError, Chain, InstrumentationLevel, MultiChain2, MultiChain2Ordered, MultiChain3, + MultiChain4, Pipeline, PipelineBuilder, PipelineConfig, +}; +pub use erased::{ErasedStep, ErasedStepCtx, TypedStep, TypedStep2}; +pub use finalize::FinalizeHook; +pub use handles::{ + BranchInputHandle, HeldRetry, OutputQueueSet, Tuple2View, Tuple3View, Tuple4View, + TwoInputHandles, Unpushed, +}; +pub use header::{AlreadySetError, HeaderHandle}; +pub use held::HeldSlot; +pub use item::{HeapSize, Ordered}; +pub use outputs::{ + MAX_ARITY, OrderedBytesSingle, OrderedBytesTuple2, OrderedBytesTuple3, Single, StepOutputs, +}; +pub use queues::{ByteBoundedQueue, CountBoundedQueue, ItemQueue, QueueSpec, UnboundedQueue}; +pub use reorder::{BranchOrdering, ReorderStage, Sequenced}; +pub use signal::{CancelHandle, PipelineError, PipelineSignal}; +pub use step::{ + Affinity, DetachedGroup, InputHandle, OutputHandles, OutputsViewAny, Step, Step2, StepCtx, + StepCtx2, StepKind, StepOutcome, StepProfile, +}; +pub use topology::{BranchIdx, ChainGraph, StepIdx}; diff --git a/crates/fgumi-pipeline-core/src/outputs.rs b/crates/fgumi-pipeline-core/src/outputs.rs new file mode 100644 index 000000000..f84edbc45 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/outputs.rs @@ -0,0 +1,525 @@ +//! `StepOutputs`: type-level description of a step's outputs. +//! +//! Single-output steps declare `type Outputs = Single;`. +//! Multi-output steps declare `type Outputs = (A, B, C);` (positional access). +//! Sinks declare `type Outputs = ();`. +//! +//! **Maximum arity: 4.** [`MAX_ARITY`] is the largest tuple shape +//! that has a `StepOutputs` impl. A user step that declares +//! `type Outputs = (A, B, C, D, E);` (arity 5) will fail to compile with a +//! "trait `StepOutputs` is not implemented" error. Larger arities require +//! adding the additional impls in this file plus the matching +//! `build_tupleN_queues` constructor in `handles.rs`. +//! +//! Each `StepOutputs` impl provides: +//! - `arity()` — number of independent output channels. +//! - `build_queues(specs, ordering, level) -> (OutputQueueSet, OutputsViewAny)` — +//! constructs the typed queues + reorder operators (where applicable) +//! and the type-erased view the framework stores. Implemented in +//! `handles.rs` (one impl per arity). +//! +//! The `specs` and `ordering` slices both have length `arity()`, sourced +//! from `StepProfile::output_queues` and `StepProfile::branch_ordering`. + +use std::marker::PhantomData; + +use super::handles::OutputQueueSet; +use super::item::{HeapSize, Ordered}; +use super::queues::QueueSpec; +use super::reorder::BranchOrdering; +use super::step::OutputsViewAny; + +/// The largest tuple `Outputs` shape that has a `StepOutputs` impl. +/// `Single` and `()` are also valid output shapes. +pub const MAX_ARITY: usize = 4; + +/// Marker trait for a step's `Outputs` associated type. +pub trait StepOutputs: Send + 'static { + /// Number of independent output channels. + fn arity() -> usize; + + /// Construct the typed queues + type-erased view for this Outputs shape. + /// + /// `specs[i]` and `ordering[i]` together describe branch `i`. Panics if + /// `specs.len() != Self::arity()` or `ordering.len() != Self::arity()` + /// (the builder ensures this invariant before calling). + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny); + + /// Mark all output branches drained, dispatching through the typed + /// `OutputHandles::mark_all_drained` method. Implemented for + /// each per-arity variant in this file; called by `TypedStep` from + /// `mark_outputs_drained` in the worker loop's drain-propagation path. + fn mark_all_drained(handles: &super::step::OutputHandles) + where + Self: Sized; +} + +/// Wrapper for single-output steps. `type Outputs = Single;` +pub struct Single(PhantomData T>); + +impl StepOutputs for Single { + #[inline] + fn arity() -> usize { + 1 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_single_queues::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +/// Wrapper for single-output steps where the item type is heap-aware AND +/// carries its own ordinal. The canonical Phase 3 BAM step output shape: +/// every BAM step's output type impls both `HeapSize` (for byte-bounded +/// queues) and `Ordered` (so a `batch_serial: u64` field carries record- +/// read order through every Parallel transform). +/// +/// Steps that need byte-bounded outputs but don't have item-carried serials +/// use `Single` with `BranchOrdering::None` or `ByOrdinal`. Steps that +/// need item-ordinal ordering but not byte-bounded queues are uncommon +/// (and currently not supported as a separate shape) — they can use this +/// shape with `QueueSpec::CountBounded` since `T: HeapSize` is required by +/// the bound but not consulted by count-bounded queues. +pub struct OrderedBytesSingle(PhantomData T>); + +impl StepOutputs for OrderedBytesSingle { + #[inline] + fn arity() -> usize { + 1 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_single_queues_ordered_bytes::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +impl StepOutputs for (A, B) { + #[inline] + fn arity() -> usize { + 2 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_tuple2_queues::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +/// Ordered + byte-bounded tuple-2 outputs. Use when a step fans out to +/// two branches that both need `BranchOrdering::ByItemOrdinal` + +/// byte-bounded queues — e.g., `filter` emitting kept and rejected +/// records as parallel ordered streams that downstream +/// `BgzfCompress` / `WriteBgzfFile` sinks can consume. +/// +/// The plain `(A, B)` `StepOutputs` impl only bounds `A: HeapSize` +/// (suitable for `Process2`'s `BranchOrdering::None` case). To use +/// `ByItemOrdinal` on either branch, both branches must satisfy +/// `Ordered + HeapSize` — that's what this shape encodes at the type +/// level. +pub struct OrderedBytesTuple2(PhantomData (A, B)>) +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static; + +impl StepOutputs for OrderedBytesTuple2 +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, +{ + #[inline] + fn arity() -> usize { + 2 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_tuple2_queues_ordered_bytes::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +/// Ordered + byte-bounded tuple-3 outputs. The 3-branch analog of +/// [`OrderedBytesTuple2`]: use when a step fans out to three branches that all +/// need `BranchOrdering::ByItemOrdinal` + byte-bounded queues — e.g. paired +/// FASTQ output splitting one record batch into R1 / R2 / other byte streams, +/// each feeding an ordered `WriteRawFile` sink. +/// +/// The plain `(A, B, C)` `StepOutputs` impl only bounds each branch `HeapSize`. +/// To use `ByItemOrdinal` on any branch, all three must satisfy +/// `Ordered + HeapSize` — that's what this shape encodes at the type level. +#[allow(clippy::type_complexity)] +pub struct OrderedBytesTuple3(PhantomData (A, B, C)>) +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, + C: Send + HeapSize + Ordered + 'static; + +impl StepOutputs for OrderedBytesTuple3 +where + A: Send + HeapSize + Ordered + 'static, + B: Send + HeapSize + Ordered + 'static, + C: Send + HeapSize + Ordered + 'static, +{ + #[inline] + fn arity() -> usize { + 3 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_tuple3_queues_ordered_bytes::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +impl StepOutputs for (A, B, C) +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, +{ + #[inline] + fn arity() -> usize { + 3 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_tuple3_queues::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +impl StepOutputs for (A, B, C, D) +where + A: Send + HeapSize + 'static, + B: Send + HeapSize + 'static, + C: Send + HeapSize + 'static, + D: Send + HeapSize + 'static, +{ + #[inline] + fn arity() -> usize { + 4 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_tuple4_queues::(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +impl StepOutputs for () { + #[inline] + fn arity() -> usize { + 0 + } + + fn build_queues( + specs: &[QueueSpec], + ordering: &[BranchOrdering], + level: crate::builder::InstrumentationLevel, + ) -> (OutputQueueSet, OutputsViewAny) { + super::handles::build_unit_queues(specs, ordering, level) + } + + fn mark_all_drained(handles: &super::step::OutputHandles) { + handles.mark_all_drained(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::step::InputHandle; + + #[test] + fn single_arity_is_one() { + assert_eq!( as StepOutputs>::arity(), 1); + } + + #[test] + fn tuple_2_arity_is_two() { + assert_eq!(<(u32, u64) as StepOutputs>::arity(), 2); + } + + #[test] + fn tuple_3_arity_is_three() { + assert_eq!(<(u32, u64, String) as StepOutputs>::arity(), 3); + } + + #[derive(Clone, Copy)] + struct OrdU64(u64); + impl crate::item::HeapSize for OrdU64 {} + impl crate::item::Ordered for OrdU64 { + fn ordinal(&self) -> u64 { + self.0 + } + } + + #[test] + fn ordered_bytes_tuple_3_arity_is_three() { + // OrderedBytesTuple3 is the ordered + byte-bounded 3-way fan-out shape. + // Its three branches carry Ordered + HeapSize items (here the u64/u32 + // stand-ins just need to satisfy the bounds at the type level). + fn assert_arity() -> usize { + O::arity() + } + assert_eq!(assert_arity::>(), 3); + } + + #[test] + fn tuple_4_arity_is_four() { + assert_eq!(<(u32, u64, String, Vec) as StepOutputs>::arity(), 4); + } + + #[test] + fn unit_arity_is_zero() { + assert_eq!(<() as StepOutputs>::arity(), 0); + } + + #[test] + fn ordered_bytes_tuple_2_arity_is_two() { + fn assert_arity() -> usize { + O::arity() + } + assert_eq!(assert_arity::>(), 2); + } + + // ───────────────────────────────────────────────────────────────────────── + // `build_queues` / `mark_all_drained` round-trips, one test per output shape. + // + // Every branch a shape declares must come back from `build_queues` as its + // own live edge: `n_branches()` matches `arity()`, each branch's input handle + // transports the item type its position declares, each starts OPEN, and + // `mark_all_drained` closes all of them. The open-before / closed-after pair + // is what makes these discriminating — asserting only the closed-after state + // would pass just as well if `build_queues` returned branches already closed + // and `mark_all_drained` did nothing. + // + // These stay separate tests rather than an `#[rstest]` case table: each shape + // is a distinct type with distinct per-branch item types, so the cases cannot + // share a function signature. Every shape with a `StepOutputs` impl has one — + // `Single`, `OrderedBytesSingle`, the 2-/3-/4-tuples, `OrderedBytesTuple2`, + // `OrderedBytesTuple3`, and `()`. Adding a shape without adding its + // round-trip leaves its `build_queues` / `mark_all_drained` pair unexercised. + // ───────────────────────────────────────────────────────────────────────── + + fn specs(n: usize) -> Vec { + vec![QueueSpec::CountBounded { capacity: 4 }; n] + } + + fn orderings(n: usize) -> Vec { + vec![BranchOrdering::None; n] + } + + /// Build a shape's queues and hand back the branch set plus a typed + /// `OutputHandles` view, the pair the runtime hands to a step. + fn build(n: usize) -> (OutputQueueSet, crate::step::OutputHandles) { + let (queues, view) = + O::build_queues(&specs(n), &orderings(n), crate::builder::InstrumentationLevel::Off); + (queues, crate::step::OutputHandles::::new(view)) + } + + #[test] + fn single_builds_one_branch_and_drains() { + type Shape = Single; + let (mut queues, handles) = build::(1); + assert_eq!(queues.n_branches(), 1, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + assert!(!a.is_drained(), "branch starts open"); + + ::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + } + + /// The only shape whose `build_queues` routes to + /// `build_single_queues_ordered_bytes`, so without this its drain path has no + /// coverage here at all. + #[test] + fn ordered_bytes_single_builds_one_branch_and_drains() { + type Shape = OrderedBytesSingle; + let (mut queues, handles) = build::(1); + assert_eq!(queues.n_branches(), 1, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + assert!(!a.is_drained(), "branch starts open"); + + ::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + } + + #[test] + fn tuple_2_builds_two_independent_branches() { + type Shape = (u32, u64); + let (mut queues, handles) = build::(2); + assert_eq!(queues.n_branches(), 2, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + let b = queues.take_typed_input::(1); + assert!(!a.is_drained(), "branches start open"); + assert!(!b.is_drained()); + + ::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + assert!(b.is_drained(), "mark_all_drained closes branch 1"); + } + + #[test] + fn tuple_3_builds_three_independent_branches() { + let (mut queues, handles) = build::<(u32, u64, String)>(3); + assert_eq!(queues.n_branches(), 3, "one branch per declared output"); + + // Each branch's input handle must downcast to that position's type — + // a mis-wired builder would panic here or hand back the wrong branch. + let a = queues.take_typed_input::(0); + let b = queues.take_typed_input::(1); + let c = queues.take_typed_input::(2); + assert!(!a.is_drained(), "branches start open"); + assert!(!b.is_drained()); + assert!(!c.is_drained()); + + <(u32, u64, String) as StepOutputs>::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + assert!(b.is_drained(), "mark_all_drained closes branch 1"); + assert!(c.is_drained(), "mark_all_drained closes branch 2"); + } + + #[test] + fn tuple_4_builds_four_independent_branches() { + let (mut queues, handles) = build::<(u32, u64, String, Vec)>(4); + assert_eq!(queues.n_branches(), 4, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + let b = queues.take_typed_input::(1); + let c = queues.take_typed_input::(2); + let d = queues.take_typed_input::>(3); + assert!(!a.is_drained(), "branches start open"); + assert!(!b.is_drained()); + assert!(!c.is_drained()); + assert!(!d.is_drained()); + + <(u32, u64, String, Vec) as StepOutputs>::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + assert!(b.is_drained(), "mark_all_drained closes branch 1"); + assert!(c.is_drained(), "mark_all_drained closes branch 2"); + assert!(d.is_drained(), "mark_all_drained closes branch 3"); + } + + #[test] + fn ordered_bytes_tuple_2_builds_two_independent_branches() { + type Shape = OrderedBytesTuple2; + let (mut queues, handles) = build::(2); + assert_eq!(queues.n_branches(), 2, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + let b = queues.take_typed_input::(1); + assert!(!a.is_drained(), "branches start open"); + assert!(!b.is_drained()); + + // Identity, not just type. Both branches carry the SAME item type, so the + // per-position downcast cannot tell them apart, and `mark_all_drained` + // closes every branch — a builder that aliased branch 0's queue onto both + // positions would satisfy every other assertion here. Route a distinct + // value through each and read it back off its own edge. + let view = handles.view(); + view.a.push(OrdU64(10)).expect("branch 0 accepts one item"); + view.b.push(OrdU64(11)).expect("branch 1 accepts one item"); + assert_eq!(a.pop().map(|v| v.0), Some(10), "branch 0 is its own edge"); + assert_eq!(b.pop().map(|v| v.0), Some(11), "branch 1 is its own edge"); + + ::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + assert!(b.is_drained(), "mark_all_drained closes branch 1"); + } + + #[test] + fn ordered_bytes_tuple_3_builds_three_independent_branches() { + type Shape = OrderedBytesTuple3; + let (mut queues, handles) = build::(3); + assert_eq!(queues.n_branches(), 3, "one branch per declared output"); + + let a = queues.take_typed_input::(0); + let b = queues.take_typed_input::(1); + let c = queues.take_typed_input::(2); + assert!(!a.is_drained(), "branches start open"); + assert!(!b.is_drained()); + assert!(!c.is_drained()); + + // Identity, not just type — see the tuple-2 test above for why the + // same-item-type shapes need this. + let view = handles.view(); + view.a.push(OrdU64(10)).expect("branch 0 accepts one item"); + view.b.push(OrdU64(11)).expect("branch 1 accepts one item"); + view.c.push(OrdU64(12)).expect("branch 2 accepts one item"); + assert_eq!(a.pop().map(|v| v.0), Some(10), "branch 0 is its own edge"); + assert_eq!(b.pop().map(|v| v.0), Some(11), "branch 1 is its own edge"); + assert_eq!(c.pop().map(|v| v.0), Some(12), "branch 2 is its own edge"); + + ::mark_all_drained(&handles); + assert!(a.is_drained(), "mark_all_drained closes branch 0"); + assert!(b.is_drained(), "mark_all_drained closes branch 1"); + assert!(c.is_drained(), "mark_all_drained closes branch 2"); + } + + /// A sink declares no outputs, so its queue set is empty and + /// `mark_all_drained` is a no-op rather than a panic. + #[test] + fn unit_builds_no_branches_and_drains_without_panicking() { + let (queues, handles) = build::<()>(0); + assert_eq!(queues.n_branches(), 0, "a sink owns no output branches"); + <() as StepOutputs>::mark_all_drained(&handles); + } +} diff --git a/crates/fgumi-pipeline-core/src/queues.rs b/crates/fgumi-pipeline-core/src/queues.rs new file mode 100644 index 000000000..faa5f0931 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/queues.rs @@ -0,0 +1,815 @@ +//! Transport-layer queue trait + three concrete impls. +//! +//! Concerns: pure transport (push/pop, drained signal). **Not** ordering — +//! see [`crate::reorder`] for the `ReorderStage` operator that adds +//! ordinal-based reordering on top of any `ItemQueue`. **Not** memory +//! bookkeeping at the trait level — `ByteBoundedQueue` is a +//! concrete impl that knows about heap size, but the trait surface is +//! type-uniform. +//! +//! Backpressure is expressed as `try_push -> Result<(), T>`: `Err(item)` +//! returns the rejected item back to the producer (which holds it in a +//! `HeldSlot` and re-pushes on the next worker iteration). No blocking, +//! no awaiting — pure non-blocking surface. +//! +//! Drained-signal protocol: +//! - Producer (output side) calls `mark_drained()` exactly once when the +//! producing step returns `StepOutcome::Finished` (counter-gated for +//! `Parallel` so only the last clone closes the shared queue). Subsequent +//! `try_push` calls panic — in every build, not just debug (a contract +//! violation: producer pushed after declaring done, and the item would be +//! silently lost). See `assert_not_drained`. +//! - Consumer (input side) checks `is_drained() && is_empty()` to detect +//! end-of-stream. Once both are true, no further items will arrive. + +use crossbeam_queue::{ArrayQueue, SegQueue}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use super::item::HeapSize; +use super::runtime::metrics::EdgeMetrics; + +/// Transport-layer queue trait. Type-uniform across queue impls: the +/// `try_push` surface accepts any `T` regardless of whether the impl uses +/// item-count or memory bookkeeping internally. +/// +/// `Send + Sync`: queues are shared between worker threads via `Arc`. +pub trait ItemQueue: Send + Sync { + /// Non-blocking push. `Err(item)` returns the rejected item to the + /// caller; the framework holds it in a `HeldSlot` and retries. + /// + /// # Errors + /// + /// Returns `Err(item)` when the queue is at its backpressure limit + /// (item-count or byte-budget, depending on the impl). + fn try_push(&self, item: T) -> Result<(), T>; + + /// Non-blocking pop. `None` means the queue is currently empty (which + /// is *not* the same as drained — combine with `is_drained()`). + fn try_pop(&self) -> Option; + + /// True when no items are currently buffered. May race with concurrent + /// pushes; consumers that need a quiescent check combine with + /// `is_drained()`. + fn is_empty(&self) -> bool; + + /// Mark the queue drained (producer-side: "I'm done pushing"). Idempotent. + /// A `try_push` after `mark_drained` panics — see `assert_not_drained`. + fn mark_drained(&self); + + /// True if `mark_drained` has been called. + fn is_drained(&self) -> bool; +} + +/// Panic if `try_push` is called after `mark_drained`. +/// +/// The consumer treats a drained queue as closed, so an item pushed afterwards +/// may never be popped: the item is silently lost and the loss surfaces (if at +/// all) as a short output far from its cause. That makes this a framework +/// contract violation rather than a recoverable condition, so — like +/// `BranchOutputHandle::retry`'s `Ordered` + `ordinal = None` arm — it fails +/// loudly in **every** build. It was `debug_assert!`-only, which left release +/// builds performing exactly the silent push the message warns about. +/// +/// `Relaxed` is sufficient here and is the cheaper load on a per-item path. +/// `drained` is monotonic — its only write anywhere is `store(true, Release)` in +/// `mark_drained` — so a `Relaxed` load can return a stale `false` (a missed +/// detection when the producer races the close on another thread) but never a +/// spurious `true`. It cannot panic a correct program. +#[inline] +fn assert_not_drained(drained: &AtomicBool, queue_kind: &'static str) { + assert!( + !drained.load(Ordering::Relaxed), + "{queue_kind}::try_push after mark_drained — producer contract violation" + ); +} + +/// One entry per output branch in `StepProfile::output_queues`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueueSpec { + /// Item-count bounded. `try_push` rejects when `len() >= capacity`. + /// Best for fixed-size items (parsed records, compressed BGZF blocks + /// of known size, etc). + CountBounded { capacity: usize }, + /// Memory-bounded. `try_push` rejects when adding the item would push + /// the running byte counter past `limit_bytes`. Requires `T: HeapSize`. + /// Best for variable-size BAM batches and FASTQ batches. + ByteBounded { limit_bytes: u64 }, + /// No backpressure. `try_push` always succeeds. Use only when the + /// branch is naturally rate-limited upstream (e.g., a header-once + /// emit on pipeline start). + Unbounded, +} + +// ───────────────────────────────────────────────────────────────────────────── +// CountBoundedQueue +// ───────────────────────────────────────────────────────────────────────────── + +/// Item-count bounded transport. Backed by `crossbeam_queue::ArrayQueue`. +pub struct CountBoundedQueue { + inner: ArrayQueue, + drained: AtomicBool, + /// `Some` only on an instrumented edge (`--pipeline-trace`); `None` keeps the + /// hot path metric-free. Producer-push counts are recorded here; consumer-pop + /// counts are recorded at the `BranchInputHandle` (see `handles.rs`). + metrics: Option>, +} + +impl CountBoundedQueue { + /// Construct a count-bounded transport with the given capacity. + /// + /// # Panics + /// + /// Panics if `capacity == 0` (a zero-capacity queue would always reject). + #[must_use] + pub fn new(capacity: usize) -> Self { + Self::build(capacity, None) + } + + /// Like [`new`](Self::new) but recording producer-push metrics into `metrics` + /// (an instrumented edge). The non-blocking `try_*` surface is unchanged. + /// + /// # Panics + /// + /// Panics if `capacity == 0`. + #[must_use] + pub fn new_instrumented(capacity: usize, metrics: Arc) -> Self { + Self::build(capacity, Some(metrics)) + } + + /// [`new`](Self::new) when `metrics` is `None`, [`new_instrumented`](Self::new_instrumented) + /// when `Some`. Lets branch builders thread an optional metrics handle uniformly. + /// + /// # Panics + /// + /// Panics if `capacity == 0`. + #[must_use] + pub fn maybe_instrumented(capacity: usize, metrics: Option>) -> Self { + Self::build(capacity, metrics) + } + + fn build(capacity: usize, metrics: Option>) -> Self { + assert!(capacity > 0, "CountBoundedQueue capacity must be > 0"); + Self { inner: ArrayQueue::new(capacity), drained: AtomicBool::new(false), metrics } + } +} + +impl ItemQueue for CountBoundedQueue { + fn try_push(&self, item: T) -> Result<(), T> { + assert_not_drained(&self.drained, "CountBoundedQueue"); + if let Err(item) = self.inner.push(item) { + if let Some(m) = &self.metrics { + m.record_reject(); + } + return Err(item); + } + if let Some(m) = &self.metrics { + m.record_push(0); // count-bounded: items only, no byte size + } + Ok(()) + } + + fn try_pop(&self) -> Option { + let item = self.inner.pop()?; + Some(item) + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn mark_drained(&self) { + self.drained.store(true, Ordering::Release); + } + + fn is_drained(&self) -> bool { + self.drained.load(Ordering::Acquire) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// ByteBoundedQueue +// ───────────────────────────────────────────────────────────────────────────── + +/// Backing slot capacity for `ByteBoundedQueue`. Since the queue's real +/// gate is the byte budget, this just needs to be large enough that the +/// count never matters for any sane workload. 1024 slots is well past +/// the working set of any single pipeline edge — even for the smallest +/// items the byte cap (default 4 MiB) imposes a tighter bound. +/// +/// Sized in pages of `crossbeam_queue::ArrayQueue` storage (one +/// pre-allocated slot array, no per-push allocation). Mirrors the +/// `ArrayQueue::new(queue_capacity)` strategy the legacy pipeline used — it +/// also used a fixed-capacity `ArrayQueue` everywhere for the same +/// reason: `SegQueue` allocates segments on demand under load, and +/// the resulting allocator churn shows up as `mi_*` overhead in +/// profiles (≈260 samples vs legacy on CODEC 8M). +const BYTE_BOUNDED_QUEUE_SLOT_CAPACITY: usize = 1024; + +/// Memory-bounded transport. Backed by +/// `crossbeam_queue::ArrayQueue<(T, u64)>` plus an atomic byte counter. +/// `try_push` rejects when the running byte counter has already reached +/// `limit_bytes`. Requires `T: HeapSize`. +/// +/// ## Concurrency / ordering +/// +/// The check-then-add is two atomics, so two concurrent pushes can both +/// observe `cur < limit` and both succeed, yielding a small overshoot. +/// The next push will see the overshoot and reject; the budget is +/// enforced as "approximate within one item's worth per producer." This +/// trade-off avoids a CAS loop and is fine for backpressure semantics. +/// +/// `current_bytes` is **only** a backpressure heuristic — it's not used +/// to synchronize handoff of the items themselves. The handoff is the +/// `ArrayQueue`'s job; `ArrayQueue`'s internal atomics provide the +/// happens-before relationship between `inner.push` and `inner.pop`. We +/// therefore use `Relaxed` ordering on every `current_bytes` access: +/// the worst case is a slightly stale reading of the budget, never an +/// observability violation on the items. +/// +/// ## Cached size at push +/// +/// The size is stored alongside the item in the inner queue +/// (`ArrayQueue<(T, u64)>`) so `try_pop` doesn't need to recompute +/// `T::heap_size()` for the budget update. For types whose +/// `heap_size()` is O(items inside) (e.g. `BatchedRawPositionGroups`, +/// `OrderedRawPositionGroup`) this avoids recomputing a O(group) +/// walk on every pop. Mirrors the legacy `ReorderBuffer`'s +/// cached-size storage strategy (`(T, usize)` there; `(T, u64)` here, +/// matching `inner`'s `ArrayQueue<(T, u64)>` above). +pub struct ByteBoundedQueue { + inner: ArrayQueue<(T, u64)>, + current_bytes: AtomicU64, + /// Mutable byte-budget cap. The rebalancer (when enabled via + /// `PipelineConfig::queue_memory_total`) updates this atomic at + /// runtime to shift budget across queues based on observed + /// fullness. Producers read it on every `try_push`; the + /// `Relaxed` ordering matches `current_bytes` (this is a + /// best-effort backpressure heuristic, not a correctness gate). + limit_bytes: AtomicU64, + drained: AtomicBool, + /// Per-instance one-shot guard so the "slot cap hit before byte budget" + /// warning (see `try_push`) is emitted at most once *per queue*, not once + /// per process. A process-global flag would silence the warning for every + /// later queue (e.g. a second `runall` stage, or many pipelines in one + /// long-lived host / test harness) after the first occurrence. The hot-path + /// cost is a single relaxed swap after the first hit. + slot_cap_warned: AtomicBool, + /// `Some` only on an instrumented edge; producer-push (items + bytes) and + /// rejections are recorded here. Consumer-pop is recorded at the + /// `BranchInputHandle` (see `handles.rs`). + metrics: Option>, +} + +impl ByteBoundedQueue { + /// Construct a byte-bounded transport with the given memory limit. + /// + /// # Panics + /// + /// Panics if `limit_bytes == 0` (a zero-budget queue would always reject). + #[must_use] + pub fn new(limit_bytes: u64) -> Self { + Self::build(limit_bytes, None) + } + + /// Like [`new`](Self::new) but recording producer-push metrics (items + bytes + /// + rejections) into `metrics`. Byte-budget semantics unchanged. + /// + /// # Panics + /// + /// Panics if `limit_bytes == 0`. + #[must_use] + pub fn new_instrumented(limit_bytes: u64, metrics: Arc) -> Self { + Self::build(limit_bytes, Some(metrics)) + } + + /// [`new`](Self::new) when `metrics` is `None`, [`new_instrumented`](Self::new_instrumented) + /// when `Some`. + /// + /// # Panics + /// + /// Panics if `limit_bytes == 0`. + #[must_use] + pub fn maybe_instrumented(limit_bytes: u64, metrics: Option>) -> Self { + Self::build(limit_bytes, metrics) + } + + fn build(limit_bytes: u64, metrics: Option>) -> Self { + assert!(limit_bytes > 0, "ByteBoundedQueue limit_bytes must be > 0"); + Self { + inner: ArrayQueue::new(BYTE_BOUNDED_QUEUE_SLOT_CAPACITY), + current_bytes: AtomicU64::new(0), + limit_bytes: AtomicU64::new(limit_bytes), + drained: AtomicBool::new(false), + slot_cap_warned: AtomicBool::new(false), + metrics, + } + } + + /// Best-effort, stale-tolerant `Relaxed` read of the running byte + /// counter. Used by the rebalancer as a budget heuristic, not as a + /// correctness gate — it may lag a concurrent `try_push`/`try_pop`. + #[must_use] + pub fn current_bytes(&self) -> u64 { + self.current_bytes.load(Ordering::Relaxed) + } + + /// Best-effort, stale-tolerant `Relaxed` read of the byte-budget cap. + /// A concurrent `set_limit_bytes` (rebalancer) may not yet be visible; + /// callers use this as a heuristic, never as a correctness gate. + #[must_use] + pub fn limit_bytes(&self) -> u64 { + self.limit_bytes.load(Ordering::Relaxed) + } + + /// Update the byte-budget cap. Called by the rebalancer when + /// reallocating budget across queues. Concurrent `try_push`es + /// see the new cap on their next read; transient overshoot + /// (pushes already in flight that read the old cap) is + /// self-correcting. + pub fn set_limit_bytes(&self, new_limit: u64) { + // Floor at 1. `try_push` rejects when `current_bytes >= limit_bytes`, so a + // limit of 0 rejects unconditionally — even on an empty edge — and wedges + // the producer permanently. `new` asserts `limit_bytes > 0` for exactly + // this reason; without a floor here that invariant could be undone after + // construction, which is the one case the constructor cannot guard. + // + // Clamped rather than asserted: this runs on a live pipeline (the budget + // pass and the rebalancer), where degrading to a 1-byte limit still makes + // progress — `try_push` admits an item whenever `current_bytes` is under + // the limit, regardless of item size — while a panic would take down a + // running pipeline over a recoverable arithmetic slip. Every current caller + // already applies its own positive per-queue floor. + self.limit_bytes.store(new_limit.max(1), Ordering::Relaxed); + } +} + +/// Type-erased handle for a byte-bounded queue. The pipeline +/// rebalancer iterates over registered handles to read fullness +/// (`current_bytes / limit_bytes`) and reallocate budget across +/// queues by calling `set_limit_bytes`. The trait deliberately +/// does not surface the queue's item type or its `ItemQueue` +/// methods — rebalancing only needs the byte counters. +pub trait BoundedQueueHandle: Send + Sync { + /// Bytes currently held in the queue. + fn current_bytes(&self) -> u64; + /// Current byte-budget cap. May change between calls if a + /// rebalancer is active. + fn limit_bytes(&self) -> u64; + /// Update the byte-budget cap. Concurrent producers see the + /// new value on their next push. + fn set_limit_bytes(&self, new_limit: u64); +} + +impl BoundedQueueHandle for ByteBoundedQueue { + fn current_bytes(&self) -> u64 { + self.current_bytes() + } + fn limit_bytes(&self) -> u64 { + self.limit_bytes() + } + fn set_limit_bytes(&self, new_limit: u64) { + self.set_limit_bytes(new_limit); + } +} + +impl ItemQueue for ByteBoundedQueue { + fn try_push(&self, item: T) -> Result<(), T> { + assert_not_drained(&self.drained, "ByteBoundedQueue"); + // Like the legacy `ReorderBufferState::can_proceed`, this + // gates on `heap_bytes < limit` — accept if currently *under* + // budget, regardless of incoming item size. Per-item-larger-than + // -limit is a real case (busy-locus position-group batches can + // be tens of MB while the queue limit is 4 MiB), so a strict + // `cur + size <= limit` would deadlock the producer. + // + // Once `cur` reaches `limit_bytes`, subsequent pushes reject + // until a consumer drains. Transient overshoot under concurrent + // pushes is self-correcting on the next round. + let cur = self.current_bytes.load(Ordering::Relaxed); + let limit = self.limit_bytes.load(Ordering::Relaxed); + if cur >= limit { + if let Some(m) = &self.metrics { + m.record_reject(); + } + return Err(item); + } + let size = item.heap_size() as u64; + // Reserve bytes before pushing so a concurrent consumer cannot pop and + // decrement the counter before we add our share, which would cause the + // counter to underflow and create permanent false backpressure. + self.current_bytes.fetch_add(size, Ordering::Relaxed); + // ArrayQueue::push returns Err((item, size)) on full; roll back the + // reservation and return the item to the caller for retry. (In practice + // the slot cap should never be hit before the byte budget triggers a + // reject above, but defend against it anyway.) + match self.inner.push((item, size)) { + Ok(()) => { + if let Some(m) = &self.metrics { + m.record_push(size); + } + Ok(()) + } + Err((item, _size)) => { + // Roll back the byte reservation — the item never entered the queue. + self.current_bytes.fetch_sub(size, Ordering::Relaxed); + if let Some(m) = &self.metrics { + m.record_reject(); + } + // The fixed 1024-slot backing was hit before the byte budget. + // This degrades byte-backpressure into a hard count cap for + // small items (heap_size ≲ limit/1024) — correctness is + // preserved (the producer retries) but throughput silently + // suffers. Surface it once so it is observable rather than a + // silent foot-gun; near-zero cost after the first hit. + if !self.slot_cap_warned.swap(true, Ordering::Relaxed) { + log::warn!( + "ByteBoundedQueue hit its {BYTE_BOUNDED_QUEUE_SLOT_CAPACITY}-slot count \ + cap before the byte budget; small items are degrading byte-backpressure \ + into a count cap (throughput, not correctness, is affected)." + ); + } + Err(item) + } + } + } + + fn try_pop(&self) -> Option { + let (item, size) = self.inner.pop()?; + self.current_bytes.fetch_sub(size, Ordering::Relaxed); + Some(item) + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn mark_drained(&self) { + self.drained.store(true, Ordering::Release); + } + + fn is_drained(&self) -> bool { + self.drained.load(Ordering::Acquire) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// UnboundedQueue +// ───────────────────────────────────────────────────────────────────────────── + +/// Unbounded transport. `try_push` always succeeds. Backed by `SegQueue`. +pub struct UnboundedQueue { + inner: SegQueue, + drained: AtomicBool, + /// `Some` only on an instrumented edge; producer-push items are recorded + /// here (unbounded → never rejects, no byte tracking). Consumer-pop is at the + /// `BranchInputHandle`. + metrics: Option>, +} + +impl UnboundedQueue { + #[must_use] + pub fn new() -> Self { + Self { inner: SegQueue::new(), drained: AtomicBool::new(false), metrics: None } + } + + /// Like [`new`](Self::new) but recording producer-push item counts into + /// `metrics`. Unbounded edges have no byte budget and never reject; depth is + /// reported as raw length only. + #[must_use] + pub fn new_instrumented(metrics: Arc) -> Self { + Self { inner: SegQueue::new(), drained: AtomicBool::new(false), metrics: Some(metrics) } + } + + /// [`new`](Self::new) when `metrics` is `None`, [`new_instrumented`](Self::new_instrumented) + /// when `Some`. + #[must_use] + pub fn maybe_instrumented(metrics: Option>) -> Self { + Self { inner: SegQueue::new(), drained: AtomicBool::new(false), metrics } + } +} + +impl Default for UnboundedQueue { + fn default() -> Self { + Self::new() + } +} + +impl ItemQueue for UnboundedQueue { + fn try_push(&self, item: T) -> Result<(), T> { + assert_not_drained(&self.drained, "UnboundedQueue"); + self.inner.push(item); + if let Some(m) = &self.metrics { + m.record_push(0); + } + Ok(()) + } + + fn try_pop(&self) -> Option { + self.inner.pop() + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn mark_drained(&self) { + self.drained.store(true, Ordering::Release); + } + + fn is_drained(&self) -> bool { + self.drained.load(Ordering::Acquire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use std::sync::Arc; + + /// A `try_push` after `mark_drained` must panic on every transport impl, in + /// every build. It was `debug_assert!`-only, so a release build pushed the + /// item into a queue the consumer had already closed — a silent loss that + /// surfaces only as a short output. `#[values]` covers all three impls so a + /// new transport that forgets the guard is caught by the same table. + #[rstest] + #[case::count_bounded(Arc::new(CountBoundedQueue::new(2)) as Arc>)] + #[case::byte_bounded(Arc::new(ByteBoundedQueue::new(1024)) as Arc>)] + #[case::unbounded(Arc::new(UnboundedQueue::new()) as Arc>)] + #[should_panic(expected = "try_push after mark_drained — producer contract violation")] + fn try_push_after_mark_drained_panics(#[case] q: Arc>) { + q.mark_drained(); + let _ = q.try_push(1); + } + + /// The guard must not fire before `mark_drained` — a plain push on a fresh + /// queue still succeeds on every impl. + #[rstest] + #[case::count_bounded(Arc::new(CountBoundedQueue::new(2)) as Arc>)] + #[case::byte_bounded(Arc::new(ByteBoundedQueue::new(1024)) as Arc>)] + #[case::unbounded(Arc::new(UnboundedQueue::new()) as Arc>)] + fn try_push_before_mark_drained_succeeds(#[case] q: Arc>) { + assert!(q.try_push(1).is_ok(), "an undrained queue must still accept a push"); + assert_eq!(q.try_pop(), Some(1)); + } + + #[test] + fn count_bounded_round_trip() { + let q: Arc> = Arc::new(CountBoundedQueue::new(2)); + assert!(q.try_push(1).is_ok()); + assert!(q.try_push(2).is_ok()); + assert_eq!(q.try_push(3), Err(3)); + assert_eq!(q.try_pop(), Some(1)); + assert_eq!(q.try_pop(), Some(2)); + assert_eq!(q.try_pop(), None); + } + + #[test] + fn count_bounded_drain_signal() { + let q = CountBoundedQueue::::new(4); + assert!(!q.is_drained()); + q.mark_drained(); + assert!(q.is_drained()); + } + + #[derive(Debug)] + struct Heavy(Vec); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + self.0.len() + } + } + + #[test] + fn byte_bounded_slot_cap_reject_is_observable() { + // Tiny (0-byte heap) items with a huge byte limit: the byte budget is + // never reached, so the fixed slot backing becomes the binding cap. + // The first SLOT_CAPACITY pushes succeed; the next rejects on the slot + // cap even though current_bytes is far below the limit. Regression for + // F02 — this path silently degraded byte-backpressure into a count cap; + // it is now warn-once observable, and this pins the reject behaviour. + let q = ByteBoundedQueue::::new(1_000_000); + for i in 0..BYTE_BOUNDED_QUEUE_SLOT_CAPACITY { + assert!(q.try_push(Heavy(Vec::new())).is_ok(), "push {i} within slot cap"); + } + assert_eq!(q.current_bytes(), 0, "0-byte items leave the byte budget unused"); + assert!( + q.try_push(Heavy(Vec::new())).is_err(), + "push #{} must reject on the slot cap, not the byte budget", + BYTE_BOUNDED_QUEUE_SLOT_CAPACITY + 1 + ); + } + + /// The slot-cap reject path reserves `size` bytes *before* pushing and rolls + /// the reservation back when `ArrayQueue::push` reports full. The sibling test + /// above fills with 0-byte items, so that `fetch_sub` runs with `size == 0` + /// and a leak or double-subtract is invisible. Fill with nonzero items under a + /// limit large enough that the slot cap still binds, and pin the byte counter + /// across the rejected push. + #[test] + fn byte_bounded_slot_cap_reject_rolls_back_reserved_bytes() { + const ITEM_BYTES: usize = 8; + // Large enough that 1024 * 8 bytes never reaches it, so the reject below + // is the slot cap and not the byte budget. + let q = ByteBoundedQueue::::new(1_000_000); + for i in 0..BYTE_BOUNDED_QUEUE_SLOT_CAPACITY { + assert!(q.try_push(Heavy(vec![0; ITEM_BYTES])).is_ok(), "push {i} within slot cap"); + } + let before = q.current_bytes(); + assert_eq!( + before, + (BYTE_BOUNDED_QUEUE_SLOT_CAPACITY * ITEM_BYTES) as u64, + "every admitted item's bytes are accounted" + ); + assert!(before < 1_000_000, "the byte budget must not be the binding cap here"); + assert!( + q.try_push(Heavy(vec![0; ITEM_BYTES])).is_err(), + "push #{} must reject on the slot cap", + BYTE_BOUNDED_QUEUE_SLOT_CAPACITY + 1 + ); + assert_eq!( + q.current_bytes(), + before, + "a slot-cap reject must roll its reservation back exactly — leaking bytes here \ + would create permanent false backpressure" + ); + } + + /// A 0 limit makes `try_push` reject unconditionally (`current_bytes >= 0` is + /// always true), wedging the producer forever. `new` asserts against it, so + /// the setter must not be able to reintroduce it after construction. Clamping + /// to 1 keeps the edge alive: `try_push` admits an item whenever + /// `current_bytes` is *under* the limit, whatever the item's size. + #[test] + fn set_limit_bytes_clamps_zero_to_one_so_the_edge_still_admits() { + let q = ByteBoundedQueue::::new(4096); + q.set_limit_bytes(0); + assert_eq!(q.limit_bytes(), 1, "a 0 limit is floored to 1, never stored as 0"); + assert!( + q.try_push(Heavy(vec![0; 64])).is_ok(), + "an empty edge must still admit one item — a 0 limit would reject forever" + ); + // Now over the 1-byte limit, so the next push rejects: still a real bound, + // not a silent promotion to unbounded. + assert!(q.try_push(Heavy(vec![0; 64])).is_err(), "the clamped limit still applies"); + } + + #[test] + fn slot_cap_warn_flag_is_per_instance_not_process_global() { + // The "slot cap hit before byte budget" warn-once guard lives on the + // queue instance, so a second queue (e.g. a later runall stage, or a new + // pipeline in a long-lived host) still warns on its own first hit — the + // signal is not silenced process-wide by an earlier queue. + let fill_to_slot_cap = |q: &ByteBoundedQueue| { + for _ in 0..BYTE_BOUNDED_QUEUE_SLOT_CAPACITY { + q.try_push(Heavy(Vec::new())).expect("push within slot cap"); + } + // This push trips the slot cap and (first time) sets the flag. + assert!(q.try_push(Heavy(Vec::new())).is_err(), "push must reject on slot cap"); + }; + + let q1 = ByteBoundedQueue::::new(1_000_000); + assert!(!q1.slot_cap_warned.load(Ordering::Relaxed)); + fill_to_slot_cap(&q1); + assert!(q1.slot_cap_warned.load(Ordering::Relaxed), "first queue must warn on its hit"); + + // A fresh queue starts un-warned even though q1 already warned, so it + // will warn on its own first hit (per-queue, not process-global). + let q2 = ByteBoundedQueue::::new(1_000_000); + assert!( + !q2.slot_cap_warned.load(Ordering::Relaxed), + "a second queue must NOT inherit the first queue's warned state" + ); + fill_to_slot_cap(&q2); + assert!( + q2.slot_cap_warned.load(Ordering::Relaxed), + "second queue must warn on its own hit" + ); + } + + #[test] + fn byte_bounded_respects_limit() { + let q = ByteBoundedQueue::::new(100); + // Empty queue accepts even an oversized item (legacy semantics: + // gate on `cur < limit`, not `cur + size <= limit`). This is the + // fix for the per-item-larger-than-limit deadlock. + assert!(q.try_push(Heavy(vec![0; 200])).is_ok()); + assert_eq!(q.current_bytes(), 200); + // Now `cur >= limit`, all subsequent pushes reject regardless + // of size. + let rejected = q.try_push(Heavy(vec![0; 1])); + assert!(rejected.is_err(), "queue at/over budget should reject"); + assert_eq!(q.current_bytes(), 200); + // After a pop drops `cur` below limit, pushes succeed again. + let _ = q.try_pop().unwrap(); + assert_eq!(q.current_bytes(), 0); + assert!(q.try_push(Heavy(vec![0; 50])).is_ok()); + assert_eq!(q.current_bytes(), 50); + } + + #[test] + fn byte_bounded_oversized_first_push_succeeds() { + // Regression: previously a single push larger than `limit_bytes` + // would always reject (`0 + size > limit`), deadlocking + // producers that emit oversized batches (e.g. busy-locus + // position-group batches). With the legacy `cur < limit` + // semantics, the oversized push goes through. + let q = ByteBoundedQueue::::new(100); + assert!(q.try_push(Heavy(vec![0; 1024])).is_ok()); + } + + #[test] + fn byte_bounded_decrements_on_pop() { + let q = ByteBoundedQueue::::new(1000); + q.try_push(Heavy(vec![0; 200])).unwrap(); + assert_eq!(q.current_bytes(), 200); + let _ = q.try_pop().unwrap(); + assert_eq!(q.current_bytes(), 0); + } + + #[test] + fn unbounded_never_rejects() { + let q = UnboundedQueue::::new(); + for i in 0..1024 { + assert!(q.try_push(i).is_ok()); + } + } + + // ── Per-edge metrics (L2-instrumentation Task 2) ───────────────────────── + + #[test] + fn instrumented_queue_counts_push_and_reject() { + let m = EdgeMetrics::new(); + let q = CountBoundedQueue::::new_instrumented(1, Arc::clone(&m)); + assert!(q.try_push(1).is_ok()); + assert_eq!(q.try_push(2), Err(2)); // full (cap 1) → reject + let s = m.snapshot(); + assert_eq!(s.pushed_items, 1, "one successful push"); + assert_eq!(s.push_rejections, 1, "one rejection"); + // Producer-push only at this layer; pop is counted at the input handle. + assert_eq!(s.popped_items, 0); + } + + #[test] + fn byte_bounded_instrumented_push_bytes_and_depth() { + let m = EdgeMetrics::new(); + let q = ByteBoundedQueue::::new_instrumented(1000, Arc::clone(&m)); + q.try_push(Heavy(vec![0; 200])).unwrap(); + let s = m.snapshot(); + assert_eq!(s.pushed_items, 1); + assert_eq!(s.pushed_bytes, 200); + } + + #[test] + fn byte_bounded_instrumented_counts_reject() { + // The byte-budget reject path increments push_rejections (distinct from + // the CountBounded slot reject above). Fill to budget, then a push rejects. + let m = EdgeMetrics::new(); + let q = ByteBoundedQueue::::new_instrumented(100, Arc::clone(&m)); + q.try_push(Heavy(vec![0; 200])).unwrap(); // accepted (cur::new(4).metrics.is_none()); + assert!(ByteBoundedQueue::::new(100).metrics.is_none()); + assert!(UnboundedQueue::::new().metrics.is_none()); + // And instrumented constructors do attach metrics. + assert!( + CountBoundedQueue::::new_instrumented(4, EdgeMetrics::new()).metrics.is_some() + ); + assert!(UnboundedQueue::::new_instrumented(EdgeMetrics::new()).metrics.is_some()); + assert!( + ByteBoundedQueue::::new_instrumented(100, EdgeMetrics::new()).metrics.is_some() + ); + + // `maybe_instrumented` is the constructor the branch builders actually + // call, and it was the only one with no coverage: one that dropped a + // `Some(metrics)` would silently produce an edge that reports nothing + // under `--pipeline-trace`, with every other test still green. Both + // directions, all three impls. + assert!(CountBoundedQueue::::maybe_instrumented(4, None).metrics.is_none()); + assert!(ByteBoundedQueue::::maybe_instrumented(100, None).metrics.is_none()); + assert!(UnboundedQueue::::maybe_instrumented(None).metrics.is_none()); + assert!( + CountBoundedQueue::::maybe_instrumented(4, Some(EdgeMetrics::new())) + .metrics + .is_some() + ); + assert!( + ByteBoundedQueue::::maybe_instrumented(100, Some(EdgeMetrics::new())) + .metrics + .is_some() + ); + assert!( + UnboundedQueue::::maybe_instrumented(Some(EdgeMetrics::new())).metrics.is_some() + ); + } +} diff --git a/crates/fgumi-pipeline-core/src/reorder.rs b/crates/fgumi-pipeline-core/src/reorder.rs new file mode 100644 index 000000000..c0f1c06b3 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/reorder.rs @@ -0,0 +1,1036 @@ +//! Reorder operator on top of an `ItemQueue>`. +//! +//! `ReorderStage` is layered between a producer's transport queue and +//! the consumer's `InputHandle`. The producer pushes items wrapped in +//! a framework-managed `Sequenced { ordinal, item }`. The reorder +//! stage buffers items until their ordinal equals `next_serial`, then +//! releases them in order. +//! +//! Smart backpressure (deadlock avoidance): +//! - Until `next_serial` is *observable in the reorder buffer*, the stage +//! MUST accept everything (refusing could deadlock — the producer of +//! `next_serial` may be one of the backpressured producers). +//! - Once `next_serial` is in the buffer, the consumer can make progress +//! and producers MAY be rejected by the underlying transport's normal +//! backpressure. +//! +//! "Observable in the reorder buffer" is the conservative test: items +//! already in transport but not yet pulled into the buffer count as +//! *not* observable. This means producers over-accept when `next_serial` is +//! sitting in transport waiting to be pulled — but it can never deadlock, +//! and the over-acceptance window closes the next time a consumer calls +//! `try_pop_in_order` (which drains transport into the buffer). +//! +//! Storage layering: +//! - "In flight" items live in the underlying `ItemQueue>` +//! transport (`CountBounded` / `Unbounded` — see PR 1 caveat below). +//! - "Stashed" items (must-accept overflow when transport rejected, or +//! items pulled but not yet at their turn) live in `state.buffer: +//! AHashMap` (ahash — the ordinals are trivial monotonic `u64` +//! keys, so the default `SipHash` buys nothing on this per-item path). +//! - The transport's backpressure budget covers only in-flight items. +//! The overflow stash has its own byte cap (`max_overflow_bytes`). The +//! framework sizes that cap thread-awarely from the per-edge transport +//! budget (see `apply_initial_queue_budget` / `set_max_overflow_bytes`), +//! clamped to a fixed ceiling — so at low thread counts the stash stays +//! small (a streaming footprint) and at high thread counts it keeps the +//! prior lookahead headroom. `next_serial` is always exempt from the cap, +//! so the stash bound is purely a memory/throughput knob, never a +//! liveness constraint (any cap ≥ 0 is deadlock-free). +//! +//! `Sequenced` impls `HeapSize` (see below), so `BranchOrdering::ByOrdinal` +//! / `ByItemOrdinal` compose with `QueueSpec::ByteBounded` — the canonical BAM +//! step output shape (`build_branch_ordered_bytes`). + +use ahash::AHashMap; +use parking_lot::Mutex; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use super::item::{HeapSize, Ordered}; +use super::queues::ItemQueue; + +/// Default cap on a `ReorderStage`'s must-accept overflow buffer, in +/// bytes. Mirrors legacy `BACKPRESSURE_THRESHOLD_BYTES / 2` (`base.rs:704`): +/// the legacy pipeline gates non-`next_seq` reorder pushes at half the +/// 512 MB threshold (= 256 MB). We use the same value per branch so a +/// multi-stage pipeline with four ordered edges peaks around 1 GB of +/// reorder overflow under heterogeneous load — far below the unbounded +/// growth Task #29 hit (23+ GB on a 53M-record group workload). Items +/// at `next_serial` are exempt to preserve liveness. +pub const DEFAULT_REORDER_OVERFLOW_BYTES: u64 = 256 * 1024 * 1024; + +/// Per-branch ordering directive in `StepProfile::branch_ordering`. +/// +/// Three modes; pick based on what the consumer needs: +/// +/// - **`None`** — FIFO by arrival. Cheapest. Consumer sees items in the +/// order workers happened to push them, which under multi-producer +/// concurrency is non-deterministic. +/// +/// - **`ByOrdinal`** — producer-allocated ordinals via a per-branch +/// `AtomicU64` counter. The framework wraps each pushed item in a +/// `Sequenced` and inserts a `ReorderStage` in front of the consumer. +/// Imposes a total order at the producer's emission point but **does +/// not preserve any pre-existing global ordering** — under multi- +/// producer Parallel concurrency, the ordinal a worker gets bears no +/// relation to the order of the input it processed. Useful for +/// single-producer steps (sources) and for cases where any deterministic +/// total order suffices. +/// +/// - **`ByItemOrdinal`** — items carry their own serial via the [`Ordered`] +/// trait. The framework reads `item.ordinal()` instead of allocating +/// one. Preserves global ordering across multi-step Parallel transforms +/// when each step propagates the input's serial onto its outputs (the +/// canonical pattern for BAM pipelines: every batch carries +/// `batch_serial: u64` from its read order, and every transform +/// preserves that serial on its output items). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BranchOrdering { + /// FIFO by arrival. + None, + /// Producer-allocated ordinal via a per-branch `AtomicU64` counter. + ByOrdinal, + /// Items carry their own ordinal via the `Ordered` trait. Requires + /// `T: Ordered` at branch construction time. + ByItemOrdinal, +} + +/// Framework-internal wrapper carrying a producer-assigned ordinal. +/// +/// Step authors never see this type; it is `pub` only because it appears in the +/// public `ItemQueue>` trait bounds that ordered queues (and their +/// tests) instantiate, so callers may construct it directly when wiring queues. +pub struct Sequenced { + /// Producer-assigned monotonic ordinal used to restore emission order. + pub ordinal: u64, + /// The wrapped payload carried alongside its `ordinal`. + pub item: T, +} + +impl Ordered for Sequenced { + fn ordinal(&self) -> u64 { + self.ordinal + } +} + +/// `Sequenced` forwards `HeapSize` to the inner item, allowing byte- +/// bounded queues to wrap ordered items. The `ordinal` field is stack- +/// only and contributes nothing to the heap budget. +impl HeapSize for Sequenced { + fn heap_size(&self) -> usize { + self.item.heap_size() + } +} + +/// Reorder operator. Wraps an `Arc>>` and +/// presents a `try_pop_in_order() -> Option` surface plus an +/// ordinal-tagged `try_push(ordinal, item)`. +pub struct ReorderStage { + transport: Arc>>, + state: Mutex>, + /// Cached "is `next_serial` currently in the buffer?" snapshot, + /// updated under the state lock by writers and read **without** the + /// lock by `try_push`'s fast path. Mirrors legacy + /// `OrderedQueue::has_next` (`queue.rs:54`). + /// + /// Semantics: + /// - `false` → `next_serial` NOT in buffer → producers MUST overflow + /// into the buffer if transport rejects (the slow path takes the + /// state lock and re-checks under it; the I1 fix from Phase 1). + /// - `true` → `next_serial` IS in buffer → consumer can drain → + /// producers CAN apply backpressure. Fast path: a lock-free + /// `transport.try_push` whose rejection surfaces as `Err`. + /// + /// Stale-`true` is harmless (the slow path is always correct). + /// Stale-`false` is also harmless (just a missed fast-path + /// opportunity; producer takes the slow path which still does the + /// right thing). + next_serial_buffered: AtomicBool, + /// Sticky drained-observed flag for `is_drained()` callers that want + /// to short-circuit on subsequent calls. Set lazily by `is_drained` + /// when it observes the drained-and-empty condition. + drained_observed: AtomicBool, + /// Byte cap on the must-accept overflow stash: once the buffered bytes + /// reach the cap, must-accept rejects new pushes for ordinals other than + /// `next_serial` (the producer then holds via its `held_slot`). + /// `next_serial` is always exempt — its producer cannot be backpressured + /// by this cap (see Task #29) — so any cap value is deadlock-free. + /// `u64::MAX` (`REORDER_OVERFLOW_UNBOUNDED`) means unbounded. + /// + /// Set once at construction (`with_max_overflow_bytes`, the no-budget + /// fallback) and re-set once before workers spawn by the budget pass + /// (`set_max_overflow_bytes` via `apply_initial_queue_budget`) to a + /// thread-aware value from the per-edge transport budget. Read only on the + /// must-accept slow path (under the `state` lock), so the atomic costs + /// nothing on the lock-free fast path. Mirrors legacy + /// `ReorderBufferState::can_proceed` (`base.rs:766-781`). + max_overflow_bytes: AtomicU64, + /// Producer-side push metrics for an **ordered** edge, recorded at THIS + /// boundary rather than on the transport queue. `try_push` can turn a + /// full-transport `Err` into a must-accept stash `Ok`, so recording on the + /// transport would miscount a stashed (accepted) item as a rejection. Here we + /// record `record_push` on every accepted push (transport OR stash) and + /// `record_reject` only on a true `Err`, keeping `pushed_items` / + /// `push_rejections` accurate under producer skew. `None` when the edge is + /// not instrumented; the pop side is recorded separately by the input handle. + push_metrics: Option>, + /// Whether `push_metrics` records each push's `heap_size` (byte-bounded edge) + /// or `0` (count/unbounded edge) — mirroring the underlying queue's own byte + /// accounting. Set from the branch's queue kind at construction. + record_item_bytes: bool, +} + +/// Unbounded sentinel for [`ReorderStage::max_overflow_bytes`]. +const REORDER_OVERFLOW_UNBOUNDED: u64 = u64::MAX; + +/// Type-erased setter for a [`ReorderStage`]'s overflow cap, so the runtime +/// can size it without naming the branch's item type `T`. The framework +/// collects one per ordered byte-bounded branch and sets the cap in +/// `apply_initial_queue_budget` (before any worker spawns). Kept separate +/// from `BoundedQueueHandle` (the transport-resize handle) so existing +/// transport-handle impls are untouched. +pub trait ReorderCapHandle: Send + Sync { + /// Set the overflow byte cap (`u64::MAX` = unbounded). + fn set_max_overflow_bytes(&self, bytes: u64); + + /// Bytes currently held in the must-accept overflow stash. The deadlock + /// monitor sums this across branches (with the transport queues) to tell a + /// real wedge (work stuck) from upstream starvation (everything empty); the + /// cap-enforcement tests use it to observe stash growth. + fn current_buffer_bytes(&self) -> u64; +} + +impl ReorderCapHandle for ReorderStage { + fn set_max_overflow_bytes(&self, bytes: u64) { + // Set once before workers spawn; `Relaxed` is sufficient because the + // worker pool's spawn establishes the happens-before edge, and the + // value is only read on the must-accept slow path under `state.lock()`. + self.max_overflow_bytes.store(bytes, Ordering::Relaxed); + } + + fn current_buffer_bytes(&self) -> u64 { + self.state.lock().buffer_bytes + } +} + +#[cfg(test)] +impl ReorderStage { + /// Read the current overflow cap (`u64::MAX` = unbounded). Test-only — + /// lets the budget-wiring test assert `apply_initial_queue_budget` set it. + pub(crate) fn current_max_overflow_bytes(&self) -> u64 { + self.max_overflow_bytes.load(Ordering::Relaxed) + } +} + +struct ReorderState { + /// Items pulled from transport but not yet at their turn, indexed by + /// ordinal. Each entry caches the item's `heap_size()` measured at + /// insert time so we don't pay an O(item) walk on every pop or + /// transport→buffer drain. Mirrors legacy `ReorderBuffer` + /// (`fgumi-bam-io/src/reorder.rs:50`) which stores `(T, usize)`. + /// The size is consumed by `buffer_bytes` accounting; for items + /// whose `heap_size()` is O(records) (e.g. position groups) this + /// caching avoids 3× the `heap_size` cost per item flowing through. + buffer: AHashMap, + /// Tracked heap bytes of items currently in `buffer`. Updated on + /// insert + on `try_pop_in_order` drain. Used to enforce + /// `max_overflow_bytes`. Mirrors legacy + /// `ReorderBufferState::heap_bytes` (`base.rs:746`). + buffer_bytes: u64, + /// Next ordinal we'll release. + next_serial: u64, +} + +impl ReorderStage { + #[must_use] + pub fn new(transport: Arc>>) -> Self { + Self { + transport, + state: Mutex::new(ReorderState { + buffer: AHashMap::new(), + buffer_bytes: 0, + next_serial: 0, + }), + next_serial_buffered: AtomicBool::new(false), + drained_observed: AtomicBool::new(false), + max_overflow_bytes: AtomicU64::new(REORDER_OVERFLOW_UNBOUNDED), + push_metrics: None, + record_item_bytes: false, + } + } + + /// Variant that caps the must-accept overflow buffer to `max_bytes` + /// of accumulated heap (computed via `T::heap_size()` per item). + /// `next_serial` is exempt (always accepted to preserve liveness); + /// other ordinals are rejected back to the producer when the buffer + /// is at the byte cap, so producers hold via their `held_slot` and + /// retry. Without a cap, heterogeneous-size workloads (e.g. one + /// large position group from a busy locus while others crank + /// through small groups) can OOM via must-accept overflow. + /// + /// Mirrors legacy `ReorderBufferState::can_proceed` semantics + /// (`base.rs:766-781`): the legacy gates non-`next_seq` pushes on + /// `heap_bytes < memory_limit / 2`. We collapse the halving into + /// the caller-supplied cap so the hot-path check is one load + one + /// compare. + /// + /// In production the framework constructs the stage with a fallback cap + /// here and then RE-SIZES it thread-awarely via [`set_max_overflow_bytes`] + /// in `apply_initial_queue_budget` (from the same per-edge budget as the + /// transport queue). So this constructor's value is the no-budget fallback; + /// the live cap tracks the transport budget. + /// + /// [`set_max_overflow_bytes`]: ReorderCapHandle::set_max_overflow_bytes + #[must_use] + pub fn with_max_overflow_bytes( + transport: Arc>>, + max_bytes: u64, + ) -> Self { + Self { + transport, + state: Mutex::new(ReorderState { + buffer: AHashMap::new(), + buffer_bytes: 0, + next_serial: 0, + }), + next_serial_buffered: AtomicBool::new(false), + drained_observed: AtomicBool::new(false), + max_overflow_bytes: AtomicU64::new(max_bytes), + push_metrics: None, + record_item_bytes: false, + } + } + + /// Attach producer-side push metrics recorded at the `ReorderStage` boundary + /// (see `push_metrics`). `record_item_bytes` is `true` + /// for a byte-bounded edge (record each push's `heap_size`) and `false` for a + /// count/unbounded edge (record `0`), matching the underlying queue's byte + /// accounting. Called once by the ordered-branch builder. + #[must_use] + pub fn with_push_metrics( + mut self, + push_metrics: Option>, + record_item_bytes: bool, + ) -> Self { + self.push_metrics = push_metrics; + self.record_item_bytes = record_item_bytes; + self + } + + /// Producer-side push. The framework allocates `ordinal` from a + /// per-branch `AtomicU64` counter (see `handles.rs`). + /// + /// Backpressure semantics: + /// - If `next_serial` is not yet in the buffer, MUST accept (overflows + /// into the buffer if transport rejects). This prevents the deadlock + /// where the producer of `next_serial` is itself backpressured. + /// - If `next_serial` is in the buffer, the transport's normal + /// `try_push` rules apply; rejection surfaces as `Err((ordinal, item))`. + /// + /// Performance: when `next_serial` is observed in the buffer (the + /// steady-state case once the consumer is keeping up), the push is + /// **lock-free** — only the transport's atomic `try_push` runs. The + /// state mutex is acquired only when overflow into the buffer might + /// be needed. + /// + /// # Errors + /// + /// Returns `Err((ordinal, item))` when the transport rejected and we + /// were not in must-accept mode (i.e., `next_serial` is already + /// observable, so the consumer can drain). + pub fn try_push(&self, ordinal: u64, item: T) -> Result<(), (u64, T)> { + // Record push-side metrics at THIS boundary (see `push_metrics`). The + // must-accept path turns a full-transport `Err` into a stash `Ok`, so the + // transport queue can't tell a stashed (accepted) push from a reject — + // only the final `Result` here can. `heap_size()` is read before `item` + // moves into the inner push. Every `Ok` (transport OR stash) is a push; + // every `Err` (backpressure or stash-cap held) is a reject. + // + // Gate the `heap_size()` call on metrics being present: on the default + // instrumentation-off path (`push_metrics == None`) the byte figure is + // never recorded, so computing it would be pure hot-path overhead. + let bytes = if self.push_metrics.is_some() && self.record_item_bytes { + item.heap_size() as u64 + } else { + 0 + }; + let result = self.try_push_inner(ordinal, item); + if let Some(m) = &self.push_metrics { + match &result { + Ok(()) => m.record_push(bytes), + Err(_) => m.record_reject(), + } + } + result + } + + /// Inner push: the transport / must-accept-stash decision, without metrics. + /// See [`try_push`](Self::try_push) for the public contract; metrics are + /// recorded there so a stashed push is not miscounted as a rejection. + fn try_push_inner(&self, ordinal: u64, item: T) -> Result<(), (u64, T)> { + // Lock-free fast path. If next_serial is in the buffer (consumer + // can drain), the producer is in pure-backpressure mode: a + // transport push that succeeds returns Ok; a rejection returns + // Err. No overflow into the buffer is possible, so we don't need + // the state lock at all. + // + // Liveness: returning `Err` here (instead of falling back to the + // slow path) is safe because `try_pop_in_order` re-derives + // `next_serial_buffered` under the state lock, and the round-robin + // worker driver guarantees a `try_pop_in_order` runs between + // producer retries — so a stale-`true` flag here is corrected on + // the next consumer poll and the producer is re-dispatched. + if self.next_serial_buffered.load(Ordering::Acquire) { + let seq = Sequenced { ordinal, item }; + return match self.transport.try_push(seq) { + Ok(()) => Ok(()), + Err(seq) => Err((seq.ordinal, seq.item)), + }; + } + + // Slow path: must_accept may apply. Hold the state lock across + // the must_accept check AND the transport push so a concurrent + // consumer can't drain `next_serial` between the two — which + // would let producers keep over-accepting into the overflow + // buffer indefinitely (I1 from the Phase 1 review). + // + // Transport pushes are non-blocking, so holding the lock briefly + // is safe (we don't risk priority inversion against blocking I/O). + let mut state = self.state.lock(); + let must_accept = !state.buffer.contains_key(&state.next_serial); + let seq = Sequenced { ordinal, item }; + + if must_accept { + // Apply the byte-aware overflow cap *before* attempting the + // transport push, not only when transport is full. The consumer's + // `try_pop_in_order` drain loop relocates the entire transport into + // the stash while `next_serial` is absent, so a cap that only fires + // on transport-full never binds — the consumer keeps transport + // non-full — and the stash grows without bound (#330 zipper OOM: + // 7.7 GB of reorder stash vs 466 MB of transport). Gating the push + // on `buffer_bytes >= cap` regardless of transport room bounds the + // stash to roughly `cap + one transport-worth`. + // + // Liveness preserved: `next_serial` is exempt (always accepted), + // so the producer of `next_serial` can never be backpressured by + // this cap and the consumer can always make progress. Any cap value + // is deadlock-free (see `concurrent_tiny_cap_drains_all_in_order`, + // which proves this with a 1-byte cap). + let landed_next = ordinal == state.next_serial; + let cap = self.max_overflow_bytes.load(Ordering::Relaxed); + if !landed_next && cap != REORDER_OVERFLOW_UNBOUNDED && state.buffer_bytes >= cap { + let Sequenced { ordinal, item } = seq; + return Err((ordinal, item)); + } + match self.transport.try_push(seq) { + Ok(()) => Ok(()), + Err(seq) => { + // Transport full: overflow into the stash. The cap was + // already checked above; `next_serial` is exempt either + // way, so the must-accept liveness guarantee holds. + // `usize → u64` is a lossless widen on every supported + // (≤64-bit) target, so this cast never truncates. + let item_bytes = seq.item.heap_size() as u64; + // A duplicate ordinal would silently drop the buffered item and + // leak its bytes into `buffer_bytes`. `ByOrdinal` serials are + // unique by construction; a hit here means a `ByItemOrdinal` + // upstream emitted two items with the same serial (a step bug). + // Fail loud in release too (like the `next_serial` overflow guard + // below): silently dropping a buffered record is a data-integrity bug. + assert!( + !state.buffer.contains_key(&seq.ordinal), + "duplicate reorder ordinal {} — ByItemOrdinal upstream serials must be unique", + seq.ordinal + ); + state.buffer.insert(seq.ordinal, (seq.item, item_bytes)); + state.buffer_bytes = state.buffer_bytes.saturating_add(item_bytes); + if landed_next { + self.next_serial_buffered.store(true, Ordering::Release); + } + Ok(()) + } + } + } else { + // Cache was stale (false) but next_serial actually IS in the + // buffer. Update the cache so subsequent producers take the + // fast path, then apply backpressure. + self.next_serial_buffered.store(true, Ordering::Release); + match self.transport.try_push(seq) { + Ok(()) => Ok(()), + Err(seq) => Err((seq.ordinal, seq.item)), + } + } + } + + /// Consumer-side pop. Returns `Some(T)` if `next_serial` is available, + /// `None` if still waiting for it. + /// + /// # Panics + /// + /// Panics if the in-order ordinal counter would overflow `u64` (i.e. + /// `next_serial == u64::MAX`). Branch ordinals start at 0 and increment, so + /// this is unreachable on any real workload (~1.8e19 items on one edge); the + /// guard exists only to fail loudly rather than silently wrap and wait + /// forever for ordinal 0. + pub fn try_pop_in_order(&self) -> Option { + self.try_pop_in_order_reporting_blocked().0 + } + + /// Like [`try_pop_in_order`](Self::try_pop_in_order), but also reports whether + /// a `None` result was *reorder-blocked* — later ordinals are buffered while + /// the stage waits for an earlier one — rather than genuinely drained. The + /// pop and the reorder-blocked check are computed under the SAME state lock, + /// so a shared (`Parallel`) consumer cannot observe a torn `(item, blocked)` + /// pair that would skew the `pop_empties` starvation metric. The flag is + /// always `false` when an item is returned. + /// + /// # Panics + /// + /// Panics if the in-order ordinal counter would overflow `u64` (i.e. + /// `next_serial == u64::MAX`). Branch ordinals start at 0 and increment, so + /// this is unreachable on any real workload (~1.8e19 items on one edge); the + /// guard exists only to fail loudly rather than silently wrap and wait + /// forever for ordinal 0. + pub fn try_pop_in_order_reporting_blocked(&self) -> (Option, bool) { + let mut state = self.state.lock(); + let next = state.next_serial; + + // Drain transport into buffer until we see next_serial or transport + // is empty. This is the only place transport → buffer movement + // happens; it's the visibility synchronizer between producer pushes + // and consumer reads. Each move costs one `heap_size()` call, + // cached alongside the item so subsequent pop / cap accounting + // doesn't pay it again. + if !state.buffer.contains_key(&next) { + while let Some(seq) = self.transport.try_pop() { + // `usize → u64` is a lossless widen on every supported + // (≤64-bit) target, so this cast never truncates. + let bytes = seq.item.heap_size() as u64; + // See the drain-path insert above: a duplicate ordinal here would + // silently drop the buffered item and corrupt `buffer_bytes`. + // Fail loud in release too (like the `next_serial` overflow guard): + // silently dropping a buffered record is a data-integrity bug. + assert!( + !state.buffer.contains_key(&seq.ordinal), + "duplicate reorder ordinal {} — ByItemOrdinal upstream serials must be unique", + seq.ordinal + ); + state.buffer.insert(seq.ordinal, (seq.item, bytes)); + state.buffer_bytes = state.buffer_bytes.saturating_add(bytes); + if state.buffer.contains_key(&next) { + break; + } + } + } + + if let Some((item, item_bytes)) = state.buffer.remove(&next) { + state.buffer_bytes = state.buffer_bytes.saturating_sub(item_bytes); + // Advance the in-order cursor. `ByOrdinal` ordinals come from an + // `AtomicU64` allocator and `ByItemOrdinal` from upstream item + // ordinals; both start at 0 and increment, so `u64::MAX` is + // unreachable on any real workload (~1.8e19 items on one edge). + // `checked_add` makes that invariant explicit: a wrap here would + // silently wait forever for ordinal 0, so we fail loudly instead. + let new_next = next.checked_add(1).expect("reorder ordinal overflow (next_serial)"); + state.next_serial = new_next; + // Update the cache: is the NEW next_serial in the buffer? + // Producers reading post-update see the right state. + let new_buffered = state.buffer.contains_key(&new_next); + self.next_serial_buffered.store(new_buffered, Ordering::Release); + (Some(item), false) + } else { + // No in-order item. Reorder-blocked (backlog, not starvation) iff the + // buffer still holds out-of-order items awaiting an earlier ordinal — + // `next` was just confirmed absent, so any remaining entry is a later + // ordinal. Computed here under the same lock as the pop above. + let reorder_blocked = !state.buffer.is_empty(); + (None, reorder_blocked) + } + } + + /// True iff transport is drained, transport is empty, and the reorder + /// buffer is empty. Sticky: once observed true, stays true. + pub fn is_drained(&self) -> bool { + if self.drained_observed.load(Ordering::Acquire) { + return true; + } + if !self.transport.is_drained() { + return false; + } + if !self.transport.is_empty() { + return false; + } + let buf_empty = self.state.lock().buffer.is_empty(); + if buf_empty { + self.drained_observed.store(true, Ordering::Release); + } + buf_empty + } + + /// Producer-side: mark transport drained. Once drained-and-empty is + /// observed by a consumer, `is_drained()` returns true. + pub fn mark_drained(&self) { + self.transport.mark_drained(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::queues::CountBoundedQueue; + + fn make_stage(transport_capacity: usize) -> ReorderStage { + let q: Arc>> = + Arc::new(CountBoundedQueue::>::new(transport_capacity)); + ReorderStage::new(q) + } + + #[test] + fn pops_in_serial_order_regardless_of_push_order() { + let s = make_stage(8); + s.try_push(2, 200).unwrap(); + s.try_push(0, 100).unwrap(); + s.try_push(1, 150).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(100)); + assert_eq!(s.try_pop_in_order(), Some(150)); + assert_eq!(s.try_pop_in_order(), Some(200)); + assert_eq!(s.try_pop_in_order(), None); + } + + /// Q1 (audit D4): a `ByItemOrdinal` upstream supplies each item's serial, so + /// a buggy step could emit two items with the same ordinal. Inserting the + /// second would silently drop the first and leak its bytes into + /// `buffer_bytes`; the always-on `assert!` must turn that into a loud failure + /// (in release builds too). + #[test] + #[should_panic(expected = "duplicate reorder ordinal")] + fn duplicate_ordinal_trips_assert() { + let s = make_stage(8); + s.try_push(1, 100).unwrap(); + s.try_push(1, 200).unwrap(); // duplicate ordinal (simulated upstream bug) + s.try_push(0, 0).unwrap(); + // Draining moves both ordinal-1 items from the transport into the + // in-order buffer; the second insert hits the duplicate guard. + let _ = s.try_pop_in_order(); + } + + /// Sibling of `duplicate_ordinal_trips_assert`, covering the OTHER always-on + /// duplicate-ordinal guard: the must-accept *stash-insert* path in + /// `try_push_inner` (transport full → overflow into `state.buffer`), not the + /// transport-drain path in `try_pop_in_order_reporting_blocked`. + /// + /// With transport capacity 1 and `next_serial` (0) still absent, every push + /// is must-accept: the first fills the transport, and each later push finds + /// the transport full and overflows into the stash. The second and third + /// pushes both carry ordinal 1, so the second stashes it and the third trips + /// the stash-insert `assert!` — no `try_pop_in_order` runs, so the drain-path + /// guard is never reached. This test fails if that stash-insert guard is + /// removed. + #[test] + #[should_panic(expected = "duplicate reorder ordinal")] + fn duplicate_ordinal_trips_stash_assert() { + let s = make_stage(1); // capacity 1 so the transport fills and later items stash + s.try_push(1, 100).unwrap(); // ordinal 1 -> transport (fills capacity-1 transport) + s.try_push(1, 200).unwrap(); // transport full -> ordinal 1 stashed into buffer + // Transport still full and ordinal 1 already in the stash: the overflow + // insert hits the duplicate guard on the push path. + let _ = s.try_push(1, 300); + } + + #[test] + fn waits_for_next_serial() { + let s = make_stage(8); + s.try_push(1, 100).unwrap(); + s.try_push(2, 200).unwrap(); + assert_eq!(s.try_pop_in_order(), None); + s.try_push(0, 0).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(0)); + assert_eq!(s.try_pop_in_order(), Some(100)); + assert_eq!(s.try_pop_in_order(), Some(200)); + } + + #[test] + fn must_accept_overflows_when_transport_full() { + // Transport capacity 1. Push 4 items in arrival order; all must be + // accepted because we're waiting for serial 0. + let s = make_stage(1); + s.try_push(3, 30).unwrap(); + s.try_push(2, 20).unwrap(); + s.try_push(1, 10).unwrap(); + s.try_push(0, 0).unwrap(); + // Now drain in order. + assert_eq!(s.try_pop_in_order(), Some(0)); + assert_eq!(s.try_pop_in_order(), Some(10)); + assert_eq!(s.try_pop_in_order(), Some(20)); + assert_eq!(s.try_pop_in_order(), Some(30)); + } + + #[test] + fn reporting_pop_flags_reorder_blocked_vs_drained() { + // The reporting pop returns the reorder-blocked flag from the same locked + // path as the pop itself, so a shared consumer sees a consistent pair. + let s = make_stage(8); + // Buffer a later ordinal while ordinal 0 is absent → reorder-blocked. + s.try_push(1, 100).unwrap(); + assert_eq!(s.try_pop_in_order_reporting_blocked(), (None, true), "blocked, not drained"); + // Ordinal 0 arrives; the pop yields it and is not blocked. + s.try_push(0, 0).unwrap(); + assert_eq!(s.try_pop_in_order_reporting_blocked(), (Some(0), false)); + assert_eq!(s.try_pop_in_order_reporting_blocked(), (Some(100), false)); + // Genuinely drained now → None and NOT reorder-blocked (a true empty pop). + assert_eq!(s.try_pop_in_order_reporting_blocked(), (None, false), "drained, not blocked"); + } + + #[test] + fn push_metrics_count_stashed_item_as_push_not_reject() { + use crate::runtime::metrics::EdgeMetrics; + // Transport capacity 1, next_serial (0) absent → must-accept. The first + // push fills the transport; the second must-accept-overflows into the + // stash, returning Ok. Recording at the ReorderStage boundary must count + // BOTH as pushes and NEITHER as a rejection — the transport's internal + // `Err` on the stashed push is not a real backpressure event. (Recording + // on the transport, as before, miscounted the stashed item as a reject.) + let m = EdgeMetrics::new(); + let q: Arc>> = Arc::new(CountBoundedQueue::new(1)); + let stage = ReorderStage::new(q).with_push_metrics(Some(Arc::clone(&m)), false); + assert!(stage.try_push(1, 10).is_ok(), "ordinal 1 into transport"); + assert!(stage.try_push(2, 20).is_ok(), "ordinal 2 stashed (transport full)"); + let s = m.snapshot(); + assert_eq!(s.pushed_items, 2, "both accepted pushes counted"); + assert_eq!(s.push_rejections, 0, "a stashed push is not a rejection"); + assert_eq!(s.pushed_bytes, 0, "count-bounded edge records 0 push bytes"); + } + + #[test] + fn push_metrics_record_item_bytes_when_byte_bounded() { + use crate::runtime::metrics::EdgeMetrics; + #[derive(Debug)] + struct Heavy; + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + 100 + } + } + // With `record_item_bytes = true` (byte-bounded edge), each accepted push + // records its `heap_size` — so `pushed_bytes` tracks the bare `T`, the + // same size the pop side records. + let m = EdgeMetrics::new(); + let q: Arc>> = Arc::new(CountBoundedQueue::new(8)); + let stage = ReorderStage::new(q).with_push_metrics(Some(Arc::clone(&m)), true); + assert!(stage.try_push(0, Heavy).is_ok()); + assert!(stage.try_push(1, Heavy).is_ok()); + let s = m.snapshot(); + assert_eq!(s.pushed_items, 2); + assert_eq!(s.pushed_bytes, 200, "byte-bounded edge records heap_size per push"); + } + + #[test] + fn push_without_metrics_skips_heap_size() { + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + // On the default instrumentation-off path (`push_metrics == None`) the + // push wrapper must NOT compute `heap_size` — that byte figure is only + // needed to record a push, so computing it would be pure hot-path cost. + #[derive(Debug)] + struct Counted(Arc); + impl HeapSize for Counted { + fn heap_size(&self) -> usize { + self.0.fetch_add(1, AtomicOrdering::Relaxed); + 0 + } + } + let calls = Arc::new(AtomicUsize::new(0)); + let q: Arc>> = Arc::new(CountBoundedQueue::new(8)); + // Byte-bounded edge (`record_item_bytes = true`) but no push metrics. + let stage = ReorderStage::new(q).with_push_metrics(None, true); + // ordinal 0 == next_serial → transport accepts (no stash), so the only + // `heap_size` call would be the wrapper's — which the gate must skip. + stage.try_push(0, Counted(Arc::clone(&calls))).unwrap(); + assert_eq!( + calls.load(AtomicOrdering::Relaxed), + 0, + "heap_size must not be computed when push metrics are disabled" + ); + } + + #[test] + fn max_overflow_bytes_caps_must_accept_buffer() { + // Use `Heavy(u32)` carrying a pretend heap size of 100 bytes per + // item. Transport capacity 1, byte cap 200. Buffer accepts items + // until `buffer_bytes >= 200`, then rejects (except for + // `next_serial`, which always gets in). + #[derive(Debug, PartialEq)] + struct Heavy(u32); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + 100 + } + } + let q: Arc>> = + Arc::new(CountBoundedQueue::>::new(1)); + let s = ReorderStage::with_max_overflow_bytes(q, 200); + + // Fill transport (capacity 1). + s.try_push(3, Heavy(30)).unwrap(); + // Overflow into buffer: ordinal 4 (100 B). buffer_bytes = 100, + // < cap=200, accepted. + s.try_push(4, Heavy(40)).unwrap(); + // Ordinal 5 lands in buffer too (buffer_bytes = 100 at the cap + // check, before insert; insert lifts it to 200). + s.try_push(5, Heavy(50)).unwrap(); + // Pushing ordinal 6: buffer_bytes = 200 >= cap. Reject. Producer's + // job to retry via held_slot. + assert_eq!(s.try_push(6, Heavy(60)), Err((6, Heavy(60)))); + // But the next_serial=0 ALWAYS gets in (liveness exemption). + s.try_push(0, Heavy(0)).unwrap(); + // Drain to advance next_serial. + assert_eq!(s.try_pop_in_order(), Some(Heavy(0))); + // next_serial is now 1; ordinal 1 IS the new next_serial → must + // always be accepted regardless of byte cap. + s.try_push(1, Heavy(10)).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(Heavy(10))); + s.try_push(2, Heavy(20)).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(Heavy(20))); + assert_eq!(s.try_pop_in_order(), Some(Heavy(30))); + assert_eq!(s.try_pop_in_order(), Some(Heavy(40))); + assert_eq!(s.try_pop_in_order(), Some(Heavy(50))); + // After buffer drains, we can finally push the rejected 6. + s.try_push(6, Heavy(60)).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(Heavy(60))); + } + + #[test] + fn drain_into_stash_respects_cap_on_success_path() { + // Regression for the zipper reorder-stash OOM (issue #330). When + // `next_serial` is withheld by a lagging producer, the consumer's + // `try_pop_in_order` drain loop relocates the *entire* transport into + // the stash hunting for the absent serial. If the must-accept push + // only consults the byte cap on the transport-FULL path, the cap never + // fires — the consumer keeps transport non-full — and the stash grows + // without bound (measured: 7.7 GB on a 60M-read zipper run). The cap + // must gate non-`next_serial` pushes regardless of transport room. + #[derive(Debug, PartialEq)] + struct Heavy(u32); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + 100 + } + } + + // Transport capacity 1 item (≤100 B in flight); stash byte cap 200 B. + let q: Arc>> = + Arc::new(CountBoundedQueue::>::new(1)); + let s = ReorderStage::with_max_overflow_bytes(q, 200); + + // Serial 0 is never pushed (the lagging worker). Push ordinals 1..=50 + // in order, polling the consumer after each push — the poll returns + // None (serial 0 absent) but drains transport into the stash as a side + // effect, which is the relocation that bypassed the cap. Track the peak + // stash size after each drain. + let mut rejected = 0u32; + let mut peak_stash = 0u64; + for ord in 1u32..=50 { + if s.try_push(u64::from(ord), Heavy(ord)).is_err() { + rejected += 1; + } + assert_eq!(s.try_pop_in_order(), None, "serial 0 absent → no item releases"); + peak_stash = peak_stash.max(s.current_buffer_bytes()); + } + + // The cap must fire (without the fix, the success path bypasses it + // entirely and `rejected` stays 0). + assert!(rejected > 0, "stash cap never fired: drain-into-stash bypassed the byte cap"); + // Peak stash is bounded by the cap (200 B) plus at most one + // transport-worth (100 B) of overshoot. + assert!(peak_stash <= 300, "stash grew past cap + one transport-worth: {peak_stash} B"); + } + + #[test] + fn setter_tightens_cap_and_tiny_cap_preserves_liveness() { + #[derive(Debug, PartialEq)] + struct Heavy(u32); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + 100 + } + } + + // (a) The `ReorderCapHandle` setter tightens an initially-unbounded + // stage's cap (this is how `apply_initial_queue_budget` sizes it). + let q: Arc>> = + Arc::new(CountBoundedQueue::>::new(1)); + let s = ReorderStage::new(q); + s.set_max_overflow_bytes(150); + s.try_push(3, Heavy(30)).unwrap(); // -> transport (cap 1) + s.try_push(4, Heavy(40)).unwrap(); // overflow buffer: 0 < 150 + s.try_push(5, Heavy(50)).unwrap(); // overflow buffer: 100 < 150 + // buffer_bytes is now 200 >= 150 → a non-next push is rejected, + // proving the setter's value took effect. + assert_eq!(s.try_push(6, Heavy(60)), Err((6, Heavy(60))), "tightened cap rejects"); + + // (b) Liveness for any cap (§10.3): a near-zero (1-byte) cap, far below + // any item, still drains every item in serial order with none lost. + // The `next_serial` exemption guarantees progress; the round bound + // catches a livelock blow-up (not just a hard deadlock). + let q2: Arc>> = + Arc::new(CountBoundedQueue::>::new(1)); + let s2 = ReorderStage::new(q2); + s2.set_max_overflow_bytes(1); + let n = 16u32; + let mut pending: Vec<(u64, Heavy)> = + (0..n).rev().map(|i| (u64::from(i), Heavy(i))).collect(); + let mut out: Vec = Vec::new(); + let mut rounds = 0u32; + while out.len() < n as usize { + rounds += 1; + assert!(rounds <= 4 * n, "tiny cap must converge (no livelock blow-up)"); + // Retry every pending push (held-slot semantics), keeping rejects. + let mut still = Vec::new(); + for (ord, item) in pending.drain(..) { + if let Err(rej) = s2.try_push(ord, item) { + still.push(rej); + } + } + pending = still; + while let Some(Heavy(v)) = s2.try_pop_in_order() { + out.push(v); + } + } + assert_eq!(out, (0..n).collect::>(), "all drain in serial order, none lost"); + } + + #[test] + fn concurrent_tiny_cap_drains_all_in_order() { + // The load-bearing liveness test (§10.3/§10.5): at t>1, deadlock-freedom + // is "not proven by the next_serial exemption alone" — the worry is all + // producers wedged on rejected later-ordinal pushes while next_serial is + // unpushed. Here N producer threads concurrently push a disjoint, + // interleaved set of ordinals into one stage with a 1-byte cap (far + // below any item), retrying rejects (held-slot semantics); a consumer + // drains. All items must emerge in serial order with none lost. Progress + // is guaranteed because the producer owning `next_serial` always has it + // as its current (ascending) push, and `next_serial` is cap-exempt. + use std::thread; + + // Heavy carries its ordinal as `usize` (identity) with a fixed heap + // size, so the byte cap binds without any narrowing casts in the test. + #[derive(Debug, PartialEq)] + struct Heavy(usize); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + 100 + } + } + + let n_threads = 4usize; + let per = 64usize; + let total = n_threads * per; + let q: Arc>> = + Arc::new(CountBoundedQueue::>::new(4)); + let stage = Arc::new(ReorderStage::::new(q)); + stage.set_max_overflow_bytes(1); // 1 byte ≪ any item + + let producers: Vec<_> = (0..n_threads) + .map(|t| { + let s = Arc::clone(&stage); + thread::spawn(move || { + // This thread owns ordinals {t, t+N, t+2N, ...}, pushed in + // ascending order (each retried until accepted). + let mine: Vec = (0..per).map(|k| t + k * n_threads).collect(); + let mut i = 0; + while i < mine.len() { + let ord = mine[i]; + match s.try_push(ord as u64, Heavy(ord)) { + Ok(()) => i += 1, + Err(_) => thread::yield_now(), // held-slot retry + } + } + }) + }) + .collect(); + + // Deadline guard: this test proves deadlock-freedom, so a liveness + // regression must *fail fast* rather than hang the whole `nextest` run + // until a global harness timeout (if any) fires. 30s is generous — + // 256 tiny items drain in milliseconds when healthy — so it only trips + // on a genuine wedge, not on a slow CI host. Mirrors the + // bounded-convergence guard the single-threaded sibling uses. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let mut out: Vec = Vec::with_capacity(total); + while out.len() < total { + assert!( + std::time::Instant::now() < deadline, + "concurrent reorder drain did not complete within 30s ({} of {total} drained) — \ + likely a deadlock/livelock regression", + out.len(), + ); + match stage.try_pop_in_order() { + Some(Heavy(v)) => out.push(v), + None => thread::yield_now(), + } + } + // Bound producer completion with the same deadline. The drain loop above + // only proves the *consumer* made progress; if a regression let + // `out.len()` reach `total` (e.g. a double-emit) while a producer is + // still stuck in its `try_push` retry loop, an unbounded `join()` would + // hang the run forever. Poll `is_finished()` against the deadline so a + // stuck producer fails fast instead. + for p in producers { + while !p.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "a producer thread did not finish within 30s — likely a stuck try_push \ + retry loop (liveness regression)", + ); + thread::yield_now(); + } + p.join().unwrap(); + } + assert_eq!(out, (0..total).collect::>(), "all drain in serial order, none lost"); + } + + #[test] + fn drained_propagates() { + let s = make_stage(4); + s.try_push(0, 0).unwrap(); + s.try_push(1, 1).unwrap(); + s.mark_drained(); + // Not drained while items remain (need at least one pop to hydrate + // the buffer/transport visibility test). + assert!(!s.is_drained()); + let _ = s.try_pop_in_order().unwrap(); + let _ = s.try_pop_in_order().unwrap(); + assert!(s.is_drained()); + } + + #[test] + fn backpressure_applies_after_next_serial_buffered() { + // Drive the stage into a state where the reorder buffer holds + // `next_serial`, then verify producers see backpressure rejection. + // + // Step 1: push ordinal 1 with next_serial=0 still pending. + // transport: [(1,1)], buf: {}, next_serial=0. + // Step 2: a consumer call can't return anything (still waiting for 0), + // but it drains transport into buf as a side effect. + // transport: [], buf: {1}, next_serial=0. + // Step 3: push ordinal 0 — must_accept; goes to transport. + // transport: [(0,0)], buf: {1}, next_serial=0. + // Step 4: pop returns 0; advances next_serial=1; buf still has 1. + // transport: [], buf: {1}, next_serial=1. + // Step 5: now buf has next_serial=1, so backpressure path is active. + // Pushes 2, 3 fill transport (capacity 2). Push 4 rejects. + let s = make_stage(2); + s.try_push(1, 100).unwrap(); + assert_eq!(s.try_pop_in_order(), None); // drains transport into buf as side effect + s.try_push(0, 0).unwrap(); + assert_eq!(s.try_pop_in_order(), Some(0)); + // buf now contains next_serial=1; backpressure path active. + s.try_push(2, 200).unwrap(); + s.try_push(3, 300).unwrap(); + // Transport full, buf has next_serial=1: producer 4 must be rejected. + let result = s.try_push(4, 400); + assert!(result.is_err(), "expected backpressure rejection"); + let (ord, item) = result.unwrap_err(); + assert_eq!((ord, item), (4, 400)); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/contexts.rs b/crates/fgumi-pipeline-core/src/runtime/contexts.rs new file mode 100644 index 000000000..1265b85ad --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/contexts.rs @@ -0,0 +1,644 @@ +//! `ChainContexts`: per-step typed input + outputs handles, constructed +//! at `Pipeline::run` start by walking `ChainGraph` and calling each +//! step's `build_input_handle` / `build_output_set`. +//! +//! Layout: +//! - `inputs[step_idx]` — `Box` carrying `BranchInputHandle` +//! (or a dummy unit handle for sources). The worker loop hands this to +//! `ErasedStepCtx.input`. +//! - `outputs[step_idx]` — `Box` carrying `OutputHandles`. +//! The worker loop hands this to `ErasedStepCtx.outputs` AND uses it +//! for the typed `mark_outputs_drained` dispatch via the step's +//! `ErasedStep::mark_outputs_drained` method. +//! +//! Since `OutputsViewAny` no longer carries per-branch drained-flag Arcs +//! (option (c) pivot — drain marking is per-branch via +//! `BranchOutputHandle::mark_drained`, dispatched through the typed view), +//! the framework's drain-propagation path goes through +//! `ErasedStep::mark_outputs_drained(outputs_any.as_ref())`. + +use std::any::Any; +use std::sync::Arc; + +use crate::erased::ErasedStep; +use crate::handles::{BranchInputHandle, OutputQueueSet}; +use crate::item::HeapSize; +use crate::topology::{BranchIdx, ChainGraph, StepIdx}; + +/// Per-step input + outputs handles. The worker loop hands these to +/// `ErasedStepCtx` for each `try_run_erased` dispatch and to +/// `ErasedStep::mark_outputs_drained` for the drain-propagation path. +pub struct ChainContexts { + /// `inputs[step_idx]` = boxed `BranchInputHandle` (or a dummy + /// drained unit handle for sources). + pub inputs: Vec>, + /// `outputs[step_idx]` = boxed `OutputHandles` (the typed + /// surface step authors push into; also dispatchable via + /// `ErasedStep::mark_outputs_drained`). + pub outputs: Vec>, + /// Registry of every byte-bounded queue in the chain. Populated + /// during `build_chain_contexts` from each `Branch`'s + /// `bounded_queue_handle`. Empty when no byte-bounded queues + /// exist (e.g. a chain composed entirely of `CountBounded` / + /// `Unbounded` steps). + pub bounded_queues: Vec, + /// Registry of every instrumented edge (`--pipeline-trace`), with its + /// `EdgeMetrics` + producer/consumer + (for byte-bounded edges) a depth + /// source. Empty when instrumentation is `Off`. Read by the occupancy + /// sampler and the end-of-run edge report. + pub edges: Vec, +} + +/// One instrumented edge: its shared [`EdgeMetrics`](crate::runtime::metrics::EdgeMetrics) +/// (push counts from the transport, pop counts from the input handle), its +/// producer + consumer steps, and a depth source for occupancy sampling. +pub struct RegisteredEdge { + pub producer_step: StepIdx, + pub producer_name: &'static str, + /// `None` for a terminal branch with no consumer (e.g. a `--rejects` tail). + pub consumer_step: Option, + pub consumer_name: Option<&'static str>, + pub branch: BranchIdx, + /// On an **ordered** edge, the push-side counters (`pushed_items` / + /// `pushed_bytes` / `push_rejections`) are recorded at the `ReorderStage` + /// boundary, not on the transport queue: the reorder stage's must-accept path + /// turns a full-transport reject into an accepted (stashed) push, so recording + /// on the transport would miscount stashed items as backpressure. Recorded at + /// the boundary, `pushed_bytes` counts the bare `T` (`heap_size`) and so + /// matches `popped_bytes` (the ordinal tag adds no heap). + pub metrics: std::sync::Arc, + /// `Some` for byte-bounded edges — the transport queue's `current_bytes` + /// (plus, for an ordered edge, the `reorder_depth` stash bytes) over + /// `limit_bytes` gives the occupancy fraction the sampler records. `None` + /// for count/unbounded edges (counters still apply; occupancy histogram is + /// skipped). + pub depth_source: Option>, + /// `Some` for an **ordered** byte-bounded edge — the `ReorderStage`'s + /// overflow-stash handle. Its buffered bytes are added to the transport's + /// `current_bytes` when sampling depth, so an ordered edge reflects total + /// buffered bytes (transport + reorder stash) rather than the transport + /// queue alone (which under-reports occupancy when items pile in the reorder + /// buffer under producer skew). `None` for direct/count/unbounded edges. + pub reorder_depth: Option>, +} + +/// One byte-bounded queue's location in the chain plus the handles for +/// updating its budget: the transport-limit setter (`handle`) and, for an +/// ordered branch, the reorder stage's overflow-cap setter (`reorder_cap`). +/// Used by the budget pass (which sets both from one per-edge budget) and +/// the queue-memory rebalancer (which shifts transport budget at runtime). +pub struct RegisteredQueue { + pub producer_step_name: &'static str, + pub producer_step: StepIdx, + pub branch: BranchIdx, + pub handle: std::sync::Arc, + /// `Some` for an ordered byte-bounded branch — its reorder overflow stash + /// is sized from the same per-edge budget as `handle`. `None` for a direct + /// (unordered) byte-bounded branch (no reorder stage). + pub reorder_cap: Option>, +} + +/// Build the chain's per-step contexts. +/// +/// Walks `graph` to discover each step's producer and pulls the typed input +/// handle out of the producer's `OutputQueueSet`. Sources get a dummy +/// pre-drained `BranchInputHandle<()>` (their input is implicitly drained +/// from the start; framework never pops from it). +/// +/// # Panics +/// +/// Panics if `graph.n_steps() != steps.len()` or if any non-source step +/// has no producer (the builder's all-wired check should prevent this). +#[must_use] +pub fn build_chain_contexts( + steps: &[Box], + graph: &ChainGraph, + level: crate::builder::InstrumentationLevel, +) -> ChainContexts { + build_chain_contexts_inner(steps, graph, false, level) +} + +/// Build the chain's per-step contexts with **direct** inter-step transports +/// (no reorder stages, profile queue bounds retained) — the wiring the +/// single-thread *fused* driver runs over. +/// +/// Identical to [`build_chain_contexts`] except every producer's output set is +/// built via [`ErasedStep::build_fused_output_set`] instead of +/// [`ErasedStep::build_output_set`]. At one worker FIFO push order is already +/// the correct order, so dropping the reorder stage is sound; the count/byte +/// bound stays, so a producer that outruns its consumer within a pass gets +/// backpressure rather than growing the edge without limit. Only call this for a +/// linear (single source → single sink, single-input) chain at `--threads 1`; +/// see [`super::run_fused_single_thread`]. +#[must_use] +pub fn build_chain_contexts_fused( + steps: &[Box], + graph: &ChainGraph, +) -> ChainContexts { + // Fusion is only chosen when instrumentation is off (see + // `should_fuse_single_thread`), and the fused path registers no edges, so + // force `Off` regardless of the run's level. + build_chain_contexts_inner(steps, graph, true, crate::builder::InstrumentationLevel::Off) +} + +/// Build the producer→consumer edge map (keyed by `(producer_step, branch)`) +/// used to label registered edges with their consumer. Empty when +/// instrumentation is off (no edges are registered, so the map is unused). +fn build_consumer_map( + steps: &[Box], + graph: &ChainGraph, + level: crate::builder::InstrumentationLevel, +) -> std::collections::HashMap<(usize, usize), usize> { + let mut m = std::collections::HashMap::new(); + if !level.is_on() { + return m; + } + for (consumer_idx, step) in steps.iter().enumerate() { + if step.is_source() { + continue; + } + match step.input_arity() { + 1 => { + let (p, b) = find_producer(graph, StepIdx(consumer_idx)); + m.insert((p.0, b.0), consumer_idx); + } + 2 => { + for (p, b) in find_all_producers(graph, StepIdx(consumer_idx)) { + m.insert((p.0, b.0), consumer_idx); + } + } + _ => {} + } + } + m +} + +fn build_chain_contexts_inner( + steps: &[Box], + graph: &ChainGraph, + direct: bool, + level: crate::builder::InstrumentationLevel, +) -> ChainContexts { + assert_eq!(steps.len(), graph.n_steps(), "chain length mismatch"); + + let n_steps = steps.len(); + let mut output_sets: Vec = Vec::with_capacity(n_steps); + let mut outputs: Vec> = Vec::with_capacity(n_steps); + let mut inputs: Vec> = Vec::with_capacity(n_steps); + + // Pass 1: build each step's output set + boxed `OutputHandles`. The + // fused driver uses direct transports with the profile's queue bounds + // (`build_fused_output_set`, which only drops the ordering); the scheduled + // path honours each step's profiled queue spec + ordering. + for step in steps { + let (set, view) = + if direct { step.build_fused_output_set(level) } else { step.build_output_set(level) }; + let outputs_box = step.wrap_outputs_view(view); + output_sets.push(set); + outputs.push(outputs_box); + } + + // Producer→consumer map for edge registration (inverse of `find_producer`, + // which Pass 2 uses consumer→producer). Empty when instrumentation is off. + let consumer_of = build_consumer_map(steps, graph, level); + + // Pass 1.5: collect byte-bounded queue handles into the registry. + // Must run before Pass 2 because `take_typed_input` (called via + // `build_input_handle`) replaces the `BranchEntry` with a fresh + // one whose `bounded_queue_handle` is `None` — by then the + // handles have been moved out of the chain. + let mut bounded_queues: Vec = Vec::new(); + let mut edges: Vec = Vec::new(); + for (step_idx_usize, set) in output_sets.iter().enumerate() { + for (branch_idx_usize, entry) in set.branches.iter().enumerate() { + if let Some(handles) = entry.bounded_queue_handle.as_ref() { + bounded_queues.push(RegisteredQueue { + producer_step_name: steps[step_idx_usize].profile().name, + producer_step: StepIdx(step_idx_usize), + branch: BranchIdx(branch_idx_usize), + handle: std::sync::Arc::clone(&handles.transport), + reorder_cap: handles.reorder_cap.clone(), + }); + } + // Instrumented edge: register its shared metrics + producer/consumer + // + (for byte-bounded edges) a depth source for occupancy sampling. + if let Some(metrics) = entry.metrics.as_ref() { + let consumer_step = + consumer_of.get(&(step_idx_usize, branch_idx_usize)).map(|c| StepIdx(*c)); + edges.push(RegisteredEdge { + producer_step: StepIdx(step_idx_usize), + producer_name: steps[step_idx_usize].profile().name, + consumer_step, + consumer_name: consumer_step.map(|c| steps[c.0].profile().name), + branch: BranchIdx(branch_idx_usize), + metrics: std::sync::Arc::clone(metrics), + depth_source: entry + .bounded_queue_handle + .as_ref() + .map(|h| std::sync::Arc::clone(&h.transport)), + reorder_depth: entry + .bounded_queue_handle + .as_ref() + .and_then(|h| h.reorder_cap.clone()), + }); + } + } + } + + // Pass 2: input handles. Sources get a dummy drained unit handle; + // single-input mid-steps and sinks pull one handle from their + // producer's `OutputQueueSet`; two-input merge steps (`Step2` + // adapters, identified by `input_arity() == 2`) pull TWO handles + // — one per consumer-input-slot — and wrap them in a + // `TwoInputHandles`. + for (consumer_idx, step) in steps.iter().enumerate() { + let input_box = if step.is_source() { + // Source step (Input = ()). Build a permanently-drained + // dummy unit handle — the worker loop never pops from it. + // Chains with multiple sources (e.g. zipper's mapped + + // unmapped subchains converging at a Step2 merger) all + // share this code path; the runtime walks every source + // independently to Finished. + dummy_unit_input_handle() + } else { + match step.input_arity() { + 1 => { + let (producer_idx, branch_idx) = find_producer(graph, StepIdx(consumer_idx)); + step.build_input_handle(&mut output_sets[producer_idx.0], branch_idx.0) + } + 2 => { + let edges = find_all_producers(graph, StepIdx(consumer_idx)); + assert_eq!( + edges.len(), + 2, + "Step2 consumer {:?} expects 2 input edges, found {}", + StepIdx(consumer_idx), + edges.len() + ); + let (p0, p0_branch) = edges[0]; + let (p1, p1_branch) = edges[1]; + step.build_two_input_handles( + &mut output_sets, + p0.0, + p0_branch.0, + p1.0, + p1_branch.0, + ) + } + n => panic!("unsupported input_arity {n} for step {:?}", StepIdx(consumer_idx)), + } + }; + inputs.push(input_box); + } + + // `output_sets` is consumed implicitly here — every branch was taken + // exactly once via `build_input_handle`, leaving placeholder slots. + drop(output_sets); + + ChainContexts { inputs, outputs, bounded_queues, edges } +} + +/// Construct a `BranchInputHandle<()>` that's already drained — for source +/// steps whose input is implicitly empty + drained from t=0. Uses the zero-state +/// `always_drained` handle (no backing queue), so building a source costs no +/// `SegQueue` allocation for a handle whose only job is to report +/// `is_drained() == true` (the worker loop never pops from a source's input). +fn dummy_unit_input_handle() -> Box { + Box::new(BranchInputHandle::<()>::always_drained()) +} + +/// Find the single (producer, branch) that produces the given single-input +/// consumer step. +/// +/// # Panics +/// +/// Panics if no producer exists, or if more than one producer branch targets +/// the consumer (the builder's all-wired check should have prevented either on +/// a built pipeline). +fn find_producer(graph: &ChainGraph, consumer: StepIdx) -> (StepIdx, BranchIdx) { + let mut found: Option<(StepIdx, BranchIdx)> = None; + for producer_usize in 0..graph.n_steps() { + let producer = StepIdx(producer_usize); + let n_branches = graph.branch_count(producer); + for branch_usize in 0..n_branches { + let branch = BranchIdx(branch_usize); + if graph.consumer(producer, branch) == Some(consumer) { + // A single-input consumer has exactly one incoming edge. Two + // producer branches wired to it would leave the second one + // unpopped: the caller takes an input handle for the first edge + // only, so the extra branch's queue is never drained and its + // producer wedges on backpressure. Reject the graph instead of + // silently keeping whichever edge was scanned first. Mirrors the + // per-slot uniqueness check in `find_all_producers`. + if let Some((prev_producer, prev_branch)) = found { + panic!( + "single-input consumer {consumer:?} is wired by more than one \ + producer branch ({prev_producer:?} branch {prev_branch:?} and \ + {producer:?} branch {branch:?}); the extra edge would never be \ + popped" + ); + } + found = Some((producer, branch)); + } + } + } + found.unwrap_or_else(|| { + panic!( + "step {consumer:?} has no producer in chain graph; \ + the builder's all-wired check should have caught this" + ) + }) +} + +/// Find all `(producer, producer_branch)` edges feeding the given +/// multi-input consumer, sorted by the consumer's input-slot index +/// (`returned[i]` feeds the consumer's input slot `i`). Used by +/// [`build_chain_contexts`] to assemble per-branch input handles +/// for `Step2` and future `StepN` consumers. +/// +/// # Panics +/// +/// Panics if any of the consumer's input slots is unwired (the +/// builder's all-wired check should have caught this on a built +/// pipeline). +fn find_all_producers(graph: &ChainGraph, consumer: StepIdx) -> Vec<(StepIdx, BranchIdx)> { + let arity = graph.input_arity(consumer); + let mut edges: Vec> = vec![None; arity]; + for producer_usize in 0..graph.n_steps() { + let producer = StepIdx(producer_usize); + let n_branches = graph.branch_count(producer); + for branch_usize in 0..n_branches { + let branch = BranchIdx(branch_usize); + if graph.consumer(producer, branch) == Some(consumer) { + let slot = graph + .consumer_input_slot(producer, branch) + .expect("consumer_input_slot must be set when consumer is wired"); + assert!( + slot < arity, + "consumer {consumer:?} edge has input slot {slot} but arity is {arity}" + ); + assert!( + edges[slot].is_none(), + "consumer {consumer:?} input slot {slot} wired more than once", + ); + edges[slot] = Some((producer, branch)); + } + } + } + edges + .into_iter() + .enumerate() + .map(|(slot, e)| { + e.unwrap_or_else(|| panic!("consumer {consumer:?} input slot {slot} has no producer")) + }) + .collect() +} + +/// Convenience: borrow the typed `BranchInputHandle` for a given step. +/// +/// # Panics +/// +/// Panics if the step's input handle doesn't downcast to `T` (a framework +/// invariant violation). +#[must_use] +pub fn input_as( + contexts: &Arc, + step: StepIdx, +) -> &BranchInputHandle { + contexts.inputs[step.0] + .downcast_ref::>() + .expect("BranchInputHandle downcast failed in input_as") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + + use proptest::prelude::*; + use rstest::rstest; + + use crate::erased::TypedStep; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{ + InputHandle, OutputHandles, Step, StepCtx, StepKind, StepOutcome, StepProfile, + }; + + #[derive(Clone)] + struct StubSource; + impl Step for StubSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Source", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::ByOrdinal], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + } + + #[derive(Clone)] + struct StubSink; + impl Step for StubSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "Sink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// A `u32 → u32` pass-through step used to grow a linear chain to an + /// arbitrary length between the source and sink. + #[derive(Clone)] + struct MiddleStep; + impl Step for MiddleStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Middle", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// Build an `n`-step linear chain `Source → Middle×(n - 2) → Sink`, + /// returning the erased step boxes alongside the fully wired graph. + /// + /// # Panics + /// + /// Panics if `n < 2` (a linear chain needs at least a source and a sink). + fn linear_chain(n: usize) -> (Vec>, ChainGraph) { + assert!(n >= 2, "linear chain needs at least a source and a sink"); + + let mut graph = ChainGraph::new(); + let mut steps: Vec> = Vec::with_capacity(n); + let mut indices: Vec = Vec::with_capacity(n); + + indices.push(graph.register_step("Source", 1)); + steps.push(Box::new(TypedStep::new(StubSource))); + for _ in 1..n - 1 { + indices.push(graph.register_step("Middle", 1)); + steps.push(Box::new(TypedStep::new(MiddleStep))); + } + indices.push(graph.register_step("Sink", 0)); + steps.push(Box::new(TypedStep::new(StubSink))); + + for pair in indices.windows(2) { + graph.wire(pair[0], BranchIdx(0), pair[1]); + } + + (steps, graph) + } + + /// Assert the invariants `build_chain_contexts` must uphold for an + /// `n`-step linear chain: one input/output slot per step, no byte-bounded + /// queues (the stubs only use `CountBounded` transport), the source's + /// input is a dummy pre-drained `BranchInputHandle<()>`, and every + /// downstream step receives a real `BranchInputHandle` wired from its + /// producer. + fn assert_linear_chain_invariants(ctx: &ChainContexts, n: usize) { + assert_eq!(ctx.inputs.len(), n); + assert_eq!(ctx.outputs.len(), n); + assert!(ctx.bounded_queues.is_empty()); + + let source = ctx.inputs[0] + .downcast_ref::>() + .expect("source must have a dummy unit input handle"); + // Pin the `always_drained()` invariant: a source's input is implicitly + // drained from t=0, so a regression that swapped it back to a real + // (never-draining) queue would be caught here, not just the downcast. + assert!( + source.is_drained(), + "source's dummy unit input handle must report drained (always_drained invariant)" + ); + // Identity, not just type: every `MiddleStep` and the sink share the + // type `BranchInputHandle`, so a `find_producer` regression that + // handed several consumers the SAME producer's branch handle would still + // downcast and still pass. Push a per-producer distinct value and assert + // it arrives on that producer's consumer. + for i in 1..n { + let input = ctx.inputs[i] + .downcast_ref::>() + .unwrap_or_else(|| panic!("step {i} must have a BranchInputHandle")); + let tag = 1000 + u32::try_from(i).expect("chain length fits in u32"); + let producer = ctx.outputs[i - 1] + .downcast_ref::>>() + .unwrap_or_else(|| panic!("step {} must have OutputHandles>", i - 1)); + producer.push(tag).expect("producer transport accepts one item"); + assert_eq!( + input.pop(), + Some(tag), + "step {i}'s input must be wired to step {}'s output branch 0", + i - 1 + ); + } + } + + /// `build_chain_contexts` wires linear chains of varying length: the + /// three-step case also exercises `find_all_producers` for a middle step. + #[rstest] + #[case(2)] + #[case(3)] + #[case(4)] + fn build_chain_contexts_linear(#[case] n: usize) { + let (steps, graph) = linear_chain(n); + let ctx = build_chain_contexts(&steps, &graph, crate::builder::InstrumentationLevel::Off); + assert_linear_chain_invariants(&ctx, n); + } + + /// `ChainGraph::wire` only rejects re-wiring the same *producer* branch, so + /// nothing stops two producers from targeting one single-input consumer. + /// `find_producer` must reject that graph rather than return the + /// first-scanned edge: the caller takes an input handle for that edge only, + /// so the second producer's branch is never popped and it wedges on + /// backpressure once its transport fills. + #[test] + #[should_panic(expected = "is wired by more than one producer branch")] + fn find_producer_rejects_duplicate_incoming_edges() { + let mut graph = ChainGraph::new(); + let source_a = graph.register_step("SourceA", 1); + let source_b = graph.register_step("SourceB", 1); + let sink = graph.register_step("Sink", 0); + graph.wire(source_a, BranchIdx(0), sink); + graph.wire(source_b, BranchIdx(0), sink); + let _ = find_producer(&graph, sink); + } + + /// The single-producer case still resolves to that one edge — the + /// uniqueness check must not have turned the happy path into a panic. + #[test] + fn find_producer_resolves_the_single_incoming_edge() { + let (_steps, graph) = linear_chain(3); + assert_eq!(find_producer(&graph, StepIdx(2)), (StepIdx(1), BranchIdx(0))); + } + + #[test] + fn registry_covers_edges_with_producer_and_consumer() { + use crate::builder::InstrumentationLevel as L; + // Source → Middle → Sink: two producing edges, each with a consumer. + let (steps, graph) = linear_chain(3); + let ctx = build_chain_contexts(&steps, &graph, L::Summary); + assert_eq!(ctx.edges.len(), 2, "every bounded producing edge is registered"); + // Identity, not just presence. Labelling each edge with its own consumer is + // the whole job of `build_consumer_map`, and a count-plus-`is_some()` check + // passes just as well if the two consumers were swapped or if both edges + // were labelled with the same one. Pin the pairs. + let mut pairs: Vec<(&str, Option<&str>)> = + ctx.edges.iter().map(|e| (e.producer_name, e.consumer_name)).collect(); + pairs.sort_unstable(); + assert_eq!( + pairs, + vec![("Middle", Some("Sink")), ("Source", Some("Middle"))], + "each edge is labelled with its own consumer" + ); + for e in &ctx.edges { + assert!( + e.consumer_step.is_some(), + "edge from {} has a resolved consumer", + e.producer_name + ); + assert!(e.consumer_name.is_some()); + // CountBounded edges expose no byte depth source (occupancy via len only). + assert!(e.depth_source.is_none(), "count-bounded edge has no byte depth source"); + } + // Off → no edges registered (zero-overhead path). + let (steps, graph) = linear_chain(3); + let off = build_chain_contexts(&steps, &graph, L::Off); + assert!(off.edges.is_empty(), "level Off registers no edges"); + } + + proptest! { + /// The typed-handle and bounded-queue invariants hold for linear + /// chains of any length, not just the hand-picked rstest cases. + #[test] + fn build_chain_contexts_linear_invariants(n in 2usize..=8) { + let (steps, graph) = linear_chain(n); + let ctx = build_chain_contexts(&steps, &graph, crate::builder::InstrumentationLevel::Off); + assert_linear_chain_invariants(&ctx, n); + } + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/detached.rs b/crates/fgumi-pipeline-core/src/runtime/detached.rs new file mode 100644 index 000000000..e8f653a85 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/detached.rs @@ -0,0 +1,775 @@ +//! Detached-step extraction and the dedicated **driver thread** that runs them +//! off the work-stealing pool. +//! +//! A [`StepKind::Detached`] step is excluded +//! from the pool ([`build_worker_storage`](crate::runtime::build_worker_storage) +//! gives every pool worker a `Skip` entry for it) and instead runs on a dedicated +//! OS thread spawned at run start alongside the deadlock-monitor / +//! queue-rebalancer and joined after the workers. This is the legacy sort's +//! "N + 2" threading: N pool workers do the parallel (compression-bound) work +//! while off-pool driver threads do the serial coordination + I/O, so neither +//! steals a pool worker slot. +//! +//! ## The driver IS a 1-thread pool ([`run_detached_driver`]) +//! +//! A driver thread does **not** have a bespoke drive loop. It runs the *same* +//! [`run_worker_loop`] the pool uses — over a +//! purpose-built storage row where its group's steps are `Owned` and every other +//! step is `Skip` ([`build_driver_storage`]) — with a +//! [`WorkerCore::driver`](crate::runtime::WorkerCore) (Park backoff, off-pool +//! stats attribution) and the [`DrainFirstScheduler`]. So a driver is literally a +//! 1-thread (or, for a group, still 1-thread over several `Owned` steps) instance +//! of the pool loop. Several detached steps sharing a +//! [`DetachedGroup::Shared`](crate::step::DetachedGroup) label are driven by ONE +//! thread that round-robins them; [`DetachedGroup::PerStep`] (the default) keeps +//! one thread per step. +//! +//! Because the group's steps and their phases are temporally disjoint (e.g. the +//! sort's phase-1 admit/sort/frame finish and leave the live set before the +//! phase-2 merge runs), one driver thread covers a whole phase's coordination +//! without oversubscription — exactly like main's single main thread. +//! +//! ## Lock-ordering acyclicity (non-negotiable, the deadlock proof) +//! +//! A driver thread must NEVER hold one queue's internal lock while parking on +//! another. `run_worker_loop` upholds that by construction: +//! +//! 1. Each `try_run_erased` is a single non-blocking call. The step body pops +//! from its input transport (a lock-free `crossbeam ArrayQueue` — `try_pop`, +//! no lock held across the call) and pushes to its outputs (`try_push`, +//! likewise); a full output / empty input is reported back as `NoProgress` / +//! `Contention`. It never *blocks* inside `try_run`. +//! 2. The only blocking a driver does is the loop's `WorkerCore::sleep_backoff` +//! (`park_timeout` under the Park policy), which holds NO queue lock. +//! 3. The loop tries EVERY live step in a pass before it parks (round-robin, +//! park only after a full no-progress pass). So when one grouped step is +//! blocked, a sibling on the same driver still runs — a park-on-first-idle +//! loop would wedge (see `driver_round_robins_all_live_before_parking`). +//! +//! So there is no cycle: a driver parks only *between* `try_run` calls, never +//! while holding a transport lock, so a two-sided step (consuming from the pool +//! AND producing to it, both bounded) cannot deadlock. The +//! `detached_two_sided_no_deadlock` test pins this with a wall-clock watchdog. + +use std::any::Any; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::erased::{ErasedStep, ErasedStepCtx}; +use crate::runtime::contexts::ChainContexts; +use crate::runtime::drain::StepDrainCounter; +use crate::runtime::driver::run_worker_loop; +use crate::runtime::scheduler::DrainFirstScheduler; +use crate::runtime::stats::PipelineStats; +use crate::runtime::storage::WorkerStepEntry; +use crate::runtime::worker_core::WorkerCore; +use crate::signal::PipelineSignal; +use crate::step::{Affinity, DetachedGroup, OutputsViewAny, StepKind, StepOutcome, StepProfile}; +use crate::topology::StepIdx; + +/// Sentinel left in the chain's `steps` vec in place of a `Detached` step after +/// its real instance has been extracted for its dedicated thread (see +/// [`extract_detached_steps`]). Keeping a same-position placeholder preserves +/// the `step_idx`-aligned indexing that `build_worker_storage` and +/// `ChainContexts` rely on. `build_worker_storage` only reads `kind()` (matches +/// `Detached` → every worker gets `Skip`) and then drops the box, so the +/// placeholder never has any other method invoked; they panic to catch a +/// framework bug if one ever is. +struct DetachedPlaceholder { + name: &'static str, + /// The real step's group, carried through the swap. + /// + /// Not consulted on the run path — `extract_detached_steps` reads the real + /// step's group *before* installing this placeholder. Preserved because + /// `extract_detached_steps` is `pub` and leaves these placeholders in the + /// caller's slice: an external caller regrouping from that slice would read + /// `PerStep` for a step that declared `Shared(..)` and split one shared + /// group across separate driver threads. `profile()` panics for the same + /// class of reason — returning fabricated metadata misinforms that caller. + group: DetachedGroup, +} + +impl ErasedStep for DetachedPlaceholder { + fn profile(&self) -> StepProfile { + // Panics like every other placeholder method rather than returning empty + // `output_queues` / `branch_ordering`. `Pipeline::run` reads + // `step.profile().kind` for the drain counters BEFORE + // `extract_detached_steps` swaps these in, and everything after reads the + // cached `kind()` / `name()` accessors — so nothing on the run path calls + // this. But `extract_detached_steps` is `pub` and leaves placeholders in + // the caller's slice: silently reporting "no outputs" for a step that + // declares them would misinform any later caller. + panic!( + "DetachedPlaceholder::profile invoked for {:?} — the real instance was extracted; \ + read the cached kind()/name() accessors instead", + self.name + ); + } + fn name(&self) -> &'static str { + self.name + } + fn kind(&self) -> StepKind { + StepKind::Detached + } + fn sticky(&self) -> bool { + false + } + fn affinity(&self) -> Affinity { + Affinity::None + } + fn detached_group(&self) -> DetachedGroup { + // See the field doc: off the run path, but must not report `PerStep` for a + // step that declared `Shared(..)`. + self.group + } + fn try_run_erased(&mut self, _ctx: &mut ErasedStepCtx<'_>) -> std::io::Result { + panic!("DetachedPlaceholder::try_run_erased invoked — the real instance was extracted"); + } + fn clone_boxed(&self) -> Box { + panic!("DetachedPlaceholder::clone_boxed invoked — placeholder is never cloned"); + } + fn build_input_handle( + &self, + _producer_set: &mut crate::handles::OutputQueueSet, + _branch_idx: usize, + ) -> Box { + panic!("DetachedPlaceholder::build_input_handle invoked"); + } + fn build_output_set( + &self, + _level: crate::builder::InstrumentationLevel, + ) -> (crate::handles::OutputQueueSet, OutputsViewAny) { + panic!("DetachedPlaceholder::build_output_set invoked"); + } + fn build_fused_output_set( + &self, + _level: crate::builder::InstrumentationLevel, + ) -> (crate::handles::OutputQueueSet, OutputsViewAny) { + panic!("DetachedPlaceholder::build_fused_output_set invoked"); + } + fn wrap_outputs_view(&self, _view: OutputsViewAny) -> Box { + panic!("DetachedPlaceholder::wrap_outputs_view invoked"); + } + fn mark_outputs_drained(&self, _outputs: &(dyn Any + Send + Sync)) { + panic!("DetachedPlaceholder::mark_outputs_drained invoked"); + } + fn is_source(&self) -> bool { + false + } +} + +/// One dedicated driver thread's worth of extracted detached steps, in chain +/// (`StepIdx`) order. The caller spawns one OS thread per group and drives it +/// with [`run_detached_driver`]. +pub struct DetachedDriverGroup { + /// The steps this one driver thread runs, in chain order. Always non-empty + /// and all [`StepKind::Detached`]. Private so those invariants — enforced by + /// [`Self::new`] — cannot be bypassed by an external caller building the + /// struct directly (which could otherwise trigger `steps[0]` panics in + /// `primary_step` / `label`, or run non-detached work on an off-pool driver). + steps: Vec<(StepIdx, Box)>, +} + +impl DetachedDriverGroup { + /// Wrap a driver thread's extracted steps, enforcing the invariants every + /// consumer relies on: the group is **non-empty** (`primary_step` / `label` + /// index `steps[0]`) and **every step is [`StepKind::Detached`]** (a + /// non-detached step must not run off-pool on a dedicated driver thread, and + /// a `Parallel` step would hang its shared output — see + /// [`build_driver_storage`]). + /// + /// # Panics + /// + /// Panics if `steps` is empty or contains a non-`Detached` step. + #[must_use] + fn new(steps: Vec<(StepIdx, Box)>) -> Self { + assert!(!steps.is_empty(), "detached driver group must be non-empty"); + assert!( + steps.iter().all(|(_, step)| step.kind() == StepKind::Detached), + "detached driver group may only contain Detached steps" + ); + Self { steps } + } + + /// The group's representative step (first in chain order). Used as the + /// driver thread's off-pool stats key (`WorkerCore::driver`) and as a stable + /// label. Non-empty by construction. + #[must_use] + pub fn primary_step(&self) -> StepIdx { + self.steps[0].0 + } + + /// The name of the group's representative step (first in chain order), used + /// to label a `PerStep` driver thread. Non-empty by construction. + #[must_use] + pub fn primary_name(&self) -> &'static str { + self.steps[0].1.name() + } + + /// The group label — derived from the steps (every step in the group reports + /// the same [`DetachedGroup`]), used to name the driver thread. Non-empty by + /// construction. + #[must_use] + pub fn label(&self) -> DetachedGroup { + self.steps[0].1.detached_group() + } + + /// Consume the group, yielding its steps for [`build_driver_storage`]. + #[must_use] + fn into_steps(self) -> Vec<(StepIdx, Box)> { + self.steps + } +} + +/// Remove every [`StepKind::Detached`] step's +/// real instance from `steps`, replacing each in place with a +/// `DetachedPlaceholder` so the surviving slots keep their `step_idx` +/// positions (which `build_worker_storage` and `ChainContexts` index by). +/// +/// Groups the extracted steps by their [`DetachedGroup`]: every +/// [`DetachedGroup::Shared`] label collects onto ONE group (one driver thread); +/// each [`DetachedGroup::PerStep`] step becomes its own singleton group (the +/// legacy one-thread-per-step behavior — the default, so non-sort chains are +/// unchanged). Within a group and across groups, order follows chain order +/// (first appearance). Called by `Pipeline::run` **before** `build_worker_storage` +/// consumes `steps`, while the (read-only) `ChainContexts` have already been +/// built from `&steps`. +#[must_use] +pub fn extract_detached_steps(steps: &mut [Box]) -> Vec { + // Accumulate each driver thread's steps as a raw vec, then wrap through + // `DetachedDriverGroup::new` so the non-empty / all-Detached invariants are + // enforced in one place rather than trusting each construction site. + let mut group_steps: Vec)>> = Vec::new(); + // Shared(label) -> index into `group_steps`, for O(1) append. PerStep steps + // never share, so they are not indexed (each starts its own group). + let mut shared_index: HashMap<&'static str, usize> = HashMap::new(); + for (idx, slot) in steps.iter_mut().enumerate() { + if slot.kind() != StepKind::Detached { + continue; + } + let group = slot.detached_group(); + let placeholder: Box = + Box::new(DetachedPlaceholder { name: slot.name(), group }); + let real = std::mem::replace(slot, placeholder); + let entry = (StepIdx(idx), real); + match group { + DetachedGroup::PerStep => group_steps.push(vec![entry]), + DetachedGroup::Shared(label) => { + if let Some(&gi) = shared_index.get(label) { + group_steps[gi].push(entry); + } else { + shared_index.insert(label, group_steps.len()); + group_steps.push(vec![entry]); + } + } + } + } + group_steps.into_iter().map(DetachedDriverGroup::new).collect() +} + +/// Build a driver thread's storage row: a full-length `Vec` +/// (length `n_total_steps`, indexed by global `step_idx` like every other row) +/// where each of `group_steps` is `Owned` and every other slot is `Skip`. The +/// driver thread runs [`run_worker_loop`] over this row exactly as a pool worker +/// runs over its own row. +/// +/// # Panics +/// +/// - if a group step's kind is `Parallel` — a 1-thread driver's +/// [`StepDrainCounter`] is init 1 (single finisher), which would never close a +/// `Parallel` step's shared output (that needs init N, all clones finishing), +/// hanging the downstream consumer; +/// - if two group steps map to the same `step_idx` (a double registration), or a +/// step index is out of range. +#[must_use] +pub fn build_driver_storage( + group_steps: Vec<(StepIdx, Box)>, + n_total_steps: usize, +) -> Vec { + let mut row: Vec = (0..n_total_steps).map(|_| WorkerStepEntry::Skip).collect(); + for (idx, step) in group_steps { + assert_ne!( + step.kind(), + StepKind::Parallel, + "driver group step `{}` is Parallel; a 1-thread driver's StepDrainCounter (init 1) \ + would never close its shared output — group only single-runner steps", + step.name() + ); + assert!( + matches!(row[idx.0], WorkerStepEntry::Skip), + "driver group step index {} registered twice (dual registration)", + idx.0 + ); + row[idx.0] = WorkerStepEntry::Owned { step }; + } + row +} + +/// Drive one [`DetachedDriverGroup`] to completion on the calling (dedicated) +/// thread — the unified "1-thread pool". Builds the group's `Owned`/`Skip` +/// storage row ([`build_driver_storage`]) and runs the *same* +/// [`run_worker_loop`] the pool uses, with a [`WorkerCore::driver`] (Park +/// backoff, off-pool stats) and the [`DrainFirstScheduler`] (drain/seal +/// downstream before producing more — frees the sort's capacity-1 arena fastest). +/// +/// `drain_counters` is the full per-step slice (init 1 for each detached step, so +/// the single finisher closes its output edges — the downstream consumer's +/// end-of-stream signal). On a cancel before `Finished`, outputs are NOT closed +/// (the run is tearing down; the recorded error/cancel is what propagates) — +/// `run_worker_loop`'s top-of-loop `is_done` break upholds this. +pub fn run_detached_driver( + group: DetachedDriverGroup, + contexts: &Arc, + drain_counters: &[Arc], + signal: &Arc, + stats: Option<&Arc>, +) { + let primary = group.primary_step(); + let mut row = build_driver_storage(group.into_steps(), contexts.inputs.len()); + let mut worker = WorkerCore::driver(primary); + run_worker_loop( + &mut worker, + &mut row, + contexts, + drain_counters, + signal, + stats, + &DrainFirstScheduler, + ); +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::time::Duration; + + use super::*; + use crate::builder::InstrumentationLevel; + use crate::erased::TypedStep; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{InputHandle, OutputHandles, Step, StepCtx, StepKind, StepProfile}; + + /// `() -> u32` source stub used only so `build_output_set` constructs the + /// transport that becomes the Detached step's input edge. Never run. + #[derive(Clone)] + struct SrcStub { + capacity: usize, + } + impl Step for SrcStub { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Src", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: self.capacity }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// `u32 -> u32` pass-through Detached step: pop one item per `try_run`, + /// push it on (holding it on output-full backpressure), report `Finished` + /// once input drains and nothing is held. + #[derive(Clone)] + struct PassThroughDetached { + held: Option, + } + impl Step for PassThroughDetached { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "PassThroughDetached", + kind: StepKind::Detached, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + // A rejected push reports `NoProgress`, not `Contention`: the driver + // treats them identically, but `Contention` means "a Serial step's + // mutex was held by another worker" and feeds `contention_count`, + // which the bottleneck verdict turns into its SPIN ratio. Using it + // for ordinary output backpressure invents contention that never + // happened. + if let Some(v) = self.held.take() { + if ctx.outputs.push(v).is_err() { + self.held = Some(v); + return Ok(StepOutcome::NoProgress); + } + return Ok(StepOutcome::Progress); + } + match ctx.input.pop() { + Some(v) => match ctx.outputs.push(v) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + }, + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// Assemble a `Source -> PassThroughDetached -> Sink` shaped context by + /// hand: build the producer's output set (= the Detached input edge), wire + /// the Detached step's input from it, build the Detached step's own output + /// set (= the downstream consumer's input edge), and return the pieces. + #[allow(clippy::type_complexity)] + fn build_one_detached( + src_capacity: usize, + ) -> ( + Box, // the detached step + Arc, // contexts for step_idx 1 + Arc>, // producer outputs (push side) + crate::handles::BranchInputHandle, // downstream consumer (pop side) + ) { + let producer: Box = + Box::new(TypedStep::new(SrcStub { capacity: src_capacity })); + let (mut producer_set, producer_view) = + producer.build_output_set(InstrumentationLevel::Off); + let producer_outputs_any = producer.wrap_outputs_view(producer_view); + + let det: Box = Box::new(TypedStep::new(PassThroughDetached { held: None })); + let det_input = det.build_input_handle(&mut producer_set, 0); + let (mut det_set, det_view) = det.build_output_set(InstrumentationLevel::Off); + let det_outputs_any = det.wrap_outputs_view(det_view); + let det_output_consumer = det_set.take_typed_input::(0); + + let contexts = Arc::new(ChainContexts { + inputs: vec![Box::new(()), det_input, Box::new(())], + outputs: vec![Box::new(()), det_outputs_any, Box::new(())], + bounded_queues: vec![], + edges: vec![], + }); + (det, contexts, Arc::new(producer_outputs_any), det_output_consumer) + } + + /// Drive a single detached `step` at `step_idx` through the unified driver — + /// a `PerStep` group of one — on the calling thread, with a full-length + /// `drain_counters` slice (init 1 each). Mirrors what `builder.rs` step 4d + /// does for a one-step group. + fn drive_single( + step: Box, + step_idx: StepIdx, + contexts: &Arc, + signal: &Arc, + ) { + let drain_counters: Vec> = + (0..contexts.inputs.len()).map(|_| StepDrainCounter::new(1)).collect(); + let group = DetachedDriverGroup::new(vec![(step_idx, step)]); + run_detached_driver(group, contexts, &drain_counters, signal, None); + } + + /// Items pushed onto a Detached step's input flow through to its output; + /// the step finishes once its input is drained and closes the output edge. + #[test] + fn detached_step_flows_items_and_finishes() { + let (det, contexts, producer_outputs_any, consumer) = build_one_detached(64); + let producer_outputs = + producer_outputs_any.downcast_ref::>>().unwrap(); + producer_outputs.push(10).unwrap(); + producer_outputs.push(20).unwrap(); + producer_outputs.push(30).unwrap(); + producer_outputs.mark_all_drained(); + + let signal = PipelineSignal::new(); + drive_single(det, StepIdx(1), &contexts, &signal); + + let mut got = Vec::new(); + while let Some(v) = consumer.pop() { + got.push(v); + } + assert_eq!(got, vec![10, 20, 30]); + assert!(InputHandle::is_drained(&consumer), "output closed on Finished"); + assert!(!signal.is_done(), "clean completion, no error"); + } + + /// Zero items in (input drained from the start): the Detached step cleanly + /// drains its output and returns. + #[test] + fn detached_step_zero_items_clean_drain() { + let (det, contexts, producer_outputs_any, consumer) = build_one_detached(64); + let producer_outputs = + producer_outputs_any.downcast_ref::>>().unwrap(); + producer_outputs.mark_all_drained(); // no items + + let signal = PipelineSignal::new(); + drive_single(det, StepIdx(1), &contexts, &signal); + + assert!(consumer.pop().is_none(), "no items produced"); + assert!(InputHandle::is_drained(&consumer), "output closed on clean empty drain"); + } + + /// A two-sided Detached step — consumer of a bounded pool-fed input AND + /// producer to a bounded pool-drained output, both tiny — completes without + /// deadlock. A wall-clock watchdog aborts (fails the test) if it wedges. + /// This is the sort merge's topology. + #[test] + fn detached_two_sided_no_deadlock() { + const N: u32 = 5_000; + + // cap-2 input AND cap-4 output both force interleaved backpressure. + let (det, contexts, producer_outputs_any, consumer) = build_one_detached(2); + let signal = PipelineSignal::new(); + + // Watchdog: a deadlock parks forever; abort so the test FAILS loudly. + let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); + { + let done = Arc::clone(&done); + std::thread::spawn(move || { + for _ in 0..200 { + std::thread::sleep(Duration::from_millis(50)); + if done.load(Ordering::SeqCst) { + return; + } + } + eprintln!("detached_two_sided_no_deadlock: WEDGED (deadlock)"); + std::process::abort(); + }); + } + + // Producer thread: push N items into the cap-2 input (backpressure), + // then close it so the Detached step's input drains. + let pushed = Arc::new(AtomicU32::new(0)); + let producer_handle = { + let producer_outputs_any = Arc::clone(&producer_outputs_any); + let pushed = Arc::clone(&pushed); + std::thread::spawn(move || { + let outputs = + producer_outputs_any.downcast_ref::>>().unwrap(); + let mut held: Option = None; + let mut next = 0u32; + loop { + if let Some(v) = held.take() { + match outputs.push(v) { + Ok(()) => { + pushed.fetch_add(1, Ordering::Relaxed); + } + Err(unpushed) => { + held = Some(unpushed.into_item()); + std::thread::yield_now(); + } + } + continue; + } + if next >= N { + break; + } + match outputs.push(next) { + Ok(()) => { + pushed.fetch_add(1, Ordering::Relaxed); + next += 1; + } + Err(unpushed) => { + // The value at `next` is now held for retry; advance + // `next` so the fresh-push branch doesn't re-emit it + // after `held` flushes (which would double-count). + held = Some(unpushed.into_item()); + next += 1; + std::thread::yield_now(); + } + } + } + outputs.mark_all_drained(); + }) + }; + + // Consumer thread: pop everything the Detached step produces. Keep the + // VALUES, not just a count — a count-only assertion passes for a step that + // emits N copies of one item, or that duplicates the held value while + // dropping a popped one, which is exactly the loss this test exists to + // catch. + let received = Arc::new(parking_lot::Mutex::new(Vec::::new())); + let consumer_handle = { + let received = Arc::clone(&received); + std::thread::spawn(move || { + loop { + if let Some(v) = consumer.pop() { + received.lock().push(v); + } else if InputHandle::is_drained(&consumer) { + break; + } else { + std::thread::yield_now(); + } + } + }) + }; + + // Drive the Detached step (as a one-step group) on this thread to + // completion via the unified driver. + drive_single(det, StepIdx(1), &contexts, &signal); + + producer_handle.join().unwrap(); + consumer_handle.join().unwrap(); + done.store(true, Ordering::SeqCst); + + assert_eq!(pushed.load(Ordering::Relaxed), N, "all items pushed"); + // Sorted multiset, not the sequence: the producer's backpressure branch + // holds `next` and advances, so a retried item can arrive after a later + // one. Order is not the invariant here; every distinct item arriving + // exactly once is. + let mut got = received.lock().clone(); + got.sort_unstable(); + assert_eq!( + got, + (0..N).collect::>(), + "every distinct item must flow through the two-sided Detached step exactly once" + ); + } + + /// A `Detached` step declaring a `Shared` group label. Used only to exercise + /// `extract_detached_steps` grouping — never actually run. + #[derive(Clone)] + struct SharedDetached { + label: &'static str, + } + impl Step for SharedDetached { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SharedDetached", + kind: StepKind::Detached, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn detached_group(&self) -> crate::step::DetachedGroup { + crate::step::DetachedGroup::Shared(self.label) + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + } + + /// A `Parallel` step — must never be placed on a 1-thread driver. + #[derive(Clone)] + struct ParallelStub; + impl Step for ParallelStub { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "ParallelStub", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + /// `extract_detached_steps` collects every `Shared(label)` onto one group, + /// keeps each `PerStep` (default) step as its own singleton, preserves chain + /// order within and across groups, and leaves non-detached steps in place. + #[test] + fn extract_groups_shared_together_and_perstep_alone() { + use crate::step::DetachedGroup; + // idx0 non-detached; idx1/idx3 Shared("coord"); idx2 PerStep; idx4 Shared("io"). + let mut steps: Vec> = vec![ + Box::new(TypedStep::new(SrcStub { capacity: 4 })), + Box::new(TypedStep::new(SharedDetached { label: "coord" })), + Box::new(TypedStep::new(PassThroughDetached { held: None })), + Box::new(TypedStep::new(SharedDetached { label: "coord" })), + Box::new(TypedStep::new(SharedDetached { label: "io" })), + ]; + let groups = extract_detached_steps(&mut steps); + + assert_eq!(groups.len(), 3, "coord{{1,3}}, perstep{{2}}, io{{4}}"); + // Order is first-appearance in chain order. + assert_eq!(groups[0].label(), DetachedGroup::Shared("coord")); + assert_eq!( + groups[0].steps.iter().map(|(i, _)| i.0).collect::>(), + vec![1, 3], + "shared group keeps both steps in chain order" + ); + assert_eq!(groups[0].primary_step(), StepIdx(1)); + assert_eq!(groups[1].label(), DetachedGroup::PerStep); + assert_eq!(groups[1].steps.iter().map(|(i, _)| i.0).collect::>(), vec![2]); + assert_eq!(groups[2].label(), DetachedGroup::Shared("io")); + + // Non-detached step survives; detached slots became placeholders that + // still report `Detached` (so `build_worker_storage` Skips them on pool). + assert_eq!(steps[0].kind(), StepKind::Exclusive); + assert_eq!(steps[1].kind(), StepKind::Detached); + assert_eq!(steps[2].kind(), StepKind::Detached); + + // Each placeholder reports the group its real step declared, not a blanket + // `PerStep`. `extract_detached_steps` is `pub` and hands this slice back, + // so a caller regrouping from it would otherwise split the `coord` group + // across separate driver threads. + assert_eq!( + steps[1].detached_group(), + DetachedGroup::Shared("coord"), + "placeholder must not downgrade a Shared group to PerStep" + ); + assert_eq!(steps[2].detached_group(), DetachedGroup::PerStep); + } + + #[test] + #[should_panic(expected = "must be non-empty")] + fn driver_group_rejects_empty() { + // An empty group would panic later in `primary_step`/`label` (`steps[0]`); + // the constructor rejects it up front. + let _ = DetachedDriverGroup::new(vec![]); + } + + #[test] + #[should_panic(expected = "only contain Detached steps")] + fn driver_group_rejects_non_detached_step() { + // `SrcStub` is Exclusive, not Detached — running it off-pool on a driver + // thread is a bug, so the constructor rejects the group. + let step: Box = Box::new(TypedStep::new(SrcStub { capacity: 1 })); + let _ = DetachedDriverGroup::new(vec![(StepIdx(0), step)]); + } + + /// `build_driver_storage` makes the group's steps `Owned` and every other + /// slot `Skip`, at the correct global indices. + #[test] + fn build_driver_storage_owns_group_skips_rest() { + let det: Box = Box::new(TypedStep::new(PassThroughDetached { held: None })); + let row = build_driver_storage(vec![(StepIdx(2), det)], 5); + assert_eq!(row.len(), 5); + assert!(matches!(row[2], WorkerStepEntry::Owned { .. }), "group step is Owned"); + for i in [0usize, 1, 3, 4] { + assert!(matches!(row[i], WorkerStepEntry::Skip), "non-group slot {i} is Skip"); + } + } + + /// G3: a `Parallel` step must never be grouped onto a 1-thread driver (its + /// init-1 counter would never close the shared output). + #[test] + #[should_panic(expected = "is Parallel")] + fn build_driver_storage_rejects_parallel_group_step() { + let par: Box = Box::new(TypedStep::new(ParallelStub)); + let _ = build_driver_storage(vec![(StepIdx(0), par)], 2); + } + + /// G3: two group steps at the same index is a dual registration — rejected. + #[test] + #[should_panic(expected = "registered twice")] + fn build_driver_storage_rejects_dual_registration() { + let a: Box = Box::new(TypedStep::new(PassThroughDetached { held: None })); + let b: Box = Box::new(TypedStep::new(PassThroughDetached { held: None })); + let _ = build_driver_storage(vec![(StepIdx(1), a), (StepIdx(1), b)], 3); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/drain.rs b/crates/fgumi-pipeline-core/src/runtime/drain.rs new file mode 100644 index 000000000..fe60db784 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/drain.rs @@ -0,0 +1,99 @@ +//! `StepDrainCounter`: coordinates last-worker-wins for closing a step's +//! shared output queue when it reports `StepOutcome::Finished`. +//! +//! For `Parallel` steps, init to N (= worker count). Each clone returns +//! `Finished` independently when the shared input edge drains; each calls +//! `observe_drain`, and the clone that takes the counter to 0 is the "last +//! worker" — only it calls `mark_outputs_drained` (closing the shared output). +//! Otherwise a clone could close the output while a sibling is still pushing. +//! +//! For `Serial` / `Exclusive` steps, init to 1. The single finisher wins on +//! its first call. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + +#[derive(Debug)] +pub struct StepDrainCounter { + remaining: AtomicUsize, +} + +impl StepDrainCounter { + /// Construct a counter with the given initial decrement budget. Returns + /// an `Arc` for sharing across worker threads. + #[must_use] + pub fn new(initial: usize) -> Arc { + Arc::new(Self { remaining: AtomicUsize::new(initial) }) + } + + /// Called by a worker when it observes drain on this step. Returns + /// `true` if this is the last worker (counter went from 1 to 0). + /// Subsequent calls (counter already 0) return `false`. + pub fn observe_drain(&self) -> bool { + // CAS-decrement loop: atomically decrement only if `> 0`. Avoids + // underflow under concurrent over-calls. + let mut prev = self.remaining.load(AtomicOrdering::Acquire); + loop { + if prev == 0 { + return false; + } + match self.remaining.compare_exchange_weak( + prev, + prev - 1, + AtomicOrdering::AcqRel, + AtomicOrdering::Acquire, + ) { + Ok(_) => return prev == 1, + Err(actual) => prev = actual, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn first_worker_with_init_one_wins() { + let counter = StepDrainCounter::new(1); + assert!(counter.observe_drain()); + assert!(!counter.observe_drain()); + } + + #[test] + fn last_worker_with_init_n_wins() { + let counter = StepDrainCounter::new(4); + assert!(!counter.observe_drain()); + assert!(!counter.observe_drain()); + assert!(!counter.observe_drain()); + assert!(counter.observe_drain()); + } + + #[test] + fn extra_calls_after_zero_return_false() { + let counter = StepDrainCounter::new(2); + counter.observe_drain(); + counter.observe_drain(); + assert!(!counter.observe_drain()); + } + + #[test] + fn concurrent_decrements_have_exactly_one_winner() { + use std::thread; + let counter = StepDrainCounter::new(8); + // Collect the handles BEFORE joining: iterator adapters are lazy, so a + // chained `.map(spawn).map(join)` pulls one element at a time and joins + // each thread before spawning the next — making this the same sequential + // path as `last_worker_with_init_n_wins`, and passing even against a + // non-atomic load/store implementation. + let handles: Vec<_> = (0..8) + .map(|_| { + let c = Arc::clone(&counter); + thread::spawn(move || c.observe_drain()) + }) + .collect(); + let winners: Vec = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert_eq!(winners.iter().filter(|&&w| w).count(), 1, "exactly one winner"); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/driver.rs b/crates/fgumi-pipeline-core/src/runtime/driver.rs new file mode 100644 index 000000000..2069d9930 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/driver.rs @@ -0,0 +1,1269 @@ +//! Worker loop body. Each worker thread runs `run_worker_loop` until +//! `signal.is_done()` or all steps are drained. +//! +//! Loop structure (per iteration): +//! 1. Check `signal.is_done()`; bail if true. +//! 2. **Sticky re-entry**: if this worker owns an `Exclusive` step that's +//! sticky, drive it to a stop (`Progress`→loop; `Finished`→done; +//! `NoProgress` or `Contention`→exit sticky), for at most +//! `STICKY_BURST_LIMIT` consecutive calls. Sticky avoids context- +//! switch overhead for source-style steps that emit in tight bursts. +//! 3. **Round-robin dispatch**: try each step in chain order. On +//! `Progress`, restart from step 0 (priority) — except for this worker's +//! sticky owner, which already had its burst in step 2. On `Finished`, the +//! step is complete — `mark_outputs_drained` (counter-gated for `Parallel`) +//! and remove it from the worklist. `NoProgress`/`Contention` are idle ticks. +//! 4. If no work happened this iteration, exponential-backoff sleep. +//! +//! Completion: every step — source, mid, or sink — terminates by returning +//! `StepOutcome::Finished` from `try_run` once its input edges are drained and +//! it holds no buffered output. The framework then closes its output edges and +//! drops it from the per-worker worklist. For a `Parallel` step the per-step +//! [`StepDrainCounter`] gates `mark_outputs_drained` so only the last clone to +//! finish closes the shared output queue (see `dispatch_one_step`). + +use std::any::Any; +use std::sync::Arc; +use std::time::Instant; + +use crate::erased::ErasedStepCtx; +use crate::runtime::contexts::ChainContexts; +use crate::runtime::drain::StepDrainCounter; +use crate::runtime::live::LiveSteps; +use crate::runtime::scheduler::{Scheduler, WalkDirection}; +use crate::runtime::stats::PipelineStats; +use crate::runtime::storage::WorkerStepEntry; +use crate::runtime::worker_core::{WorkerCore, WorkerRole}; +use crate::signal::{PipelineError, PipelineSignal}; +use crate::step::StepOutcome; +use crate::topology::StepIdx; + +/// Maximum consecutive sticky re-entries before the worker drops back to a +/// round-robin pass. +/// +/// `StepOutcome::Progress` means the step "pushed **or held** an item" (see the +/// [`crate::step`] completion contract), so a sticky step whose output is full +/// reports `Progress` on every call while moving nothing. Re-entering without a +/// bound would spin on that step forever and never dispatch the downstream step +/// that drains the output — at one worker, a hang; at several, a burnt core. +/// +/// The bound is high enough that the fast path keeps its point (a source +/// emitting a tight burst skips the outer loop's bookkeeping per item) and low +/// enough that a step stuck on a full output yields promptly. It caps starvation, +/// it is not a tuning knob: forward progress comes from this bound *together +/// with* round-robin declining to restart the walk on the sticky owner's +/// `Progress`. +const STICKY_BURST_LIMIT: usize = 1024; + +/// Run the worker loop for one worker thread. +/// +/// `entries[step_idx] = WorkerStepEntry` — this worker's storage. +/// `contexts` — shared per-step input/output handles. +/// `drain_counters[step_idx]` — the per-step `StepDrainCounter` that gates the +/// output close on `Finished` (init N for Parallel so only the last clone +/// closes the shared output; init 1 for Serial/Exclusive). +/// `signal` — error/cancel broadcast. +pub fn run_worker_loop( + worker: &mut WorkerCore, + entries: &mut [WorkerStepEntry], + contexts: &Arc, + drain_counters: &[Arc], + signal: &Arc, + stats: Option<&Arc>, + scheduler: &dyn Scheduler, +) { + // Per-worker worklist of still-dispatchable steps, in chain order. A step + // is removed when it returns `StepOutcome::Finished`; the worker exits once + // the list is empty. Build-time `Skip` placeholders (Exclusive steps owned + // by other workers, Serial steps gated out by affinity) never enter the + // worklist. + let mut live = LiveSteps::from_entries(entries); + + // Cache whether this worker's sticky owner (if any) is still dispatchable, + // so the hot sticky fast-path does not run a linear `live.contains` scan on + // every outer-loop iteration (the sticky path exists precisely to shave + // per-iteration overhead). `sticky_owner` is fixed for the worker's + // lifetime; the step leaves `live` exactly when it returns `Finished`, + // either via the sticky block below or via round-robin dispatch — both + // sites flip this flag false. A `None` sticky owner is permanently "not + // live" so the fast path is skipped entirely. + let mut sticky_live = worker.sticky_owner.is_some_and(|idx| live.contains(idx)); + + // A driver thread attributes busy time PER grouped step on the off-pool + // detached line (so a multi-step `Shared` group shows each step's real + // busy, not the whole thread's time under one name), recorded inside + // `dispatch_one_step`; a pool worker records aggregate busy by `thread_id` + // below. Idle/park stay thread-level (keyed to the group's primary step). + let is_driver = matches!(worker.role(), WorkerRole::Driver { .. }); + + loop { + if signal.is_done() { + break; + } + // Exit when this worker has nothing left to dispatch. + if live.is_empty() { + break; + } + + let mut did_work = false; + + // Time the dispatch (busy) section vs the backoff sleep (idle) below, + // per worker, only when stats are on (`Instant::now()` is otherwise not + // called — the no-stats loop stays zero-cost). + let work_start = stats.map(|_| Instant::now()); + + // 1. Sticky re-entry for sticky-owned steps (either an Exclusive + // sticky step this worker owns, or a Serial+sticky step whose + // Affinity targets this worker). The Pipeline::run path only + // sets `sticky_owner` when the step is actually sticky, so no + // per-iteration profile peek is needed. Re-enter while the + // step makes Progress; exit on Finished / NoProgress / Contention + // / Err. Remove from the worklist on Finished (source drain) or + // observed input drain (mid-step drain). Gated on the step still + // being live — once removed, the sticky fast-path is disabled. + // + // Bounded at `STICKY_BURST_LIMIT` calls: `Progress` also covers "held an + // item", so a step whose output is full reports it indefinitely without + // moving anything. An unbounded re-entry would then never reach the + // round-robin pass that dispatches the downstream step draining that + // output. + if let Some(owned_idx) = worker.sticky_owner.filter(|_| sticky_live) { + let mut mark_skip = false; + for _ in 0..STICKY_BURST_LIMIT { + if signal.is_done() { + break; + } + let entry = &mut entries[owned_idx.0]; + let Some(info) = dispatch_one_step( + entry, + owned_idx, + contexts, + &drain_counters[owned_idx.0], + signal, + stats, + is_driver, + ) else { + break; // Skip + }; + match info.result { + Ok(StepOutcome::Progress) => { + did_work = true; + // Continue sticky. + } + Ok(StepOutcome::Finished) => { + // Outputs were marked drained under the dispatch guard. + did_work = true; + mark_skip = true; + break; + } + // Nothing to do this call — yield out of the sticky loop + // back to round-robin. The step terminates via `Finished`, + // not a drain protocol. + Ok(StepOutcome::NoProgress | StepOutcome::Contention) => break, + Err(io_err) => { + signal.record_error(PipelineError::Io { step: info.name, source: io_err }); + break; + } + } + } + if mark_skip { + live.remove(owned_idx); + // The sticky owner finished here; disable the fast path. + sticky_live = false; + } + } + + // 2. Round-robin priority dispatch over all live steps. + if !signal.is_done() { + let outcome = round_robin_dispatch( + entries, + &mut live, + worker.sticky_owner, + contexts, + drain_counters, + signal, + stats, + scheduler.walk(), + is_driver, + ); + did_work |= outcome.did_work; + if outcome.removed_sticky_owner { + // The sticky owner finished during round-robin; disable the + // fast path so subsequent iterations skip the sticky block. + sticky_live = false; + } + } + + // Attribute the dispatch section's wall time to this thread's busy total. + // Pool workers sum the whole pass into the N-worker utilisation line (by + // thread_id). Driver threads instead record each grouped step's own busy + // inside `dispatch_one_step` (on the off-pool detached line, by step) so a + // multi-step `Shared` group isn't collapsed onto one name — so nothing to + // record here for a driver. + if let (Some(stats), Some(ws)) = (stats, work_start) + && let WorkerRole::Pool = worker.role() + { + let ns = u64::try_from(ws.elapsed().as_nanos()).unwrap_or(u64::MAX); + stats.record_worker_busy(worker.thread_id, ns); + } + + // 3. Exponential-backoff sleep on no-progress; reset on progress. The + // sleep is the worker's idle/blocked time — attribute it per worker so + // pool under-utilisation (cores parked while one worker drives a Serial + // step) is visible in `--pipeline-stats`. + if did_work { + worker.reset_backoff(); + } else if signal.is_done() { + break; + } else { + let sleep_start = stats.map(|_| Instant::now()); + worker.sleep_backoff(); + worker.increase_backoff(); + if let (Some(stats), Some(ss)) = (stats, sleep_start) { + let ns = u64::try_from(ss.elapsed().as_nanos()).unwrap_or(u64::MAX); + match worker.role() { + WorkerRole::Pool => stats.record_worker_idle(worker.thread_id, ns), + WorkerRole::Driver { primary_step } => { + stats.record_detached_idle(primary_step, ns); + stats.record_detached_park(primary_step); + } + } + } + } + } +} + +/// Result of one round-robin pass: whether any step did useful work (caller +/// resets backoff), and whether the worker's sticky owner finished during the +/// pass (caller clears its `sticky_live` cache so the sticky fast-path is not +/// re-attempted on a removed step). +struct RoundRobinOutcome { + did_work: bool, + removed_sticky_owner: bool, +} + +/// One pass of the round-robin dispatch over this worker's live steps, in +/// chain order. Finished steps are removed from `live` at end-of-pass (deferred +/// so the in-progress walk over `live.order()` is not mutated underneath it). +/// `sticky_owner` (if any) has two roles here: it is reported back via +/// [`RoundRobinOutcome::removed_sticky_owner`] when it finishes in this pass, so +/// the caller can disable the per-iteration sticky fast-path without a linear +/// `live.contains` scan; and its `Progress` does **not** trigger the priority +/// restart, so the walk continues to the steps downstream of it (see the +/// `Progress` arm). +#[allow(clippy::too_many_arguments)] // shared per-step state + the walk policy; a struct would not clarify +fn round_robin_dispatch( + entries: &mut [WorkerStepEntry], + live: &mut LiveSteps, + sticky_owner: Option, + contexts: &Arc, + drain_counters: &[Arc], + signal: &Arc, + stats: Option<&Arc>, + walk: WalkDirection, + is_driver: bool, +) -> RoundRobinOutcome { + let mut did_work = false; + // Steps that finished this pass, removed from `live` after the walk. A + // step is visited at most once per pass (the cursor only advances; we + // `break` on `Progress`/error, never revisit), so deferring removal is + // safe and avoids reorder-under-iteration. + let mut finished: Vec = Vec::new(); + let n = live.len(); + for i in 0..n { + if signal.is_done() { + break; + } + // The Scheduler selects the walk DIRECTION over this worker's live + // steps: `Forward` = chain order (upstream-first, favour production); + // `Reverse` = downstream-first (favour draining buffered work before + // producing more). Everything else — skip-on-contention, the sticky + // source/sink fast-path, Finished handling — is direction-agnostic. + let pos = match walk { + WalkDirection::Forward => i, + WalkDirection::Reverse => n - 1 - i, + }; + let step_idx = live.order()[pos]; + let entry = &mut entries[step_idx.0]; + let mut mark_skip = false; + let mut restart_priority = false; + let Some(info) = dispatch_one_step( + entry, + step_idx, + contexts, + &drain_counters[step_idx.0], + signal, + stats, + is_driver, + ) else { + continue; // Skip (build-time placeholder; should not appear in `live`) + }; + match info.result { + Ok(StepOutcome::Progress) => { + did_work = true; + // Restart the walk at the top so the highest-priority step runs + // again — EXCEPT for this worker's sticky owner. That step just + // had its dedicated burst in phase 1 of the worker loop, so a + // priority restart here only re-runs it. Worse, `Progress` also + // means "held an item", so a sticky step sitting on a full output + // reports it forever: breaking the pass on that outcome means the + // downstream step that would drain the output is never reached + // and the pair livelocks at one worker. Walking past the sticky + // owner is what turns the bounded burst into real forward + // progress. + restart_priority = sticky_owner != Some(step_idx); + } + // Nothing to do this call. The step terminates by returning + // `Finished` (handled below); `NoProgress`/`Contention` are idle + // ticks — there is no separate drain protocol. + Ok(StepOutcome::NoProgress | StepOutcome::Contention) => {} + Ok(StepOutcome::Finished) => { + // Any step (source, mid, or sink) may report `Finished` once + // all its inputs are drained and it holds no buffered output. + // Outputs were marked drained under the dispatch guard (and, for + // a Serial step, the shared `finished` latch was set so the + // other workers stop re-dispatching it — see `dispatch_one_step`). + did_work = true; + mark_skip = true; + } + Err(io_err) => { + signal.record_error(PipelineError::Io { step: info.name, source: io_err }); + break; + } + } + if mark_skip { + finished.push(step_idx); + } + if restart_priority { + break; + } + } + let removed_sticky_owner = sticky_owner.is_some_and(|owner| finished.contains(&owner)); + for step_idx in finished { + live.remove(step_idx); + } + RoundRobinOutcome { did_work, removed_sticky_owner } +} + +/// Outcome of dispatching one step, plus the `name` captured *during* the +/// dispatch — under the same `Shared`-mutex guard as the run itself — for +/// error reporting. +struct DispatchInfo { + result: std::io::Result, + name: &'static str, +} + +/// Dispatch one step's `try_run_erased`. Returns: +/// - `Some(DispatchInfo)` — dispatched (the `result` carries the outcome or +/// the step's `Err`); on `Finished`, outputs are already marked drained. +/// - `None` — entry is `Skip` (caller continues to next step). +/// +/// For a `Serial` (`Shared`) step the shared `finished` latch on its +/// `DrainGate` is consulted *before* acquiring the step mutex: once any worker +/// has finished the step (returned `Finished`, or completed its cooperative +/// drain), the latch is set and every other worker short-circuits to a synthetic +/// `Finished` here rather than re-`try_lock`-ing and re-running an already-done +/// step. The winning worker sets the latch under the dispatch guard before +/// `mark_outputs_drained`, so a non-idempotent flusher can never be re-entered. +fn dispatch_one_step( + entry: &mut WorkerStepEntry, + step_idx: StepIdx, + contexts: &ChainContexts, + counter: &StepDrainCounter, + signal: &Arc, + stats: Option<&Arc>, + // When true (a dedicated driver thread), attribute this dispatch's wall time + // to the off-pool detached line keyed by `step_idx` — so each grouped step + // reports its own busy. Pool workers pass `false` and record aggregate busy + // by `thread_id` in the loop instead. + is_driver: bool, +) -> Option { + let outputs_any: &(dyn Any + Send + Sync) = contexts.outputs[step_idx.0].as_ref(); + let mut ctx = + ErasedStepCtx { input: contexts.inputs[step_idx.0].as_ref(), outputs: outputs_any, signal }; + + // Time the dispatch only when stats collection is on. `Instant::now()` + // is ~20-50ns on Apple Silicon, ~50-100ns on x86_64; gating on + // `stats.is_some()` keeps the no-stats path zero-cost. + let start = stats.map(|_| Instant::now()); + + // Run the step and capture `name` *while still holding the `Shared` guard* + // (or with direct `&mut` for Owned/Exclusive). On `Finished`, mark outputs + // drained here too — under the same guard — so the caller never re-acquires + // the lock for any post-dispatch inspection. + // + // `mark_outputs_drained` is gated behind `counter.observe_drain()` (the + // per-step `StepDrainCounter`): for a `Parallel` step (counter init N) + // every clone returns `Finished` independently when the shared input edge + // is drained, but only the LAST clone to finish (the one that takes the + // counter to 0) closes the shared output queue — otherwise a clone could + // `mark_drained` while a sibling is still pushing (`try_push`-after-drained + // panic). For `Serial`/`Exclusive` (counter init 1) the single finisher + // wins on its first call, unchanged. + // + // INVARIANT: for a `Parallel` step, `counter` init == clone count == the + // worker count, and a clone leaves its worklist ONLY by returning + // `Finished`, so the counter reaches 0 exactly when every clone has + // finished. Any future scheduler change that removes a Parallel clone for + // another reason (work-stealing, per-worker early exit) — or makes a source + // `Parallel` — would leave the counter stuck above 0 and never close the + // shared output, hanging the downstream consumer. Keep the init (builder.rs) + // and this gate in lockstep. + let info: Option = match entry { + WorkerStepEntry::Owned { step } | WorkerStepEntry::Exclusive { step } => { + let result = step.try_run_erased(&mut ctx); + // `name()` returns the cached static name — no per-dispatch + // `StepProfile` (and its two `Vec`s) is built. + let name = step.name(); + if matches!(result, Ok(StepOutcome::Finished)) && counter.observe_drain() { + step.mark_outputs_drained(outputs_any); + } + Some(DispatchInfo { result, name }) + } + WorkerStepEntry::Shared { step, drain } => { + if drain.is_finished() { + // Another worker already finished this Serial step. Don't + // re-`try_lock`/re-run it — report a synthetic `Finished` so the + // caller drops it from this worker's live set. Outputs were + // already marked drained by the finishing worker. + Some(DispatchInfo { + result: Ok(StepOutcome::Finished), + name: "", + }) + } else { + match step.try_lock() { + None => Some(DispatchInfo { + result: Ok(StepOutcome::Contention), + name: "", + }), + Some(mut guard) => { + let result = guard.try_run_erased(&mut ctx); + let name = guard.name(); + if matches!(result, Ok(StepOutcome::Finished)) { + // Set the shared finished latch under the guard, + // before marking outputs drained, so a concurrent + // worker that observes the latch never re-runs the + // step nor re-marks its outputs. + drain.mark_finished(); + if counter.observe_drain() { + guard.mark_outputs_drained(outputs_any); + } + } + Some(DispatchInfo { result, name }) + } + } + } + } + WorkerStepEntry::Skip => None, + }; + + if let (Some(stats), Some(start)) = (stats, start) { + let elapsed_ns = u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX); + // Wall ns at dispatch start, relative to pipeline start. + let start_ns = stats.elapsed_ns().saturating_sub(elapsed_ns); + // `None` (Skip) attempted no work, so there is nothing to record. + if let Some(i) = info.as_ref() { + match &i.result { + Ok(outcome) => stats.record(step_idx, *outcome, start_ns, elapsed_ns), + Err(_) => stats.record_error(step_idx, start_ns, elapsed_ns), + } + // On a driver thread, this step's try_run wall is its own off-pool + // busy (excluded from the pool%); each grouped step accrues its own. + if is_driver { + stats.record_detached_busy(step_idx, elapsed_ns); + } + } + } + + info +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use super::*; + use crate::erased::{ErasedStep, TypedStep}; + use crate::handles::BranchInputHandle; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::runtime::contexts::build_chain_contexts; + use crate::runtime::storage::DrainGate; + use crate::step::{InputHandle, Step, StepCtx, StepKind, StepOutcome, StepProfile}; + use crate::topology::{BranchIdx, ChainGraph}; + use parking_lot::Mutex; + + #[test] + fn run_worker_loop_exits_on_signal_done() { + let signal = PipelineSignal::new(); + let mut entries: Vec = vec![]; + let contexts = Arc::new(ChainContexts { + inputs: vec![], + outputs: vec![], + bounded_queues: vec![], + edges: vec![], + }); + let drain_counters: Vec> = vec![]; + let _ = ChainGraph::new(); + let mut worker = WorkerCore::new(0, None, None); + + signal.cancel(); + run_worker_loop( + &mut worker, + &mut entries, + &contexts, + &drain_counters, + &signal, + None, + &crate::runtime::scheduler::ChainOrderScheduler, + ); + // If we reach this line, the loop exited cleanly. + } + + // ── Test steps for dispatch-level coverage ────────────────────────────── + + /// `() → u32` source that returns `Finished` immediately. + #[derive(Clone)] + struct SrcFinished; + impl Step for SrcFinished { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Src", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } + } + + /// `() → u32` source that returns `NoProgress` on its first `try_run` (so + /// the sticky fast-path yields back to round-robin without removing it) and + /// `Finished` on every later call (so it is removed during the round-robin + /// pass, exercising `RoundRobinOutcome::removed_sticky_owner`). + #[derive(Clone)] + struct SrcIdleThenFinish { + calls: Arc, + } + impl Step for SrcIdleThenFinish { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SrcIdleThenFinish", + kind: StepKind::Exclusive, + sticky: true, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + let n = self.calls.fetch_add(1, Ordering::Relaxed); + if n == 0 { Ok(StepOutcome::NoProgress) } else { Ok(StepOutcome::Finished) } + } + } + + /// `u32 → u32` step that always returns `Finished`. Used both as a + /// `Parallel` body (counter-gated output close) and a `Serial` body + /// (`DrainGate` short-circuit). The `runs` counter records every `try_run` + /// so the short-circuit test can prove a worker did NOT re-run the step. + #[derive(Clone)] + struct FinishStep { + kind: StepKind, + runs: Arc, + } + impl Step for FinishStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Finish", + kind: self.kind, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + self.runs.fetch_add(1, Ordering::Relaxed); + Ok(StepOutcome::Finished) + } + fn new_worker_copy(&self) -> Self { + // Clones share the `runs` counter so the test can total runs across + // every Parallel clone. + self.clone() + } + } + + #[derive(Clone)] + struct SinkStep; + impl Step for SinkStep { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "Sink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + // Drain any inputs, then finish once the upstream edge is drained so + // the worker loop can terminate (a real sink finishes on drain). + while ctx.input.pop().is_some() {} + if ctx.input.is_drained() { + Ok(StepOutcome::Finished) + } else { + Ok(StepOutcome::NoProgress) + } + } + } + + /// A non-sticky `Exclusive` sink that deliberately stays live for one extra + /// round-robin pass: it ignores its input-drain status and finishes purely + /// on an internal tick counter — `NoProgress` on the first `try_run`, + /// `Finished` after. Keeping a second step alive for one more outer + /// iteration *after* the sticky source is removed is what makes + /// `sticky_owner_removed_via_round_robin_and_loop_exits` branch-specific: a + /// plain `SinkStep` finishes in the same round-robin pass as the source + /// (its input is already drained), emptying `live` so the loop exits via + /// `live.is_empty()` even if the `removed_sticky_owner` branch had failed to + /// clear `sticky_live`. Lingering forces the extra iteration on which a + /// stale `sticky_live` would re-enter the sticky fast-path and re-invoke the + /// already-removed source. + struct LingerThenFinish { + ticks: usize, + } + impl Step for LingerThenFinish { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "Linger", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + while ctx.input.pop().is_some() {} + self.ticks += 1; + // Stay live for exactly one extra round-robin pass before finishing, + // regardless of input-drain status. + if self.ticks >= 2 { Ok(StepOutcome::Finished) } else { Ok(StepOutcome::NoProgress) } + } + } + + /// Build `Src → Finish → Sink` (Finish having the given kind) and return the + /// erased steps + the wired graph. The `Finish` step's `try_run` counter is + /// returned so tests can assert how many times it actually ran. + fn three_step_chain( + finish_kind: StepKind, + ) -> (Vec>, ChainGraph, Arc) { + let runs = Arc::new(AtomicUsize::new(0)); + let mut graph = ChainGraph::new(); + let src = graph.register_step("Src", 1); + let mid = graph.register_step("Finish", 1); + let sink = graph.register_step("Sink", 0); + graph.wire(src, BranchIdx(0), mid); + graph.wire(mid, BranchIdx(0), sink); + let steps: Vec> = vec![ + Box::new(TypedStep::new(SrcFinished)), + Box::new(TypedStep::new(FinishStep { kind: finish_kind, runs: Arc::clone(&runs) })), + Box::new(TypedStep::new(SinkStep)), + ]; + (steps, graph, runs) + } + + /// A `Parallel` step's shared output queue must be closed exactly once — by + /// the LAST clone to finish (the one that takes the `StepDrainCounter` to + /// 0). Earlier finishers must leave the downstream input un-drained so a + /// sibling could still push. + #[test] + fn parallel_last_finisher_closes_shared_output_exactly_once() { + const N: usize = 4; + let (steps, graph, _runs) = three_step_chain(StepKind::Parallel); + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + let mid = StepIdx(1); + let counter = StepDrainCounter::new(N); + let signal = PipelineSignal::new(); + + // One `Owned` clone per worker, each sharing `contexts.outputs[mid]`. + let mut clones: Vec = + (0..N).map(|_| WorkerStepEntry::Owned { step: steps[mid.0].clone_boxed() }).collect(); + + let sink_input = contexts.inputs[2].downcast_ref::>().unwrap(); + + for (i, clone) in clones.iter_mut().enumerate() { + assert!( + !InputHandle::is_drained(sink_input), + "downstream input drained before the last clone finished (after {i} of {N})" + ); + let info = + dispatch_one_step(clone, mid, &contexts, &counter, &signal, None, false).unwrap(); + assert!(matches!(info.result, Ok(StepOutcome::Finished))); + } + assert!( + InputHandle::is_drained(sink_input), + "downstream input must be drained once the last Parallel clone finished" + ); + } + + /// Once one worker finishes a `Serial` step (setting the shared `DrainGate`), + /// a second `dispatch_one_step` short-circuits to a synthetic `Finished` + /// WITHOUT re-acquiring the mutex or re-running the step. + #[test] + fn serial_drain_gate_short_circuits_second_worker() { + let (steps, graph, runs) = three_step_chain(StepKind::Serial); + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + let mid = StepIdx(1); + let counter = StepDrainCounter::new(1); + let signal = PipelineSignal::new(); + + let shared = Arc::new(Mutex::new(steps.into_iter().nth(1).unwrap())); + let drain = Arc::new(DrainGate::default()); + + // Worker 1 finishes the step: runs once, sets the latch. + let mut entry1 = + WorkerStepEntry::Shared { step: Arc::clone(&shared), drain: Arc::clone(&drain) }; + let info1 = + dispatch_one_step(&mut entry1, mid, &contexts, &counter, &signal, None, false).unwrap(); + assert!(matches!(info1.result, Ok(StepOutcome::Finished))); + assert_eq!(runs.load(Ordering::Relaxed), 1, "step ran exactly once on the first worker"); + assert!(drain.is_finished(), "first finisher must set the DrainGate latch"); + + // Worker 2 dispatches the same step: short-circuit, no re-run. + let mut entry2 = + WorkerStepEntry::Shared { step: Arc::clone(&shared), drain: Arc::clone(&drain) }; + let info2 = + dispatch_one_step(&mut entry2, mid, &contexts, &counter, &signal, None, false).unwrap(); + assert!(matches!(info2.result, Ok(StepOutcome::Finished))); + assert_eq!( + info2.name, "", + "second worker takes the latch short-circuit" + ); + assert_eq!( + runs.load(Ordering::Relaxed), + 1, + "the Serial step must NOT be re-run after the DrainGate latch is set" + ); + } + + /// A sticky-owned source driven through `run_worker_loop` completes and the + /// loop exits even though the cached `sticky_live` flag (not a per-iteration + /// `live.contains` scan) gates the fast path. Exercises the S1b-006 cache: + /// the sticky step is removed once it returns `Finished`, after which the + /// fast path must be disabled and the loop must terminate. + #[test] + fn sticky_owner_completes_and_loop_exits() { + let mut graph = ChainGraph::new(); + let src = graph.register_step("Src", 1); + let sink = graph.register_step("Sink", 0); + graph.wire(src, BranchIdx(0), sink); + let steps: Vec> = + vec![Box::new(TypedStep::new(SrcFinished)), Box::new(TypedStep::new(SinkStep))]; + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + + // Single worker; the source (idx 0) is its Exclusive sticky owner, the + // sink (idx 1) is Exclusive owned by the same worker for this 1-worker + // run. + let mut entries: Vec = vec![ + WorkerStepEntry::Exclusive { step: steps.into_iter().next().unwrap() }, + WorkerStepEntry::Exclusive { step: Box::new(TypedStep::new(SinkStep)) }, + ]; + let drain_counters = vec![StepDrainCounter::new(1), StepDrainCounter::new(1)]; + let signal = PipelineSignal::new(); + let mut worker = WorkerCore::new(0, Some(src), Some(src)); + + // The source finishes immediately; the sink then sees its input drained + // and finishes too. `run_worker_loop` must return (no hang). + run_worker_loop( + &mut worker, + &mut entries, + &contexts, + &drain_counters, + &signal, + None, + &crate::runtime::scheduler::ChainOrderScheduler, + ); + } + + /// A sticky owner that returns `NoProgress` on its first call (yielding out + /// of the sticky fast-path back to round-robin) and `Finished` later must be + /// removed via the round-robin path (`RoundRobinOutcome::removed_sticky_owner`), + /// after which the next outer iteration skips the sticky re-entry. This pins + /// the round-robin removal branch (lines around `outcome.removed_sticky_owner`), + /// not just the sticky fast-path removal exercised by + /// `sticky_owner_completes_and_loop_exits`. + /// + /// A second `LingerThenFinish` step is kept alive for one extra round-robin + /// pass *after* the source is removed, so the worker loop must run one more + /// outer iteration. That iteration is where a stale `sticky_live` would + /// wrongly re-enter the sticky fast-path and re-invoke the + /// already-removed-from-`live` source — making the `== 2` source-call + /// assertion below uniquely diagnostic of the `removed_sticky_owner` branch. + /// (Without the linger, a plain sink would finish in the same pass as the + /// source, emptying `live` so the loop exits via `live.is_empty()` whether + /// or not `sticky_live` was cleared — and `== 2` would not be branch-specific.) + #[test] + fn sticky_owner_removed_via_round_robin_and_loop_exits() { + let mut graph = ChainGraph::new(); + let src = graph.register_step("SrcIdleThenFinish", 1); + let linger = graph.register_step("Linger", 0); + graph.wire(src, BranchIdx(0), linger); + + let calls = Arc::new(AtomicUsize::new(0)); + let steps: Vec> = vec![ + Box::new(TypedStep::new(SrcIdleThenFinish { calls: Arc::clone(&calls) })), + Box::new(TypedStep::new(LingerThenFinish { ticks: 0 })), + ]; + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + + let mut entries: Vec = vec![ + WorkerStepEntry::Exclusive { step: steps.into_iter().next().unwrap() }, + WorkerStepEntry::Exclusive { + step: Box::new(TypedStep::new(LingerThenFinish { ticks: 0 })), + }, + ]; + let drain_counters = vec![StepDrainCounter::new(1), StepDrainCounter::new(1)]; + let signal = PipelineSignal::new(); + let mut worker = WorkerCore::new(0, Some(src), Some(src)); + + // First sticky call → NoProgress (yield to round-robin); the source then + // returns Finished during a round-robin pass, which must remove it and + // disable the sticky fast-path so the loop terminates rather than hangs. + // The `Linger` step stays alive for one more iteration, forcing the + // post-removal outer iteration that exercises the cleared fast path. + run_worker_loop( + &mut worker, + &mut entries, + &contexts, + &drain_counters, + &signal, + None, + &crate::runtime::scheduler::ChainOrderScheduler, + ); + + // The source must have been called EXACTLY twice: call 1 = idle in the + // sticky fast-path (`NoProgress`, which does NOT remove it there — that + // block only reaps a `Finished`), call 2 = `Finished` during the + // round-robin pass. The lingering second step guarantees one more outer + // iteration after that removal, so `== 2` is the branch-specific signal + // for the `removed_sticky_owner` path: if that branch had failed to clear + // `sticky_live`, the extra iteration's sticky fast-path would re-invoke + // the (already-removed-from-`live`) source — the sticky block dispatches + // `entries[owned_idx]` directly, not gated on `live` membership — + // producing a third call. `== 2` therefore proves removal happened via + // round-robin AND that it correctly disabled the fast path. + assert_eq!( + calls.load(Ordering::Relaxed), + 2, + "source must be called exactly twice (sticky idle, then round-robin finish); \ + a different count means the removed_sticky_owner branch did not gate the fast path", + ); + } + + // ── G1: the load-bearing driver invariant ─────────────────────────────── + // + // A driver thread (`WorkerCore::driver`) is just `run_worker_loop` over a + // few `Owned` steps. Its no-deadlock property rests ENTIRELY on the loop + // trying EVERY live step in a pass before it parks. A naive "drive the first + // live step until it yields, then park" loop would wedge the coordination + // driver: it parks on a step whose input isn't ready yet while a *sibling* + // step on the same driver holds the work that would unblock it (e.g. park on + // `FindBoundariesAndSort` while `SpillGather` holds the chunk that frees the + // capacity-1 arena). These two steps pin that the whole-pass discipline holds. + + /// `Owned` step wedged on `NoProgress` until `gate` is flipped by a sibling, + /// then `Finished`. Placed FIRST in the walk so a park-on-first-NoProgress + /// loop would never let the sibling run — the gate never flips — and hang. + #[derive(Clone)] + struct WedgedUntilGate { + gate: Arc, + } + impl Step for WedgedUntilGate { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "WedgedUntilGate", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.gate.load(Ordering::Acquire) { + Ok(StepOutcome::Finished) + } else { + Ok(StepOutcome::NoProgress) + } + } + } + + /// `Owned` sibling that flips `gate` and finishes on its first dispatch — + /// reached only if the loop tries all live steps in a pass rather than + /// parking on the wedged step's `NoProgress`. + #[derive(Clone)] + struct GateOpener { + gate: Arc, + } + impl Step for GateOpener { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "GateOpener", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + self.gate.store(true, Ordering::Release); + Ok(StepOutcome::Finished) + } + } + + /// A driver (`WorkerCore::driver`, Park backoff) driving `[Wedged, Opener]` + /// as two `Owned` steps must drive the sibling that unblocks the wedged step + /// and terminate. A watchdog aborts the process on a wedge so the failure is + /// loud rather than a silent hang. + #[test] + fn driver_round_robins_all_live_before_parking() { + let mut graph = ChainGraph::new(); + let wedged = graph.register_step("WedgedUntilGate", 1); + let opener = graph.register_step("GateOpener", 0); + graph.wire(wedged, BranchIdx(0), opener); + + let gate = Arc::new(AtomicBool::new(false)); + let steps: Vec> = vec![ + Box::new(TypedStep::new(WedgedUntilGate { gate: Arc::clone(&gate) })), + Box::new(TypedStep::new(GateOpener { gate: Arc::clone(&gate) })), + ]; + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + + // Hand-built driver row: both steps Owned on the one driver thread. + let mut entries: Vec = + steps.into_iter().map(|step| WorkerStepEntry::Owned { step }).collect(); + let drain_counters = vec![StepDrainCounter::new(1), StepDrainCounter::new(1)]; + let signal = PipelineSignal::new(); + + // Watchdog: a wedge parks forever; abort so the test FAILS loudly. + let done = Arc::new(AtomicBool::new(false)); + { + let done = Arc::clone(&done); + std::thread::spawn(move || { + for _ in 0..200 { + std::thread::sleep(std::time::Duration::from_millis(25)); + if done.load(Ordering::SeqCst) { + return; + } + } + eprintln!("driver_round_robins_all_live_before_parking: WEDGED"); + std::process::abort(); + }); + } + + // Forward walk (ChainOrderScheduler) tries `wedged` (idx 0) first: it + // yields NoProgress, and the loop MUST proceed to `opener` in the same + // pass, flip the gate, then finish `wedged` on the next pass. + let mut worker = WorkerCore::driver(wedged); + run_worker_loop( + &mut worker, + &mut entries, + &contexts, + &drain_counters, + &signal, + None, + &crate::runtime::scheduler::ChainOrderScheduler, + ); + done.store(true, Ordering::SeqCst); + + assert!(gate.load(Ordering::Acquire), "the sibling opener must have run"); + assert!(!signal.is_done(), "clean completion, no error"); + } + + // ── Sticky forward-progress ───────────────────────────────────────────── + // + // `StepOutcome::Progress` means "pushed OR HELD an item", so a sticky step + // whose output is full keeps reporting `Progress` while moving nothing. Two + // things must hold for the pair below to make progress at ONE worker: the + // sticky burst is bounded, and round-robin does not restart the walk on the + // sticky owner's `Progress` (which would break the pass before the sink is + // ever reached). Removing either one hangs `sticky_holding_source_yields_to_ + // its_draining_consumer`. + + /// Sticky source over a capacity-1 output. Emits `remaining` items; when the + /// transport rejects a push it *holds* the item and reports `Progress` — the + /// contract's "pushed or held" case, and the outcome that makes an unbounded + /// sticky loop spin forever. + struct StickyHoldingSource { + remaining: u32, + held: Option, + calls: Arc, + } + impl Step for StickyHoldingSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "StickyHoldingSource", + kind: StepKind::Exclusive, + sticky: true, + // Capacity 1 so the second item in a burst always backs up. + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + self.calls.fetch_add(1, Ordering::Relaxed); + // Flush-first: retry the held item before producing a new one. + if let Some(item) = self.held.take() { + return match ctx.outputs.push(item) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::Progress) + } + }; + } + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + let item = self.remaining; + self.remaining -= 1; + match ctx.outputs.push(item) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::Progress) + } + } + } + } + + /// Sink that pops one item per dispatch — the only thing that frees a slot in + /// the source's capacity-1 output. + struct DrainingSink { + received: Arc>>, + } + impl Step for DrainingSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "DrainingSink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(item) => { + self.received.lock().push(item); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// One worker owning a sticky source (capacity-1 output) and the sink that + /// drains it must deliver every item and terminate. A watchdog aborts on a + /// wedge so the failure is loud rather than a silent hang. + #[test] + fn sticky_holding_source_yields_to_its_draining_consumer() { + const N_ITEMS: u32 = 4; + + let mut graph = ChainGraph::new(); + let src = graph.register_step("StickyHoldingSource", 1); + let sink = graph.register_step("DrainingSink", 0); + graph.wire(src, BranchIdx(0), sink); + + let calls = Arc::new(AtomicUsize::new(0)); + let received = Arc::new(Mutex::new(Vec::new())); + let steps: Vec> = vec![ + Box::new(TypedStep::new(StickyHoldingSource { + remaining: N_ITEMS, + held: None, + calls: Arc::clone(&calls), + })), + Box::new(TypedStep::new(DrainingSink { received: Arc::clone(&received) })), + ]; + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + + let mut entries: Vec = + steps.into_iter().map(|step| WorkerStepEntry::Exclusive { step }).collect(); + let drain_counters = vec![StepDrainCounter::new(1), StepDrainCounter::new(1)]; + let signal = PipelineSignal::new(); + + // Watchdog: an unbounded sticky loop never returns, so abort loudly. + let done = Arc::new(AtomicBool::new(false)); + { + let done = Arc::clone(&done); + std::thread::spawn(move || { + for _ in 0..400 { + std::thread::sleep(std::time::Duration::from_millis(25)); + if done.load(Ordering::SeqCst) { + return; + } + } + eprintln!( + "sticky_holding_source_yields_to_its_draining_consumer: WEDGED — the \ + sticky fast-path is starving its downstream consumer" + ); + std::process::abort(); + }); + } + + // Forward walk: the source is idx 0 and is this worker's sticky owner. + let mut worker = WorkerCore::new(0, Some(src), Some(src)); + run_worker_loop( + &mut worker, + &mut entries, + &contexts, + &drain_counters, + &signal, + None, + &crate::runtime::scheduler::ChainOrderScheduler, + ); + done.store(true, Ordering::SeqCst); + + assert_eq!( + *received.lock(), + (1..=N_ITEMS).rev().collect::>(), + "every item must reach the sink, in emission order" + ); + assert!(!signal.is_done(), "clean completion, no error"); + // Each item costs at most one full sticky burst plus a round-robin + // dispatch, so the source cannot have been called an unbounded number of + // times. Loose on purpose — it pins "bounded", not an exact schedule. + let n_calls = calls.load(Ordering::Relaxed); + let ceiling = (usize::try_from(N_ITEMS).unwrap() + 2) * (STICKY_BURST_LIMIT + 2); + assert!( + n_calls <= ceiling, + "source dispatches must stay bounded by the sticky burst limit: \ + {n_calls} calls > {ceiling}" + ); + } + + /// A step that spends a measurable, nonzero span inside `try_run` so its + /// dispatch busy-time rounds above 0 ns. + #[derive(Clone)] + struct SlowFinish; + impl Step for SlowFinish { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SlowFinish", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + std::thread::sleep(std::time::Duration::from_micros(200)); + Ok(StepOutcome::Finished) + } + } + + /// A driver dispatch (`is_driver = true`) records the step's busy on the + /// off-pool detached line keyed by THAT step's own index — so a multi-step + /// `Shared` group attributes each member's real time, not the whole thread's + /// under one name. A pool dispatch (`is_driver = false`) records nothing there. + #[test] + fn driver_dispatch_records_detached_busy_by_own_step() { + let mut graph = ChainGraph::new(); + let a = graph.register_step("SlowFinish", 1); + let sink = graph.register_step("Sink", 0); + graph.wire(a, BranchIdx(0), sink); + let steps: Vec> = + vec![Box::new(TypedStep::new(SlowFinish)), Box::new(TypedStep::new(SinkStep))]; + let contexts = Arc::new(build_chain_contexts( + &steps, + &graph, + crate::builder::InstrumentationLevel::Off, + )); + let mid = StepIdx(0); + let counter = StepDrainCounter::new(1); + let signal = PipelineSignal::new(); + + // Driver dispatch of step 0 → its busy lands on the detached line keyed + // to step 0 (not some group primary). + let stats = Arc::new(PipelineStats::new(vec!["SlowFinish", "Sink"])); + let mut entry = WorkerStepEntry::Owned { step: Box::new(TypedStep::new(SlowFinish)) }; + let _ = + dispatch_one_step(&mut entry, mid, &contexts, &counter, &signal, Some(&stats), true); + let snap = stats.snapshot(); + assert!( + snap.detached.iter().any(|&(step, name, busy, ..)| { + step == mid.0 && name == "SlowFinish" && busy > 0 + }), + "driver dispatch must record detached busy for the dispatched step itself" + ); + + // Pool dispatch (is_driver=false) records nothing on the detached line. + // A FRESH counter: the driver dispatch above consumed the first one, so + // reusing it would make `observe_drain()` return false here and skip + // `mark_outputs_drained` — the second dispatch would silently stop + // exercising the same output-close path as the first. + let counter_pool = StepDrainCounter::new(1); + let stats_pool = Arc::new(PipelineStats::new(vec!["SlowFinish", "Sink"])); + let mut entry_pool = WorkerStepEntry::Owned { step: Box::new(TypedStep::new(SlowFinish)) }; + let _ = dispatch_one_step( + &mut entry_pool, + mid, + &contexts, + &counter_pool, + &signal, + Some(&stats_pool), + false, + ); + assert!( + stats_pool.snapshot().detached.is_empty(), + "pool dispatch must not record on the off-pool detached line" + ); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/fused.rs b/crates/fgumi-pipeline-core/src/runtime/fused.rs new file mode 100644 index 000000000..7bd57cccd --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/fused.rs @@ -0,0 +1,846 @@ +//! Single-thread *fused* execution mode (issue #330). +//! +//! At `--threads 1` a forward-wired `source → … → sink` chain gains nothing from +//! the scheduled worker pool: there is one worker, so the inter-step bounded +//! queues, round-robin polling, held-slot retries, and reorder bookkeeping are +//! pure overhead (profiling showed ~2/3 of `try_run` calls do no useful work). +//! +//! This module is the structural fix. Fusion is **not** a per-command rewrite +//! — it is an execution mode of the existing pipeline. [`is_fusible_chain`] +//! detects a fusible chain (forward-wired, fan-out allowed); +//! [`run_fused_single_thread`] then drives the chain's +//! own type-erased steps inline, in topological order, over **direct** buffers +//! (built by [`build_chain_contexts_fused`]). FIFO push order is already the +//! correct order at one worker, so the reorder stage is dropped; the profile's +//! count/byte bound is kept, because the driver runs a producer before its +//! consumer and a step emitting more per `try_run` than its consumer removes +//! would otherwise grow the edge on every pass. The runtime's +//! generic wiring (`build_chain_contexts`) and dispatch (`try_run_erased`) are +//! reused verbatim — no step logic is duplicated. +//! +//! Non-linear chains (two-input `Step2` merges like zipper, multi-output splits +//! like `correct --rejects`) and `--threads ≥ 2` are not eligible and fall back +//! to the scheduled [`run_worker_loop`](super::driver::run_worker_loop). + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use super::contexts::build_chain_contexts_fused; +use super::stats::PipelineStats; +use crate::builder::InstrumentationLevel; +use crate::erased::{ErasedStep, ErasedStepCtx}; +use crate::signal::{PipelineError, PipelineSignal}; +use crate::step::StepOutcome; +use crate::topology::{BranchIdx, ChainGraph, StepIdx}; + +/// Returns `true` iff `steps` (in chain-construction order) form a chain the +/// fused driver can run on one worker: at least two steps, exactly one source +/// (at index 0), no two-input (`Step2`) merge, and every output branch wired +/// **forward** to a later step. +/// +/// "Forward-wired" allows fan-out — a step may have more than one output branch +/// (e.g. the kept/rejects split of `--rejects`) as long as each branch feeds a +/// later step. Build order equals a topological order (`append_source` then +/// `append_step`/`append_step2`, producers always before consumers), so the +/// driver can walk `steps` by index in a single pass and every producer runs +/// before its consumer. A chain may therefore have **multiple sinks** (one per +/// fan-out leaf), each a zero-branch step at some index; the only structural +/// requirement is that the last step has no unwired-forward branch, which forces +/// at least one terminal sink. +/// +/// Excluded: single-step chains (`n < 2` — nothing to fuse; the scheduled +/// single-thread path is already optimal), and any chain with a `Step2` merge +/// (two input streams — a single-worker inline drive assumes one source). Those +/// fall back to the scheduled worker pool. +#[must_use] +pub fn is_fusible_chain(steps: &[Box], graph: &ChainGraph) -> bool { + let n = steps.len(); + if n < 2 { + return false; + } + // Exactly one source, and it must be the first step. + if !steps[0].is_source() || steps[1..].iter().any(|s| s.is_source()) { + return false; + } + for i in 0..n { + let idx = StepIdx(i); + // No two-input (`Step2`) consumers: a single-worker inline drive follows + // one input stream. Sources register arity 0 (their input is implicit), + // single-input steps 1, and `Step2` 2 — so anything above 1 is a merge. + if graph.input_arity(idx) > 1 { + return false; + } + // Every output branch (one for a linear step, ≥2 for a fan-out like the + // `--rejects` split) must be wired to a strictly later step. A sink has + // zero branches, so its loop body is skipped; the last step necessarily + // has no forward target, so it must be a sink. + for b in 0..graph.branch_count(idx) { + match graph.consumer(idx, BranchIdx(b)) { + Some(StepIdx(j)) if j > i => {} + _ => return false, + } + } + } + true +} + +/// Whether `build_chain`'s fused single-thread fast path may be taken for this +/// chain. +/// +/// The fast path drives a fusible chain inline over direct buffers, skipping the +/// scheduler — and with it the per-edge instrumentation the scheduled path sets +/// up (edge [`EdgeMetrics`](super::metrics::EdgeMetrics), the occupancy sampler, +/// and the `snapshot_with_edges` bottleneck verdict). An instrumented run +/// (`InstrumentationLevel != Off`) must therefore NOT fuse, or its +/// `--pipeline-stats` / `--pipeline-trace` output would silently omit all edge / +/// occupancy data. When instrumentation is `Off` (the default), fusion is taken +/// whenever the chain is single-thread and fusible (zero-overhead fast path +/// preserved). +#[must_use] +pub fn should_fuse_single_thread( + n_threads: usize, + instrumentation: InstrumentationLevel, + steps: &[Box], + graph: &ChainGraph, +) -> bool { + n_threads == 1 && !instrumentation.is_on() && is_fusible_chain(steps, graph) +} + +/// Drive a fusible chain to completion on the calling thread, fused. +/// +/// Builds the chain's per-step contexts with direct inter-step transports (no +/// reorder stage, profile queue bounds retained — see +/// [`ErasedStep::build_fused_output_set`]), then repeatedly walks the steps in +/// topological order — popping +/// from each step's input, pushing to its output(s) — until **every** step has +/// reported [`StepOutcome::Finished`]. On a step's `Finished` the driver marks +/// all its output branches drained so downstream steps see their inputs closed +/// (the same drain propagation the scheduled driver performs, minus the +/// `Parallel` counter gate — there is exactly one instance of each step at one +/// worker). Waiting for *all* steps (not just the last) is what lets a fan-out +/// chain (e.g. the `--rejects` split) finish both its sink subchains. +/// +/// Callers must have confirmed [`is_fusible_chain`] first. +/// +/// `queue_memory_total` is the run's `--queue-memory-total`, if set. Because the +/// fused contexts are built here rather than by the caller, this function is the +/// only place that budget can be applied to the fused transports. +/// +/// `deadlock_timeout_secs` is the run's stall patience. A pass in which no step +/// progresses is retried with a backoff until this budget is exhausted, and only +/// then reported as a stall; `0` selects a built-in default rather than +/// "unbounded" (see the constant's comment for why). +/// +/// # Errors +/// +/// Returns [`PipelineError::Io`] if any step's `try_run` returned `Err` (the +/// first such error wins, carrying the originating step's name), or +/// [`PipelineError::Cancelled`] if the run was cancelled via the pipeline's +/// [`CancelHandle`](crate::signal::CancelHandle) — matching +/// [`crate::builder::Pipeline::run`]'s contract. +pub fn run_fused_single_thread( + mut steps: Vec>, + graph: &ChainGraph, + signal: &Arc, + stats: Option<&Arc>, + queue_memory_total: Option, + deadlock_timeout_secs: u64, +) -> Result<(), PipelineError> { + // A single no-progress pass is NOT a stall. `NoProgress` is the transient + // "input momentarily empty but not drained" outcome — a source waiting on a + // background reader thread returns it legitimately — so failing on the first + // idle pass aborts a healthy run and truncates its output. Tolerate idling + // until this wall-clock budget is exhausted, which is what the scheduled path + // does via `WorkerCore::sleep_backoff`. That path has no stall limit at all + // because the deadlock monitor catches wedges for it; the fused path is not + // monitored, so it needs its own bound rather than hanging forever. + // + // Budgeted by TIME, not by a pass count: `thread::sleep` granularity varies by + // platform (a requested 50µs can round up to ~1ms), so a fixed number of + // passes would mean wildly different real budgets across hosts. + // + // The budget comes from `PipelineConfig::deadlock_timeout_secs`, so a chain + // with a genuinely slow source (a network-backed reader, say) can raise it + // instead of having a library-chosen default abort a legitimate run. + // + // `deadlock_timeout_secs == 0` — the config default, meaning "monitor + // disarmed" on the scheduled path — falls back to `DEFAULT_STALL_BUDGET` here + // rather than meaning "unbounded". That asymmetry is deliberate and is the + // whole reason this bound exists: the scheduled path has a deadlock monitor to + // arm, and this one does not, so disabling the bound would leave a fused wedge + // with nothing at all to detect it. A caller wanting more patience raises the + // number; there is deliberately no way to remove the bound. + // Deliberately generous. This bound exists to catch a PERMANENT wedge, not to + // police a slow source, and it is the value most runs get (`deadlock_timeout_secs` + // defaults to 0). A source whose background reader blocks on a cold page cache or + // network-backed input can legitimately idle for tens of seconds, and the idle + // timer only resets on progress — so a tight default trades a real risk of failing + // a healthy run for a few seconds off the report of a wedge that has already hung. + // The costs are asymmetric; err long. + const DEFAULT_STALL_BUDGET: Duration = Duration::from_secs(60); + // Long enough to stop pegging the calling thread at 100%, short enough that it + // adds no meaningful latency to a source that is about to produce. + const IDLE_BACKOFF: Duration = Duration::from_micros(50); + + let stall_budget = if deadlock_timeout_secs > 0 { + Duration::from_secs(deadlock_timeout_secs) + } else { + DEFAULT_STALL_BUDGET + }; + + let n = steps.len(); + let contexts = build_chain_contexts_fused(&steps, graph); + // Honour `--queue-memory-total` here as the scheduled path does. The fused + // transports keep each step's profiled byte bound (see + // `ErasedStep::build_fused_output_set`), so without this the user's budget + // would be silently ignored in favour of the per-step defaults. These + // contexts are local to this call, so the budget cannot be applied by + // `Pipeline::run` on its behalf. + if let Some(total) = queue_memory_total { + crate::builder::apply_initial_queue_budget(&contexts.bounded_queues, total); + } + let mut finished = vec![false; n]; + // `Some(t)` while the driver has been idle since `t`; cleared by any pass that + // makes progress. + let mut idle_since: Option = None; + + 'drive: loop { + // Bail promptly on an external cancel or a prior-pass error. + if signal.is_done() { + break; + } + let mut progressed = false; + for i in 0..n { + if finished[i] { + continue; + } + let outputs_any = contexts.outputs[i].as_ref(); + let mut ctx = + ErasedStepCtx { input: contexts.inputs[i].as_ref(), outputs: outputs_any, signal }; + + // Time the dispatch only when stats collection is on (mirrors + // `dispatch_one_step`): `Instant::now()` is non-trivial on the hot + // path, so gate it on `stats.is_some()`. + let start = stats.map(|_| Instant::now()); + let result = steps[i].try_run_erased(&mut ctx); + if let (Some(stats), Some(start)) = (stats, start) { + let elapsed_ns = u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX); + let start_ns = stats.elapsed_ns().saturating_sub(elapsed_ns); + match &result { + Ok(outcome) => stats.record(StepIdx(i), *outcome, start_ns, elapsed_ns), + Err(_) => stats.record_error(StepIdx(i), start_ns, elapsed_ns), + } + } + + match result { + Ok(StepOutcome::Progress) => progressed = true, + Ok(StepOutcome::Finished) => { + // Close this step's output branches so downstream drains. + // No `StepDrainCounter` gate: one instance per step here. + steps[i].mark_outputs_drained(outputs_any); + finished[i] = true; + progressed = true; + } + Ok(StepOutcome::NoProgress | StepOutcome::Contention) => {} + Err(io_err) => { + signal + .record_error(PipelineError::Io { step: steps[i].name(), source: io_err }); + break 'drive; + } + } + } + // Done when every step has finished. For a fan-out chain (`--rejects`) + // that means BOTH sink subchains have drained — checking only the last + // step would break before an earlier-indexed reject sink had flushed. + if finished.iter().all(|&f| f) { + break; + } + if progressed { + idle_since = None; + } else { + // Idle pass. Back off and retry the same state until the budget runs + // out — only a sustained run of idle passes is evidence of a wedge. + let idle_start = *idle_since.get_or_insert_with(Instant::now); + if idle_start.elapsed() < stall_budget { + std::thread::sleep(IDLE_BACKOFF); + continue 'drive; + } + // Budget exhausted. For a well-formed fusible chain this is + // unreachable: the source either progresses or finishes, and + // `Finished` cascades drain downstream so some step always advances + // until every sink closes. Guard against an infinite spin rather than + // trusting that invariant blindly. + debug_assert!( + false, + "fused single-thread driver stalled: no step progressed for {stall_budget:?} \ + and not all steps have finished" + ); + // In release builds the `debug_assert!` is compiled out, so record + // the stall as an error before breaking. Otherwise the loop would + // exit and map to `Ok`, silently truncating the output instead of + // surfacing the broken invariant. Name the still-unfinished steps so + // the error points at the wedged step(s) rather than just the + // synthetic "fused-driver" name. + let stalled_steps: Vec<&'static str> = finished + .iter() + .enumerate() + .filter(|(_, f)| !**f) + .map(|(i, _)| steps[i].name()) + .collect(); + signal.record_error(PipelineError::Io { + step: "fused-driver", + source: std::io::Error::other(format!( + "fused single-thread driver stalled: no step progressed for {stall_budget:?} \ + and not all steps have finished; unfinished step(s): {}", + stalled_steps.join(", "), + )), + }); + break 'drive; + } + } + + // Map the recorded outcome to the run result (same shape as `Pipeline::run`). + // `to_result` reconstructs the non-`Clone` `PipelineError` and synthesizes + // `Cancelled` from the state when an external cancel published the terminal + // state but its `OnceLock` payload is not yet visible to this thread. + signal.to_result() +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::{Arc, Mutex}; + + use rstest::rstest; + + use super::*; + use crate::erased::TypedStep; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{Step, StepCtx, StepKind, StepProfile}; + + // ── Stub steps: source → +100 mid → collecting sink ────────────────── + + /// Source emitting `0, 1, …, count-1` then `Finished`. + struct CountSource { + next: u32, + count: u32, + } + impl Step for CountSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "CountSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::Unbounded], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.next >= self.count { + return Ok(StepOutcome::Finished); + } + let _ = ctx.outputs.push(self.next); + self.next += 1; + Ok(StepOutcome::Progress) + } + } + + /// Mid: pops a `u32`, pushes `+100`; `Finished` once its input is drained. + #[derive(Clone)] + struct AddHundred; + impl Step for AddHundred { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "AddHundred", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::Unbounded], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(v) => { + let _ = ctx.outputs.push(v + 100); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// Sink: collects popped values into a shared `Vec`; `Finished` on drain. + struct CollectSink { + out: Arc>>, + } + impl Step for CollectSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "CollectSink", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(v) => { + self.out.lock().unwrap().push(v); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// Build a linear `source → mid → sink` chain (graph + boxed steps). + fn linear_chain( + count: u32, + out: &Arc>>, + ) -> (Vec>, ChainGraph) { + let mut graph = ChainGraph::new(); + let s = graph.register_step("CountSource", 1); + let m = graph.register_step("AddHundred", 1); + let k = graph.register_step("CollectSink", 0); + graph.wire(s, BranchIdx(0), m); + graph.wire(m, BranchIdx(0), k); + let steps: Vec> = vec![ + Box::new(TypedStep::new(CountSource { next: 0, count })), + Box::new(TypedStep::new(AddHundred)), + Box::new(TypedStep::new(CollectSink { out: Arc::clone(out) })), + ]; + (steps, graph) + } + + /// Fan-out split: routes even values to branch 0, odd to branch 1 — the + /// shape of a `--rejects` kept/rejects split. + struct EvenOddSplit; + impl Step for EvenOddSplit { + type Input = u32; + type Outputs = (u32, u32); + fn profile(&self) -> StepProfile { + StepProfile { + name: "EvenOddSplit", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::Unbounded, QueueSpec::Unbounded], + branch_ordering: vec![BranchOrdering::None, BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(v) => { + let view = ctx.outputs.view(); + if v % 2 == 0 { + let _ = view.a.push(v); + } else { + let _ = view.b.push(v); + } + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + /// Build a fan-out `source → split → (even sink, odd sink)` chain. + fn fan_out_chain( + count: u32, + even: &Arc>>, + odd: &Arc>>, + ) -> (Vec>, ChainGraph) { + let mut graph = ChainGraph::new(); + let s = graph.register_step("CountSource", 1); + let m = graph.register_step("EvenOddSplit", 2); + let k0 = graph.register_step("CollectSink", 0); + let k1 = graph.register_step("CollectSink", 0); + graph.wire(s, BranchIdx(0), m); + graph.wire(m, BranchIdx(0), k0); + graph.wire(m, BranchIdx(1), k1); + let steps: Vec> = vec![ + Box::new(TypedStep::new(CountSource { next: 0, count })), + Box::new(TypedStep::new(EvenOddSplit)), + Box::new(TypedStep::new(CollectSink { out: Arc::clone(even) })), + Box::new(TypedStep::new(CollectSink { out: Arc::clone(odd) })), + ]; + (steps, graph) + } + + #[test] + fn is_fusible_detects_source_mid_sink() { + let out = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = linear_chain(3, &out); + assert!(is_fusible_chain(&steps, &graph)); + } + + #[test] + fn is_fusible_rejects_single_step() { + // A self-contained source+sink (Input=(), no output branches) is one + // step — nothing to fuse, so not eligible. + let mut graph = ChainGraph::new(); + graph.register_step("CountSource", 0); + let steps: Vec> = + vec![Box::new(TypedStep::new(CountSource { next: 0, count: 0 }))]; + assert!(!is_fusible_chain(&steps, &graph)); + } + + #[test] + fn is_fusible_rejects_empty() { + let steps: Vec> = vec![]; + let graph = ChainGraph::new(); + assert!(!is_fusible_chain(&steps, &graph)); + } + + #[test] + fn is_fusible_accepts_fan_out() { + // A fan-out (the `--rejects` shape: one step with two output branches, + // each wired forward to its own sink) IS fusible — both branches feed + // strictly later steps. + let even = Arc::new(Mutex::new(Vec::new())); + let odd = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = fan_out_chain(0, &even, &odd); + assert!(is_fusible_chain(&steps, &graph)); + } + + // A fusible chain fuses ONLY when it is single-thread AND uninstrumented: the + // fused fast path skips the scheduled path's edge metrics / occupancy sampler + // / bottleneck verdict, so any instrumentation level (or ≥2 threads) must fall + // through to the scheduled path instead of silently dropping that output. + #[rstest] + #[case::off_single_thread(1, InstrumentationLevel::Off, true)] + #[case::summary_single_thread(1, InstrumentationLevel::Summary, false)] + #[case::timeline_single_thread(1, InstrumentationLevel::Timeline, false)] + #[case::deep_single_thread(1, InstrumentationLevel::Deep, false)] + #[case::off_multi_thread(2, InstrumentationLevel::Off, false)] + fn should_fuse_only_when_uninstrumented_single_thread( + #[case] n_threads: usize, + #[case] level: InstrumentationLevel, + #[case] expected: bool, + ) { + let out = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = linear_chain(4, &out); + assert!(is_fusible_chain(&steps, &graph), "linear chain is fusible"); + assert_eq!(should_fuse_single_thread(n_threads, level, &steps, &graph), expected); + } + + #[test] + fn drive_runs_chain_to_completion_in_order() { + let out = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = linear_chain(5, &out); + let signal = PipelineSignal::new(); + run_fused_single_thread(steps, &graph, &signal, None, None, 0).expect("clean run"); + // Source emits 0..5, mid adds 100, sink collects in FIFO order. + assert_eq!(*out.lock().unwrap(), vec![100, 101, 102, 103, 104]); + } + + /// Source with a *byte-bounded* output, so a fused chain built from it + /// registers a real bounded transport. Emits `0, 1, …, count-1`, holding and + /// retrying whatever the transport rejects — the contract every step already + /// owes the scheduled path. + struct ByteBoundedSource { + next: u32, + count: u32, + held: Option, + } + impl Step for ByteBoundedSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "ByteBoundedSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { limit_bytes: PROFILE_LIMIT_BYTES }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if let Some(item) = self.held.take() { + if let Err(unpushed) = ctx.outputs.push(item) { + self.held = Some(unpushed.into_item()); + } + return Ok(StepOutcome::Progress); + } + if self.next >= self.count { + return Ok(StepOutcome::Finished); + } + let item = self.next; + self.next += 1; + if let Err(unpushed) = ctx.outputs.push(item) { + self.held = Some(unpushed.into_item()); + } + Ok(StepOutcome::Progress) + } + } + + /// Deliberately unusual so the assertions below cannot pass by matching a + /// default or a budget-derived value. + const PROFILE_LIMIT_BYTES: u64 = 7 * 1024 * 1024; + + fn byte_bounded_chain( + count: u32, + out: &Arc>>, + ) -> (Vec>, ChainGraph) { + let mut graph = ChainGraph::new(); + let src = graph.register_step("ByteBoundedSource", 1); + let sink = graph.register_step("CollectSink", 0); + graph.wire(src, BranchIdx(0), sink); + let steps: Vec> = vec![ + Box::new(TypedStep::new(ByteBoundedSource { next: 0, count, held: None })), + Box::new(TypedStep::new(CollectSink { out: Arc::clone(out) })), + ]; + (steps, graph) + } + + /// The fused path must honour `--queue-memory-total`, as the scheduled path + /// does. Two facts are load-bearing and both were false before fused + /// transports kept their profile bounds: the fused contexts register a + /// bounded queue at all, and the budget resizes it off the profile default. + #[test] + fn fused_contexts_register_bounded_queues_and_take_the_budget() { + let out = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = byte_bounded_chain(4, &out); + let contexts = build_chain_contexts_fused(&steps, &graph); + + assert_eq!( + contexts.bounded_queues.len(), + 1, + "a byte-bounded fused edge must be registered, else there is nothing \ + for the queue-memory budget to apply to" + ); + assert_eq!( + contexts.bounded_queues[0].handle.limit_bytes(), + PROFILE_LIMIT_BYTES, + "the fused transport starts at the profile's declared bound" + ); + + // 64 MiB over one queue → 64 MiB per queue, well above the 1 MiB floor, + // and distinct from the profile default so the assert cannot pass by + // accident. + let total = 64 * 1024 * 1024; + crate::builder::apply_initial_queue_budget(&contexts.bounded_queues, total); + assert_eq!( + contexts.bounded_queues[0].handle.limit_bytes(), + total, + "the user's budget must override the per-step default on the fused path" + ); + } + + /// End-to-end: a byte-bounded fused chain run with a budget still delivers + /// every item in order. Pins that threading the budget through does not wedge + /// the driver — a bounded fused edge relies on the producer's hold-and-retry. + #[rstest] + #[case::no_budget(None)] + #[case::with_budget(Some(2 * 1024 * 1024))] + fn fused_byte_bounded_chain_completes(#[case] queue_memory_total: Option) { + let out = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = byte_bounded_chain(5, &out); + let signal = PipelineSignal::new(); + run_fused_single_thread(steps, &graph, &signal, None, queue_memory_total, 0) + .expect("clean run"); + assert_eq!(*out.lock().unwrap(), vec![0, 1, 2, 3, 4]); + } + + /// Source that reports `NoProgress` for its first few dispatches before it + /// starts emitting — the shape of a source waiting on a background reader. + struct IdleThenEmitSource { + idle_left: u32, + remaining: u32, + dispatches: Arc>, + } + impl Step for IdleThenEmitSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "IdleThenEmitSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + *self.dispatches.lock().unwrap() += 1; + if self.idle_left > 0 { + self.idle_left -= 1; + // Legitimate transient: nothing to hand over *yet*. + return Ok(StepOutcome::NoProgress); + } + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + let item = self.remaining; + self.remaining -= 1; + let _ = ctx.outputs.push(item); + Ok(StepOutcome::Progress) + } + } + + /// A pass in which no step progressed is not a stall — `NoProgress` is the + /// transient "input momentarily empty but not drained" outcome. The driver + /// used to fail the run on the FIRST such pass, recording `PipelineError::Io` + /// and truncating the output (and panicking through the `debug_assert!` in + /// test builds). It must back off and retry instead, so a source that idles + /// before producing still completes. + #[test] + fn drive_tolerates_transient_no_progress_passes() { + const IDLE_PASSES: u32 = 5; + const N_ITEMS: u32 = 3; + + let mut graph = ChainGraph::new(); + let src = graph.register_step("IdleThenEmitSource", 1); + let sink = graph.register_step("CollectSink", 0); + graph.wire(src, BranchIdx(0), sink); + + let out = Arc::new(Mutex::new(Vec::new())); + let dispatches = Arc::new(Mutex::new(0)); + let steps: Vec> = vec![ + Box::new(TypedStep::new(IdleThenEmitSource { + idle_left: IDLE_PASSES, + remaining: N_ITEMS, + dispatches: Arc::clone(&dispatches), + })), + Box::new(TypedStep::new(CollectSink { out: Arc::clone(&out) })), + ]; + let signal = PipelineSignal::new(); + run_fused_single_thread(steps, &graph, &signal, None, None, 0) + .expect("transient NoProgress must not fail the run"); + + assert_eq!( + *out.lock().unwrap(), + vec![3, 2, 1], + "every item must still be delivered after the idle passes" + ); + assert!(!signal.is_done(), "no error recorded for a transient idle pass"); + assert!( + *dispatches.lock().unwrap() > IDLE_PASSES, + "the driver must have retried past the idle passes, not given up on the first" + ); + } + + /// Source that never progresses and never finishes — a permanent wedge, the + /// case the stall bound exists for. + struct WedgedSource; + impl Step for WedgedSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "WedgedSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// The stall budget is `deadlock_timeout_secs`, and `0` selects the built-in + /// default rather than "unbounded" — a fused wedge has no deadlock monitor to + /// fall back on, so the bound must not be disableable. A 1-second budget is + /// used here to keep the test fast; the assertion is that the run fails inside + /// its own budget and names the wedged step. + /// + /// `#[should_panic]` rather than an error assertion: the `debug_assert!` on the + /// stall path fires first in a test build, which is itself the contract (a + /// genuine wedge must be loud in tests). + #[test] + #[should_panic(expected = "fused single-thread driver stalled")] + fn drive_reports_a_real_stall_within_the_configured_budget() { + let mut graph = ChainGraph::new(); + let src = graph.register_step("WedgedSource", 1); + let sink = graph.register_step("CollectSink", 0); + graph.wire(src, BranchIdx(0), sink); + + let out = Arc::new(Mutex::new(Vec::new())); + let steps: Vec> = vec![ + Box::new(TypedStep::new(WedgedSource)), + Box::new(TypedStep::new(CollectSink { out })), + ]; + let signal = PipelineSignal::new(); + let started = Instant::now(); + let result = run_fused_single_thread(steps, &graph, &signal, None, None, 1); + // Only reached in a release test build, where the `debug_assert!` is gone. + assert!(result.is_err(), "a permanent wedge must not report success"); + assert!( + started.elapsed() < Duration::from_secs(30), + "the configured 1s budget must bound the wait, not the default" + ); + } + + #[test] + fn drive_runs_fan_out_to_completion() { + // Both sink subchains of a fan-out must drain — the driver waits for ALL + // steps to finish, not just the last-indexed one. + let even = Arc::new(Mutex::new(Vec::new())); + let odd = Arc::new(Mutex::new(Vec::new())); + let (steps, graph) = fan_out_chain(6, &even, &odd); + let signal = PipelineSignal::new(); + run_fused_single_thread(steps, &graph, &signal, None, None, 0).expect("clean run"); + // Source emits 0..6; evens route to branch 0, odds to branch 1. + assert_eq!(*even.lock().unwrap(), vec![0, 2, 4]); + assert_eq!(*odd.lock().unwrap(), vec![1, 3, 5]); + } + + #[test] + fn drive_propagates_step_error() { + /// Mid that errors on the first item. + struct Boom; + impl Step for Boom { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Boom", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::Unbounded], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if ctx.input.pop().is_some() { + return Err(io::Error::other("boom")); + } + if ctx.input.is_drained() { + Ok(StepOutcome::Finished) + } else { + Ok(StepOutcome::NoProgress) + } + } + } + + let out = Arc::new(Mutex::new(Vec::new())); + let mut graph = ChainGraph::new(); + let s = graph.register_step("CountSource", 1); + let m = graph.register_step("Boom", 1); + let k = graph.register_step("CollectSink", 0); + graph.wire(s, BranchIdx(0), m); + graph.wire(m, BranchIdx(0), k); + let steps: Vec> = vec![ + Box::new(TypedStep::new(CountSource { next: 0, count: 3 })), + Box::new(TypedStep::new(Boom)), + Box::new(TypedStep::new(CollectSink { out })), + ]; + let signal = PipelineSignal::new(); + let err = + run_fused_single_thread(steps, &graph, &signal, None, None, 0).expect_err("must error"); + assert!(matches!(err, PipelineError::Io { step: "Boom", .. })); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/live.rs b/crates/fgumi-pipeline-core/src/runtime/live.rs new file mode 100644 index 000000000..4600e3b8b --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/live.rs @@ -0,0 +1,154 @@ +//! `LiveSteps`: per-worker worklist of still-dispatchable steps. +//! +//! Each worker dispatches steps by walking a `Vec` of the steps it +//! still has work for, in chain order. A step is **removed** from this list +//! when it returns `StepOutcome::Finished`. The worker exits once the list is +//! empty. +//! +//! This replaces the previous scheme of mutating each finished step's +//! `WorkerStepEntry` to `WorkerStepEntry::Skip` in place and re-scanning the +//! full `entries` vec every pass (`entries.iter().all(|e| !e.is_dispatchable())` +//! for the exit check, plus a per-pass `continue` over every inert `Skip` +//! slot). The worklist: +//! +//! - never re-visits a finished or build-time-excluded step, +//! - makes "done" an honest removal rather than an in-place inert variant, and +//! - keeps `StepIdx` stable as the canonical identity into the parallel +//! `entries` / `contexts.inputs` / `contexts.outputs` / `drain_counters` +//! arrays (the worklist holds indices *into* that stable storage; it does +//! not renumber anything). +//! +//! Removal is **stable** (`Vec::remove`, not `swap_remove`): the chain-order +//! invariant the round-robin dispatch relies on (attempt upstream steps before +//! downstream, restart from the front on `Progress`) must survive a removal. At +//! the handful of steps in a chain the linear `remove`/`position` cost is +//! irrelevant. + +use crate::runtime::storage::WorkerStepEntry; +use crate::topology::StepIdx; + +/// One per worker: the steps this worker can still dispatch, in chain order. +pub struct LiveSteps { + order: Vec, +} + +impl LiveSteps { + /// Build from this worker's storage. Every dispatchable entry (i.e. not a + /// build-time `WorkerStepEntry::Skip` placeholder for an Exclusive step + /// owned by another worker, or a Serial step this worker's affinity gates + /// out) enters the worklist, in `StepIdx` (chain) order. + #[must_use] + pub fn from_entries(entries: &[WorkerStepEntry]) -> Self { + let order = entries + .iter() + .enumerate() + .filter(|(_, entry)| entry.is_dispatchable()) + .map(|(idx, _)| StepIdx(idx)) + .collect(); + Self { order } + } + + /// The steps this worker can still dispatch, in chain order. + #[must_use] + pub fn order(&self) -> &[StepIdx] { + &self.order + } + + /// Number of still-dispatchable steps. + #[must_use] + pub fn len(&self) -> usize { + self.order.len() + } + + /// `true` once this worker has nothing left to dispatch (loop exit). + #[must_use] + pub fn is_empty(&self) -> bool { + self.order.is_empty() + } + + /// `true` if `step_idx` is still dispatchable by this worker. + #[must_use] + pub fn contains(&self, step_idx: StepIdx) -> bool { + self.order.contains(&step_idx) + } + + /// Stably remove a finished step. No-op if already absent (idempotent). + /// Preserves the relative order of the remaining steps. + pub fn remove(&mut self, step_idx: StepIdx) { + if let Some(pos) = self.order.iter().position(|&s| s == step_idx) { + self.order.remove(pos); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use parking_lot::Mutex; + + use crate::erased::{ErasedStep, TypedStep}; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::runtime::storage::DrainGate; + use crate::step::{Step, StepCtx, StepKind, StepOutcome, StepProfile}; + + #[derive(Clone)] + struct Nop; + impl Step for Nop { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Nop", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> std::io::Result { + Ok(StepOutcome::NoProgress) + } + } + + fn shared_entry() -> WorkerStepEntry { + let step: Box = Box::new(TypedStep::new(Nop)); + WorkerStepEntry::Shared { + step: Arc::new(Mutex::new(step)), + drain: Arc::new(DrainGate::default()), + } + } + + #[test] + fn from_entries_excludes_build_time_skips() { + let entries = + vec![shared_entry(), WorkerStepEntry::Skip, shared_entry(), WorkerStepEntry::Skip]; + let live = LiveSteps::from_entries(&entries); + assert_eq!(live.order(), &[StepIdx(0), StepIdx(2)]); + assert!(!live.is_empty()); + assert_eq!(live.len(), 2); + } + + #[test] + fn remove_is_stable_and_idempotent() { + let entries = vec![shared_entry(), shared_entry(), shared_entry()]; + let mut live = LiveSteps::from_entries(&entries); + assert_eq!(live.order(), &[StepIdx(0), StepIdx(1), StepIdx(2)]); + + // Remove the middle; the survivors keep their relative order. + live.remove(StepIdx(1)); + assert_eq!(live.order(), &[StepIdx(0), StepIdx(2)]); + assert!(!live.contains(StepIdx(1))); + + // Removing an already-absent step is a no-op. + live.remove(StepIdx(1)); + assert_eq!(live.order(), &[StepIdx(0), StepIdx(2)]); + + live.remove(StepIdx(0)); + live.remove(StepIdx(2)); + assert!(live.is_empty()); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/metrics.rs b/crates/fgumi-pipeline-core/src/runtime/metrics.rs new file mode 100644 index 000000000..7d4981380 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/metrics.rs @@ -0,0 +1,308 @@ +//! Per-edge instrumentation counters + occupancy histogram. +//! +//! An [`EdgeMetrics`] is the edge-side complement of the step-side +//! [`PipelineStats`](super::stats::PipelineStats): one per instrumented queue +//! edge, shared (`Arc`) between the producer's transport (push/reject counts) +//! and the consumer's input handle (pop/empty counts), and sampled periodically +//! for occupancy. All counters are `Relaxed` atomics — these are statistics, not +//! synchronization (staleness is fine), mirroring `ByteBoundedQueue::current_bytes`. +//! +//! What the histogram alone can classify is [`RawOccupancy`]; the richer +//! `Empty`-vs-`Starved` / `Full`-vs-`Backpressured` split needs the reject/empty +//! *rates* and is done by the renderer, not here. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Number of occupancy histogram buckets (depth fraction `0.0..=1.0` split into +/// `OCCUPANCY_BUCKETS` equal bands; the top band captures exactly-full). +pub const OCCUPANCY_BUCKETS: usize = 8; + +/// Per-edge counters + occupancy histogram. Construct with [`EdgeMetrics::new`] +/// (returns an `Arc` so the producer transport and consumer input handle share +/// one instance). All methods are lock-free `Relaxed` atomic updates. +#[derive(Debug)] +pub struct EdgeMetrics { + /// Items the producer successfully pushed. + pushed_items: AtomicU64, + /// Bytes pushed (only meaningful for byte-bounded edges; `0` otherwise). + pushed_bytes: AtomicU64, + /// Items the consumer popped. + popped_items: AtomicU64, + /// Bytes popped (byte-bounded edges only). + popped_bytes: AtomicU64, + /// `try_push` rejections — backpressure events (producer wanted to push, the + /// edge was full). + push_rejections: AtomicU64, + /// Consumer `pop` on an empty edge — starvation events. + pop_empties: AtomicU64, + /// Number of occupancy samples taken (sampler ticks). + depth_samples: AtomicU64, + /// Histogram of occupancy fraction at sample time. + occupancy_buckets: [AtomicU64; OCCUPANCY_BUCKETS], + /// Σ of `depth_fraction * 1000` over all samples, for the mean. + occupancy_sum_milli: AtomicU64, + /// Σ of raw occupied **bytes** over all samples. Kept alongside the fraction + /// sum so the mean occupancy can be reported in absolute bytes — which, + /// unlike `fraction × final_limit`, stays correct when the byte limit is + /// rebalanced at runtime (`queue_memory_total`). For an ordered edge the + /// sampled bytes include the `ReorderStage` overflow stash. + occupancy_bytes_sum: AtomicU64, +} + +impl EdgeMetrics { + /// Construct a fresh metrics instance, shared via `Arc`. + #[must_use] + pub fn new() -> Arc { + Arc::new(Self { + pushed_items: AtomicU64::new(0), + pushed_bytes: AtomicU64::new(0), + popped_items: AtomicU64::new(0), + popped_bytes: AtomicU64::new(0), + push_rejections: AtomicU64::new(0), + pop_empties: AtomicU64::new(0), + depth_samples: AtomicU64::new(0), + occupancy_buckets: std::array::from_fn(|_| AtomicU64::new(0)), + occupancy_sum_milli: AtomicU64::new(0), + occupancy_bytes_sum: AtomicU64::new(0), + }) + } + + /// Record a successful producer push of `bytes` (pass `0` for count-bounded + /// / unbounded edges that don't track bytes). + pub fn record_push(&self, bytes: u64) { + self.pushed_items.fetch_add(1, Ordering::Relaxed); + self.pushed_bytes.fetch_add(bytes, Ordering::Relaxed); + } + + /// Record a successful consumer pop of `bytes`. + pub fn record_pop(&self, bytes: u64) { + self.popped_items.fetch_add(1, Ordering::Relaxed); + self.popped_bytes.fetch_add(bytes, Ordering::Relaxed); + } + + /// Record a `try_push` rejection (backpressure). + pub fn record_reject(&self) { + self.push_rejections.fetch_add(1, Ordering::Relaxed); + } + + /// Record a consumer pop that found the edge empty (starvation). + pub fn record_empty(&self) { + self.pop_empties.fetch_add(1, Ordering::Relaxed); + } + + /// Record one occupancy sample from the edge's current `occupied_bytes` and + /// its byte `limit_bytes`. The histogram bucket and fraction use + /// `occupied_bytes / limit_bytes` clamped to `0.0..=1.0`; the **raw** + /// `occupied_bytes` (un-clamped — so it stays truthful when an ordered + /// edge's reorder stash pushes total buffered bytes past the transport + /// limit) is summed for the byte-accurate mean the latency estimate uses. A + /// `limit_bytes` of `0` is treated as fraction `0.0` defensively (the + /// sampler already skips count/unbounded edges). + // Casts are bounded statistics: `f ∈ [0,1]` so `f * BUCKETS` ∈ [0,8] and + // `f * 1000` ∈ [0,1000] — no meaningful truncation/sign loss/precision loss. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] + pub fn record_depth(&self, occupied_bytes: u64, limit_bytes: u64) { + let f = if limit_bytes == 0 { + 0.0 + } else { + (occupied_bytes as f32 / limit_bytes as f32).clamp(0.0, 1.0) + }; + let bucket = ((f * OCCUPANCY_BUCKETS as f32) as usize).min(OCCUPANCY_BUCKETS - 1); + self.occupancy_buckets[bucket].fetch_add(1, Ordering::Relaxed); + self.occupancy_sum_milli.fetch_add((f * 1000.0) as u64, Ordering::Relaxed); + self.occupancy_bytes_sum.fetch_add(occupied_bytes, Ordering::Relaxed); + self.depth_samples.fetch_add(1, Ordering::Relaxed); + } + + /// Take a consistent-enough point-in-time snapshot of the counters. + // Mean is a display statistic; u64→f32 precision loss past 2^23 samples is + // irrelevant to a 0..1 occupancy mean. + #[allow(clippy::cast_precision_loss)] + #[must_use] + pub fn snapshot(&self) -> EdgeMetricsSnapshot { + let buckets: [u64; OCCUPANCY_BUCKETS] = + std::array::from_fn(|i| self.occupancy_buckets[i].load(Ordering::Relaxed)); + let depth_samples = self.depth_samples.load(Ordering::Relaxed); + let mean_occupancy = if depth_samples == 0 { + 0.0 + } else { + (self.occupancy_sum_milli.load(Ordering::Relaxed) as f32 / 1000.0) + / depth_samples as f32 + }; + let mean_occupancy_bytes = if depth_samples == 0 { + 0.0 + } else { + self.occupancy_bytes_sum.load(Ordering::Relaxed) as f64 / depth_samples as f64 + }; + EdgeMetricsSnapshot { + pushed_items: self.pushed_items.load(Ordering::Relaxed), + pushed_bytes: self.pushed_bytes.load(Ordering::Relaxed), + popped_items: self.popped_items.load(Ordering::Relaxed), + popped_bytes: self.popped_bytes.load(Ordering::Relaxed), + push_rejections: self.push_rejections.load(Ordering::Relaxed), + pop_empties: self.pop_empties.load(Ordering::Relaxed), + depth_samples, + raw_occupancy: RawOccupancy::from_buckets(&buckets, depth_samples), + mean_occupancy, + mean_occupancy_bytes, + } + } +} + +/// A point-in-time read of an [`EdgeMetrics`]. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct EdgeMetricsSnapshot { + /// Items successfully pushed onto the edge by the producer. + pub pushed_items: u64, + /// Bytes successfully pushed onto the edge (0 for count/unbounded edges). + pub pushed_bytes: u64, + /// Items successfully popped off the edge by the consumer. + pub popped_items: u64, + /// Bytes successfully popped off the edge (0 for count/unbounded edges). + pub popped_bytes: u64, + /// Push attempts rejected by backpressure (edge at its limit). + pub push_rejections: u64, + /// Pop attempts that found the edge empty (starvation signal). + pub pop_empties: u64, + /// Number of occupancy samples the background sampler took on this edge. + pub depth_samples: u64, + /// What the occupancy histogram alone says (no rate-based refinement). + pub raw_occupancy: RawOccupancy, + /// Mean occupancy fraction `0.0..=1.0` over all samples (`0.0` if none). + pub mean_occupancy: f32, + /// Mean occupied **bytes** over all samples (`0.0` if none). Unlike + /// `mean_occupancy` (a fraction that must be multiplied by a limit to + /// recover bytes), this is measured directly at sample time, so the + /// Little's-Law latency estimate stays correct even when `queue_memory_total` + /// rebalances the byte limit mid-run. For an ordered edge it includes the + /// `ReorderStage` overflow stash. + pub mean_occupancy_bytes: f64, +} + +/// Occupancy classification derivable from the histogram **alone** (no +/// reject/empty rates). The renderer refines `MostlyEmpty`→`Empty`/`Starved` +/// and `MostlyFull`→`Full`/`Backpressured` using the rates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RawOccupancy { + /// Insufficient samples to classify. + Unknown, + /// ≥80% of samples in the bottom bucket. + MostlyEmpty, + /// ≥80% of samples in the top bucket. + MostlyFull, + /// Bottom and top buckets each hold >25% of samples (bursty coupling). + Bimodal, + /// Spread across the middle — neither side bound. + Healthy, +} + +impl RawOccupancy { + /// Classify from the bucket histogram and total sample count. + // Ratios are display statistics; u64→f64 precision loss is irrelevant to the + // 0.25/0.80 thresholds. + #[allow(clippy::cast_precision_loss)] + #[must_use] + pub fn from_buckets(buckets: &[u64; OCCUPANCY_BUCKETS], samples: u64) -> Self { + if samples == 0 { + return Self::Unknown; + } + let s = samples as f64; + let bottom = buckets[0] as f64 / s; + let top = buckets[OCCUPANCY_BUCKETS - 1] as f64 / s; + if bottom >= 0.80 { + Self::MostlyEmpty + } else if top >= 0.80 { + Self::MostlyFull + } else if bottom > 0.25 && top > 0.25 { + Self::Bimodal + } else { + Self::Healthy + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[test] + fn edge_metrics_counts_are_exact() { + let m = EdgeMetrics::new(); + for _ in 0..3 { + m.record_push(100); + } + m.record_reject(); + for _ in 0..2 { + m.record_pop(100); + } + m.record_empty(); + let s = m.snapshot(); + assert_eq!(s.pushed_items, 3); + assert_eq!(s.pushed_bytes, 300); + assert_eq!(s.push_rejections, 1); + assert_eq!(s.popped_items, 2); + assert_eq!(s.popped_bytes, 200); + assert_eq!(s.pop_empties, 1); + } + + // Each case records a depth pattern, then asserts the classification (and, + // where the pattern pins one, the mean occupancy within a tolerance). The + // `mostly_empty` case reuses the `(mean, tol)` form as `(0.0, 0.02)`, i.e. + // `mean < 0.02`, since occupancy is non-negative. `fill` is a non-capturing + // closure coerced to a `fn` pointer so it can ride as an rstest case value. + #[rstest] + // Samples are `(occupied_bytes, limit_bytes)`; a 1000-byte limit makes the + // occupied byte count read directly as the depth fraction ×1000. + #[case::mostly_full(|m: &EdgeMetrics| for _ in 0..100 { m.record_depth(1000, 1000); }, RawOccupancy::MostlyFull, Some((1.0, 0.02)))] + #[case::mostly_empty(|m: &EdgeMetrics| for _ in 0..100 { m.record_depth(0, 1000); }, RawOccupancy::MostlyEmpty, Some((0.0, 0.02)))] + #[case::bimodal(|m: &EdgeMetrics| for _ in 0..50 { m.record_depth(0, 1000); m.record_depth(1000, 1000); }, RawOccupancy::Bimodal, None)] + // Spread uniformly across the middle buckets → mean ≈ 0.45. + #[case::healthy(|m: &EdgeMetrics| for i in 0..100u32 { m.record_depth(300 + u64::from(i % 4) * 100, 1000); }, RawOccupancy::Healthy, Some((0.45, 0.05)))] + #[case::unknown(|_m: &EdgeMetrics| {}, RawOccupancy::Unknown, None)] + fn occupancy_classification( + #[case] fill: fn(&EdgeMetrics), + #[case] expected: RawOccupancy, + #[case] mean_within: Option<(f32, f32)>, + ) { + let m = EdgeMetrics::new(); + fill(&m); + let s = m.snapshot(); + assert_eq!(s.raw_occupancy, expected); + if let Some((mean, tol)) = mean_within { + assert!( + (s.mean_occupancy - mean).abs() < tol, + "mean_occupancy {} not within {tol} of {mean}", + s.mean_occupancy + ); + } + } + + #[test] + fn mean_occupancy_bytes_tracks_absolute_bytes_independent_of_limit() { + // Sampled occupied bytes are averaged in absolute terms, not as a + // fraction of the limit — so a later limit change can't distort the mean + // (the property the derived-latency estimate relies on). Two 600-byte + // samples average to 600 bytes regardless of the 1000-byte limit used. + let m = EdgeMetrics::new(); + m.record_depth(600, 1000); + m.record_depth(600, 1000); + let s = m.snapshot(); + assert!((s.mean_occupancy_bytes - 600.0).abs() < 1e-9, "{}", s.mean_occupancy_bytes); + assert!((s.mean_occupancy - 0.6).abs() < 0.01, "{}", s.mean_occupancy); + } + + #[test] + fn mean_occupancy_bytes_keeps_raw_bytes_when_stash_exceeds_limit() { + // An ordered edge's reorder stash can push total buffered bytes past the + // transport limit; the fraction saturates at 1.0 but the byte mean stays + // truthful (1500 bytes), so latency is not under-counted. + let m = EdgeMetrics::new(); + m.record_depth(1500, 1000); // 150% of limit + let s = m.snapshot(); + assert!((s.mean_occupancy - 1.0).abs() < 1e-6, "fraction clamps to 1.0"); + assert!((s.mean_occupancy_bytes - 1500.0).abs() < 1e-9, "raw bytes preserved"); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/mod.rs b/crates/fgumi-pipeline-core/src/runtime/mod.rs new file mode 100644 index 000000000..39143a3ac --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/mod.rs @@ -0,0 +1,30 @@ +//! Runtime: per-worker step storage, chain contexts, drain coordination, +//! worker pool, worker loop body. Built on top of Phase 1's trait surface. + +pub mod contexts; +pub mod detached; +pub mod drain; +pub mod driver; +pub mod fused; +pub mod live; +pub mod metrics; +pub mod pool; +pub mod sampler; +pub mod scheduler; +pub mod stats; +pub mod storage; +pub mod worker_core; + +pub use contexts::{ChainContexts, build_chain_contexts, build_chain_contexts_fused}; +pub use detached::{ + DetachedDriverGroup, build_driver_storage, extract_detached_steps, run_detached_driver, +}; +pub use drain::StepDrainCounter; +pub use driver::run_worker_loop; +pub use fused::{is_fusible_chain, run_fused_single_thread, should_fuse_single_thread}; +pub use live::LiveSteps; +pub use pool::{assign_exclusive_owners, assign_sticky_owners}; +pub use scheduler::{ChainOrderScheduler, DrainFirstScheduler, Scheduler, WalkDirection}; +pub use stats::{PipelineStats, StatsSnapshot, StepStatsSnapshot}; +pub use storage::{WorkerStepEntry, build_worker_storage}; +pub use worker_core::{BackoffPolicy, WorkerCore, WorkerRole}; diff --git a/crates/fgumi-pipeline-core/src/runtime/pool.rs b/crates/fgumi-pipeline-core/src/runtime/pool.rs new file mode 100644 index 000000000..d991e5577 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/pool.rs @@ -0,0 +1,294 @@ +//! Exclusive owner assignment. + +use crate::erased::ErasedStep; +use crate::signal::PipelineError; +use crate::step::StepKind; +use crate::topology::StepIdx; + +/// Assign Exclusive steps to specific worker threads in chain declaration +/// order. Returns `Ok(owners)` where `owners[step_idx] == Some(worker_id)` +/// for Exclusive steps and `None` otherwise. +/// +/// # Errors +/// +/// Returns `PipelineError::NotEnoughThreads` if more Exclusive steps exist +/// than worker threads available. +pub fn assign_exclusive_owners( + steps: &[Box], + n_threads: usize, +) -> Result>, PipelineError> { + let total_exclusive = steps.iter().filter(|s| s.kind() == StepKind::Exclusive).count(); + if total_exclusive > n_threads { + return Err(PipelineError::NotEnoughThreads { + required: total_exclusive, + available: n_threads, + }); + } + + let mut owners: Vec> = vec![None; steps.len()]; + let mut next_owner = 0usize; + for (idx, step) in steps.iter().enumerate() { + if step.kind() == StepKind::Exclusive { + owners[idx] = Some(next_owner); + next_owner += 1; + } + } + Ok(owners) +} + +/// Compute each worker's sticky-driven step (the step its `WorkerCore::sticky_owner` +/// will hold). A step contributes to a worker's `sticky_owner` iff it's flagged +/// `sticky=true` in its profile AND that worker is its sole eligible dispatcher: +/// - `Exclusive sticky` step → sticky owner is the worker assigned by +/// `assign_exclusive_owners` (read from `exclusive_owners[step_idx]`). +/// - `Serial sticky` + `Affinity::Reader` → sticky owner is worker 0. +/// - `Serial sticky` + `Affinity::Writer` → sticky owner is worker `N-1`. +/// - `Serial sticky` + `Affinity::Worker(idx)` → sticky owner is worker `idx`. +/// - `Serial sticky` + `Affinity::None` → no sticky owner (every worker +/// is eligible, so no single worker can drive sticky without starving +/// the others; we silently drop the sticky hint here). +/// - `Parallel` steps are never sticky-driven (each worker has its own +/// clone — sticky drive on one would not gate others). +/// +/// If two steps' sticky-ownership rules collide on the same worker, the first +/// writer to that worker's slot wins — the framework only models one +/// sticky-owned step per worker today. The Exclusive pass runs before the Serial +/// pass and each pass only fills empty slots, so an Exclusive owner beats a +/// later Serial-sticky-Affinity target on the same worker; the same first-wins +/// rule resolves Exclusive-vs-Exclusive and Serial-vs-Serial collisions too. +/// Returns `None` for workers without any sticky-owned step. +#[must_use] +pub fn assign_sticky_owners( + steps: &[Box], + exclusive_owners: &[Option], + n_workers: usize, +) -> Vec> { + debug_assert_eq!(steps.len(), exclusive_owners.len()); + let mut sticky: Vec> = vec![None; n_workers]; + + // First pass: Exclusive-sticky owners (highest priority). + for (step_usize, step) in steps.iter().enumerate() { + if step.kind() == StepKind::Exclusive + && step.sticky() + && let Some(owner) = exclusive_owners[step_usize] + && owner < n_workers + && sticky[owner].is_none() + { + sticky[owner] = Some(StepIdx(step_usize)); + } + } + + // Second pass: Serial-sticky-Affinity owners (only fill empty slots). + for (step_usize, step) in steps.iter().enumerate() { + // Resolve through the shared `Affinity::target_worker` helper rather + // than re-matching the variants here: `Affinity::eligible` gates which + // worker may dispatch the step, and a local copy of that mapping could + // drift and hand a sticky owner to a worker that `Skip`s the step. + if step.kind() == StepKind::Serial + && step.sticky() + && let Some(target) = step.affinity().target_worker(n_workers) + && target < n_workers + && sticky[target].is_none() + { + sticky[target] = Some(StepIdx(step_usize)); + } + } + + sticky +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + + use rstest::rstest; + + use crate::erased::TypedStep; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{Affinity, Step, StepCtx, StepOutcome, StepProfile}; + + fn stub_step(kind: StepKind) -> Box { + #[derive(Clone)] + struct StubStep(StepKind); + impl Step for StubStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Stub", + kind: self.0, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + Box::new(TypedStep::new(StubStep(kind))) + } + + #[test] + fn no_exclusives_returns_all_none() { + let steps = vec![stub_step(StepKind::Parallel), stub_step(StepKind::Serial)]; + let owners = assign_exclusive_owners(&steps, 4).unwrap(); + assert_eq!(owners, vec![None, None]); + } + + #[test] + fn one_exclusive_assigned_to_thread_zero() { + let steps = vec![stub_step(StepKind::Exclusive), stub_step(StepKind::Parallel)]; + let owners = assign_exclusive_owners(&steps, 4).unwrap(); + assert_eq!(owners, vec![Some(0), None]); + } + + #[test] + fn two_exclusives_assigned_zero_and_one() { + let steps = vec![ + stub_step(StepKind::Exclusive), + stub_step(StepKind::Parallel), + stub_step(StepKind::Exclusive), + ]; + let owners = assign_exclusive_owners(&steps, 4).unwrap(); + assert_eq!(owners, vec![Some(0), None, Some(1)]); + } + + #[test] + fn too_many_exclusives_returns_not_enough_threads() { + let steps = vec![ + stub_step(StepKind::Exclusive), + stub_step(StepKind::Exclusive), + stub_step(StepKind::Exclusive), + ]; + let result = assign_exclusive_owners(&steps, 2); + assert!(matches!( + result, + Err(PipelineError::NotEnoughThreads { required: 3, available: 2 }) + )); + } + + fn sticky_exclusive_step() -> Box { + #[derive(Clone)] + struct StickyExclusive; + impl Step for StickyExclusive { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "StickyExclusive", + kind: StepKind::Exclusive, + sticky: true, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + Box::new(TypedStep::new(StickyExclusive)) + } + + fn sticky_serial_step(affinity: crate::step::Affinity) -> Box { + struct StickySerial(crate::step::Affinity); + impl Step for StickySerial { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "StickySerial", + kind: StepKind::Serial, + sticky: true, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn affinity(&self) -> crate::step::Affinity { + self.0 + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + Box::new(TypedStep::new(StickySerial(affinity))) + } + + /// Assert that `sticky` holds `Some(StepIdx(0))` in exactly `expected_slot` + /// (when `Some`) and `None` everywhere else across `n_workers` slots. + fn assert_only_slot( + sticky: &[Option], + n_workers: usize, + expected_slot: Option, + ) { + for (slot, &got) in sticky.iter().enumerate().take(n_workers) { + let want = if Some(slot) == expected_slot { Some(StepIdx(0)) } else { None }; + assert_eq!(got, want, "slot {slot}; expected owner slot {expected_slot:?}"); + } + } + + #[rstest] + #[case::in_range(2, Some(2))] + #[case::out_of_range(10, None)] + fn sticky_exclusive_owner_maps_to_slot( + #[case] owner: usize, + #[case] expected_slot: Option, + ) { + // A sticky-exclusive step's slot is its (in-range) owner worker; an + // out-of-range owner is skipped, leaving every slot empty. + let steps = vec![sticky_exclusive_step(), stub_step(StepKind::Parallel)]; + let exclusive_owners = vec![Some(owner), None]; + let sticky = assign_sticky_owners(&steps, &exclusive_owners, 4); + assert_only_slot(&sticky, 4, expected_slot); + } + + #[test] + fn sticky_exclusive_occupied_slot_not_overwritten() { + // Two sticky-exclusive steps competing for the same worker slot. + let steps = vec![sticky_exclusive_step(), sticky_exclusive_step()]; + let exclusive_owners = vec![Some(0_usize), Some(0_usize)]; + let sticky = assign_sticky_owners(&steps, &exclusive_owners, 4); + // First step wins; second is skipped because slot[0] is already occupied. + assert_eq!(sticky[0], Some(StepIdx(0))); + } + + #[rstest] + #[case::reader(Affinity::Reader, Some(0))] + #[case::writer(Affinity::Writer, Some(3))] + #[case::worker_in_range(Affinity::Worker(2), Some(2))] + #[case::worker_out_of_range(Affinity::Worker(10), None)] + #[case::none(Affinity::None, None)] + fn sticky_serial_affinity_maps_to_slot( + #[case] affinity: Affinity, + #[case] expected_slot: Option, + ) { + // Serial affinity resolves to a single worker slot: Reader→0, + // Writer→last, Worker(i)→i; an out-of-range Worker index and None are + // skipped, leaving every slot empty. + let steps = vec![sticky_serial_step(affinity)]; + let exclusive_owners = vec![None]; + let sticky = assign_sticky_owners(&steps, &exclusive_owners, 4); + assert_only_slot(&sticky, 4, expected_slot); + } + + #[test] + fn sticky_exclusive_beats_sticky_serial_on_same_slot() { + // Exclusive pass runs first; serial pass only fills empty slots. + let exc = sticky_exclusive_step(); + let ser = sticky_serial_step(crate::step::Affinity::Reader); // also targets slot 0 + let steps: Vec> = vec![exc, ser]; + let exclusive_owners = vec![Some(0_usize), None]; + let sticky = assign_sticky_owners(&steps, &exclusive_owners, 4); + // Exclusive (step 0) wins slot 0; Serial (step 1) is blocked. + assert_eq!(sticky[0], Some(StepIdx(0))); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/sampler.rs b/crates/fgumi-pipeline-core/src/runtime/sampler.rs new file mode 100644 index 000000000..63c080c23 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/sampler.rs @@ -0,0 +1,441 @@ +//! Background occupancy sampler for `--pipeline-trace`. +//! +//! When instrumentation is on, `Pipeline::run` spawns one +//! [`run_occupancy_sampler`] thread (same lifecycle slot as the deadlock +//! monitor / queue rebalancer) that periodically reads each byte-bounded edge's +//! depth (`current_bytes / limit_bytes`) and feeds it to the edge's +//! [`EdgeMetrics`](super::metrics::EdgeMetrics) occupancy histogram. Only edges +//! with a `depth_source` (byte-bounded) are sampled — count/unbounded edges still +//! get their push/pop counters, just no occupancy histogram. +//! +//! Reads are cheap but not free. A direct byte-bounded edge costs one `Relaxed` +//! load per tick. An **ordered** edge additionally reads its `ReorderStage` +//! overflow stash, and `ReorderCapHandle::current_buffer_bytes` takes the stage's +//! `state` mutex — the same one every must-accept push and every +//! `try_pop_in_order` holds. So an ordered edge does briefly touch a worker-hot +//! lock once per tick (at the default interval, negligible against per-item +//! traffic, but not zero). +//! +//! Each tick therefore reads every edge's depth **once**, via `read_depths`, +//! and hands the result to both consumers (`record_depths` and the timeline +//! writer). Letting each consumer read for itself would take that mutex twice per +//! ordered edge per tick and could record two different depths for one tick. + +use std::fmt::Write as _; +use std::io::{BufWriter, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use super::contexts::RegisteredEdge; + +/// Default sampling interval. Two milliseconds is cheap (one atomic load per +/// edge) yet fine-grained enough to resolve a chain's phase structure over a +/// multi-second run. +pub const DEFAULT_SAMPLE_INTERVAL: Duration = Duration::from_millis(2); + +/// Poll each byte-bounded edge's occupancy into its histogram until `stop` is +/// set. Edges without a `depth_source` (count/unbounded) are skipped. When +/// `trace_path` is `Some` (the `Timeline` level), also append one TSV row per +/// tick — `t_ms` plus, per edge, its depth fraction and cumulative +/// pushed/popped item counts — so the run's phase structure can be plotted. +pub fn run_occupancy_sampler( + stop: &AtomicBool, + edges: &[RegisteredEdge], + interval: Duration, + trace_path: Option, +) { + let mut trace = trace_path.and_then(|p| TraceWriter::open(&p, edges)); + let start = Instant::now(); + let mut sampled_in_loop = false; + while !stop.load(Ordering::Relaxed) { + // One read per tick, shared by the histogram and the timeline row. + let depths = read_depths(edges); + record_depths(edges, &depths); + if let Some(t) = trace.as_mut() { + t.write_row(edges, &depths, start.elapsed()); + } + sampled_in_loop = true; + std::thread::sleep(interval); + } + // Guard the final sample: only take it when the loop never sampled (a run + // so short `stop` was already set before the first iteration). Sampling + // unconditionally here would add an extra occupancy point + timeline row + // taken AFTER the pipeline already drained, biasing the mean toward the + // empty final state. + if !sampled_in_loop { + let depths = read_depths(edges); + record_depths(edges, &depths); + if let Some(t) = trace.as_mut() { + t.write_row(edges, &depths, start.elapsed()); + } + } + if let Some(mut t) = trace { + t.flush(); + } +} + +/// Bytes buffered in an ordered edge's `ReorderStage` overflow stash (0 for a +/// direct/count/unbounded edge). Added to the transport queue's `current_bytes` +/// when sampling depth so an ordered edge reflects total buffered bytes rather +/// than reading empty while items pile in the reorder buffer awaiting an earlier +/// ordinal. +fn reorder_stash_bytes(edge: &RegisteredEdge) -> u64 { + edge.reorder_depth.as_ref().map_or(0, |r| r.current_buffer_bytes()) +} + +/// One edge's depth for a single tick: `(occupied_bytes, limit_bytes)`, or `None` +/// for a count/unbounded edge (no `depth_source`) or one whose limit reads 0. +type EdgeDepth = Option<(u64, u64)>; + +/// Read every edge's depth once, for one tick. +/// +/// Called once per tick and shared by [`record_depths`] and the timeline writer +/// so that (a) an ordered edge's `ReorderStage` mutex is taken once per tick +/// rather than once per consumer, and (b) the histogram sample and the timeline +/// row for a given tick always report the same number. +fn read_depths(edges: &[RegisteredEdge]) -> Vec { + edges + .iter() + .map(|edge| { + let src = edge.depth_source.as_ref()?; + let limit = src.limit_bytes(); + if limit == 0 { + return None; + } + Some((src.current_bytes().saturating_add(reorder_stash_bytes(edge)), limit)) + }) + .collect() +} + +/// Feed one tick's depths into each edge's occupancy histogram. +fn record_depths(edges: &[RegisteredEdge], depths: &[EdgeDepth]) { + for (edge, depth) in edges.iter().zip(depths) { + if let Some((occupied, limit)) = *depth { + edge.metrics.record_depth(occupied, limit); + } + } +} + +/// Column-name prefix for one edge's timeline columns. Includes the producer +/// step index and output branch so fan-out edges (one producer, several +/// branches) and repeated step names produce distinct, collision-free headers — +/// a bare `producer__consumer` prefix duplicates columns whenever two edges +/// share both names. `(producer_step, branch)` uniquely identifies an edge. +fn edge_column_prefix(e: &RegisteredEdge) -> String { + format!( + "{}__{}#{}b{}", + e.producer_name, + e.consumer_name.unwrap_or("sink"), + e.producer_step.0, + e.branch.0, + ) +} + +/// Per-tick TSV writer for the `Timeline` level. Best-effort: a write error is +/// logged once and further rows are dropped (instrumentation never aborts a run). +struct TraceWriter { + writer: BufWriter, + failed: bool, +} + +impl TraceWriter { + /// Open `path` and write the header (`t_ms` + three columns per edge). + /// Returns `None` (with a warning) if the file can't be created. + fn open(path: &std::path::Path, edges: &[RegisteredEdge]) -> Option { + match std::fs::File::create(path) { + Ok(file) => { + let mut writer = BufWriter::new(file); + let mut header = String::from("t_ms"); + for e in edges { + let edge = edge_column_prefix(e); + let _ = write!(header, "\t{edge}.depth\t{edge}.pushed\t{edge}.popped"); + } + if writeln!(writer, "{header}").is_err() { + log::warn!( + "pipeline-trace: failed to write timeline header to {}", + path.display() + ); + return None; + } + Some(Self { writer, failed: false }) + } + Err(e) => { + log::warn!("pipeline-trace: cannot create timeline file {}: {e}", path.display()); + None + } + } + } + + /// `depths` is this tick's depths from [`read_depths`], shared with + /// [`record_depths`] so the row and the histogram agree and each ordered + /// edge's reorder mutex is taken once per tick. + #[allow(clippy::cast_precision_loss)] + fn write_row(&mut self, edges: &[RegisteredEdge], depths: &[EdgeDepth], elapsed: Duration) { + if self.failed { + return; + } + let mut row = format!("{}", elapsed.as_millis()); + for (e, depth) in edges.iter().zip(depths) { + // Count/unbounded edges (no `depth_source`) are unsampled — emit `NA` + // rather than `0.000`, which would misread as "empty" instead of + // "not measured". Byte-bounded edges report total buffered depth + // (transport + reorder stash) as a fraction of the limit. + let depth = depth.map_or_else( + || "NA".to_string(), + |(occupied, limit)| format!("{:.3}", occupied as f32 / limit as f32), + ); + let ms = e.metrics.snapshot(); + let _ = write!(row, "\t{depth}\t{}\t{}", ms.pushed_items, ms.popped_items); + } + if writeln!(self.writer, "{row}").is_err() { + log::warn!("pipeline-trace: timeline write failed; dropping further rows"); + self.failed = true; + } + } + + fn flush(&mut self) { + if self.failed { + return; + } + // A dropped flush error can silently lose buffered rows after every + // write appeared to succeed — warn and mark the writer failed, matching + // `write_row`'s best-effort error handling. + if self.writer.flush().is_err() { + log::warn!("pipeline-trace: timeline flush failed; buffered rows may be lost"); + self.failed = true; + } + } +} + +/// One sampling sweep over all edges. Extracted so tests can drive a single +/// deterministic tick without the sleep loop. +/// +/// The sampler loop does not call this — it uses `read_depths` once per tick +/// and shares the result with the timeline writer, so both record the same +/// numbers from a single read (see the module doc). +pub fn sample_once(edges: &[RegisteredEdge]) { + record_depths(edges, &read_depths(edges)); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + + use crate::item::HeapSize; + use crate::queues::{BoundedQueueHandle, ByteBoundedQueue, ItemQueue}; + use crate::runtime::metrics::EdgeMetrics; + use crate::topology::{BranchIdx, StepIdx}; + + #[derive(Debug)] + struct Heavy(Vec); + impl HeapSize for Heavy { + fn heap_size(&self) -> usize { + self.0.len() + } + } + + fn edge_over( + metrics: Arc, + depth_source: Option>, + ) -> RegisteredEdge { + RegisteredEdge { + producer_step: StepIdx(0), + producer_name: "producer", + consumer_step: Some(StepIdx(1)), + consumer_name: Some("consumer"), + branch: BranchIdx(0), + metrics, + depth_source, + reorder_depth: None, + } + } + + #[test] + fn sample_once_records_occupancy_from_depth_source() { + let m = EdgeMetrics::new(); + let q = Arc::new(ByteBoundedQueue::::new(1000)); + q.try_push(Heavy(vec![0; 500])).unwrap(); // 50% of the 1000-byte budget + let edge = edge_over(Arc::clone(&m), Some(Arc::clone(&q) as Arc)); + for _ in 0..10 { + sample_once(std::slice::from_ref(&edge)); + } + let s = m.snapshot(); + assert_eq!(s.depth_samples, 10); + assert!((s.mean_occupancy - 0.5).abs() < 0.05, "mean ≈ 0.5, got {}", s.mean_occupancy); + } + + #[test] + fn count_edge_without_depth_source_is_skipped() { + // An edge with no depth_source (count/unbounded) records no occupancy. + let m = EdgeMetrics::new(); + let edge = edge_over(Arc::clone(&m), None); + for _ in 0..5 { + sample_once(std::slice::from_ref(&edge)); + } + assert_eq!(m.snapshot().depth_samples, 0, "no depth source → no occupancy samples"); + } + + #[test] + fn run_occupancy_sampler_stops_and_records() { + let m = EdgeMetrics::new(); + let q = Arc::new(ByteBoundedQueue::::new(1000)); + q.try_push(Heavy(vec![0; 800])).unwrap(); + let edges = + vec![edge_over(Arc::clone(&m), Some(Arc::clone(&q) as Arc))]; + let stop = Arc::new(AtomicBool::new(false)); + let stop_c = Arc::clone(&stop); + let handle = std::thread::spawn(move || { + run_occupancy_sampler(&stop_c, &edges, Duration::from_millis(1), None); + }); + std::thread::sleep(Duration::from_millis(30)); + stop.store(true, Ordering::Relaxed); + handle.join().unwrap(); + let s = m.snapshot(); + assert!(s.depth_samples > 0, "sampler recorded at least one tick"); + assert!((s.mean_occupancy - 0.8).abs() < 0.1, "mean ≈ 0.8, got {}", s.mean_occupancy); + } + + #[test] + fn edge_columns_are_unique_when_step_names_collide() { + // Regression: two edges that share producer AND consumer names (fan-out, + // or duplicate step names) must still produce distinct TSV columns — a + // bare `producer__consumer` prefix would emit duplicate column headers. + let m0 = EdgeMetrics::new(); + let m1 = EdgeMetrics::new(); + let e0 = RegisteredEdge { + producer_step: StepIdx(0), + producer_name: "dup", + consumer_step: Some(StepIdx(1)), + consumer_name: Some("sink"), + branch: BranchIdx(0), + metrics: m0, + depth_source: None, + reorder_depth: None, + }; + // Same names, different (producer_step, branch): a fan-out sibling. + let e1 = RegisteredEdge { + producer_step: StepIdx(0), + producer_name: "dup", + consumer_step: Some(StepIdx(2)), + consumer_name: Some("sink"), + branch: BranchIdx(1), + metrics: m1, + depth_source: None, + reorder_depth: None, + }; + let p0 = edge_column_prefix(&e0); + let p1 = edge_column_prefix(&e1); + assert_ne!(p0, p1, "colliding names must yield distinct column prefixes"); + assert_eq!(p0, "dup__sink#0b0"); + assert_eq!(p1, "dup__sink#0b1"); + } + + #[test] + fn ordered_edge_occupancy_includes_reorder_stash() { + use crate::queues::CountBoundedQueue; + use crate::reorder::{ReorderCapHandle, ReorderStage, Sequenced}; + // The transport (occupancy depth source) is empty, but the reorder stash + // holds 400 buffered bytes waiting for an earlier ordinal. Sampled + // occupancy must reflect the stash (400/1000 = 0.4), not read empty — + // otherwise a producer-skewed ordered edge looks idle while backed up. + let m = EdgeMetrics::new(); + let transport = Arc::new(ByteBoundedQueue::::new(1000)); + + let reorder_transport: Arc>> = + Arc::new(CountBoundedQueue::>::new(8)); + let stage = Arc::new(ReorderStage::new(reorder_transport)); + stage.try_push(1, Heavy(vec![0; 400])).unwrap(); // out-of-order → stashed + assert!(stage.try_pop_in_order().is_none(), "ordinal 0 absent → nothing pops"); + assert!(stage.current_buffer_bytes() >= 400, "stash holds the buffered bytes"); + + let edge = RegisteredEdge { + producer_step: StepIdx(0), + producer_name: "p", + consumer_step: Some(StepIdx(1)), + consumer_name: Some("c"), + branch: BranchIdx(0), + metrics: Arc::clone(&m), + depth_source: Some(Arc::clone(&transport) as Arc), + reorder_depth: Some(Arc::clone(&stage) as Arc), + }; + for _ in 0..10 { + sample_once(std::slice::from_ref(&edge)); + } + let s = m.snapshot(); + assert_eq!(s.depth_samples, 10); + assert!( + (s.mean_occupancy - 0.4).abs() < 0.05, + "occupancy reflects the reorder stash, got {}", + s.mean_occupancy + ); + assert!( + (s.mean_occupancy_bytes - 400.0).abs() < 1.0, + "byte mean equals the stashed bytes, got {}", + s.mean_occupancy_bytes + ); + } + + #[test] + fn timeline_tsv_has_header_and_rows() { + let m = EdgeMetrics::new(); + let q = Arc::new(ByteBoundedQueue::::new(1000)); + q.try_push(Heavy(vec![0; 400])).unwrap(); + let edges = + vec![edge_over(Arc::clone(&m), Some(Arc::clone(&q) as Arc))]; + let dir = std::env::temp_dir(); + let path = dir.join(format!("fgumi-trace-test-{}.tsv", std::process::id())); + let stop = Arc::new(AtomicBool::new(false)); + let stop_c = Arc::clone(&stop); + let path_c = path.clone(); + let handle = std::thread::spawn(move || { + run_occupancy_sampler(&stop_c, &edges, Duration::from_millis(2), Some(path_c)); + }); + std::thread::sleep(Duration::from_millis(30)); + stop.store(true, Ordering::Relaxed); + handle.join().unwrap(); + + let content = std::fs::read_to_string(&path).expect("trace file written"); + let _ = std::fs::remove_file(&path); + let mut lines = content.lines(); + let header = lines.next().expect("header row"); + assert!(header.starts_with("t_ms"), "header begins with t_ms"); + assert!(header.contains("producer__consumer#0b0.depth"), "per-edge depth column"); + let rows: Vec<&str> = lines.collect(); + assert!(!rows.is_empty(), "at least one data row"); + // First field of a data row is a monotonic t_ms integer. + let first_t: u128 = rows[0].split('\t').next().unwrap().parse().expect("t_ms is an int"); + let last_t: u128 = rows.last().unwrap().split('\t').next().unwrap().parse().unwrap(); + assert!(last_t >= first_t, "t_ms is monotonic"); + } + + #[test] + fn timeline_tsv_marks_unsampled_edge_na() { + // A count/unbounded edge (no depth_source) is unsampled: its depth column + // must read `NA`, not `0.000` (which would misread as an empty byte edge). + let m = EdgeMetrics::new(); + let edges = vec![edge_over(Arc::clone(&m), None)]; + let dir = std::env::temp_dir(); + let path = dir.join(format!("fgumi-trace-na-{}.tsv", std::process::id())); + let stop = Arc::new(AtomicBool::new(false)); + let stop_c = Arc::clone(&stop); + let path_c = path.clone(); + let handle = std::thread::spawn(move || { + run_occupancy_sampler(&stop_c, &edges, Duration::from_millis(2), Some(path_c)); + }); + std::thread::sleep(Duration::from_millis(30)); + stop.store(true, Ordering::Relaxed); + handle.join().unwrap(); + + let content = std::fs::read_to_string(&path).expect("trace file written"); + let _ = std::fs::remove_file(&path); + let mut lines = content.lines(); + let _header = lines.next().expect("header row"); + let row = lines.next().expect("at least one data row"); + // Columns: t_ms, .depth, .pushed, .popped. + let depth = row.split('\t').nth(1).expect("depth column"); + assert_eq!(depth, "NA", "unsampled edge's depth column is NA, row: {row}"); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/scheduler.rs b/crates/fgumi-pipeline-core/src/runtime/scheduler.rs new file mode 100644 index 000000000..0fde6eb00 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/scheduler.rs @@ -0,0 +1,106 @@ +//! Pluggable per-worker dispatch-order policy for the round-robin pool driver. +//! +//! The worker loop ([`run_worker_loop`](crate::runtime::run_worker_loop)) walks +//! each worker's *live* steps once per pass and runs the first that makes +//! progress. A [`Scheduler`] decides the ORDER of that walk — the only thing it +//! controls; it never changes which steps exist, the sticky source/sink +//! fast-path, or the Serial/Exclusive contention rules. +//! +//! Two policies ship: +//! +//! - [`ChainOrderScheduler`] (the default) — walk **upstream-first** (chain +//! order). A worker attempts the earliest-in-chain step with work, favouring +//! production. This is the historical behaviour; every command keeps it unless +//! it opts out, so existing pipelines are byte-for-byte unaffected. +//! - [`DrainFirstScheduler`] — walk **downstream-first** (reverse chain order). +//! A worker attempts the deepest step with work first, favouring *draining* +//! buffered work before producing more. Combined with skip-on-Serial-contention +//! this self-balances: for a Serial step fed by an N-way Parallel producer, one +//! worker grabs the Serial drain (mutex) while the rest find it contended, skip, +//! and fall through to the producer — so the drain overlaps production instead +//! of starving behind it on the shared pool. A sticky step is **not** exempt +//! from the walk: it takes its bounded burst on the sticky fast-path first and +//! is then still visited in the walk itself (under `Reverse`, last rather than +//! first). Only its `Progress` priority restart is suppressed, so the walk +//! continues past it to the steps that drain its output — see +//! `super::driver::round_robin_dispatch`. +//! +//! This mirrors main's `Scheduler`-trait design (`BalancedChaseDrainScheduler` +//! among others), but generically over an arbitrary step list rather than a +//! fixed set of named BAM stages: the only lever exposed here is walk direction, +//! which is all the generic driver needs to express drain-first scheduling. + +/// The order in which a worker attempts its live steps in one round-robin pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WalkDirection { + /// Chain order (upstream → downstream): favour production. + Forward, + /// Reverse chain order (downstream → upstream): favour draining. + Reverse, +} + +/// A per-worker dispatch-order policy. Selected per pipeline via +/// [`PipelineConfig::with_scheduler`](crate::builder::PipelineConfig::with_scheduler) +/// and shared across all workers (the shipped policies are stateless). +pub trait Scheduler: Send + Sync + std::fmt::Debug { + /// Direction to walk this worker's live steps this pass. Called once per + /// round-robin pass, so it must be cheap. + fn walk(&self) -> WalkDirection; + + /// Human-readable name for diagnostics / `--pipeline-stats`. + fn name(&self) -> &'static str; +} + +/// Default upstream-first (chain-order) walk. Preserves the historical dispatch +/// behaviour for every pipeline that does not opt into a different policy. +#[derive(Debug, Default, Clone, Copy)] +pub struct ChainOrderScheduler; + +impl Scheduler for ChainOrderScheduler { + #[inline] + fn walk(&self) -> WalkDirection { + WalkDirection::Forward + } + fn name(&self) -> &'static str { + "chain-order" + } +} + +/// Downstream-first (reverse chain-order) walk — drain buffered work before +/// producing more. Opt-in per pipeline (e.g. the sort chain, to overlap the +/// serial boundary/key scan with the parallel inflate instead of starving it on +/// the shared pool). +#[derive(Debug, Default, Clone, Copy)] +pub struct DrainFirstScheduler; + +impl Scheduler for DrainFirstScheduler { + #[inline] + fn walk(&self) -> WalkDirection { + WalkDirection::Reverse + } + fn name(&self) -> &'static str { + "drain-first" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_is_forward() { + assert_eq!(ChainOrderScheduler.walk(), WalkDirection::Forward); + assert_eq!(ChainOrderScheduler.name(), "chain-order"); + } + + #[test] + fn drain_first_is_reverse() { + assert_eq!(DrainFirstScheduler.walk(), WalkDirection::Reverse); + assert_eq!(DrainFirstScheduler.name(), "drain-first"); + } + + // The forward/reverse position→step arithmetic these policies drive is + // exercised end-to-end against the real dispatcher in driver.rs + // (`driver_round_robins_all_live_before_parking`); a standalone test here + // would only re-derive the formula, not the driver's actual mapping. +} diff --git a/crates/fgumi-pipeline-core/src/runtime/stats.rs b/crates/fgumi-pipeline-core/src/runtime/stats.rs new file mode 100644 index 000000000..b04d57890 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/stats.rs @@ -0,0 +1,1392 @@ +//! Per-step pipeline statistics: dispatch counts, outcome breakdown, and +//! cumulative `try_run` time. Indexed by `StepIdx` so it works for any +//! chain shape. +//! +//! Stats are opt-in. When `PipelineConfig::stats` is `Some(arc)`, the +//! worker loop times each `dispatch_one_step` call and records the +//! outcome. When `None`, the loop pays no extra cost. +//! +//! ## What's recorded +//! +//! Per step: +//! - `try_run_total` — total `try_run_erased` dispatches that returned +//! a result (excluding `Skip` entries). +//! - `progress_count` — `StepOutcome::Progress`. +//! - `no_progress_count` — `StepOutcome::NoProgress`. +//! - `contention_count` — `StepOutcome::Contention` (Serial step mutex +//! held by another worker; or skipped via `try_lock`). Always 0 under the +//! fused single-thread driver, which holds no mutex and never contends. +//! - `finished_count` — `StepOutcome::Finished` (any step on end-of-stream: +//! source, mid, or sink all record `Finished` once their inputs drain). +//! - `error_count` — `try_run_erased` returned `Err`. +//! - `total_run_ns` — cumulative wall time across all dispatches. +//! +//! Per edge, when instrumentation is on: +//! - Queue-depth samples. [`crate::runtime::sampler::sample_once`] reads each +//! registered edge's occupancy and feeds +//! [`EdgeMetrics::record_depth`](crate::runtime::metrics::EdgeMetrics::record_depth); +//! [`PipelineStats::snapshot_with_edges`] turns the histogram into the +//! per-edge occupancy on [`EdgeStatsSnapshot`]. Plain +//! [`snapshot`](PipelineStats::snapshot) leaves `edges` empty. +//! +//! ## What's *not* recorded yet +//! +//! - Per-thread step counts. Legacy carries per-`(thread, step)` counters +//! for bottleneck attribution. Useful for the rebalancer (#17) port, +//! not for first-cut observability. Deferred until the rebalancer +//! work lands. + +use std::fmt; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use crate::step::StepOutcome; +use crate::topology::StepIdx; + +/// Atomic counters for a single step. +#[derive(Debug)] +pub struct StepStats { + /// Total `try_run` dispatches (every outcome, including errors). + pub try_run_total: AtomicU64, + /// Dispatches that returned `StepOutcome::Progress`. + pub progress_count: AtomicU64, + /// Dispatches that returned `StepOutcome::NoProgress`. + pub no_progress_count: AtomicU64, + /// Dispatches that returned `StepOutcome::Contention`. + pub contention_count: AtomicU64, + /// Dispatches that returned `StepOutcome::Finished`. + pub finished_count: AtomicU64, + /// Dispatches that returned an error from `try_run_erased`. + pub error_count: AtomicU64, + /// Cumulative wall-ns spent inside `try_run` across all dispatches. + pub total_run_ns: AtomicU64, + /// Wall ns from pipeline start to the start of this step's FIRST + /// Progress dispatch. `u64::MAX` until set. Lets us see when each + /// step actually first did useful work. + pub first_progress_ns: AtomicU64, + /// Wall ns from pipeline start to the END of this step's LAST + /// Progress dispatch. Updated on every Progress (monotonic max). + /// `0` if the step never made progress. + pub last_progress_ns: AtomicU64, +} + +impl Default for StepStats { + fn default() -> Self { + Self { + try_run_total: AtomicU64::new(0), + progress_count: AtomicU64::new(0), + no_progress_count: AtomicU64::new(0), + contention_count: AtomicU64::new(0), + finished_count: AtomicU64::new(0), + error_count: AtomicU64::new(0), + total_run_ns: AtomicU64::new(0), + first_progress_ns: AtomicU64::new(u64::MAX), + last_progress_ns: AtomicU64::new(0), + } + } +} + +impl StepStats { + fn snapshot(&self) -> StepStatsSnapshot { + StepStatsSnapshot { + try_run_total: self.try_run_total.load(Ordering::Relaxed), + progress_count: self.progress_count.load(Ordering::Relaxed), + no_progress_count: self.no_progress_count.load(Ordering::Relaxed), + contention_count: self.contention_count.load(Ordering::Relaxed), + finished_count: self.finished_count.load(Ordering::Relaxed), + error_count: self.error_count.load(Ordering::Relaxed), + total_run_ns: self.total_run_ns.load(Ordering::Relaxed), + first_progress_ns: self.first_progress_ns.load(Ordering::Relaxed), + last_progress_ns: self.last_progress_ns.load(Ordering::Relaxed), + } + } +} + +/// Upper bound on per-worker utilization slots. Worker `thread_id`s index the +/// `worker_busy_ns` / `worker_idle_ns` arrays; ids at or above this are not +/// tracked (a no-op, never a panic). Sized well above any realistic +/// `--threads`, so a fixed array avoids threading `num_workers` through every +/// `PipelineStats::new` call site. +const MAX_TRACKED_WORKERS: usize = 512; + +/// Per-step counter container. Sized to match the pipeline's chain length; +/// callers obtain one via `Pipeline::stats()`. +#[derive(Debug)] +pub struct PipelineStats { + steps: Box<[StepStats]>, + step_names: Box<[&'static str]>, + /// Per-worker wall-ns spent dispatching (the sticky + round-robin work + /// section of `run_worker_loop`), indexed by `WorkerCore::thread_id`. + worker_busy_ns: Box<[AtomicU64]>, + /// Per-worker wall-ns spent in the no-progress backoff sleep (idle/blocked + /// waiting for upstream work), indexed by `thread_id`. Together with + /// `worker_busy_ns` this answers "are workers utilised, or blocked". + worker_idle_ns: Box<[AtomicU64]>, + /// Per-step wall-ns a `StepKind::Detached` step's dedicated thread spent + /// inside `try_run` (busy) and parked on its backoff (idle), indexed by + /// `step_idx`. Detached threads have no `WorkerCore::thread_id`, so they are + /// tracked here separately and are intentionally EXCLUDED from the pool + /// utilisation line (which only sums `worker_busy_ns` / `worker_idle_ns`). + /// Reported on their own line so the legacy "N + 2" split is visible without + /// diluting the N-worker pool%. + detached_busy_ns: Box<[AtomicU64]>, + detached_idle_ns: Box<[AtomicU64]>, + /// Per-step count of backoff-park events on a `StepKind::Detached` step's + /// dedicated thread (one per [`backoff_park`](crate::runtime::detached) call), + /// indexed by `step_idx`. With `detached_idle_ns` this gives the average park + /// duration and the park-to-progress ratio — the signal for whether the + /// backoff (vs a precise per-slot condvar) adds latency on the merge's path. + detached_park_events: Box<[AtomicU64]>, + /// Anchor for first/last-progress timestamps. Set at `PipelineStats` + /// construction; all `first_progress_ns` / `last_progress_ns` values + /// are wall-ns elapsed from this `Instant`. + pipeline_start: Instant, +} + +impl PipelineStats { + /// Construct a stats container sized to `step_names.len()`. Each step's + /// counters start at zero. Wrap in `Arc` to share across worker threads. + #[must_use] + pub fn new(step_names: Vec<&'static str>) -> Self { + let steps = (0..step_names.len()).map(|_| StepStats::default()).collect::>(); + let worker_busy_ns = + (0..MAX_TRACKED_WORKERS).map(|_| AtomicU64::new(0)).collect::>(); + let worker_idle_ns = + (0..MAX_TRACKED_WORKERS).map(|_| AtomicU64::new(0)).collect::>(); + let n_steps = steps.len(); + let detached_busy_ns = (0..n_steps).map(|_| AtomicU64::new(0)).collect::>(); + let detached_idle_ns = (0..n_steps).map(|_| AtomicU64::new(0)).collect::>(); + let detached_park_events = (0..n_steps).map(|_| AtomicU64::new(0)).collect::>(); + Self { + steps: steps.into_boxed_slice(), + step_names: step_names.into_boxed_slice(), + worker_busy_ns: worker_busy_ns.into_boxed_slice(), + worker_idle_ns: worker_idle_ns.into_boxed_slice(), + detached_busy_ns: detached_busy_ns.into_boxed_slice(), + detached_idle_ns: detached_idle_ns.into_boxed_slice(), + detached_park_events: detached_park_events.into_boxed_slice(), + pipeline_start: Instant::now(), + } + } + + /// Accumulate `ns` of `try_run` (busy) time for a `StepKind::Detached` + /// step's dedicated thread, keyed by `step_idx`. Excluded from pool + /// utilisation; reported on the Detached line. Out-of-range steps are a + /// silent no-op. + #[inline] + pub fn record_detached_busy(&self, step: StepIdx, ns: u64) { + if let Some(c) = self.detached_busy_ns.get(step.0) { + c.fetch_add(ns, Ordering::Relaxed); + } + } + + /// Accumulate `ns` of backoff-park (idle) time for a `StepKind::Detached` + /// step's dedicated thread, keyed by `step_idx`. Excluded from pool + /// utilisation; reported on the Detached line. + #[inline] + pub fn record_detached_idle(&self, step: StepIdx, ns: u64) { + if let Some(c) = self.detached_idle_ns.get(step.0) { + c.fetch_add(ns, Ordering::Relaxed); + } + } + + /// Increment the backoff-park event count for a `StepKind::Detached` step's + /// dedicated thread (called once per park). Out-of-range steps are a silent + /// no-op. + #[inline] + pub fn record_detached_park(&self, step: StepIdx) { + if let Some(c) = self.detached_park_events.get(step.0) { + c.fetch_add(1, Ordering::Relaxed); + } + } + + /// Accumulate `ns` of dispatch (busy) time for `worker` (its `thread_id`). + /// Ids `>= MAX_TRACKED_WORKERS` are silently dropped. + #[inline] + pub fn record_worker_busy(&self, worker: usize, ns: u64) { + if let Some(c) = self.worker_busy_ns.get(worker) { + c.fetch_add(ns, Ordering::Relaxed); + } + } + + /// Accumulate `ns` of backoff-sleep (idle) time for `worker`. + #[inline] + pub fn record_worker_idle(&self, worker: usize, ns: u64) { + if let Some(c) = self.worker_idle_ns.get(worker) { + c.fetch_add(ns, Ordering::Relaxed); + } + } + + /// Wall ns elapsed since pipeline start. Used by the driver to stamp + /// per-step first/last progress timestamps. + #[must_use] + pub fn elapsed_ns(&self) -> u64 { + u64::try_from(self.pipeline_start.elapsed().as_nanos()).unwrap_or(u64::MAX) + } + + #[must_use] + pub fn n_steps(&self) -> usize { + self.steps.len() + } + + #[must_use] + pub fn step_name(&self, step: StepIdx) -> &'static str { + self.step_names[step.0] + } + + /// Record a successful `try_run_erased` outcome for the given step. + /// Hot path: relaxed atomics, no allocation, no locking. `start_ns` + /// is wall-ns at dispatch start (relative to `pipeline_start`); + /// `elapsed_ns` is the dispatch duration. On `Progress` we stamp + /// the step's first/last active timestamps. + /// + /// # Panics + /// + /// Panics if `step` is outside `self.steps` — it indexes directly rather than + /// using `.get()`. That is deliberate and asymmetric with the detached + /// recorders (`record_detached_busy` and friends), which silently no-op on an + /// out-of-range index: an out-of-range `StepIdx` reaching *this* path is a + /// driver bug that would otherwise silently lose every sample for the step, so + /// it should be loud. Do not assume the detached recorders' no-op contract + /// here. + #[inline] + pub fn record(&self, step: StepIdx, outcome: StepOutcome, start_ns: u64, elapsed_ns: u64) { + let s = &self.steps[step.0]; + s.try_run_total.fetch_add(1, Ordering::Relaxed); + s.total_run_ns.fetch_add(elapsed_ns, Ordering::Relaxed); + match outcome { + StepOutcome::Progress => { + s.progress_count.fetch_add(1, Ordering::Relaxed); + // CAS-min first_progress_ns (initially u64::MAX). + let mut cur = s.first_progress_ns.load(Ordering::Relaxed); + while start_ns < cur { + match s.first_progress_ns.compare_exchange_weak( + cur, + start_ns, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(seen) => cur = seen, + } + } + // last_progress_ns = max(last, start + elapsed). + let end_ns = start_ns.saturating_add(elapsed_ns); + let mut cur = s.last_progress_ns.load(Ordering::Relaxed); + while end_ns > cur { + match s.last_progress_ns.compare_exchange_weak( + cur, + end_ns, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(seen) => cur = seen, + } + } + } + StepOutcome::NoProgress => { + s.no_progress_count.fetch_add(1, Ordering::Relaxed); + } + StepOutcome::Contention => { + s.contention_count.fetch_add(1, Ordering::Relaxed); + } + StepOutcome::Finished => { + s.finished_count.fetch_add(1, Ordering::Relaxed); + } + } + } + + /// Record an error returned by `try_run_erased`. Counted toward + /// `try_run_total` and `total_run_ns`; outcome buckets are not bumped. + /// + /// # Panics + /// + /// Panics if `step` is outside `self.steps` — it indexes directly rather than + /// using `.get()`. That is deliberate and asymmetric with the detached + /// recorders (`record_detached_busy` and friends), which silently no-op on an + /// out-of-range index: an out-of-range `StepIdx` reaching *this* path is a + /// driver bug that would otherwise silently lose every sample for the step, so + /// it should be loud. Do not assume the detached recorders' no-op contract + /// here. + #[inline] + pub fn record_error(&self, step: StepIdx, _start_ns: u64, elapsed_ns: u64) { + let s = &self.steps[step.0]; + s.try_run_total.fetch_add(1, Ordering::Relaxed); + s.error_count.fetch_add(1, Ordering::Relaxed); + s.total_run_ns.fetch_add(elapsed_ns, Ordering::Relaxed); + } + + /// Snapshot all per-step counters into an owned, lock-free struct + /// suitable for printing or further analysis. + #[must_use] + pub fn snapshot(&self) -> StatsSnapshot { + let steps = self + .steps + .iter() + .zip(self.step_names.iter()) + .map(|(stats, &name)| (name, stats.snapshot())) + .collect(); + // Only workers that recorded any activity (busy or idle) — the array is + // sized to MAX_TRACKED_WORKERS but typically few slots are live. + let workers = (0..self.worker_busy_ns.len()) + .map(|w| { + ( + w, + self.worker_busy_ns[w].load(Ordering::Relaxed), + self.worker_idle_ns[w].load(Ordering::Relaxed), + ) + }) + .filter(|(_, busy, idle)| *busy != 0 || *idle != 0) + .collect(); + StatsSnapshot { steps, workers, detached: self.detached_snapshot(), edges: Vec::new() } + } + + /// Collect `(step, step_name, busy_ns, idle_ns, park_events)` for every step + /// whose Detached thread recorded any activity. Shared by both snapshot + /// builders so the Detached line renders identically with or without + /// `--pipeline-trace`. The leading `StepIdx` lets consumers match entries by + /// identity instead of by (possibly duplicated) name. + fn detached_snapshot(&self) -> Vec<(usize, &'static str, u64, u64, u64)> { + (0..self.detached_busy_ns.len()) + .map(|s| { + ( + s, + self.step_names[s], + self.detached_busy_ns[s].load(Ordering::Relaxed), + self.detached_idle_ns[s].load(Ordering::Relaxed), + self.detached_park_events[s].load(Ordering::Relaxed), + ) + }) + .filter(|(_, _, busy, idle, _)| *busy != 0 || *idle != 0) + .collect() + } + + /// Like [`snapshot`](Self::snapshot) but also derives per-edge throughput / + /// occupancy / latency from the chain's instrumented `edges` over a + /// `wall_ns` run. Used by the `--pipeline-trace` end-of-run report. + #[must_use] + pub fn snapshot_with_edges( + &self, + edges: &[crate::runtime::contexts::RegisteredEdge], + wall_ns: u64, + ) -> StatsSnapshot { + let mut snap = self.snapshot(); + snap.edges = edges + .iter() + .map(|e| { + let ms = e.metrics.snapshot(); + let limit_bytes = e.depth_source.as_ref().map(|s| s.limit_bytes()); + compute_edge_stats( + e.producer_name, + e.consumer_name, + e.producer_step.0, + e.consumer_step.map(|s| s.0), + &ms, + limit_bytes, + wall_ns, + ) + }) + .collect(); + snap + } +} + +/// Plain (non-atomic) snapshot of `PipelineStats` at a moment in time. +#[derive(Debug, Clone)] +pub struct StatsSnapshot { + pub steps: Vec<(&'static str, StepStatsSnapshot)>, + /// `(thread_id, busy_ns, idle_ns)` for each worker that did anything. + pub workers: Vec<(usize, u64, u64)>, + /// `(step, step_name, busy_ns, idle_ns, park_events)` for each + /// `StepKind::Detached` step's dedicated thread that did anything. Tracked + /// separately from `workers` and EXCLUDED from the pool utilisation line + /// (legacy "N + 2" — the merge / writer threads are not pool workers); + /// rendered on their own line. `park_events` is the backoff-park count + /// (latency signal). + /// + /// `step` is the entry's `StepIdx`, carried so consumers can match a + /// Detached step by identity rather than by name — two steps may share a + /// name, and `bottleneck_verdict`'s SPIN exemption must not leak from one to + /// the other. Entries are filtered, so the indices are not contiguous. + pub detached: Vec<(usize, &'static str, u64, u64, u64)>, + /// Per-edge throughput / occupancy / latency. Empty unless the snapshot was + /// built via [`PipelineStats::snapshot_with_edges`] (i.e. instrumentation on). + pub edges: Vec, +} + +/// Occupancy classification refined from the histogram's [`RawOccupancy`](crate::runtime::metrics::RawOccupancy) plus +/// the reject/empty rates: `MostlyEmpty` + high empty-rate → `Starved`, +/// `MostlyFull` + high reject-rate → `Backpressured`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OccupancyClass { + /// No occupancy samples (count/unbounded edge, or run too short). + Unknown, + /// Mostly empty, low empty-rate — idle, not starved. + Empty, + /// Mostly empty AND consumer frequently found it empty — producer-starved. + Starved, + /// Spread across the middle — neither side bound. + Healthy, + /// Mostly full AND producer frequently rejected — consumer-backpressured. + Backpressured, + /// Mostly full, low reject-rate — full but not actively rejecting. + Full, + /// Oscillates empty↔full (bursty / batch coupling). + Bimodal, +} + +/// Rate above which a `MostlyEmpty`/`MostlyFull` edge is reclassified as +/// `Starved`/`Backpressured`. 10% of attempts hitting the wall is a clear signal. +const RATE_REFINE_THRESHOLD: f64 = 0.10; + +/// Per-edge derived statistics for the end-of-run report. +#[derive(Debug, Clone)] +pub struct EdgeStatsSnapshot { + pub producer: &'static str, + pub consumer: Option<&'static str>, + /// Producer step index. The bottleneck verdict attributes edges to steps by + /// this identity (not by `producer`/`consumer` name), so fan-out, fan-in, or + /// duplicate step names are not misattributed to whichever edge matches a + /// name first. + pub producer_step: usize, + /// Consumer step index, `None` for a terminal edge with no consumer. + pub consumer_step: Option, + pub items_per_s: f64, + /// Consumer throughput in MiB/s (popped bytes / wall time / 2^20). + pub mibytes_per_s: f64, + pub class: OccupancyClass, + pub mean_occupancy: f32, + /// Fraction of push attempts rejected (backpressure signal). + pub reject_rate: f64, + /// Fraction of pop attempts that found the edge empty (starvation signal). + pub empty_rate: f64, + /// Derived residence time (Little's Law `L/X`), byte-bounded edges only; + /// `None` for count/unbounded edges (no occupancy) or zero throughput. + pub derived_latency_ms: Option, +} + +/// Refine the histogram's raw occupancy class using the reject/empty rates. +#[must_use] +fn refine_occupancy( + raw: crate::runtime::metrics::RawOccupancy, + reject_rate: f64, + empty_rate: f64, +) -> OccupancyClass { + use crate::runtime::metrics::RawOccupancy; + match raw { + RawOccupancy::Unknown => OccupancyClass::Unknown, + RawOccupancy::Healthy => OccupancyClass::Healthy, + RawOccupancy::Bimodal => OccupancyClass::Bimodal, + RawOccupancy::MostlyEmpty if empty_rate >= RATE_REFINE_THRESHOLD => OccupancyClass::Starved, + RawOccupancy::MostlyEmpty => OccupancyClass::Empty, + RawOccupancy::MostlyFull if reject_rate >= RATE_REFINE_THRESHOLD => { + OccupancyClass::Backpressured + } + RawOccupancy::MostlyFull => OccupancyClass::Full, + } +} + +/// Little's-Law residence time (`L/X`), byte-bounded edges only. Mean occupancy +/// in ITEMS = `mean_occupancy_bytes / mean_item_bytes`. +/// +/// `mean_occupancy_bytes` is sampled directly (absolute bytes at each tick), NOT +/// reconstructed as `fraction × limit`. That matters because `queue_memory_total` +/// can rebalance an edge's byte limit at runtime: the limit at teardown may +/// differ from the limits in force during sampling, so `fraction × final_limit` +/// would yield a materially wrong byte figure — and hence a wrong latency. +/// `limit_bytes` is retained only as the byte-bounded gate (`None` for +/// count/unbounded edges, which have no occupancy and thus no derived latency). +#[must_use] +#[allow(clippy::cast_precision_loss)] +fn derived_latency_ms( + ms: &crate::runtime::metrics::EdgeMetricsSnapshot, + limit_bytes: Option, + items_per_s: f64, +) -> Option { + if limit_bytes.is_none() || ms.pushed_items == 0 || items_per_s <= 0.0 { + return None; + } + let mean_item_bytes = ms.pushed_bytes as f64 / ms.pushed_items as f64; + if mean_item_bytes <= 0.0 { + return None; + } + let mean_items = ms.mean_occupancy_bytes / mean_item_bytes; + Some(1000.0 * mean_items / items_per_s) +} + +/// Compute one edge's derived stats from its metrics snapshot + (for a +/// byte-bounded edge) its byte budget, over a `wall_ns` run. Pure — unit-tested +/// directly. `limit_bytes` is `None` for count/unbounded edges (no occupancy → +/// no derived latency). +#[must_use] +#[allow(clippy::cast_precision_loss)] +pub(crate) fn compute_edge_stats( + producer: &'static str, + consumer: Option<&'static str>, + producer_step: usize, + consumer_step: Option, + ms: &crate::runtime::metrics::EdgeMetricsSnapshot, + limit_bytes: Option, + wall_ns: u64, +) -> EdgeStatsSnapshot { + let wall_secs = (wall_ns as f64) / 1e9; + let items_per_s = if wall_secs > 0.0 { ms.popped_items as f64 / wall_secs } else { 0.0 }; + // Divisor is 1 MiB (2^20), so the field/column is MiB/s, not MB/s. + let mibytes_per_s = + if wall_secs > 0.0 { (ms.popped_bytes as f64 / wall_secs) / 1_048_576.0 } else { 0.0 }; + let push_attempts = ms.pushed_items + ms.push_rejections; + let reject_rate = + if push_attempts > 0 { ms.push_rejections as f64 / push_attempts as f64 } else { 0.0 }; + let pop_attempts = ms.popped_items + ms.pop_empties; + let empty_rate = + if pop_attempts > 0 { ms.pop_empties as f64 / pop_attempts as f64 } else { 0.0 }; + + EdgeStatsSnapshot { + producer, + consumer, + producer_step, + consumer_step, + items_per_s, + mibytes_per_s, + class: refine_occupancy(ms.raw_occupancy, reject_rate, empty_rate), + mean_occupancy: ms.mean_occupancy, + reject_rate, + empty_rate, + derived_latency_ms: derived_latency_ms(ms, limit_bytes, items_per_s), + } +} + +#[derive(Debug, Clone, Copy)] +pub struct StepStatsSnapshot { + /// Total `try_run` dispatches (every outcome, including errors). + pub try_run_total: u64, + /// Dispatches that returned `StepOutcome::Progress`. + pub progress_count: u64, + /// Dispatches that returned `StepOutcome::NoProgress`. + pub no_progress_count: u64, + /// Dispatches that returned `StepOutcome::Contention`. + pub contention_count: u64, + /// Dispatches that returned `StepOutcome::Finished`. + pub finished_count: u64, + /// Dispatches that returned an error from `try_run_erased`. + pub error_count: u64, + /// Cumulative wall-ns spent inside `try_run` across all dispatches. + pub total_run_ns: u64, + /// Wall ns (from pipeline start) of this step's first Progress + /// dispatch start. `u64::MAX` if no Progress was ever recorded. + pub first_progress_ns: u64, + /// Wall ns (from pipeline start) of this step's last Progress + /// dispatch end. `0` if no Progress was ever recorded. + pub last_progress_ns: u64, +} + +impl StepStatsSnapshot { + #[must_use] + pub fn avg_run_ns(&self) -> Option { + if self.try_run_total == 0 { None } else { Some(self.total_run_ns / self.try_run_total) } + } +} + +/// Severity of a [`Finding`] from the bottleneck verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Severity { + /// The single rate-limiting step (full input edge + empty output edge). + Primary, + /// A contributing issue (spin, starvation) that is not the prime mover. + Secondary, + /// Whole-chain observation (e.g. latency-bound, not throughput-bound). + Info, +} + +/// One mechanically-derived diagnosis from [`bottleneck_verdict`]. +#[derive(Debug, Clone)] +pub(crate) struct Finding { + /// Triage classification. Set by [`bottleneck_verdict`] and asserted on by + /// the crate's tests; the human-readable report renders only `message`, so + /// the lib build never reads this field. + #[allow(dead_code)] + pub severity: Severity, + pub message: String, +} + +/// Rate above which a Serial step's `contention/tries` ratio is flagged as spin. +const SPIN_THRESHOLD: f64 = 0.20; + +/// Mechanically locate the chain's bottleneck and contributing issues from a +/// snapshot's step + edge stats. Rules (all from already-collected numbers): +/// - **Primary**: a step whose input edge is `Full`/`Backpressured` AND output +/// edge is `Empty`/`Starved` — work piles into it, it can't fill downstream. +/// Tagged CPU-bound if it dominates `total_run_ns`, else coordination. +/// - **Secondary (spin)**: a step with a high `contention/tries` ratio. +/// - **Secondary (starvation)**: an edge whose consumer frequently finds it +/// empty — the producer (upstream) can't keep up. +/// - **Info**: if no primary and no full/empty edge, the chain is +/// latency/coordination-bound, not throughput-bound. +#[must_use] +#[allow(clippy::cast_precision_loss)] +pub(crate) fn bottleneck_verdict(snap: &StatsSnapshot) -> Vec { + use OccupancyClass::{Backpressured, Empty, Full, Starved}; + let mut findings = Vec::new(); + let total_cpu_ns: u64 = snap.steps.iter().map(|(_, s)| s.total_run_ns).sum(); + // Attribute edges to steps by step IDENTITY (index), not by name. `snap.steps` + // is in `StepIdx` order, so a step's index is its id. A step's input is "full" + // if ANY input edge (whose consumer is this step) is Full/Backpressured, and + // its output is "empty" if ANY output edge (whose producer is this step) is + // Empty/Starved — aggregating over all matching edges rather than the first + // name match, so fan-out / fan-in / duplicate step names aren't misattributed. + let input_full = |step: usize| { + snap.edges + .iter() + .filter(|e| e.consumer_step == Some(step)) + .any(|e| matches!(e.class, Full | Backpressured)) + }; + let output_empty = |step: usize| { + snap.edges + .iter() + .filter(|e| e.producer_step == step) + .any(|e| matches!(e.class, Empty | Starved)) + }; + + // Primary bottleneck: full input edge + empty output edge. + for (step, (name, s)) in snap.steps.iter().enumerate() { + if input_full(step) && output_empty(step) { + let cpu_share = + if total_cpu_ns > 0 { s.total_run_ns as f64 / total_cpu_ns as f64 } else { 0.0 }; + let cause = if cpu_share >= 0.30 { + format!("CPU-bound ({:.0}% of pipeline try_run time)", cpu_share * 100.0) + } else { + "coordination-bound (low CPU share — likely a serialization stall)".to_string() + }; + findings.push(Finding { + severity: Severity::Primary, + message: format!( + "BOTTLENECK: step `{name}` (input edge full, output edge empty) — {cause}" + ), + }); + } + } + + // Secondary: Serial-step spin (contention thrash). Skip Detached steps — + // their dedicated-thread backoff loop records a NoProgress/Contention on + // every idle poll, which inflates the contention ratio, and "Detach + // candidate" is meaningless for a step that is already Detached. + // Match by `StepIdx`, not by name — as the edge attribution above does. Two + // steps may share a name, and a name match would let one Detached step's + // exemption silence a genuinely thrashing pool step that happens to share it. + let detached_steps: std::collections::HashSet = + snap.detached.iter().map(|&(step, ..)| step).collect(); + for (step, (name, s)) in snap.steps.iter().enumerate() { + if detached_steps.contains(&step) { + continue; + } + if s.try_run_total > 0 { + let spin = s.contention_count as f64 / s.try_run_total as f64; + if spin >= SPIN_THRESHOLD { + findings.push(Finding { + severity: Severity::Secondary, + message: format!( + "SPIN: step `{name}` contended on {:.0}% of dispatches — affinity / fuse / Detach candidate", + spin * 100.0 + ), + }); + } + } + } + + // Secondary: starvation (consumer of an edge frequently finds it empty). + for e in &snap.edges { + if matches!(e.class, Starved) { + findings.push(Finding { + severity: Severity::Secondary, + message: format!( + "STARVATION: `{}` is starved by upstream `{}` (empty {:.0}% of pops) — look upstream", + e.consumer.unwrap_or("(sink)"), + e.producer, + e.empty_rate * 100.0 + ), + }); + } + } + + // Info: no clear bottleneck and no full/empty edge → latency-bound. + let any_extreme = + snap.edges.iter().any(|e| matches!(e.class, Full | Backpressured | Empty | Starved)); + if findings.is_empty() && !snap.edges.is_empty() && !any_extreme { + findings.push(Finding { + severity: Severity::Info, + message: "No throughput bottleneck — edges have headroom; the chain is \ + latency/coordination-bound (look at residence time, not service time)." + .to_string(), + }); + } + findings +} + +impl fmt::Display for StatsSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Pipeline stats ({} step{}):", self.steps.len(), pluralize(self.steps.len()))?; + // Total dispatch time across steps — the denominator for each step's + // `cpu%` share (the Amdahl ranking: which step to optimize first) and + // for the headroom line. + let total_cpu_ns: u64 = self.steps.iter().map(|(_, s)| s.total_run_ns).sum(); + self.write_steps(f, total_cpu_ns)?; + self.write_utilization(f, total_cpu_ns)?; + // Per-edge throughput / occupancy / latency + the bottleneck verdict + // (both only when instrumented). + self.write_edges(f)?; + self.write_verdict(f) + } +} + +impl StatsSnapshot { + /// Render the per-step counter table, including each step's `cpu%` share of + /// total dispatch time (the per-step cost ranking). `total_cpu_ns` is the + /// sum of all steps' `total_run_ns`, passed in to avoid recomputing. + fn write_steps(&self, f: &mut fmt::Formatter<'_>, total_cpu_ns: u64) -> fmt::Result { + writeln!( + f, + " {:<28} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10} {:>14} {:>7} {:>12} {:>12}", + "step", + "tries", + "progress", + "noprog", + "content", + "fin", + "err", + "total_ms", + "cpu%", + "first_ms", + "last_ms", + )?; + for (name, s) in &self.steps { + // ns -> ms with three decimal places. The `as f64` cast can lose + // precision above 2^52 ns (~52 days of cumulative run time per + // step), well beyond any real pipeline run. + #[allow(clippy::cast_precision_loss)] + let total_ms = (s.total_run_ns as f64) / 1_000_000.0; + let first_str = if s.first_progress_ns == u64::MAX { + "-".to_string() + } else { + #[allow(clippy::cast_precision_loss)] + let ms = (s.first_progress_ns as f64) / 1_000_000.0; + format!("{ms:.1}") + }; + let last_str = if s.last_progress_ns == 0 { + "-".to_string() + } else { + #[allow(clippy::cast_precision_loss)] + let ms = (s.last_progress_ns as f64) / 1_000_000.0; + format!("{ms:.1}") + }; + // Share of total dispatch time — the per-step cost ranking. The + // chain-order rows stay readable; this column gives the magnitude. + #[allow(clippy::cast_precision_loss)] + let cpu_pct = if total_cpu_ns == 0 { + 0.0 + } else { + s.total_run_ns as f64 / total_cpu_ns as f64 * 100.0 + }; + writeln!( + f, + " {:<28} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10} {:>14.3} {:>6.1}% {:>12} {:>12}", + name, + s.try_run_total, + s.progress_count, + s.no_progress_count, + s.contention_count, + s.finished_count, + s.error_count, + total_ms, + cpu_pct, + first_str, + last_str, + )?; + } + Ok(()) + } + + /// Render per-worker utilisation, the pool-level utilisation, and the + /// squeeze-headroom synthesis. No-op when no worker recorded activity. + /// + /// Per-worker: busy = dispatch time, idle = backoff-sleep time. A high + /// idle% (cores parked while one worker drives a Serial step) is the + /// signature of pool under-utilisation. The headroom line reads straight off + /// these numbers: the pool idle% is recoverable via better step overlap; + /// once the pool saturates (no idle left) the only remaining wins are fewer + /// cycles/item or more cores, and the hottest step is the Amdahl target. + fn write_utilization(&self, f: &mut fmt::Formatter<'_>, total_cpu_ns: u64) -> fmt::Result { + if self.workers.is_empty() { + // No pool workers recorded activity, but a Detached thread may still + // have run (e.g. a degenerate chain). Render its line if so. + self.write_detached(f)?; + return Ok(()); + } + #[allow(clippy::cast_precision_loss)] + let ms = |ns: u64| (ns as f64) / 1_000_000.0; + let (mut sum_busy, mut sum_idle) = (0u64, 0u64); + writeln!(f, " {:<8} {:>14} {:>14} {:>8}", "worker", "busy_ms", "idle_ms", "busy%")?; + for &(id, busy, idle) in &self.workers { + sum_busy += busy; + sum_idle += idle; + let pct = if busy + idle == 0 { + 0.0 + } else { + #[allow(clippy::cast_precision_loss)] + let p = busy as f64 / (busy + idle) as f64 * 100.0; + p + }; + writeln!(f, " {id:<8} {:>14.3} {:>14.3} {pct:>7.1}%", ms(busy), ms(idle))?; + } + let pool_pct = if sum_busy + sum_idle == 0 { + 0.0 + } else { + #[allow(clippy::cast_precision_loss)] + let p = sum_busy as f64 / (sum_busy + sum_idle) as f64 * 100.0; + p + }; + writeln!( + f, + " pool utilisation: {pool_pct:.1}% busy across {} worker{} ({:.3} ms busy / {:.3} ms idle)", + self.workers.len(), + pluralize(self.workers.len()), + ms(sum_busy), + ms(sum_idle), + )?; + if let Some((hot_name, hot)) = self.steps.iter().max_by_key(|(_, s)| s.total_run_ns) { + #[allow(clippy::cast_precision_loss)] + let hot_pct = if total_cpu_ns == 0 { + 0.0 + } else { + hot.total_run_ns as f64 / total_cpu_ns as f64 * 100.0 + }; + writeln!( + f, + " headroom: {:.1}% pool idle (recoverable via overlap); hottest step `{hot_name}` = {hot_pct:.1}% of dispatch time (Amdahl target once pool saturates)", + 100.0 - pool_pct, + )?; + } + self.write_detached(f) + } + + /// Render the `StepKind::Detached` threads' busy/idle on their own line(s). + /// These threads are NOT pool workers (legacy "N + 2"), so they are reported + /// separately and never folded into the pool utilisation %. No-op when no + /// Detached thread recorded activity (every non-sort chain). + fn write_detached(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.detached.is_empty() { + return Ok(()); + } + #[allow(clippy::cast_precision_loss)] + let ms = |ns: u64| (ns as f64) / 1_000_000.0; + for &(_step, name, busy, idle, parks) in &self.detached { + let pct = if busy + idle == 0 { + 0.0 + } else { + #[allow(clippy::cast_precision_loss)] + let p = busy as f64 / (busy + idle) as f64 * 100.0; + p + }; + // avg park = idle / parks: a few long parks (idle at end) is benign; + // many short parks during productive phases is the backoff-latency + // signal (the deferred per-slot condvar would eliminate them). + #[allow(clippy::cast_precision_loss)] + let avg_park_us = if parks == 0 { 0.0 } else { (idle as f64 / parks as f64) / 1000.0 }; + writeln!( + f, + " detached `{name}`: {:.3} ms busy / {:.3} ms idle ({pct:.1}% busy, off pool); {parks} parks (avg {avg_park_us:.1}µs)", + ms(busy), + ms(idle), + )?; + } + Ok(()) + } + + /// Render the mechanically-derived bottleneck verdict. No-op when there are + /// no instrumented edges (nothing to diagnose). + fn write_verdict(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.edges.is_empty() { + return Ok(()); + } + let findings = bottleneck_verdict(self); + if findings.is_empty() { + return Ok(()); + } + writeln!(f, "Bottleneck verdict:")?; + for finding in findings { + writeln!(f, " {}", finding.message)?; + } + Ok(()) + } + + /// Render the per-edge table (throughput / occupancy class / rates / + /// latency). No-op when there are no instrumented edges. + fn write_edges(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.edges.is_empty() { + return Ok(()); + } + writeln!(f, "Pipeline edges ({}):", self.edges.len())?; + writeln!( + f, + " {:<40} {:>12} {:>10} {:>14} {:>6} {:>6} {:>10}", + "producer→consumer", "items/s", "MiB/s", "class", "rej%", "empt%", "lat_ms", + )?; + for e in &self.edges { + let edge = format!("{}→{}", e.producer, e.consumer.unwrap_or("(none)")); + let lat = e.derived_latency_ms.map_or_else(|| "-".to_string(), |l| format!("{l:.2}")); + writeln!( + f, + " {:<40} {:>12.0} {:>10.1} {:>14} {:>5.1}% {:>5.1}% {:>10}", + edge, + e.items_per_s, + e.mibytes_per_s, + format!("{:?}", e.class), + e.reject_rate * 100.0, + e.empty_rate * 100.0, + lat, + )?; + } + writeln!( + f, + " (note: throughput is depressed under tracing — confirm wall/RSS with --pipeline-trace off)" + ) + } +} + +fn pluralize(n: usize) -> &'static str { + if n == 1 { "" } else { "s" } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[test] + fn record_increments_progress_bucket() { + let stats = PipelineStats::new(vec!["A", "B"]); + stats.record(StepIdx(0), StepOutcome::Progress, 0, 100); + stats.record(StepIdx(0), StepOutcome::Progress, 200, 50); + stats.record(StepIdx(1), StepOutcome::NoProgress, 0, 25); + + let snap = stats.snapshot(); + assert_eq!(snap.steps.len(), 2); + + let (name_a, a) = &snap.steps[0]; + assert_eq!(*name_a, "A"); + assert_eq!(a.try_run_total, 2); + assert_eq!(a.progress_count, 2); + assert_eq!(a.no_progress_count, 0); + assert_eq!(a.total_run_ns, 150); + assert_eq!(a.first_progress_ns, 0); + assert_eq!(a.last_progress_ns, 250); + + let (name_b, b) = &snap.steps[1]; + assert_eq!(*name_b, "B"); + assert_eq!(b.try_run_total, 1); + assert_eq!(b.no_progress_count, 1); + assert_eq!(b.total_run_ns, 25); + assert_eq!(b.first_progress_ns, u64::MAX); + assert_eq!(b.last_progress_ns, 0); + } + + #[test] + fn record_buckets_each_outcome() { + let stats = PipelineStats::new(vec!["S"]); + stats.record(StepIdx(0), StepOutcome::Progress, 0, 1); + stats.record(StepIdx(0), StepOutcome::NoProgress, 10, 2); + stats.record(StepIdx(0), StepOutcome::Contention, 20, 3); + stats.record(StepIdx(0), StepOutcome::Finished, 30, 4); + stats.record_error(StepIdx(0), 40, 5); + + let snap = stats.snapshot(); + let (_name, s) = &snap.steps[0]; + assert_eq!(s.try_run_total, 5); + assert_eq!(s.progress_count, 1); + assert_eq!(s.no_progress_count, 1); + assert_eq!(s.contention_count, 1); + assert_eq!(s.finished_count, 1); + assert_eq!(s.error_count, 1); + assert_eq!(s.total_run_ns, 1 + 2 + 3 + 4 + 5); + } + + #[test] + fn avg_run_ns_handles_empty_step() { + let snap = StepStatsSnapshot { + try_run_total: 0, + progress_count: 0, + no_progress_count: 0, + contention_count: 0, + finished_count: 0, + error_count: 0, + total_run_ns: 0, + first_progress_ns: u64::MAX, + last_progress_ns: 0, + }; + assert_eq!(snap.avg_run_ns(), None); + + let snap2 = StepStatsSnapshot { try_run_total: 4, total_run_ns: 1000, ..snap }; + assert_eq!(snap2.avg_run_ns(), Some(250)); + } + + #[allow(clippy::too_many_arguments)] // test builder: one arg per snapshot field + fn edge_ms( + pushed_items: u64, + pushed_bytes: u64, + popped_items: u64, + pop_empties: u64, + push_rejections: u64, + raw: crate::runtime::metrics::RawOccupancy, + mean_occupancy: f32, + mean_occupancy_bytes: f64, + ) -> crate::runtime::metrics::EdgeMetricsSnapshot { + crate::runtime::metrics::EdgeMetricsSnapshot { + pushed_items, + pushed_bytes, + popped_items, + popped_bytes: pushed_bytes, + push_rejections, + pop_empties, + depth_samples: 100, + raw_occupancy: raw, + mean_occupancy, + mean_occupancy_bytes, + } + } + + #[test] + fn edge_stats_byte_bounded_latency_is_dimensionally_correct() { + use crate::runtime::metrics::RawOccupancy; + // 1000 items/s; mean_item_bytes = 100_000/1000 = 100; sampled mean + // occupancy 1000 bytes → mean_items 10 → latency = 1000*10/1000 = 10ms. + // (The byte figure is sampled directly, so the limit only gates that the + // edge is byte-bounded — its value no longer feeds the latency.) + let ms = edge_ms(1000, 100_000, 1000, 0, 0, RawOccupancy::Healthy, 0.5, 1000.0); + let e = compute_edge_stats("p", Some("c"), 0, Some(1), &ms, Some(2000), 1_000_000_000); + assert!((e.items_per_s - 1000.0).abs() < 1.0, "items/s ≈ 1000, got {}", e.items_per_s); + let lat = e.derived_latency_ms.expect("byte edge has derived latency"); + assert!((lat - 10.0).abs() < 0.5, "latency ≈ 10ms, got {lat}"); + } + + #[test] + fn edge_stats_count_edge_has_no_latency() { + use crate::runtime::metrics::RawOccupancy; + let ms = edge_ms(500, 0, 500, 0, 0, RawOccupancy::Unknown, 0.0, 0.0); + let e = compute_edge_stats("p", Some("c"), 0, Some(1), &ms, None, 1_000_000_000); + assert!(e.derived_latency_ms.is_none(), "count/unbounded edge → no derived latency"); + assert!((e.items_per_s - 500.0).abs() < 1.0); + } + + // The raw occupancy class is refined by the empty-rate and reject-rate the + // edge actually observed; each case pins one (raw class, rates, byte bound) + // combination to its refined `OccupancyClass`. + #[rstest] + // MostlyEmpty + high empty-rate (90/100 pops empty) → Starved. + #[case::starved( + edge_ms(10, 0, 10, 90, 0, crate::runtime::metrics::RawOccupancy::MostlyEmpty, 0.0, 0.0), + None, + OccupancyClass::Starved + )] + // MostlyEmpty + low empty-rate → Empty (idle, not starved). + #[case::empty( + edge_ms(100, 0, 100, 0, 0, crate::runtime::metrics::RawOccupancy::MostlyEmpty, 0.0, 0.0), + None, + OccupancyClass::Empty + )] + // MostlyFull + high reject-rate → Backpressured. + #[case::backpressured( + edge_ms(10, 0, 10, 0, 90, crate::runtime::metrics::RawOccupancy::MostlyFull, 1.0, 1000.0), + Some(1000), + OccupancyClass::Backpressured + )] + fn occupancy_class_refined_by_rates( + #[case] ms: crate::runtime::metrics::EdgeMetricsSnapshot, + #[case] limit_bytes: Option, + #[case] expected: OccupancyClass, + ) { + assert_eq!( + compute_edge_stats("p", None, 0, None, &ms, limit_bytes, 1_000_000_000).class, + expected + ); + } + + fn step_stat( + name: &'static str, + total_run_ns: u64, + contention: u64, + tries: u64, + ) -> (&'static str, StepStatsSnapshot) { + ( + name, + StepStatsSnapshot { + try_run_total: tries, + progress_count: tries, + no_progress_count: 0, + contention_count: contention, + finished_count: 1, + error_count: 0, + total_run_ns, + first_progress_ns: 0, + last_progress_ns: total_run_ns, + }, + ) + } + + fn edge_stat( + producer: &'static str, + consumer: Option<&'static str>, + producer_step: usize, + consumer_step: Option, + class: OccupancyClass, + empty_rate: f64, + ) -> EdgeStatsSnapshot { + EdgeStatsSnapshot { + producer, + consumer, + producer_step, + consumer_step, + items_per_s: 1000.0, + mibytes_per_s: 1.0, + class, + mean_occupancy: 0.5, + reject_rate: 0.0, + empty_rate, + derived_latency_ms: Some(1.0), + } + } + + #[test] + fn verdict_locates_cpu_bound_bottleneck() { + // `Slow`'s input edge is Full and output edge is Empty, and it dominates + // CPU → Primary, CPU-bound. + let snap = StatsSnapshot { + steps: vec![step_stat("Slow", 1000, 0, 10)], + workers: vec![], + detached: vec![], + // `Slow` is step 0. Its input edge (consumer_step 0) is Full; its + // output edge (producer_step 0) is Empty. `Up`/`Down` are non-step + // ids (1/1) so only `Slow` matches by identity. + edges: vec![ + edge_stat("Up", Some("Slow"), 1, Some(0), OccupancyClass::Full, 0.0), + edge_stat("Slow", Some("Down"), 0, Some(1), OccupancyClass::Empty, 0.0), + ], + }; + let v = bottleneck_verdict(&snap); + let primary: Vec<_> = v.iter().filter(|f| f.severity == Severity::Primary).collect(); + assert_eq!(primary.len(), 1, "exactly one primary bottleneck"); + assert!(primary[0].message.contains("Slow")); + assert!(primary[0].message.contains("CPU-bound")); + } + + #[test] + fn verdict_matches_edges_by_step_identity_across_fan_in() { + // Regression for name-based misattribution: `Merge` (step 2) has TWO + // input edges (fan-in from steps 0 and 1). The first by iteration order + // is Healthy; the second is Full. Matching the FIRST edge by consumer + // name would see only the Healthy input and report no bottleneck; + // matching by step identity aggregates both inputs and correctly flags + // `Merge` (input full + output empty). + let snap = StatsSnapshot { + steps: vec![ + step_stat("A", 100, 0, 10), + step_stat("B", 100, 0, 10), + step_stat("Merge", 1000, 0, 10), + ], + workers: vec![], + detached: vec![], + edges: vec![ + edge_stat("A", Some("Merge"), 0, Some(2), OccupancyClass::Healthy, 0.0), + edge_stat("B", Some("Merge"), 1, Some(2), OccupancyClass::Full, 0.0), + edge_stat("Merge", Some("Sink"), 2, Some(3), OccupancyClass::Empty, 0.0), + ], + }; + let v = bottleneck_verdict(&snap); + let primary: Vec<_> = v.iter().filter(|f| f.severity == Severity::Primary).collect(); + assert_eq!(primary.len(), 1, "fan-in bottleneck detected via identity: {v:?}"); + assert!(primary[0].message.contains("Merge")); + } + + #[test] + fn verdict_flags_spin_and_starvation() { + let snap = StatsSnapshot { + // 5/10 dispatches contended → spin. + steps: vec![step_stat("Serializer", 100, 5, 10)], + workers: vec![], + detached: vec![], + // A starved edge (steps 1→2, not the `Serializer` step 0): consumer + // frequently finds it empty. + edges: vec![edge_stat( + "Producer", + Some("Consumer"), + 1, + Some(2), + OccupancyClass::Starved, + 0.7, + )], + }; + let v = bottleneck_verdict(&snap); + assert!( + v.iter().any(|f| f.severity == Severity::Secondary && f.message.contains("SPIN")), + "spin finding present" + ); + assert!( + v.iter().any(|f| f.message.contains("STARVATION") && f.message.contains("Consumer")), + "starvation finding present" + ); + } + + #[test] + fn verdict_skips_spin_for_detached_steps() { + // A Detached step's backoff loop records Contention on every idle poll, + // so its contention/tries ratio is high — but "SPIN: … Detach candidate" + // is nonsensical for an already-Detached step, so it must be skipped. + let snap = StatsSnapshot { + // 8/10 "contended" — would trip SPIN if it were a pool step. + steps: vec![step_stat("SortMerge", 100, 8, 10)], + workers: vec![], + detached: vec![(0, "SortMerge", 900_000_000, 100_000_000, 4_200)], + edges: vec![], + }; + let v = bottleneck_verdict(&snap); + assert!( + !v.iter().any(|f| f.message.contains("SPIN")), + "no SPIN finding for a Detached step: {v:?}" + ); + } + + /// The exemption is per step IDENTITY, not per name. Two steps sharing a + /// name — one Detached, one a thrashing pool step — must be judged + /// independently: exempting by name silenced the pool step's SPIN finding. + #[test] + fn verdict_reports_spin_for_a_pool_step_sharing_a_detached_step_name() { + let snap = StatsSnapshot { + // Step 0 is the Detached merge; step 1 is a pool step that happens to + // carry the same name and is genuinely thrashing (8/10 contended). + steps: vec![step_stat("SortMerge", 100, 8, 10), step_stat("SortMerge", 100, 8, 10)], + workers: vec![], + detached: vec![(0, "SortMerge", 900_000_000, 100_000_000, 4_200)], + edges: vec![], + }; + let v = bottleneck_verdict(&snap); + assert_eq!( + v.iter().filter(|f| f.message.contains("SPIN")).count(), + 1, + "exactly the pool step (index 1) reports SPIN; the Detached step (index 0) is \ + exempt: {v:?}" + ); + } + + #[test] + fn verdict_reports_latency_bound_when_no_extremes() { + let snap = StatsSnapshot { + steps: vec![step_stat("A", 100, 0, 10)], + workers: vec![], + detached: vec![], + edges: vec![edge_stat("A", Some("B"), 0, Some(1), OccupancyClass::Healthy, 0.0)], + }; + let v = bottleneck_verdict(&snap); + assert_eq!(v.len(), 1); + assert_eq!(v[0].severity, Severity::Info); + assert!(v[0].message.contains("latency")); + } + + #[test] + fn snapshot_display_renders_without_edges() { + // Back-compat: the edge-less snapshot() still renders (empty edges). + let stats = PipelineStats::new(vec!["A"]); + let out = format!("{}", stats.snapshot()); + assert!(out.contains("Pipeline stats")); + assert!(!out.contains("Pipeline edges"), "no edge section when edges empty"); + } + + #[test] + fn display_renders_cpu_share_and_headroom() { + // Hot owns 750/1000 = 75% of dispatch time; Cool owns the rest. The pool + // is 80% busy (800 ms / 1000 ms) → 20% idle headroom. + let snap = StatsSnapshot { + steps: vec![step_stat("Hot", 750, 0, 10), step_stat("Cool", 250, 0, 10)], + workers: vec![(0, 800, 200)], + detached: vec![], + edges: vec![], + }; + let out = format!("{snap}"); + assert!(out.contains("cpu%"), "per-step table has a cpu% column: {out}"); + assert!(out.contains("75.0%"), "Hot step shows its 75% CPU share: {out}"); + assert!(out.contains("pool utilisation: 80.0%"), "pool utilisation line present: {out}"); + assert!( + out.contains("headroom:") && out.contains("20.0% pool idle") && out.contains("`Hot`"), + "headroom line names the hottest step and the idle %: {out}" + ); + } + + #[test] + fn utilization_section_absent_without_workers() { + // No worker activity → no pool/headroom lines (fused or stats-off runs). + let snap = StatsSnapshot { + steps: vec![step_stat("Solo", 100, 0, 10)], + workers: vec![], + detached: vec![], + edges: vec![], + }; + let out = format!("{snap}"); + assert!(!out.contains("pool utilisation"), "no pool line without workers: {out}"); + assert!(!out.contains("headroom:"), "no headroom line without workers: {out}"); + } + + #[test] + fn display_renders_header_and_rows() { + let stats = PipelineStats::new(vec!["StepA", "StepB"]); + stats.record(StepIdx(0), StepOutcome::Progress, 0, 10); + let s = format!("{}", stats.snapshot()); + assert!(s.contains("Pipeline stats (2 steps):")); + assert!(s.contains("StepA")); + assert!(s.contains("StepB")); + assert!(s.contains("step")); + } + + /// L2.4: a Detached thread's busy/idle is recorded separately, renders on + /// its own line, and is EXCLUDED from the pool utilisation % (legacy + /// "N + 2"). The snapshot must render without panicking. + #[test] + fn detached_busy_excluded_from_pool_and_on_own_line() { + let stats = PipelineStats::new(vec!["Source", "SortMerge"]); + // One pool worker: 800 ms busy / 200 ms idle → pool% = 80%. + stats.record_worker_busy(0, 800_000_000); + stats.record_worker_idle(0, 200_000_000); + // The Detached merge: 900 ms busy / 100 ms idle (off pool). + stats.record_detached_busy(StepIdx(1), 900_000_000); + stats.record_detached_idle(StepIdx(1), 100_000_000); + + let snap = stats.snapshot(); + assert_eq!(snap.detached.len(), 1, "one Detached thread recorded"); + assert_eq!( + (snap.detached[0].0, snap.detached[0].1), + (1, "SortMerge"), + "the entry carries its StepIdx alongside the name" + ); + + let out = format!("{snap}"); + // Pool% reflects ONLY the worker (80%), NOT the Detached thread (90%). + assert!( + out.contains("pool utilisation: 80.0%"), + "pool% must exclude the Detached thread: {out}" + ); + // The Detached thread is reported on its own line. + assert!( + out.contains("detached `SortMerge`") && out.contains("90.0% busy, off pool"), + "Detached line present with its own busy%: {out}" + ); + } + + /// L2.4: the Detached line renders even when no pool worker recorded + /// activity (the `write_utilization` early-return path), without panicking. + #[test] + fn detached_line_renders_with_no_pool_workers() { + let stats = PipelineStats::new(vec!["OnlyDetached"]); + stats.record_detached_busy(StepIdx(0), 5_000_000); + let out = format!("{}", stats.snapshot()); + assert!(out.contains("detached `OnlyDetached`"), "Detached line present: {out}"); + assert!(!out.contains("pool utilisation"), "no pool line without workers: {out}"); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/storage.rs b/crates/fgumi-pipeline-core/src/runtime/storage.rs new file mode 100644 index 000000000..3ab10d59d --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/storage.rs @@ -0,0 +1,431 @@ +//! `WorkerStepEntry`: per-worker per-step storage shape. Determined by +//! `StepKind` at run start; immutable thereafter. + +use parking_lot::Mutex; +use std::sync::{Arc, OnceLock}; + +use crate::erased::ErasedStep; +use crate::step::{Affinity, StepKind}; + +/// Shared "this step has finished" latch for a `Serial` step. +/// +/// A `Serial` step is one shared instance behind a `Mutex` across all workers, +/// but each worker tracks its own worklist. Without a shared latch, after one +/// worker runs the step to `Finished` the others still hold it in their +/// worklist and would re-`try_lock`/re-run an already-finished step. The +/// finishing worker sets this latch under the dispatch guard (before +/// `mark_outputs_drained`); every other worker observes it and short-circuits +/// to a synthetic `Finished` instead of re-running the step. +#[derive(Default)] +pub struct DrainGate { + finished: OnceLock<()>, +} + +impl DrainGate { + /// True once the step has run to `StepOutcome::Finished` on some worker. + pub fn is_finished(&self) -> bool { + self.finished.get().is_some() + } + + /// Latch the step as finished (idempotent). Set under the dispatch guard, + /// before `mark_outputs_drained`, so a concurrent worker that observes the + /// latch never re-runs the step. + pub fn mark_finished(&self) { + let _ = self.finished.set(()); + } +} + +/// One per (worker, step) cell. +pub enum WorkerStepEntry { + /// `Parallel` step: this worker owns its private clone via `clone_boxed`. + /// Direct `&mut` access; no locking on `try_run` dispatch. Completion is + /// coordinated by the per-step `StepDrainCounter` (init N) in the driver: + /// every clone returns `Finished` when the input drains, but only the last + /// to finish closes the shared output. + Owned { step: Box }, + /// `Serial` step: shared instance, mutex-protected. Any worker can acquire. + /// The shared `DrainGate` finished-latch lets a worker that finishes the + /// step stop the others from re-running it. + Shared { step: Arc>>, drain: Arc }, + /// `Exclusive` step: only this worker (the owner) ever runs it. + /// Stored locally on the owner; other workers have `Skip`. + Exclusive { step: Box }, + /// This worker doesn't run this step (it's an `Exclusive` step owned by + /// another worker). The worker loop skips it in dispatch. + Skip, +} + +impl WorkerStepEntry { + /// True if this worker should attempt to dispatch this step. + #[must_use] + pub fn is_dispatchable(&self) -> bool { + !matches!(self, Self::Skip) + } +} + +/// Assemble per-worker step storage. +/// +/// `steps`: the chain in order (consumed). +/// `exclusive_owners[step_idx] == Some(worker_id)` for each Exclusive step. +/// `n_workers`: number of worker threads. +/// +/// Returns `entries[worker_id][step_idx] = WorkerStepEntry`. +/// +/// # Panics +/// +/// Panics if `exclusive_owners.len() != steps.len()` or if an Exclusive +/// step has no owner assignment (`assign_exclusive_owners` must run first). +#[must_use] +pub fn build_worker_storage( + steps: Vec>, + exclusive_owners: &[Option], + n_workers: usize, +) -> Vec> { + assert_eq!( + steps.len(), + exclusive_owners.len(), + "exclusive_owners length must match step count" + ); + assert!(n_workers > 0, "build_worker_storage requires at least one worker"); + + let mut entries: Vec> = + (0..n_workers).map(|_| Vec::with_capacity(steps.len())).collect(); + + for (step_idx, step) in steps.into_iter().enumerate() { + let kind = step.kind(); + match kind { + StepKind::Parallel => { + // Each worker gets its own clone; the original goes to worker N-1. + for entries_for_worker in entries.iter_mut().take(n_workers - 1) { + entries_for_worker.push(WorkerStepEntry::Owned { step: step.clone_boxed() }); + } + entries[n_workers - 1].push(WorkerStepEntry::Owned { step }); + } + StepKind::Serial => { + // Snapshot the affinity hint before moving `step` into + // the shared `Arc>` — affinity is part of the + // static `Step` description, so calling it here is a + // single virtual dispatch. + let affinity = step.affinity(); + // Always-on (not `debug_assert!`): an out-of-range affinity + // gates the Serial step out of every worker, so it is never + // dispatched and the pipeline deadlocks. The check is a cheap + // pure predicate, so it stays enabled in release builds too. + assert!( + affinity_in_range(affinity, n_workers), + "Serial step affinity {affinity:?} requests a worker out of range \ + (n_workers = {n_workers}); pipeline would deadlock" + ); + let drain = Arc::new(DrainGate::default()); + let shared = Arc::new(Mutex::new(step)); + for (worker_id, entries_for_worker) in entries.iter_mut().enumerate() { + if affinity.eligible(worker_id, n_workers) { + entries_for_worker.push(WorkerStepEntry::Shared { + step: Arc::clone(&shared), + drain: Arc::clone(&drain), + }); + } else { + // This worker is gated out by the Serial step's + // affinity hint. It will never `try_lock` the step's + // mutex, eliminating the `Contention` thrash that + // pure `Serial` exhibits at high thread counts. + entries_for_worker.push(WorkerStepEntry::Skip); + } + } + } + StepKind::Exclusive => { + let owner = exclusive_owners[step_idx].expect( + "Exclusive step has no owner assignment; \ + assign_exclusive_owners must run before build_worker_storage", + ); + // Always-on, matching the Serial arm's affinity check above: an + // out-of-range owner would otherwise surface as an opaque slice + // index panic on `entries[owner]`. `assign_sticky_owners` + // explicitly tolerates an out-of-range owner (and its + // `out_of_range` test case treats one as reachable input), so the + // two functions must not disagree about whether it can happen. + assert!( + owner < n_workers, + "Exclusive step owner {owner} is out of range (n_workers = {n_workers}); \ + the step would never be dispatched and the pipeline would deadlock" + ); + // Push Skip placeholders for all workers, then replace owner's slot. + for entries_for_worker in &mut entries { + entries_for_worker.push(WorkerStepEntry::Skip); + } + entries[owner][step_idx] = WorkerStepEntry::Exclusive { step }; + } + StepKind::Detached => { + // A Detached step never runs on the pool. Every real Detached + // instance is extracted by `extract_detached_steps` BEFORE this + // function runs (it drives the step on its own dedicated + // thread) and is replaced in `steps` by a `DetachedPlaceholder` + // whose `kind()` still reports `StepKind::Detached`. So on every + // real sort run this arm IS reached — once per placeholder — and + // its job is exactly this: give every pool worker a `Skip` entry + // and drop the placeholder (it holds no state to run). + for entries_for_worker in &mut entries { + entries_for_worker.push(WorkerStepEntry::Skip); + } + drop(step); + } + } + } + + entries +} + +/// Validate that a Serial step's affinity refers to a worker that exists. +/// `Affinity::None` / `Reader` / `Writer` always resolve to a real worker +/// when `n_workers >= 1` (which the runtime's `assert!(n_workers > 0)` +/// already guarantees). `Worker(idx)` is only valid when `idx < n_workers`. +/// +/// Routed through [`Affinity::target_worker`] rather than re-matching the +/// variants here: that method is the single source of truth for the +/// affinity→worker mapping, and the dispatch gate resolves the target the same +/// way. A future variant therefore cannot make this range check and the gate +/// disagree about which worker a step lands on. +#[inline] +fn affinity_in_range(affinity: Affinity, n_workers: usize) -> bool { + affinity.target_worker(n_workers).is_none_or(|target| target < n_workers) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + + use rstest::rstest; + + use crate::erased::TypedStep; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + use crate::step::{Affinity, Step, StepCtx, StepOutcome, StepProfile}; + + fn profile_for(name: &'static str, kind: StepKind, sticky: bool) -> StepProfile { + StepProfile { + name, + kind, + sticky, + output_queues: vec![QueueSpec::CountBounded { capacity: 4 }], + branch_ordering: vec![BranchOrdering::None], + } + } + + #[derive(Clone)] + struct ParallelStep; + impl Step for ParallelStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + profile_for("Par", StepKind::Parallel, false) + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + fn new_worker_copy(&self) -> Self { + self.clone() + } + } + + #[derive(Clone)] + struct SerialStep; + impl Step for SerialStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + profile_for("Ser", StepKind::Serial, false) + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[derive(Clone)] + struct ExclusiveStep; + impl Step for ExclusiveStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + profile_for("Excl", StepKind::Exclusive, false) + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[derive(Clone)] + struct DetachedStep; + impl Step for DetachedStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + profile_for("Det", StepKind::Detached, false) + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[test] + fn skip_is_not_dispatchable() { + assert!(!WorkerStepEntry::Skip.is_dispatchable()); + } + + /// L2.1: a `Detached` step is excluded from every pool worker's dispatch + /// list (all workers get `Skip`). The dedicated-thread extraction is L2.3; + /// here we only prove the pool never sees it. + #[test] + fn detached_step_is_skipped_on_all_workers() { + let steps: Vec> = vec![Box::new(TypedStep::new(DetachedStep))]; + let owners = vec![None]; + let entries = build_worker_storage(steps, &owners, 4); + assert_eq!(entries.len(), 4); + for w in &entries { + assert_eq!(w.len(), 1); + assert!( + matches!(w[0], WorkerStepEntry::Skip), + "Detached step must be Skip on every pool worker" + ); + } + } + + #[test] + fn parallel_step_yields_owned_per_worker() { + let steps: Vec> = vec![Box::new(TypedStep::new(ParallelStep))]; + let owners = vec![None]; + let entries = build_worker_storage(steps, &owners, 4); + assert_eq!(entries.len(), 4); + for w in &entries { + assert_eq!(w.len(), 1); + assert!(matches!(w[0], WorkerStepEntry::Owned { .. })); + } + } + + #[test] + fn serial_step_yields_shared_arc_for_all_workers() { + let steps: Vec> = vec![Box::new(TypedStep::new(SerialStep))]; + let owners = vec![None]; + let entries = build_worker_storage(steps, &owners, 3); + for w in &entries { + assert!(matches!(w[0], WorkerStepEntry::Shared { .. })); + } + let arc0 = if let WorkerStepEntry::Shared { step, .. } = &entries[0][0] { + Arc::clone(step) + } else { + panic!() + }; + let arc1 = if let WorkerStepEntry::Shared { step, .. } = &entries[1][0] { + Arc::clone(step) + } else { + panic!() + }; + assert!(Arc::ptr_eq(&arc0, &arc1)); + } + + #[test] + fn exclusive_step_owner_gets_exclusive_others_skip() { + let steps: Vec> = vec![Box::new(TypedStep::new(ExclusiveStep))]; + let owners = vec![Some(2)]; + let entries = build_worker_storage(steps, &owners, 4); + assert!(matches!(entries[0][0], WorkerStepEntry::Skip)); + assert!(matches!(entries[1][0], WorkerStepEntry::Skip)); + assert!(matches!(entries[2][0], WorkerStepEntry::Exclusive { .. })); + assert!(matches!(entries[3][0], WorkerStepEntry::Skip)); + } + + struct AffinitySerialStep(crate::step::Affinity); + impl Step for AffinitySerialStep { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + profile_for("AffinitySer", StepKind::Serial, false) + } + fn affinity(&self) -> crate::step::Affinity { + self.0 + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[rstest] + #[case::reader(Affinity::Reader, 0)] + #[case::writer(Affinity::Writer, 2)] // last of 3 workers + #[case::worker_idx(Affinity::Worker(1), 1)] + fn serial_affinity_eligible_only_for_target_worker( + #[case] affinity: Affinity, + #[case] eligible: usize, + ) { + // A Serial step's affinity makes exactly one worker eligible (`Shared`); + // every other worker `Skip`s it. Reader→0, Writer→last, Worker(i)→i. + let steps: Vec> = + vec![Box::new(TypedStep::new(AffinitySerialStep(affinity)))]; + let owners = vec![None]; + let entries = build_worker_storage(steps, &owners, 3); + for (worker, entry) in entries.iter().enumerate() { + if worker == eligible { + assert!( + matches!(entry[0], WorkerStepEntry::Shared { .. }), + "worker {worker} should be eligible (Shared) for {affinity:?}" + ); + } else { + assert!( + matches!(entry[0], WorkerStepEntry::Skip), + "worker {worker} should Skip for {affinity:?}" + ); + } + } + } + + #[test] + #[should_panic(expected = "Serial step affinity")] + fn serial_out_of_range_worker_panics_in_storage() { + let steps: Vec> = + vec![Box::new(TypedStep::new(AffinitySerialStep(Affinity::Worker(99))))]; + let owners = vec![None]; + // build_worker_storage checks affinity in range via assert!; should panic. + let _ = build_worker_storage(steps, &owners, 3); + } + + /// The Exclusive arm's owner-range check is an always-on `assert!` (not a + /// `debug_assert!`) because an out-of-range owner would otherwise surface as + /// an opaque `entries[owner]` slice-index panic, and `assign_sticky_owners` + /// explicitly tolerates one. Without this case only the Serial arm was + /// covered, so a refactor could weaken this one to `debug_assert!` and CI + /// would stay green. + #[test] + #[should_panic(expected = "Exclusive step owner")] + fn exclusive_out_of_range_owner_panics_in_storage() { + let steps: Vec> = vec![Box::new(TypedStep::new(ExclusiveStep))]; + let owners = vec![Some(99)]; + let _ = build_worker_storage(steps, &owners, 3); + } + + #[test] + fn mixed_chain_assigns_correctly() { + let steps: Vec> = vec![ + Box::new(TypedStep::new(ExclusiveStep)), + Box::new(TypedStep::new(ParallelStep)), + Box::new(TypedStep::new(SerialStep)), + Box::new(TypedStep::new(ExclusiveStep)), + ]; + let owners = vec![Some(0), None, None, Some(1)]; + let entries = build_worker_storage(steps, &owners, 4); + + assert!(matches!(entries[0][0], WorkerStepEntry::Exclusive { .. })); + assert!(matches!(entries[1][0], WorkerStepEntry::Skip)); + + for w in &entries { + assert!(matches!(w[1], WorkerStepEntry::Owned { .. })); + } + + for w in &entries { + assert!(matches!(w[2], WorkerStepEntry::Shared { .. })); + } + + assert!(matches!(entries[1][3], WorkerStepEntry::Exclusive { .. })); + assert!(matches!(entries[0][3], WorkerStepEntry::Skip)); + } +} diff --git a/crates/fgumi-pipeline-core/src/runtime/worker_core.rs b/crates/fgumi-pipeline-core/src/runtime/worker_core.rs new file mode 100644 index 000000000..2274419a0 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/runtime/worker_core.rs @@ -0,0 +1,247 @@ +//! `WorkerCore`: per-thread state carried by the worker loop. + +use std::time::Duration; + +use crate::topology::StepIdx; + +// Pool worker idle bounds: a pool worker is never unparked, so it sleeps and can +// ramp to a coarse cap without hurting wake latency. +const SLEEP_INITIAL_US: u64 = 1; +const SLEEP_MAX_US: u64 = 50_000; // 50 milliseconds + +// Dedicated-driver idle bounds: a driver drives a small step subset off the pool +// and must stay responsive to its peers (the pool filling/draining its edges), so +// it parks with a tight cap. `park_timeout` also lets a producer holding the +// thread handle unpark it early; the cap is the bounded fallback either way. +const PARK_INITIAL_US: u64 = 10; +const PARK_MAX_US: u64 = 500; + +/// How a worker idles on a no-progress tick. Selected per thread at construction: +/// the N-worker pool uses [`Sleep`](BackoffPolicy::Sleep); each dedicated driver +/// thread (the unified "1-thread pool") uses [`Park`](BackoffPolicy::Park). This +/// is the *only* behavioral difference between a pool worker and a driver — both +/// run the same [`run_worker_loop`](crate::runtime::run_worker_loop). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackoffPolicy { + /// `thread::sleep`, ramp 1µs→50ms. For pool workers (never unparked). + Sleep, + /// `thread::park_timeout`, ramp 10µs→500µs. For dedicated driver threads: a + /// producer *may* unpark early, and the tight cap bounds wake latency when it + /// does not. With no unparker this behaves as a bounded sleep. + Park, +} + +impl BackoffPolicy { + #[inline] + fn initial_us(self) -> u64 { + match self { + Self::Sleep => SLEEP_INITIAL_US, + Self::Park => PARK_INITIAL_US, + } + } + + #[inline] + fn max_us(self) -> u64 { + match self { + Self::Sleep => SLEEP_MAX_US, + Self::Park => PARK_MAX_US, + } + } +} + +/// Whether a `run_worker_loop` thread is an N-pool worker or a dedicated driver +/// (the unified "1-thread pool" for a set of off-pool steps). Controls only how +/// the thread's busy/idle time is attributed in `--pipeline-stats`, always +/// excluded from the pool% so the "N + 2" split stays visible: +/// - pool threads sum their whole-pass busy/idle into the N-worker utilisation +/// line, by `thread_id`; +/// - driver threads record each grouped step's own busy on the off-pool +/// "detached" line (by that step's index — so a multi-step `Shared` group +/// shows each member's real time, not the whole thread's under one name), and +/// attribute the thread-level idle/park to the group's `primary_step`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerRole { + /// N-worker pool thread; busy/idle keyed by `thread_id`. + Pool, + /// Dedicated driver thread. Per-step busy is recorded by each step's own + /// index in `dispatch_one_step`; thread-level idle/park is keyed to + /// `primary_step` (the group's representative). + Driver { primary_step: StepIdx }, +} + +pub struct WorkerCore { + /// `0..n_workers` for pool threads; unused (0) for driver threads. + pub thread_id: usize, + /// If this worker owns an `Exclusive` step, that step's index. + /// Used by the runtime to skip `WorkerStepEntry::Skip` placeholders + /// for Exclusive steps owned by other workers. + pub exclusive_owner: Option, + /// If this worker is the sole eligible dispatcher for a `sticky` step + /// (either an `Exclusive sticky` step it owns, or a `Serial + sticky` + /// step whose `Affinity` targets this worker), the step's index. + /// The driver drives this step in a tight inner loop until it returns + /// `NoProgress` / `Contention` / `Finished`, then yields to round- + /// robin. Mirrors the legacy pipeline's sticky read. + pub sticky_owner: Option, + /// Pool worker vs dedicated driver. Determines both the idle backoff policy + /// (`Pool → Sleep`, `Driver → Park`, via [`Self::policy`]) and how the + /// thread's aggregate busy/idle time is attributed in `--pipeline-stats`. + role: WorkerRole, + /// Backoff duration in microseconds. Doubled on no-progress; reset on progress. + /// Bounds come from `role`'s policy. + backoff_us: u64, +} + +impl WorkerCore { + /// A pool worker (`Sleep` backoff). Existing callers are unchanged. + #[must_use] + pub fn new( + thread_id: usize, + exclusive_owner: Option, + sticky_owner: Option, + ) -> Self { + Self { + thread_id, + exclusive_owner, + sticky_owner, + role: WorkerRole::Pool, + backoff_us: BackoffPolicy::Sleep.initial_us(), + } + } + + /// A dedicated driver thread (the unified "1-thread pool"): `Driver` role + + /// `Park` backoff (derived from the role). Drives a set of `Owned` steps off + /// the pool; its aggregate busy/idle is attributed to `primary_step` on the + /// off-pool detached line. `thread_id` is unused for drivers; it never owns + /// Exclusive or sticky steps. + #[must_use] + pub fn driver(primary_step: StepIdx) -> Self { + let mut worker = Self::new(0, None, None); + worker.role = WorkerRole::Driver { primary_step }; + worker.backoff_us = worker.policy().initial_us(); + worker + } + + /// This thread's pool/driver role (drives stats attribution in the loop). + #[must_use] + pub fn role(&self) -> WorkerRole { + self.role + } + + /// The idle backoff policy implied by this thread's role: pool workers + /// `Sleep` (never unparked), driver threads `Park`. + #[must_use] + pub fn policy(&self) -> BackoffPolicy { + match self.role { + WorkerRole::Pool => BackoffPolicy::Sleep, + WorkerRole::Driver { .. } => BackoffPolicy::Park, + } + } + + pub fn reset_backoff(&mut self) { + self.backoff_us = self.policy().initial_us(); + } + + pub fn sleep_backoff(&self) { + let dur = Duration::from_micros(self.backoff_us); + match self.policy() { + BackoffPolicy::Sleep => std::thread::sleep(dur), + BackoffPolicy::Park => std::thread::park_timeout(dur), + } + } + + pub fn increase_backoff(&mut self) { + self.backoff_us = self.backoff_us.saturating_mul(2).min(self.policy().max_us()); + } + + #[cfg(test)] + fn current_backoff_us(&self) -> u64 { + self.backoff_us + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[test] + fn fresh_backoff_is_initial() { + let w = WorkerCore::new(0, None, None); + assert_eq!(w.current_backoff_us(), SLEEP_INITIAL_US); + } + + // One doubling-then-cap sequence, exercised for both policies: the pool's + // `Sleep` bounds and a driver's tight `Park` bounds. Asserting the whole + // sequence (not just the final cap) would fail a buggy `increase_backoff` + // that jumped straight to the cap or incremented by a constant; the closing + // reset confirms it returns to *this* policy's initial, not the other's. + #[rstest] + #[case::sleep( + WorkerCore::new(0, None, None), + SLEEP_INITIAL_US, + SLEEP_MAX_US, + BackoffPolicy::Sleep + )] + #[case::park(WorkerCore::driver(StepIdx(0)), PARK_INITIAL_US, PARK_MAX_US, BackoffPolicy::Park)] + fn backoff_doubles_then_caps( + #[case] mut w: WorkerCore, + #[case] initial: u64, + #[case] max: u64, + #[case] policy: BackoffPolicy, + ) { + assert_eq!(w.policy(), policy); + assert_eq!(w.current_backoff_us(), initial); + let mut expected = initial; + for _ in 0..20 { + w.increase_backoff(); + expected = (expected * 2).min(max); + assert_eq!( + w.current_backoff_us(), + expected, + "backoff must double each step until it saturates at the cap" + ); + } + assert_eq!(w.current_backoff_us(), max); + w.reset_backoff(); + assert_eq!(w.current_backoff_us(), initial); + } + + #[test] + fn reset_after_progress() { + let mut w = WorkerCore::new(0, None, None); + for _ in 0..5 { + w.increase_backoff(); + } + w.reset_backoff(); + assert_eq!(w.current_backoff_us(), SLEEP_INITIAL_US); + } + + #[test] + fn sleep_and_park_have_distinct_caps() { + // Guard against the two policies drifting to the same bound. + assert_ne!(SLEEP_MAX_US, PARK_MAX_US); + assert_ne!(SLEEP_INITIAL_US, PARK_INITIAL_US); + // Policy is derived from role: pool worker → Sleep, driver → Park. + assert_eq!(WorkerCore::new(0, None, None).policy(), BackoffPolicy::Sleep); + assert_eq!(WorkerCore::driver(StepIdx(0)).policy(), BackoffPolicy::Park); + } + + #[test] + fn worker_with_exclusive_role() { + let w = WorkerCore::new(2, Some(StepIdx(7)), Some(StepIdx(7))); + assert_eq!(w.thread_id, 2); + assert_eq!(w.exclusive_owner, Some(StepIdx(7))); + assert_eq!(w.sticky_owner, Some(StepIdx(7))); + } + + #[test] + fn worker_with_sticky_serial_owner_only() { + // E.g., the Serial+Affinity::Reader source — sticky_owner set, + // exclusive_owner None. + let w = WorkerCore::new(0, None, Some(StepIdx(0))); + assert_eq!(w.exclusive_owner, None); + assert_eq!(w.sticky_owner, Some(StepIdx(0))); + } +} diff --git a/crates/fgumi-pipeline-core/src/signal.rs b/crates/fgumi-pipeline-core/src/signal.rs new file mode 100644 index 000000000..eda14a32e --- /dev/null +++ b/crates/fgumi-pipeline-core/src/signal.rs @@ -0,0 +1,429 @@ +//! Pipeline-wide error broadcast and cancellation, folded into a single +//! shared atomic so the worker loop pays one relaxed load per iteration. + +use std::io; +use std::sync::Arc; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU8, Ordering as AtomicOrdering}; + +const STATE_OK: u8 = 0; +const STATE_CANCELLED: u8 = 1; +const STATE_ERROR: u8 = 2; + +/// Errors a pipeline run can return to its caller. +#[derive(Debug)] +pub enum PipelineError { + /// A step's `try_run` returned `Err`. Carries the originating step's + /// name and the underlying I/O error. + Io { step: &'static str, source: io::Error }, + /// `cancel_handle().cancel()` was called from outside the pipeline. + Cancelled, + /// The chain has more `Exclusive` steps than the configured thread count. + NotEnoughThreads { required: usize, available: usize }, + /// The deadlock monitor observed no global progress for `stalled_secs` + /// while work was still stuck in queues/reorder buffers — a wedge. The + /// pipeline is failed fast rather than left to hang forever. + TimedOut { stalled_secs: u64 }, + /// The pipeline was built with the deadlock monitor armed + /// (`deadlock_timeout_secs > 0`), but `step` declares a non-`ByteBounded` + /// output transport (`spec`, e.g. `CountBounded`/`Unbounded`). The + /// `in_flight_bytes` probe cannot see a wedge on such an edge, so fail-fast + /// would be silently disabled there. The pipeline is rejected at startup + /// rather than allowed to hang. This is a chain-construction error, not a + /// runtime condition: production chains wire only `ByteBounded` transports. + MonitorBlindTransport { step: &'static str, spec: String }, +} + +impl PipelineError { + /// Reconstruct an owned copy of this error. + /// + /// `PipelineError` is not `Clone` because its `Io` variant holds an + /// [`io::Error`], which is not `Clone`; this rebuilds that source from its + /// `kind()` + display string. Used to turn the borrowed `signal.outcome()` + /// into the owned error returned from the run drivers (`Pipeline::run` and + /// the fused single-thread driver), which previously open-coded the same + /// per-variant rematch in two places. + pub(crate) fn reconstruct(&self) -> PipelineError { + match self { + Self::Cancelled => Self::Cancelled, + Self::Io { step, source } => { + Self::Io { step, source: io::Error::new(source.kind(), format!("{source}")) } + } + Self::NotEnoughThreads { required, available } => { + Self::NotEnoughThreads { required: *required, available: *available } + } + Self::TimedOut { stalled_secs } => Self::TimedOut { stalled_secs: *stalled_secs }, + Self::MonitorBlindTransport { step, spec } => { + Self::MonitorBlindTransport { step, spec: spec.clone() } + } + } + } +} + +impl std::fmt::Display for PipelineError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io { step, source } => write!(f, "step {step:?} failed: {source}"), + Self::Cancelled => write!(f, "pipeline cancelled"), + Self::NotEnoughThreads { required, available } => write!( + f, + "pipeline requires {required} threads for Exclusive steps, only {available} available" + ), + Self::TimedOut { stalled_secs } => write!( + f, + "pipeline deadlock detected: no progress for {stalled_secs}s with work still in flight" + ), + Self::MonitorBlindTransport { step, spec } => write!( + f, + "deadlock monitor is armed but step {step:?} declares a {spec} output transport, \ + which is invisible to the in_flight_bytes probe — a wedge on that edge would \ + silently disable fail-fast. Production transports must be QueueSpec::ByteBounded." + ), + } + } +} + +impl std::error::Error for PipelineError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + _ => None, + } + } +} + +/// Single shared per-pipeline state covering both error broadcast and cancel. +/// Workers check `is_done()` once per iteration — one relaxed atomic load. +#[derive(Default)] +pub struct PipelineSignal { + state: AtomicU8, + payload: OnceLock, +} + +impl PipelineSignal { + #[must_use] + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Hot-path: workers call this once per loop iteration. + /// + /// Uses `Relaxed` because the only thing a worker does on `true` is stop + /// polling — workers never read the payload. The payload is read only by + /// the run drivers via `to_result`, after every internal writer + /// (workers, deadlock monitor) has been joined, so the join supplies the + /// happens-before that makes a recorded error's payload visible. + /// + /// The one writer not covered by a join is an external + /// [`CancelHandle::cancel`], which races the driver's terminal read with no + /// synchronizing edge; `to_result` derives `Cancelled` from the + /// (coherence-visible) state rather than the payload to close that window. + #[inline] + #[must_use] + pub fn is_done(&self) -> bool { + self.state.load(AtomicOrdering::Relaxed) != STATE_OK + } + + #[inline] + #[must_use] + pub fn is_cancelled(&self) -> bool { + self.state.load(AtomicOrdering::Relaxed) == STATE_CANCELLED + } + + /// First writer wins; later writers are silently dropped (the + /// `compare_exchange` rejects the state transition and the + /// `OnceLock::set` rejects the payload write). + pub fn record_error(&self, err: PipelineError) { + // Set the payload only if we won the state CAS, so the error surfaced by + // `outcome()` always matches the writer that transitioned the state. + // Otherwise writer A could win the state CAS while writer B wins the + // `OnceLock::set`, leaving `state == ERROR` reporting B's error — + // violating the documented "first writer wins" contract. + if self + .state + .compare_exchange( + STATE_OK, + STATE_ERROR, + AtomicOrdering::Release, + AtomicOrdering::Relaxed, + ) + .is_ok() + { + let _ = self.payload.set(err); + } + } + + /// First writer wins; later writers (including a `record_error` after + /// `cancel`) are silently dropped. + pub fn cancel(&self) { + // Only publish the payload if THIS call won the state transition. + // Setting it unconditionally races a concurrent `record_error`: if + // that call wins the CAS (state → ERROR) but has not yet set its + // payload, an unconditional `payload.set(Cancelled)` here can win the + // `OnceLock` and leave `state == ERROR` while `outcome()` reports + // `Cancelled` — an inconsistent state/payload pair. Guarding on the CAS + // (as `record_error` does) keeps the two consistent: whoever wins the + // state transition is the one that sets the payload. + // (See the loom NOTE at the end of the tests module for why this + // race-guard is not exercised by a unit test.) + if self + .state + .compare_exchange( + STATE_OK, + STATE_CANCELLED, + AtomicOrdering::Release, + AtomicOrdering::Relaxed, + ) + .is_ok() + { + let _ = self.payload.set(PipelineError::Cancelled); + } + } + + /// Read the recorded error payload, if any. + /// + /// The `OnceLock::get` acquire fence pairs with the `OnceLock::set` release + /// on the writer side, so a reader synchronized with the writer (via the + /// worker/monitor join in the run drivers) sees a recorded error's payload. + /// + /// Beware the gap this does NOT cover: `is_done() == true` does not imply + /// `outcome().is_some()`. `cancel()`/`record_error()` publish the terminal + /// `state` (CAS) before `payload.set()`, so a reader can observe the + /// terminal state while the payload `OnceLock` is still empty — most + /// reachably for an external `CancelHandle::cancel`, whose writer is never + /// joined against the driver's read. Map an outcome to a run result through + /// `to_result`, which handles that window, not by branching on + /// `outcome()` directly. + #[must_use] + pub fn outcome(&self) -> Option<&PipelineError> { + self.payload.get() + } + + /// Map the recorded outcome to a run `Result`, as returned by the run + /// drivers ([`crate::builder::Pipeline::run`] and the fused single-thread + /// driver). + /// + /// A recorded error's payload is always visible here: every `record_error` + /// writer (workers, the deadlock monitor, the fused driver itself) is + /// joined before this read, so the join orders its `payload.set()` ahead of + /// the read. + /// + /// An external [`CancelHandle::cancel`] is the exception — its writer is + /// never joined against this read, so there is no happens-before edge to + /// make its `OnceLock` payload visible, and it can leave `state == + /// CANCELLED` with `outcome() == None`. A naive `match outcome()` would then + /// map a genuinely cancelled run (workers observed `is_done()` and stopped + /// early) to `Ok(())`. `Cancelled` carries no payload data, so synthesize it + /// from the coherence-visible `state` via [`Self::is_cancelled`] instead. + pub(crate) fn to_result(&self) -> Result<(), PipelineError> { + match self.outcome() { + Some(err) => Err(err.reconstruct()), + None if self.is_cancelled() => Err(PipelineError::Cancelled), + // Terminal state with no payload published yet. `record_error` wins + // the state CAS *before* it runs `payload.set`, so there is a window + // in which `state == STATE_ERROR` and `outcome()` is still `None` — + // the same publish window `is_cancelled` rescues above, on the error + // path instead of the cancel path. Falling through to `Ok(())` here + // would report a failed run as successful. + // + // The argument that every `record_error` writer is joined before this + // read is a property of the drivers (pool workers AND the detached + // driver threads, on every early-return path), not of this type. Derive + // the answer from the terminal state instead, so the guarantee does not + // depend on that staying true. A successful run never reaches this arm: + // `is_done()` is `state != STATE_OK`, so it is false unless something + // already transitioned the state away from OK. + None if self.is_done() => Err(PipelineError::Io { + step: "unknown", + source: io::Error::other( + "pipeline failed but its error payload was never published; the \ + recording thread was still in the publish window when the run \ + result was read", + ), + }), + None => Ok(()), + } + } +} + +/// External handle for cancelling an in-flight pipeline. +#[derive(Clone)] +pub struct CancelHandle { + signal: Arc, +} + +impl CancelHandle { + /// Construct a `CancelHandle` from a shared signal. Used by + /// `Pipeline::cancel_handle` (see `builder.rs`) to hand a handle to the + /// caller of a built pipeline, before `Pipeline::run` kicks off worker + /// threads. + pub(crate) fn from_signal(signal: Arc) -> Self { + Self { signal } + } + + pub fn cancel(&self) { + self.signal.cancel(); + } + + #[must_use] + pub fn is_cancelled(&self) -> bool { + self.signal.is_cancelled() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fresh_signal_is_not_done() { + let signal = PipelineSignal::new(); + assert!(!signal.is_done()); + assert!(!signal.is_cancelled()); + } + + #[test] + fn record_error_marks_done() { + let signal = PipelineSignal::new(); + signal.record_error(PipelineError::Io { + step: "test_step", + source: io::Error::other("boom"), + }); + assert!(signal.is_done()); + assert!(!signal.is_cancelled()); + assert!(matches!(signal.outcome(), Some(PipelineError::Io { step: "test_step", .. }))); + } + + #[test] + fn cancel_marks_done_and_cancelled() { + let signal = PipelineSignal::new(); + signal.cancel(); + assert!(signal.is_done()); + assert!(signal.is_cancelled()); + assert!(matches!(signal.outcome(), Some(PipelineError::Cancelled))); + } + + #[test] + fn first_recorded_error_wins() { + let signal = PipelineSignal::new(); + signal.record_error(PipelineError::Io { step: "first", source: io::Error::other("first") }); + signal + .record_error(PipelineError::Io { step: "second", source: io::Error::other("second") }); + match signal.outcome().unwrap() { + PipelineError::Io { step, .. } => assert_eq!(*step, "first"), + other => panic!("expected Io, got {other:?}"), + } + } + + #[test] + fn cancel_handle_propagates() { + let signal = PipelineSignal::new(); + let handle = CancelHandle::from_signal(Arc::clone(&signal)); + assert!(!handle.is_cancelled()); + handle.cancel(); + assert!(handle.is_cancelled()); + assert!(signal.is_cancelled()); + } + + #[test] + fn reconstruct_preserves_every_variant() { + // `reconstruct` rebuilds an owned error from a borrow (PipelineError + // isn't Clone). Each variant must round-trip; the `Io` source is + // rebuilt from kind + display, the rest are copied verbatim. + let io = PipelineError::Io { step: "s", source: io::Error::other("boom") }.reconstruct(); + match io { + PipelineError::Io { step, source } => { + assert_eq!(step, "s"); + assert_eq!(source.kind(), io::ErrorKind::Other); + assert_eq!(source.to_string(), "boom"); + } + other => panic!("expected Io, got {other:?}"), + } + + assert!(matches!(PipelineError::Cancelled.reconstruct(), PipelineError::Cancelled)); + + let net = PipelineError::NotEnoughThreads { required: 4, available: 2 }.reconstruct(); + assert!(matches!(net, PipelineError::NotEnoughThreads { required: 4, available: 2 })); + + let to = PipelineError::TimedOut { stalled_secs: 60 }.reconstruct(); + assert!(matches!(to, PipelineError::TimedOut { stalled_secs: 60 })); + + let blind = PipelineError::MonitorBlindTransport { + step: "s", + spec: "QueueSpec::Unbounded".to_string(), + } + .reconstruct(); + assert!( + matches!(blind, PipelineError::MonitorBlindTransport { step: "s", spec } if spec == "QueueSpec::Unbounded") + ); + } + + #[test] + fn to_result_maps_clean_error_and_cancel() { + let clean = PipelineSignal::new(); + assert!(clean.to_result().is_ok()); + + let errored = PipelineSignal::new(); + errored.record_error(PipelineError::Io { step: "s", source: io::Error::other("boom") }); + assert!(matches!(errored.to_result(), Err(PipelineError::Io { step: "s", .. }))); + + let cancelled = PipelineSignal::new(); + cancelled.cancel(); + assert!(matches!(cancelled.to_result(), Err(PipelineError::Cancelled))); + } + + #[test] + fn to_result_reports_cancel_when_payload_not_yet_published() { + // Reproduce the external-cancel window: a `CancelHandle::cancel` on an + // un-joined thread has published the terminal `state` (the CAS) but has + // not yet run `payload.set(Cancelled)`. A driver reading the outcome in + // that window sees `is_done() == true` but `outcome() == None`. Branching + // on `outcome()` alone would map this genuinely cancelled run to `Ok(())`; + // `to_result` must instead synthesize `Cancelled` from the state. + let signal = PipelineSignal::new(); + signal.state.store(STATE_CANCELLED, AtomicOrdering::Release); + assert!(signal.is_done(), "terminal state must read as done"); + assert!(signal.is_cancelled()); + assert!(signal.outcome().is_none(), "payload not set in this window"); + assert!( + matches!(signal.to_result(), Err(PipelineError::Cancelled)), + "a cancel observed before its payload is published must not map to Ok" + ); + } + + /// The sibling of the test above, on the error path. `record_error` wins the + /// state CAS before it runs `payload.set`, so `state == STATE_ERROR` with + /// `outcome() == None` is reachable. Branching on `outcome()` and + /// `is_cancelled()` alone mapped that to `Ok(())` — reporting a failed run as + /// successful, the worst possible direction for this to be wrong in. + #[test] + fn to_result_reports_failure_when_error_payload_not_yet_published() { + let signal = PipelineSignal::new(); + signal.state.store(STATE_ERROR, AtomicOrdering::Release); + assert!(signal.is_done(), "terminal state must read as done"); + assert!(!signal.is_cancelled(), "this is the error path, not the cancel path"); + assert!(signal.outcome().is_none(), "payload not set in this window"); + assert!( + matches!(signal.to_result(), Err(PipelineError::Io { step: "unknown", .. })), + "an error observed before its payload is published must not map to Ok" + ); + } + + /// The guard above must not fire for a run that never failed: `is_done()` is + /// `state != STATE_OK`, so an untouched signal still reports success. + #[test] + fn to_result_is_ok_for_an_untouched_signal() { + assert!(PipelineSignal::new().to_result().is_ok()); + } + + // NOTE: the `cancel` vs `record_error` state/payload-consistency fix (guarding + // `payload.set` behind the CAS) is deliberately NOT covered by a unit test. + // The bug is a true concurrent interleaving — `cancel` losing the CAS in the + // window between `record_error`'s CAS-win and its own `payload.set` — and + // `OnceLock` masks the inconsistency in every *sequential* ordering, so no + // single-threaded test can distinguish fixed from buggy. A spawned-thread + // stress loop does not reliably hit the two-instruction window either (a + // 2000-trial loop passed against the buggy code). Deterministically forcing + // the interleaving would require `loom`; until that dependency is justified, + // the fix rests on the inline argument at the `cancel` call site. +} diff --git a/crates/fgumi-pipeline-core/src/step.rs b/crates/fgumi-pipeline-core/src/step.rs new file mode 100644 index 000000000..bf8577e10 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/step.rs @@ -0,0 +1,616 @@ +//! Step trait, profile types, and context. +//! +//! Completion contract (full detail on [`StepOutcome::Finished`]): +//! +//! - Every step — source, mid, or sink — returns `StepOutcome::Finished` from +//! `try_run` once its input edges are drained (empty + upstream closed) and +//! it holds no buffered output. The framework then closes the step's output +//! edges (`mark_outputs_drained`) and drops it from the worklist. +//! - `try_run` returns `Progress` when it pushed or held an item, `NoProgress` +//! when there's nothing useful to do this call (input empty but not drained, +//! no held items), and `Contention` when a Serial-step mutex is held by +//! another worker; the scheduler reroutes. +//! +//! **Last-worker barrier for `Parallel` steps.** A `Parallel` step has N +//! per-worker `Clone`s sharing one output queue and a single drained input +//! edge. When the input drains, every clone returns `Finished`, but only the +//! LAST clone to finish (the one that takes the per-step `StepDrainCounter` to +//! 0) closes the shared output queue. Otherwise a clone could `mark_drained` +//! while a sibling is still pushing — a `try_push`-after-`mark_drained` +//! violation, which `ItemQueue::try_push` rejects with a panic in every build. +//! The gate lives in the driver's `dispatch_one_step`. + +/// Concurrency profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StepKind { + /// Any number of workers may be inside `try_run` concurrently. Each + /// worker holds its own `Clone` of the step. + Parallel, + /// At most one worker at a time. Framework holds a per-step mutex; any + /// worker can acquire it; one instance shared. + Serial, + /// Exactly one worker (the owner) ever runs this step. Framework + /// assigns owners at run start in chain declaration order. Other + /// workers skip the step entirely. + Exclusive, + /// Runs on a **dedicated OS thread**, spawned at run start alongside the + /// deadlock-monitor / queue-rebalancer — never dispatched by the + /// work-stealing pool. The dedicated thread drives the step with the + /// *same* `run_worker_loop` the pool uses (via + /// `runtime::detached::run_detached_driver`): a `WorkerCore::driver` (Park + /// backoff) over an `Owned`-for-its-group row. The step body still uses + /// non-blocking `try_pop`/`try_push` and never blocks inside `try_run`, so + /// it consumes no pool worker slot. A driver is literally a 1-thread pool. + /// + /// This mirrors the legacy sort's "N + 2" threading: N pool workers do + /// the parallel (compression-bound) work while off-pool driver threads do + /// the serial coordination + I/O, keeping the pool saturated. Several + /// detached steps sharing a [`DetachedGroup::Shared`] label are driven by + /// ONE thread; [`DetachedGroup::PerStep`] (the default) keeps one thread + /// per step. + /// + /// **Opt-in.** No existing step declares `Detached` unless the sort chain + /// wires it. A single shared instance (like `Serial`/`Exclusive`, never + /// `new_worker_copy`'d) runs on the driver thread; every pool worker gets a + /// `Skip` entry for it. + Detached, +} + +/// Optional scheduling hint for `Serial` steps. Restricts which worker(s) +/// are eligible to attempt the step's mutex on each round-robin pass — +/// non-eligible workers `Skip` the step entirely, eliminating the +/// `try_lock` thrash that pure `Serial` exhibits at high thread counts. +/// +/// The framework's per-step mutex is still in place (so a step author +/// can rely on single-thread-at-a-time semantics), but with a non-`None` +/// affinity only the hinted worker(s) ever acquire it. +/// +/// Mirrors the legacy framework's per-thread `exclusive_step_owned` +/// mapping, where T0 is the reader, T(N-1) is the writer, and interior +/// exclusive steps fan out from both ends. +/// +/// Default `None` keeps the existing pure-mutex-shared `Serial` behavior. +/// Ignored for `Parallel`, `Exclusive`, and `Detached` kinds: +/// `build_worker_storage` reads affinity only in its `StepKind::Serial` arm, so +/// a `Detached` step declaring `Affinity::Worker(99)` is silently ignored and — +/// unlike the `Serial` case — never range-checked at run start. A `Detached` +/// step runs on its own driver thread, so no pool worker gates on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Affinity { + /// Any worker may attempt this Serial step. (Default; current behavior.) + None, + /// Restrict attempts to worker 0. Other workers `Skip` this step in + /// dispatch — no `try_lock` thrash. Use for I/O sources where keeping + /// reads on a single thread improves the kernel readahead pattern. + Reader, + /// Restrict attempts to worker `N - 1` (the last worker). Other + /// workers `Skip`. Use for I/O sinks where the writer's `BufWriter` + /// benefits from thread locality. + Writer, + /// Restrict attempts to a specific worker index. Out-of-range values + /// (`>= n_threads`) trigger an assertion failure (panic) at run start + /// via an always-on `assert!` in `build_worker_storage`. + Worker(usize), +} + +impl Affinity { + /// The single worker this affinity pins a step to, or `None` when the step + /// is not pinned ([`Affinity::None`] — every worker is eligible). + /// + /// The sole source of truth for the affinity → worker mapping, shared by + /// [`Self::eligible`] (which gates dispatch) and + /// `pool::assign_sticky_owners` (which picks each worker's sticky-driven + /// step). Keeping one mapping means a future variant cannot make those two + /// disagree and assign a sticky owner to a worker that would `Skip` the step. + /// + /// The returned index may be `>= n_workers` for an out-of-range + /// [`Affinity::Worker`]; callers must range-check before indexing. + #[must_use] + pub fn target_worker(self, n_workers: usize) -> Option { + match self { + Self::None => None, + Self::Reader => Some(0), + Self::Writer => Some(n_workers.saturating_sub(1)), + Self::Worker(idx) => Some(idx), + } + } + + /// Returns `true` if `worker_id` is eligible to attempt this Serial + /// step under this affinity hint. + #[must_use] + pub fn eligible(self, worker_id: usize, n_workers: usize) -> bool { + match self.target_worker(n_workers) { + // Unpinned: every worker may attempt the step. + None => true, + // `Writer` maps to `n_workers - 1`, so a saturating target of 0 with + // `n_workers == 0` has no eligible worker — and there are none to ask. + Some(target) => worker_id == target, + } + } +} + +/// Which dedicated driver thread a [`StepKind::Detached`] step runs on. +/// +/// Detached steps run off the work-stealing pool on dedicated OS threads. By +/// default each detached step gets its own thread ([`PerStep`](Self::PerStep)) +/// — the legacy "one dedicated thread per detached step" behavior. Steps that +/// declare the same [`Shared`](Self::Shared) label instead share ONE dedicated +/// driver thread that round-robins them through the same `run_worker_loop` the +/// pool uses (the unified "1-thread pool"). This realizes the true N+2 model: +/// the sort chain groups its serial coordination steps onto one driver and its +/// writers onto another, keeping the N pool workers on pure (de)compression. +/// +/// Ignored for non-`Detached` kinds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DetachedGroup { + /// One dedicated thread for this step alone (default; legacy behavior). + PerStep, + /// Share one dedicated driver thread with every other detached step that + /// declares the same label. + Shared(&'static str), +} + +/// Static description of how a step is scheduled, plus per-output queue +/// configuration. +/// +/// `output_queues[i]` selects the transport-layer queue type for branch `i` +/// (count-bounded, byte-bounded, or unbounded). `branch_ordering[i]` selects +/// whether the framework inserts a `ReorderStage` in front of the consumer's +/// input handle (so consumers see items in producer-emitted ordinal order). +/// +/// Both vectors must have length equal to `S::Outputs::arity()`. +#[derive(Debug, Clone)] +pub struct StepProfile { + pub name: &'static str, + pub kind: StepKind, + pub sticky: bool, + pub output_queues: Vec, + pub branch_ordering: Vec, +} + +/// Outcome of a single `try_run` call. +/// +/// **`Finished` contract.** Any step — source, mid, or sink — returns +/// `Finished` once it will never push to its output again: all its input edges +/// are drained (empty + upstream closed) and it holds no buffered output. The +/// framework then closes the step's output edges (`mark_outputs_drained`) and +/// drops it from the worklist. The step must already have flushed every item +/// (via `try_run`'s flush-first path) before returning `Finished` — returning +/// it with work still buffered loses data. For a `Parallel` step the per-step +/// `StepDrainCounter` gates the output close so only the last clone to finish +/// closes the shared queue (a sibling could still be pushing). Sources are the +/// degenerate case: their (unit) input edge is drained from birth, so they +/// return `Finished` on EOF. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StepOutcome { + /// Step did useful work (pushed an item or held one for later). + Progress, + /// Step had nothing to do this call (input empty, no held work). + NoProgress, + /// Step's Serial-step mutex was contended; scheduler reroutes. + Contention, + /// The step has drained all its input and holds no buffered output — it + /// will never push again. The framework marks its output queues drained + /// (counter-gated for `Parallel`) and drops it. See the `Finished` + /// contract in the enum docs above. + Finished, +} + +/// Type-erased holder for the per-step outputs view, downcast to a typed +/// view (`SingleOutputsView`, `Tuple{2,3,4}OutputsView`, `UnitOutputsView`) +/// inside the `TypedStep` adapter. Defined here so the `Step` trait +/// declaration can reference it; concrete views live in `handles.rs`. +/// +/// Drain marking is per-branch via `BranchOutputHandle::mark_drained`, +/// exposed by typed `mark_all_drained()` methods on each `*OutputsView`. +/// `TypedStep` (Phase 1 Task 15) downcasts `inner` to the right typed +/// view and calls `mark_all_drained` from the worker loop's drain path. +pub struct OutputsViewAny { + pub(crate) inner: Box, +} + +use std::io; + +use super::item::HeapSize; +use super::outputs::StepOutputs; + +/// Handle to this step's input queue. +/// +/// Implemented by `OrderedQueueInputHandle` in `handles.rs`. Steps see +/// only the trait surface; the concrete handle is constructed by the +/// framework at chain build time. +pub trait InputHandle: Send + Sync { + /// Pop the next item, or `None` if the queue is empty right now. + /// Non-blocking. + fn pop(&self) -> Option; + + /// Returns `true` if upstream has marked the queue drained AND the + /// queue is currently empty. Once `is_drained()` returns true, no + /// further items will arrive. Used by mid-steps and sinks to detect + /// when to stop pulling. + fn is_drained(&self) -> bool; +} + +/// Handle to this step's output queues, shaped by `S::Outputs`. +/// +/// Concrete shape is the per-arity view: `SingleOutputsView` for +/// `Single` outputs, `Tuple2OutputsView` for `(A, B)`, etc. +/// Per-arity extension impls in `handles.rs` provide typed `push` methods +/// (e.g., `OutputHandles>::push(&self, item: T) -> Result<(), T>`). +pub struct OutputHandles { + pub(crate) inner: OutputsViewAny, + /// `PhantomData O>` (not `PhantomData`) so `OutputHandles` + /// is always `Send + Sync` regardless of `O` — required so the runtime + /// can box it as `Box` without forcing a `Sync` + /// bound on item types. + pub(crate) _phantom: std::marker::PhantomData O>, +} + +impl OutputHandles { + /// Wrap a type-erased outputs view into a typed `OutputHandles`. + /// + /// Called from the `ErasedStep::wrap_outputs_view` impls in `erased.rs` — + /// `TypedStep` for one-input steps and `TypedStep2` for the two-input + /// (zipper) shape — while `build_chain_contexts` assembles one context per + /// step. That happens on the thread calling `Pipeline::run`, *before* any + /// worker is spawned; workers never construct one, they borrow the box built + /// here (which is what `TypedStep::resolve_outputs`' cache relies on). + pub(crate) fn new(inner: OutputsViewAny) -> Self { + Self { inner, _phantom: std::marker::PhantomData } + } +} + +/// Step trait — every chain link implements this. +/// +/// `Send + 'static`: +/// - `Send`: instances move across threads (workers run on dedicated threads). +/// - `'static`: no borrowed references in step state. +/// +/// `Clone` is **not** a super-trait. Per-worker copies for `Parallel` steps +/// go through [`Step::new_worker_copy`] instead, which only `Parallel` +/// authors need to implement; `Serial`/`Exclusive` step authors inherit the +/// default panic and pay nothing. +/// +/// # Per-worker copy patterns by step kind +/// +/// **`Parallel` steps** must override [`Step::new_worker_copy`]. Each worker +/// thread calls it once during `build_worker_storage` to materialize its +/// private instance. The typical impl forwards to a regular `Clone` impl +/// (most parallel steps either `#[derive(Clone)]` or hand-roll a `Clone` +/// that resets per-worker scratch state). For example: +/// +/// ```ignore +/// #[derive(Clone)] +/// pub struct ParseBamRecords; +/// +/// impl Step for ParseBamRecords { +/// // ... +/// fn new_worker_copy(&self) -> Self { self.clone() } +/// } +/// ``` +/// +/// Closure-driven `Parallel` steps (`process(fn)`, `serialize(fn)`) wrap +/// their closure in `Arc` so the per-worker copy is one atomic +/// increment. +/// +/// **`Serial` and `Exclusive` steps** do not override +/// [`Step::new_worker_copy`]. The framework holds a single instance (behind +/// a `Mutex` for `Serial`, pinned to one worker for `Exclusive`) and never +/// asks for additional copies. The default impl panics with a descriptive +/// message — if it ever fires, that's a framework bug. +pub trait Step: Send + Sized + 'static { + type Input: Send + HeapSize + 'static; + type Outputs: StepOutputs; + + /// Static description of how this step is scheduled. + fn profile(&self) -> StepProfile; + + /// Optional scheduling hint for `Serial` kinds. Defaults to + /// `Affinity::None` (any worker may attempt the step). Override for + /// I/O sources/sinks that benefit from thread locality — see the + /// [`Affinity`] enum docs. + /// + /// Ignored for `Parallel` and `Exclusive` kinds (those have their + /// own per-worker dispatch model). + fn affinity(&self) -> Affinity { + Affinity::None + } + + /// Which dedicated driver thread this step runs on, for [`StepKind::Detached`] + /// steps. Defaults to [`DetachedGroup::PerStep`] (its own thread — legacy + /// behavior). Override to [`DetachedGroup::Shared`] to co-locate several + /// detached steps on one driver thread (the true N+2 model). Ignored for + /// non-`Detached` kinds. + fn detached_group(&self) -> DetachedGroup { + DetachedGroup::PerStep + } + + /// Step body. Pop from `ctx.input`, push to `ctx.outputs`. Returns + /// `Progress` / `NoProgress` / `Contention` / `Finished`. Errors propagate via `Err`. + /// + /// # Errors + /// + /// Returns the underlying I/O error from any failed read, write, or + /// codec operation inside the step body. The framework records the + /// first error via `PipelineSignal::record_error` and broadcasts to + /// other workers. + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result; + + /// Construct a fresh per-worker copy of this step. Only `Parallel` + /// steps need to override this — the framework calls it during + /// `build_worker_storage` to materialize one instance per worker. + /// `Serial`/`Exclusive` steps inherit the default panic; the framework + /// guarantees it is never invoked on them (one shared instance behind + /// a `Mutex` for `Serial`, pinned to a single owner worker for + /// `Exclusive`). + /// + /// # Panics + /// + /// Default impl panics with the step name + kind. Hitting it indicates + /// a framework bug (the runtime tried to clone a non-`Parallel` step). + #[must_use] + fn new_worker_copy(&self) -> Self { + let p = self.profile(); + panic!( + "Step::new_worker_copy invoked on '{}' (kind = {:?}); \ + only Parallel steps need to override this. The framework \ + never clones Serial/Exclusive steps — file a bug.", + p.name, p.kind + ); + } +} + +/// Context passed to `try_run`. +pub struct StepCtx<'a, S: Step> { + pub input: &'a dyn InputHandle, + pub outputs: &'a OutputHandles, +} + +/// Two-input variant of [`Step`]. Used by merge steps (zipper, +/// AAM-aligner-output + original-record-stream) that need to pop +/// from two upstream queues independently. +/// +/// Sits **alongside** [`Step`]; single-input steps are unaffected. +/// The framework's [`TypedStep2`](crate::erased::TypedStep2) adapter +/// (`crate::erased::TypedStep2`) is the +/// dual of [`TypedStep`](crate::erased::TypedStep), implementing the same [`ErasedStep`](crate::erased::ErasedStep) +/// contract so the runtime sees no difference between single- and +/// two-input steps at dispatch time. +/// +/// The two input branches can be the same type (`InputA == InputB` +/// — what zipper uses, both `RawRecord`) or different types +/// (heterogeneous — also supported, no special case in the +/// framework). +pub trait Step2: Send + Sized + 'static { + type InputA: Send + HeapSize + 'static; + type InputB: Send + HeapSize + 'static; + type Outputs: StepOutputs; + + fn profile(&self) -> StepProfile; + + /// Same semantics as [`Step::affinity`]. Defaults to + /// `Affinity::None`. + fn affinity(&self) -> Affinity { + Affinity::None + } + + /// Same semantics as [`Step::detached_group`]. Defaults to + /// [`DetachedGroup::PerStep`]. + fn detached_group(&self) -> DetachedGroup { + DetachedGroup::PerStep + } + + /// Step body. Pop from `ctx.a` (input branch 0) and `ctx.b` + /// (input branch 1), push to `ctx.outputs`. + /// + /// # Errors + /// + /// Same handling as [`Step::try_run`]. + fn try_run(&mut self, ctx: &mut StepCtx2<'_, Self>) -> io::Result; + + /// Same semantics as [`Step::new_worker_copy`]. Default panics; + /// only `Parallel` Step2 impls need to override. + /// + /// # Panics + /// + /// Default impl panics with the step name + kind. Hitting it + /// indicates a framework bug (the runtime tried to clone a + /// non-`Parallel` step). + #[must_use] + fn new_worker_copy(&self) -> Self { + let p = self.profile(); + panic!( + "Step2::new_worker_copy invoked on '{}' (kind = {:?}); \ + only Parallel steps need to override this. The framework \ + never clones Serial/Exclusive steps — file a bug.", + p.name, p.kind + ); + } +} + +/// Context passed to [`Step2::try_run`]. Holds independent input handles +/// for each branch. +pub struct StepCtx2<'a, S: Step2> { + pub a: &'a dyn InputHandle, + pub b: &'a dyn InputHandle, + pub outputs: &'a OutputHandles, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + + #[test] + fn step_profile_constructs() { + let p = StepProfile { + name: "Test", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 64 }], + branch_ordering: vec![BranchOrdering::None], + }; + assert_eq!(p.name, "Test"); + assert_eq!(p.kind, StepKind::Parallel); + assert!(!p.sticky); + assert_eq!(p.output_queues.len(), 1); + assert!(matches!(p.output_queues[0], QueueSpec::CountBounded { capacity: 64 })); + assert_eq!(p.branch_ordering, vec![BranchOrdering::None]); + } + + #[test] + fn step_kind_is_comparable() { + assert_eq!(StepKind::Parallel, StepKind::Parallel); + assert_ne!(StepKind::Parallel, StepKind::Serial); + assert_ne!(StepKind::Serial, StepKind::Exclusive); + } + + #[test] + fn step_outcome_includes_finished() { + let o = StepOutcome::Finished; + assert_eq!(o, StepOutcome::Finished); + assert_ne!(StepOutcome::Finished, StepOutcome::Progress); + assert_ne!(StepOutcome::Finished, StepOutcome::NoProgress); + assert_ne!(StepOutcome::Finished, StepOutcome::Contention); + } +} + +#[cfg(test)] +mod step_trait_compile_tests { + use super::*; + use crate::outputs::Single; + use crate::queues::QueueSpec; + use crate::reorder::BranchOrdering; + + /// Stub step that does nothing — exercises the trait declaration. + #[derive(Clone)] + struct NopStep; + + impl Step for NopStep { + type Input = u32; + type Outputs = Single; + + fn profile(&self) -> StepProfile { + StepProfile { + name: "Nop", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[test] + fn nop_step_advertises_profile() { + let s = NopStep; + let p = s.profile(); + assert_eq!(p.name, "Nop"); + assert_eq!(p.kind, StepKind::Parallel); + } + + /// Stub multi-output step — exercises tuple Outputs. + #[derive(Clone)] + struct FanOutStep; + + impl Step for FanOutStep { + type Input = u32; + type Outputs = (u32, String); + + fn profile(&self) -> StepProfile { + StepProfile { + name: "FanOut", + kind: StepKind::Parallel, + sticky: false, + output_queues: vec![ + QueueSpec::CountBounded { capacity: 1 }, + QueueSpec::CountBounded { capacity: 1 }, + ], + branch_ordering: vec![BranchOrdering::None, BranchOrdering::None], + } + } + + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + #[test] + fn fan_out_step_advertises_profile() { + let s = FanOutStep; + assert_eq!(s.profile().name, "FanOut"); + } + + /// Default `new_worker_copy` panics on Serial steps. The framework + /// guarantees it never invokes this on Serial/Exclusive in practice; + /// this test pins the safety-net behavior in case a future refactor + /// accidentally calls it. The panic message must include the step + /// name so a real-world hit is debuggable. + #[test] + #[should_panic(expected = "SerialStub")] + fn new_worker_copy_default_panics_for_serial() { + struct SerialStub; + impl Step for SerialStub { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SerialStub", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + let _ = SerialStub.new_worker_copy(); + } + + /// A two-input step that overrides nothing beyond the required items, so + /// the trait defaults are what the framework will read. + struct Step2Stub; + impl Step2 for Step2Stub { + type InputA = u32; + type InputB = u64; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "Step2Stub", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx2<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } + } + + /// `Step2` mirrors `Step`'s defaults: any worker may attempt a Serial step, + /// and a step that opts into `Detached` gets its own driver thread unless it + /// asks to share one. Pinning these keeps the two traits from drifting. + #[test] + fn step2_defaults_match_step_defaults() { + let s = Step2Stub; + assert!(matches!(s.affinity(), Affinity::None)); + assert!(matches!(s.detached_group(), DetachedGroup::PerStep)); + } + + /// Same safety net as `new_worker_copy_default_panics_for_serial`, for the + /// two-input trait: the framework never clones a non-`Parallel` step, so + /// reaching this default is a framework bug and must name the step. + #[test] + #[should_panic(expected = "Step2Stub")] + fn step2_new_worker_copy_default_panics_for_serial() { + let _ = Step2Stub.new_worker_copy(); + } +} diff --git a/crates/fgumi-pipeline-core/src/tests.rs b/crates/fgumi-pipeline-core/src/tests.rs new file mode 100644 index 000000000..ba06eadd6 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/tests.rs @@ -0,0 +1,1236 @@ +//! Phase 1 cross-module tests. Validates that the type vocabulary, queue +//! handles, and `TypedStep` adapter compose into a working chain when +//! wired manually. +//! +//! The runtime that wires these automatically (worker pool, worker loop) +//! lives in Phase 2. These tests confirm the Phase 1 pieces support the +//! unified report-`Finished` completion contract end-to-end. + +use std::collections::VecDeque; +use std::io; +use std::sync::Arc; + +use super::erased::{ErasedStep, ErasedStepCtx}; +use super::outputs::Single; +use super::queues::QueueSpec; +use super::reorder::BranchOrdering; +use super::signal::PipelineSignal; +use super::step::{Step, StepCtx, StepKind, StepOutcome, StepProfile}; + +// ───────────────────────────────────────────────────────────────────────────── +// F1' — Step2 (multi-input merge) tests. +// +// Validates that: +// * `TypedStep2` adapter dispatches correctly through `ErasedStep`. +// * `TwoInputHandles` round-trips per-branch handles via the +// ChainContexts construction path. +// * Both branches' drain signals propagate so `is_input_drained` +// reports drained only when BOTH branches are drained. +// ───────────────────────────────────────────────────────────────────────────── + +use super::erased::TypedStep2; +use super::step::{InputHandle as _, Step2, StepCtx2}; + +/// Test `Step2`: sum of `(a, b)` → `u64`. Emits `a + b as u64` once both +/// branches have yielded an item, `NoProgress` otherwise. Mirrors what zipper +/// does at the level of "pop from two queues in lockstep, combine". +/// +/// Buffers a popped item whose sibling branch was empty rather than popping both +/// eagerly: `(ctx.a.pop(), ctx.b.pop())` discards any `Some` that lands in the +/// non-pair arm, which is silent data loss. This test's producers happen to be +/// pre-loaded with equal counts so the hazard never fires here — which is +/// exactly why it must not be modelled this way: a step author copying the +/// pattern into real code loses records, and a future change to the test's input +/// counts would turn the bug into a silently passing test. Same shape as +/// `PairSummer` below. +#[derive(Default)] +struct SumPairStep { + pending_a: Option, + pending_b: Option, +} + +impl Step2 for SumPairStep { + type InputA = u32; + type InputB = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SumPair", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx2<'_, Self>) -> io::Result { + if self.pending_a.is_none() { + self.pending_a = ctx.a.pop(); + } + if self.pending_b.is_none() { + self.pending_b = ctx.b.pop(); + } + match (self.pending_a.take(), self.pending_b.take()) { + (Some(a), Some(b)) => { + ctx.outputs + .push(u64::from(a) + u64::from(b)) + .expect("capacity-8 output must accept the pair"); + Ok(StepOutcome::Progress) + } + // Hold whichever branch did yield until its sibling catches up. + (a, b) => { + self.pending_a = a; + self.pending_b = b; + Ok(StepOutcome::NoProgress) + } + } + } +} + +#[test] +fn step2_typed_dispatch_pairs_both_branches() { + // Build two single-output sub-chains feeding into a SumPair Step2. + // Producer A and producer B each output u32 on branch 0. We manually + // wire the TypedStep2's TwoInputHandles via + // build_two_input_handles, then drive it through ErasedStep. + + let sum_step: Box = Box::new(TypedStep2::new(SumPairStep::default())); + + // Producer A's output set + handles. + let (a_set, a_view) = as super::outputs::StepOutputs>::build_queues( + &[QueueSpec::CountBounded { capacity: 4 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let a_outputs: super::step::OutputHandles> = + super::step::OutputHandles::new(a_view); + + // Producer B's output set + handles. + let (b_set, b_view) = as super::outputs::StepOutputs>::build_queues( + &[QueueSpec::CountBounded { capacity: 4 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let b_outputs: super::step::OutputHandles> = + super::step::OutputHandles::new(b_view); + + // Build the merge step's TwoInputHandles by handing the + // ErasedStep adapter both producer sets. + let mut producer_sets = vec![a_set, b_set]; + let merge_input_any = sum_step.build_two_input_handles(&mut producer_sets, 0, 0, 1, 0); + let _ = producer_sets; // both branches taken by build_two_input_handles + + // Sanity: input_arity reports 2. + assert_eq!(sum_step.input_arity(), 2); + + // Merge step's output set. + let (mut merge_outset, merge_view) = + sum_step.build_output_set(crate::builder::InstrumentationLevel::Off); + let merge_outputs_any = sum_step.wrap_outputs_view(merge_view); + let merge_consumer_input = merge_outset.take_typed_input::(0); + + // Push a couple of items on each producer. + a_outputs.push(10).unwrap(); + a_outputs.push(20).unwrap(); + b_outputs.push(1).unwrap(); + b_outputs.push(2).unwrap(); + + let signal = PipelineSignal::new(); + // Drive the merge step twice — once per pair. + let mut sum_step = sum_step; + for _ in 0..2 { + let mut ctx = ErasedStepCtx { + input: merge_input_any.as_ref(), + outputs: merge_outputs_any.as_ref(), + signal: &signal, + }; + let outcome = sum_step.try_run_erased(&mut ctx).unwrap(); + assert_eq!(outcome, StepOutcome::Progress); + } + // No more pairs available — both branches non-drained but empty. + { + let mut ctx = ErasedStepCtx { + input: merge_input_any.as_ref(), + outputs: merge_outputs_any.as_ref(), + signal: &signal, + }; + let outcome = sum_step.try_run_erased(&mut ctx).unwrap(); + assert_eq!(outcome, StepOutcome::NoProgress); + } + + // Verify the merge step pushed 11, 22 (10+1, 20+2) in order. + assert_eq!(merge_consumer_input.pop(), Some(11)); + assert_eq!(merge_consumer_input.pop(), Some(22)); + assert_eq!(merge_consumer_input.pop(), None); +} + +#[test] +fn step2_build_two_input_handles_rejects_same_producer_and_branch() { + // When both `Step2` inputs share ONE producer step, + // `build_two_input_handles` asserts the two branches differ — it does NOT + // require distinct producer indices. Taking the same branch twice is the + // illegal case; pin the panic so a refactor that drops the branch check + // surfaces immediately. (The legal same-producer/distinct-branch case is + // covered by the test below.) + let sum_step: Box = Box::new(TypedStep2::new(SumPairStep::default())); + let (a_set, _a_view) = as super::outputs::StepOutputs>::build_queues( + &[QueueSpec::CountBounded { capacity: 4 }], + &[BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let mut producer_sets = vec![a_set]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + sum_step.build_two_input_handles(&mut producer_sets, 0, 0, 0, 0) + })); + assert!(result.is_err(), "expected panic on same producer AND same branch"); +} + +/// Like `SumPairStep` but with a **non-commutative** combiner, so a test can tell +/// input A from input B. `SumPairStep` cannot: `a + b` makes `10 + 1` and `1 + 10` +/// both 11, so a swapped branch-to-input wiring produces an identical result and +/// is invisible. `a * 1000 + b` distinguishes them (10001 vs 1010). +#[derive(Default)] +struct AsymmetricPairStep { + pending_a: Option, + pending_b: Option, +} + +impl Step2 for AsymmetricPairStep { + type InputA = u32; + type InputB = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "AsymmetricPair", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx2<'_, Self>) -> io::Result { + if self.pending_a.is_none() { + self.pending_a = ctx.a.pop(); + } + if self.pending_b.is_none() { + self.pending_b = ctx.b.pop(); + } + match (self.pending_a, self.pending_b) { + (Some(a), Some(b)) => { + self.pending_a = None; + self.pending_b = None; + ctx.outputs + .push(u64::from(a) * 1000 + u64::from(b)) + .expect("capacity-8 output must accept the pair"); + Ok(StepOutcome::Progress) + } + // Hold whichever branch did yield until its sibling catches up. + _ => Ok(StepOutcome::NoProgress), + } + } +} + +/// A fan-out producer feeding BOTH of a `Step2`'s inputs from its two distinct +/// branches is legal, and had no coverage — the duplicate-producer test above +/// passes only because both its branches are `0`, so a regression that took the +/// same branch twice would not be caught by it. +#[test] +fn step2_build_two_input_handles_accepts_one_producer_with_distinct_branches() { + // Asymmetric combiner on purpose: `SumPairStep`'s `a + b` is commutative, so a + // builder that wired branch 1 to input A and branch 0 to input B would produce + // the same 11 and this test would still pass while asserting the mapping. + let sum_step: Box = Box::new(TypedStep2::new(AsymmetricPairStep::default())); + + // One producer, two branches: `(A, B)` is the two-branch output shape. + let (fanout_set, fanout_view) = <(u32, u32) as super::outputs::StepOutputs>::build_queues( + &[QueueSpec::CountBounded { capacity: 4 }, QueueSpec::CountBounded { capacity: 4 }], + &[BranchOrdering::None, BranchOrdering::None], + crate::builder::InstrumentationLevel::Off, + ); + let fanout_outputs: super::step::OutputHandles<(u32, u32)> = + super::step::OutputHandles::new(fanout_view); + + // Same producer index (0) for both inputs, distinct branches 0 and 1. + let mut producer_sets = vec![fanout_set]; + let merge_input_any = sum_step.build_two_input_handles(&mut producer_sets, 0, 0, 0, 1); + + let (mut merge_outset, merge_view) = + sum_step.build_output_set(crate::builder::InstrumentationLevel::Off); + let merge_outputs_any = sum_step.wrap_outputs_view(merge_view); + let merge_consumer_input = merge_outset.take_typed_input::(0); + + // Branch 0 feeds input A, branch 1 feeds input B. + let view = fanout_outputs.view(); + view.a.push(10).unwrap(); + view.b.push(1).unwrap(); + + let signal = PipelineSignal::new(); + let mut sum_step = sum_step; + let mut ctx = ErasedStepCtx { + input: merge_input_any.as_ref(), + outputs: merge_outputs_any.as_ref(), + signal: &signal, + }; + assert_eq!(sum_step.try_run_erased(&mut ctx).unwrap(), StepOutcome::Progress); + assert_eq!( + merge_consumer_input.pop(), + Some(10 * 1000 + 1), + "branch 0 must feed input A and branch 1 input B — a swapped mapping yields 1010" + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// F1' integration test — end-to-end Pipeline::run with TWO sources joined +// at a Step2, demonstrating that the entire chain-builder + runtime path +// (PipelineBuilder.chain ×2 → MultiChain2::from_chains → MultiChain2::join +// → Chain::chain → PipelineBuilder::build → Pipeline::run) works through +// the real worker loop. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +#[allow(clippy::too_many_lines)] +fn step2_end_to_end_pipeline_pairs_two_sources_through_runtime() { + use super::builder::{MultiChain2, Pipeline, PipelineConfig}; + + /// Source A: emits 1..=5 then Finished. + #[derive(Clone)] + struct SourceA { + remaining: u32, + } + impl Step for SourceA { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SourceA", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + let n = self.remaining; + self.remaining -= 1; + ctx.outputs.push(n).expect("capacity-8 output must accept all 5 items"); + Ok(StepOutcome::Progress) + } + } + + /// Source B: emits 10..=50 (step 10) then Finished — same count + /// as Source A so every paired emit consumes one from each. + #[derive(Clone)] + struct SourceB { + remaining: u32, + } + impl Step for SourceB { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "SourceB", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + let n = self.remaining * 10; + self.remaining -= 1; + ctx.outputs.push(n).expect("capacity-8 output must accept all 5 items"); + Ok(StepOutcome::Progress) + } + } + + /// Step2 merger: sum each pair (a, b) → u64. Buffers a popped + /// item from one branch if the sibling branch is empty — + /// otherwise the popped item would be dropped when the sibling + /// pop returns None, and we'd lose data. + struct PairSummer { + pending_a: Option, + pending_b: Option, + } + impl Step2 for PairSummer { + type InputA = u32; + type InputB = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "PairSummer", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 8 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx2<'_, Self>) -> io::Result { + if self.pending_a.is_none() { + self.pending_a = ctx.a.pop(); + } + if self.pending_b.is_none() { + self.pending_b = ctx.b.pop(); + } + match (self.pending_a, self.pending_b) { + (Some(a), Some(b)) => { + self.pending_a = None; + self.pending_b = None; + ctx.outputs + .push(u64::from(a) + u64::from(b)) + .expect("capacity-8 output must accept all 5 pairs"); + Ok(StepOutcome::Progress) + } + // Both inputs drained AND nothing buffered: genuinely done. + _ if ctx.a.is_drained() + && ctx.b.is_drained() + && self.pending_a.is_none() + && self.pending_b.is_none() => + { + Ok(StepOutcome::Finished) + } + // Both inputs drained but one branch item is still buffered — + // the two branches emitted different counts. Reporting + // `Finished` here would silently drop that item, exactly the + // data loss the `PairSummer` doc warns step authors about, so + // fail loudly instead of modelling the bug. + _ if ctx.a.is_drained() && ctx.b.is_drained() => { + panic!("PairSummer: unpaired item left buffered after both branches drained") + } + _ => Ok(StepOutcome::NoProgress), + } + } + } + + /// Sink: records received items so the test can assert pairing. + #[derive(Clone)] + struct CollectSink { + received: Arc>>, + } + impl Step for CollectSink { + type Input = u64; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "CollectSink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(n) => { + self.received.lock().push(n); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + let received: Arc>> = Arc::new(parking_lot::Mutex::new(Vec::new())); + let received_for_run = Arc::clone(&received); + + run_with_deadlock_timeout("Step2 PairSummer join at --threads 4", move || { + let builder = Pipeline::builder(); + let chain_a = builder.chain(SourceA { remaining: 5 }); + let chain_b = builder.chain(SourceB { remaining: 5 }); + MultiChain2::::from_chains(chain_a, chain_b) + .join(PairSummer { pending_a: None, pending_b: None }) + .chain(CollectSink { received: received_for_run }) + .into_sink_marker(); + + let pipeline = builder.build().expect("pipeline build"); + // 4 threads: 2 Exclusive sources own 2 workers, the Exclusive sink + // owns a third, leaving 1 free worker for the Serial PairSummer. + pipeline.run(PipelineConfig { threads: 4, ..Default::default() }).expect("pipeline run"); + }); + + // Both sources emit 5 items in step (a: 5,4,3,2,1; b: 50,40,30,20,10), so + // pair sums are 55, 44, 33, 22, 11 — and that ORDER is guaranteed, not just + // the multiset. Each branch queue is FIFO, `PairSummer` pops at most one + // item per branch per tick into `pending_a`/`pending_b`, so A's k-th item + // always pairs with B's k-th; and the step is `Serial`, so pairs enter the + // output queue in emit order and the Exclusive sink pops them in that order. + // Sorting here would discard an ordering guarantee the transport does + // provide, letting a reordering regression in Serial dispatch pass. + let collected = received.lock().clone(); + assert_eq!( + collected, + vec![55, 44, 33, 22, 11], + "Serial dispatch must preserve FIFO pairing and output order" + ); +} + +/// Sink: records every value it receives so tests can assert no records were +/// dropped, duplicated, or corrupted — not just that the count matched. +#[derive(Clone)] +struct DrainReproSink { + received: Arc>>, +} +impl Step for DrainReproSink { + type Input = u32; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "DrainReproSink", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(n) => { + self.received.lock().push(n); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Unified model (issue #330): a mid-chain step (not just sources) reports +// `StepOutcome::Finished` once its input is drained and it holds no buffered +// output, folding its final flush into `try_run`'s flush-first path. A +// `NoProgress` tick mid-flush (full output queue) is just an idle yield — the +// step keeps flushing across re-dispatches until it reports `Finished`. At >1 +// thread the shared `finished` latch must stop the other workers from +// re-running the finished Serial step. +// ───────────────────────────────────────────────────────────────────────────── + +/// Source emitting `0..count` then `Finished`, with an output queue wide enough +/// to hold every item so `push` never hits backpressure: a Serial source +/// hammered by N workers must not silently drop on a full queue, which is why +/// the push below is asserted rather than discarded. Capacity 256 covers the +/// test's 64 items. +#[derive(Clone)] +struct WideQueueSource { + remaining: u32, +} +impl Step for WideQueueSource { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "WideQueueSource", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 256 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + self.remaining -= 1; + ctx.outputs.push(self.remaining).expect("wide source queue must never reject"); + Ok(StepOutcome::Progress) + } +} + +/// Mid (Serial): buffers every input item, then flushes the whole buffer in +/// `try_run`'s flush-first path through a capacity-1 output queue, and reports +/// `Finished` once input is drained and the buffer is empty — the unified +/// completion contract. +#[derive(Clone)] +struct ReportsFinishedBuffer { + // A `VecDeque` so the flush-first drain pops the FIFO front in O(1). Real + // steps process millions of records: a `Vec` with `remove(0)` would be O(n) + // per flushed item (quadratic drain). Step authors copying this worked + // example must keep the front-drain O(1) — never `Vec::remove(0)`. + buffered: VecDeque, +} +impl Step for ReportsFinishedBuffer { + type Input = u32; + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "ReportsFinishedBuffer", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + // 1. Flush-first: push held items until the queue rejects one. + let mut pushed = false; + while let Some(&item) = self.buffered.front() { + match ctx.outputs.push(item) { + Ok(()) => { + self.buffered.pop_front(); + pushed = true; + } + Err(_unpushed) => break, // queue full — yield, retry next pass + } + } + if pushed { + return Ok(StepOutcome::Progress); + } + // 2. Consume input. + if let Some(n) = ctx.input.pop() { + self.buffered.push_back(n); + return Ok(StepOutcome::Progress); + } + // 3. Completion: input drained AND nothing held — never push again. + if ctx.input.is_drained() && self.buffered.is_empty() { + return Ok(StepOutcome::Finished); + } + // Input drained but buffer still non-empty (queue was full this call): + // NoProgress is just an idle tick — the round-robin moves on to the + // sink, which drains the queue, and we flush more on the next pass, + // eventually emptying the buffer and reporting Finished above. There is + // no drain protocol to misfire here. + Ok(StepOutcome::NoProgress) + } + // No `new_worker_copy` override: this step is `Serial`, so the framework + // holds ONE shared instance behind a mutex and never clones it. An override + // here would be dead code implying `buffered` could be split per worker. +} + +/// Run `build_and_run` on its own thread and fail the test if it has not +/// returned within ten seconds. +/// +/// Every end-to-end test in this module must go through here. These pipelines' +/// primary failure modes are deadlock and stall, and calling `Pipeline::run` +/// directly on the test thread turns either one into a hung harness with no +/// message — the run never returns, so no assertion is ever reached. `context` +/// names the pipeline in the timeout message. +/// +/// A panic inside `build_and_run` is re-raised as itself, so an assertion +/// failure inside the run reports its own message rather than a spurious +/// deadlock. +fn run_with_deadlock_timeout(context: &str, build_and_run: impl FnOnce() + Send + 'static) { + let (tx, rx) = std::sync::mpsc::channel::<()>(); + let handle = std::thread::spawn(move || { + build_and_run(); + let _ = tx.send(()); + }); + match rx.recv_timeout(std::time::Duration::from_secs(10)) { + Ok(()) => handle.join().expect("worker thread panicked"), + // The sender dropped without sending, so `build_and_run` panicked — + // typically a failed assertion inside the run. Re-raise that panic + // instead of blaming a deadlock: reporting "DEADLOCKED or stalled" here + // would invert the diagnosis for every assertion failure in a pipeline. + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + let panic = handle.join().expect_err("a disconnect implies the closure panicked"); + std::panic::resume_unwind(panic); + } + // Nothing sent and the sender is still alive: a genuine stall. + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + panic!("{context} DEADLOCKED or stalled") + } + } +} + +fn run_reports_finished_pipeline(threads: usize, n_items: u32) { + let received: Arc>> = Arc::new(parking_lot::Mutex::new(Vec::new())); + let received_for_run = Arc::clone(&received); + + run_with_deadlock_timeout( + &format!("reports_finished mid-step pipeline at --threads {threads}"), + move || { + use crate::{Pipeline, PipelineConfig}; + let builder = Pipeline::builder(); + builder + .chain(WideQueueSource { remaining: n_items }) + .chain(ReportsFinishedBuffer { buffered: VecDeque::new() }) + .chain(DrainReproSink { received: received_for_run }) + .into_sink_marker(); + let pipeline = builder.build().expect("pipeline build"); + pipeline.run(PipelineConfig { threads, ..Default::default() }).expect("pipeline run"); + }, + ); + // WideQueueSource emits exactly 0..n_items. Assert the multiset (not just the + // count) so a dropped, duplicated, or value-corrupted record is caught, not + // only an off-by-N total. + let mut got = received.lock().clone(); + if threads == 1 { + // At one worker the sequence is fully determined, so sorting would throw + // away a checkable guarantee: `WideQueueSource` emits `n_items-1..0`, + // every transport is FIFO, and the single `ReportsFinishedBuffer` drains + // its `VecDeque` front-first — so a reordering regression in the + // flush-first path is only visible here. Assert identity, not multiset. + let expected_order: Vec = (0..n_items).rev().collect(); + assert_eq!( + got, expected_order, + "single worker must deliver items in emit order (flush-first is FIFO)" + ); + return; + } + // Above one worker, order is not asserted: the Serial step's re-dispatch + // interleaving across workers is a valid scheduling detail. + got.sort_unstable(); + let expected: Vec = (0..n_items).collect(); + assert_eq!( + got, expected, + "sink must receive every item 0..{n_items} exactly once (no drop/dup/corruption) \ + at --threads {threads}" + ); +} + +#[test] +fn reports_finished_mid_step_single_thread_no_premature_drain() { + // Lone worker: returning NoProgress mid-flush must not lose buffered output; + // the round-robin must reach the sink so the full queue drains (no spin), + // and the step eventually reports Finished. (The old BUG #5 drain-spin shape.) + run_reports_finished_pipeline(1, 5); +} + +#[test] +fn reports_finished_mid_step_multi_thread_shared_latch_stops_redispatch() { + // 4 workers: the mid step is Serial + Affinity::None, so it sits in every + // worker's live set. When one worker finishes it, the shared `finished` + // latch must stop the others from re-running it (a non-idempotent flusher + // re-entered would push into a drained queue → panic). A larger item count + // widens the window for a concurrent re-dispatch. + run_reports_finished_pipeline(4, 64); +} + +// ───────────────────────────────────────────────────────────────────────────── +// `MultiChain2Ordered::{from_chains, join}` smoke test. Mirrors +// `step2_end_to_end_pipeline_pairs_two_sources_through_runtime` but uses +// `OrderedBytesSingle<_>` source outputs — the chain wrapper real BAM/FASTQ +// source subchains produce — so the converge method is exercised on its +// native input shape. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +#[allow(clippy::too_many_lines)] +fn multi_chain2_ordered_pairs_two_byte_bounded_sources() { + use super::builder::{MultiChain2Ordered, Pipeline, PipelineConfig}; + use super::item::{HeapSize, Ordered}; + use super::outputs::OrderedBytesSingle; + use super::step::Step2; + use super::step::StepCtx2; + + /// Bytes of heap payload each test item carries. Non-zero so the + /// `ByteBounded` transports below actually account for these items: + /// `ByteBoundedQueue` sums `HeapSize::heap_size()` only (it never counts + /// `size_of::()`), so a zero-heap item would make every push free, the + /// byte cap unreachable, and this test blind to a regression that let a + /// `ByteBounded` queue grow without bound — the memory-bound failure mode it + /// exists to cover. + const PAYLOAD_BYTES: usize = 1024; + + /// Byte budget per output edge. Deliberately smaller than + /// `5 * PAYLOAD_BYTES` so the sources hit the cap partway through and must + /// hold the rejected item and retry — exercising backpressure rather than + /// just fitting everything in one go. + const EDGE_LIMIT_BYTES: usize = 2 * PAYLOAD_BYTES; + + /// Minimal ordered, byte-bounded item type for the test. + #[derive(Debug, Clone)] + struct OrderedU32 { + ordinal: u64, + value: u32, + payload: Vec, + } + + impl OrderedU32 { + fn new(ordinal: u64, value: u32) -> Self { + Self { ordinal, value, payload: vec![0u8; PAYLOAD_BYTES] } + } + } + + impl HeapSize for OrderedU32 { + fn heap_size(&self) -> usize { + self.payload.capacity() + } + } + + impl Ordered for OrderedU32 { + fn ordinal(&self) -> u64 { + self.ordinal + } + } + + /// Ordered source: emits items with monotonic `ordinal` carrying + /// values N..=1 then `Finished`. + /// + /// `branch_ordering` is `BranchOrdering::None` deliberately, despite the + /// `OrderedBytesSingle` output shape and the `Ordered` items. The shape is + /// chosen for its `HeapSize + Ordered` bounds and its byte-bounded transport, + /// not to engage the reorder stage: `build_branch_ordered_bytes` maps + /// `(ByteBounded, None)` to a **direct** branch, so these steps exercise + /// byte-backpressure over a plain FIFO edge, which is exactly what the + /// ordered-output assertion below reasons from (FIFO branch queues plus one + /// pop per branch per tick). Declaring `ByOrdinal`/`ByItemOrdinal` here would + /// change what the test covers, not strengthen it. + /// + /// Reorder-stage restoration is covered where it belongs — see + /// `erased::tests::step2_serial_byitemordinal_output_is_reordered_not_collapsed`, + /// `handles::handle_tests::ordered_branch_preserves_ordinal_across_retry`, and + /// the `reorder` module's own tests. + /// + /// Holds an item the byte-bounded output + /// rejected and retries it on a later tick — dropping it would punch a hole + /// in the ordinal sequence while still reporting `Progress`, so the sink's + /// count assertion would fail with no indication of the cause. + #[derive(Clone)] + struct OrderedSource { + next_ordinal: u64, + remaining: u32, + held: Option, + } + + impl Step for OrderedSource { + type Input = (); + type Outputs = OrderedBytesSingle; + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrderedSource", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { + limit_bytes: EDGE_LIMIT_BYTES as u64, + }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + // Flush-first: retry the item a previous tick's full queue rejected. + // A rejected push reports `NoProgress`, not `Contention`: the driver + // treats the two identically, but `Contention` means "a Serial step's + // mutex was held by another worker" and feeds `contention_count`, from + // which the bottleneck verdict derives its SPIN ratio. Reporting it + // for ordinary output backpressure would invent mutex contention that + // never happened and can trip a bogus SPIN finding. + if let Some(item) = self.held.take() { + return match ctx.outputs.push(item) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + }; + } + if self.remaining == 0 { + return Ok(StepOutcome::Finished); + } + let value = self.remaining; + self.remaining -= 1; + let ordinal = self.next_ordinal; + self.next_ordinal += 1; + match ctx.outputs.push(OrderedU32::new(ordinal, value)) { + Ok(()) => Ok(StepOutcome::Progress), + // Hold, never drop: the ordinal was already consumed above, so + // discarding the item would leave a hole in the sequence. + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + } + } + } + + /// Step2 merger over two ordered inputs. Sums per-pair, emits + /// `OrderedU64`. Branch queues are FIFO and this step pops at most one item + /// per branch per tick, so pair order is deterministic — the test asserts + /// the exact output sequence, not just the multiset. Backpressure changes + /// only *when* each item moves, never the order. + #[derive(Debug, Clone)] + struct OrderedU64 { + ordinal: u64, + value: u64, + payload: Vec, + } + impl OrderedU64 { + fn new(ordinal: u64, value: u64) -> Self { + Self { ordinal, value, payload: vec![0u8; PAYLOAD_BYTES] } + } + } + impl HeapSize for OrderedU64 { + fn heap_size(&self) -> usize { + self.payload.capacity() + } + } + impl Ordered for OrderedU64 { + fn ordinal(&self) -> u64 { + self.ordinal + } + } + + struct OrderedPairSummer { + pending_a: Option, + pending_b: Option, + /// A summed pair the byte-bounded output rejected, retried on a later + /// tick. As in `OrderedSource`, dropping it would consume an out-ordinal + /// and lose a record while still reporting `Progress`. + held: Option, + next_out_ordinal: u64, + } + impl Step2 for OrderedPairSummer { + type InputA = OrderedU32; + type InputB = OrderedU32; + type Outputs = OrderedBytesSingle; + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrderedPairSummer", + kind: StepKind::Serial, + sticky: false, + output_queues: vec![QueueSpec::ByteBounded { + limit_bytes: EDGE_LIMIT_BYTES as u64, + }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, ctx: &mut StepCtx2<'_, Self>) -> io::Result { + // Flush-first: retry a pair the full output rejected earlier. + // `NoProgress` rather than `Contention` on a rejected push, for the + // reason given in `OrderedSource::try_run`. + if let Some(out) = self.held.take() { + return match ctx.outputs.push(out) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + }; + } + if self.pending_a.is_none() { + self.pending_a = ctx.a.pop(); + } + if self.pending_b.is_none() { + self.pending_b = ctx.b.pop(); + } + match (self.pending_a.as_ref(), self.pending_b.as_ref()) { + (Some(_), Some(_)) => { + let a = self.pending_a.take().unwrap(); + let b = self.pending_b.take().unwrap(); + let out = OrderedU64::new( + self.next_out_ordinal, + u64::from(a.value) + u64::from(b.value), + ); + self.next_out_ordinal += 1; + match ctx.outputs.push(out) { + Ok(()) => Ok(StepOutcome::Progress), + Err(unpushed) => { + self.held = Some(unpushed.into_item()); + Ok(StepOutcome::NoProgress) + } + } + } + // Same completion guard as `PairSummer`: both inputs drained and + // neither branch item buffered. (`held` is always `None` here — + // the flush-first block above returns when it is `Some`.) + _ if ctx.a.is_drained() + && ctx.b.is_drained() + && self.pending_a.is_none() + && self.pending_b.is_none() => + { + Ok(StepOutcome::Finished) + } + _ if ctx.a.is_drained() && ctx.b.is_drained() => { + panic!( + "OrderedPairSummer: unpaired item left buffered after both branches drained" + ) + } + _ => Ok(StepOutcome::NoProgress), + } + } + } + + /// Records `(ordinal, value)`, not just `value`. + /// + /// The out-ordinal is the thing `OrderedPairSummer`'s hold-and-retry path + /// exists to protect: it assigns `next_out_ordinal` *before* the push, so a + /// rejected push must retry the SAME ordinal. Recording only `value` left that + /// invariant unasserted — a regression that reassigned or skipped an ordinal on + /// the retry path leaves the value sequence intact and the test still passes. + #[derive(Clone)] + struct OrderedSink { + received: Arc>>, + } + impl Step for OrderedSink { + type Input = OrderedU64; + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "OrderedSink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, ctx: &mut StepCtx<'_, Self>) -> io::Result { + match ctx.input.pop() { + Some(item) => { + self.received.lock().push((item.ordinal, item.value)); + Ok(StepOutcome::Progress) + } + None if ctx.input.is_drained() => Ok(StepOutcome::Finished), + None => Ok(StepOutcome::NoProgress), + } + } + } + + // Pin the byte budget's teeth deterministically, independent of how the + // scheduler happens to interleave the run below: an edge must admit at least + // one item (or the pipeline would wedge) and must reject before all five fit + // (or the cap would never bind and the test would prove nothing about byte + // bounding). Both follow from PAYLOAD_BYTES vs EDGE_LIMIT_BYTES, so a future + // edit to either constant that silently removes backpressure fails here. + { + use crate::queues::{ByteBoundedQueue, ItemQueue}; + let q = ByteBoundedQueue::::new(EDGE_LIMIT_BYTES as u64); + let admitted = (0..5u32) + .take_while(|i| q.try_push(OrderedU32::new(u64::from(*i), *i)).is_ok()) + .count(); + assert!(admitted >= 1, "a byte-bounded edge must always admit one item, else it wedges"); + assert!( + admitted < 5, + "the byte cap must bind before all 5 items fit — otherwise this test \ + exercises no byte bounding at all (admitted {admitted})" + ); + } + + let received: Arc>> = + Arc::new(parking_lot::Mutex::new(Vec::new())); + let received_for_run = Arc::clone(&received); + + run_with_deadlock_timeout("MultiChain2Ordered byte-bounded join at --threads 4", move || { + let builder = Pipeline::builder(); + let chain_a = builder.chain(OrderedSource { next_ordinal: 0, remaining: 5, held: None }); + let chain_b = builder.chain(OrderedSource { next_ordinal: 0, remaining: 5, held: None }); + MultiChain2Ordered::::from_chains(chain_a, chain_b) + .join(OrderedPairSummer { + pending_a: None, + pending_b: None, + held: None, + next_out_ordinal: 0, + }) + .chain(OrderedSink { received: received_for_run }) + .into_sink_marker(); + + let pipeline = builder.build().expect("pipeline build"); + pipeline.run(PipelineConfig { threads: 4, ..Default::default() }).expect("pipeline run"); + }); + + let collected = received.lock().clone(); + // Both sources emit values 5,4,3,2,1 in step → pair sums 10,8,6,4,2, in that + // order for the same reason as the `PairSummer` test above: FIFO branch + // queues, one pop per branch per tick, and a `Serial` merge feeding an + // `Exclusive` sink. Backpressure (the byte cap is smaller than the total + // payload) changes only WHEN each item moves, never the order. + assert_eq!( + collected, + vec![(0, 10), (1, 8), (2, 6), (3, 4), (4, 2)], + "Serial dispatch must preserve FIFO pairing, output order, AND the out-ordinal \ + sequence under backpressure — the retry path must reuse an ordinal, not skip it" + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Flattened public API. +// +// `lib.rs` re-exports the step-author surface at the crate root so an +// implementor writes `fgumi_pipeline_core::StepProfile`, not +// `::step::StepProfile`. A type reachable only through its module path is an +// asymmetry a step author trips over, and nothing else catches a dropped +// `pub use` — everything in-crate refers to these by module path anyway, and +// the crate has no in-tree consumer to break. +// ───────────────────────────────────────────────────────────────────────────── + +/// Every type a step author needs must resolve at the crate root. This is a +/// compile-time assertion: naming them in type position fails to build if a +/// `pub use` is dropped or a new peer shape is added without one. +/// +/// `DetachedGroup`, the ordered tuple shapes and `InstrumentationLevel` are the +/// ones this caught — `Step::detached_group` returns `DetachedGroup` while `Step` +/// itself was flattened, `OrderedBytesTuple2`/`3` sat beside an exported `Single` +/// and `OrderedBytesSingle`, and `InstrumentationLevel` appears in +/// `StepOutputs::build_queues`' signature while the trait itself was flattened. +/// +/// `InstrumentationLevel` also turned up a whole class the earlier passes had +/// only sampled: it is the *parameter* of `build_queues`, and sweeping every +/// `pub` type named by a fully-public signature found the **return** side +/// unflattened too — `OutputQueueSet`, the `Tuple*View`s from +/// `OutputHandles::view()`, and `MultiChain2Ordered` from `Chain::into_multi()`. +/// Hence the sweep, not just the one symbol, is what this test now pins. +/// +/// Deliberately NOT asserted: `queues::BoundedQueueHandle` and +/// `reorder::ReorderCapHandle`. They are runtime budget plumbing, named only via +/// `runtime::contexts::RegisteredQueue`, which is not flattened either — and +/// `ByteBoundedQueue` exposes inherent `limit_bytes` / `set_limit_bytes` / +/// `current_bytes`, so a direct user never needs the trait in scope. Promoting +/// them would make the root surface *less* coherent, not more. +/// +/// Also deliberately NOT asserted: `handles::BranchOutputHandle`. No public +/// signature anywhere in the crate mentions it — a step author reaches its +/// methods through `OutputHandles` / the `Tuple*View`s and never names the type — +/// so there is no signature it has to be nameable in. +/// +/// The rule this test encodes is therefore "nameable because a public signature +/// names it", not "`pub`, therefore flattened". `BranchInputHandle` is asserted +/// under exactly that rule: `OutputQueueSet::take_typed_input` returns it, so +/// flattening `OutputQueueSet` here brought it into reach of a root-only import +/// and it had to follow. +#[test] +fn crate_root_reexports_the_step_author_surface() { + use crate as api; + use std::marker::PhantomData; + + fn arity_of() -> usize { + O::arity() + } + + // Step / Step2 vocabulary. + let _: fn() -> api::StepKind = || api::StepKind::Serial; + let _: fn() -> api::StepOutcome = || api::StepOutcome::Progress; + let _: fn() -> api::Affinity = || api::Affinity::None; + let _: fn() -> api::DetachedGroup = || api::DetachedGroup::PerStep; + let _: fn() -> api::QueueSpec = || api::QueueSpec::Unbounded; + let _: fn() -> api::BranchOrdering = || api::BranchOrdering::None; + // `StepOutputs::build_queues` names this in its public signature, so anyone + // implementing that trait by hand has to be able to name it too. + let _: fn() -> api::InstrumentationLevel = || api::InstrumentationLevel::Off; + + // Return types of public methods, which a step author has to name to store + // one or to write a helper that takes it. `PhantomData` rather than a value: + // naming the type in a position that checks its bounds is the whole + // assertion, and none of these are constructible from outside the runtime. + // `build_queues` returns `(OutputQueueSet, OutputsViewAny)` — the second half + // was already flattened, the first was not. + let _: PhantomData = PhantomData; + // `OutputHandles::view()`, one per fan-out arity. + let _: PhantomData> = PhantomData; + let _: PhantomData> = PhantomData; + let _: PhantomData> = PhantomData; + // `Chain::into_multi()`, the ordered sibling of the exported `MultiChain2`. + let _: PhantomData, api::Sequenced>> = + PhantomData; + // `OutputQueueSet::take_typed_input()` — flattening `OutputQueueSet` is what + // put this one within reach of a root-only import. + let _: PhantomData> = PhantomData; + // Naming it with its type is the whole assertion — a comparison against a + // literal would be a constant expression, not a check. + let _: usize = api::MAX_ARITY; + + // Output shapes: one arity per declared shape, so a new shape added without + // a re-export shows up here. + assert_eq!(arity_of::>(), 1); + assert_eq!(arity_of::>>(), 1); + assert_eq!(arity_of::<(u32, u32)>(), 2); + assert_eq!( + arity_of::, api::Sequenced>>(), + 2, + "OrderedBytesTuple2 must be nameable at the crate root" + ); + assert_eq!( + arity_of::< + api::OrderedBytesTuple3, api::Sequenced, api::Sequenced>, + >(), + 3, + "OrderedBytesTuple3 must be nameable at the crate root" + ); +} + +/// The crate docs claim the dependency graph "stays light" and then enumerate +/// it. That list is a promise about the whole graph, so it has to be exhaustive +/// — a reader weighing this crate as a dependency reads the list, not the +/// manifest. Nothing else notices when the two drift: adding a dependency +/// compiles fine, and the prose keeps asserting the old, shorter graph. +/// +/// `anyhow` is the one this caught. It backs `FinalizeHook::finalize`'s return +/// type and went undocumented, so the list understated the graph by one crate. +/// +/// Dev-dependencies are deliberately out of scope: the claim is about what a +/// consumer links, and `proptest` / `rstest` / `trybuild` are not that. +/// Platform-gated tables are in scope, because a consumer on that platform does +/// link them — `fgumi-sort` already carries `[target.'cfg(unix)'.dependencies]`, +/// so reading only the plain `[dependencies]` table would fail *open* the day +/// this crate grows one. +#[test] +fn crate_docs_enumerate_every_runtime_dependency() { + // `include_str!` resolves against this file's directory, so both paths are + // the real files the claim is made in and about — not a copy that can drift. + let manifest = include_str!("../Cargo.toml"); + let crate_docs = include_str!("lib.rs"); + + // `[dependencies]` plus every `[target.'cfg(..)'.dependencies]`. + // `[dev-dependencies]` and `[build-dependencies]` end in `-dependencies`, + // so neither form matches them, at top level or under a `target` table. + let is_runtime_table = + |header: &str| header == "dependencies" || header.ends_with(".dependencies"); + + let mut dependencies: Vec<&str> = Vec::new(); + let mut in_runtime_dependencies = false; + for line in manifest.lines() { + if let Some(header) = line.trim().strip_prefix('[').and_then(|h| h.strip_suffix(']')) { + in_runtime_dependencies = is_runtime_table(header); + // A crate may instead declare itself in the header, as + // `[dependencies.log]` or `[target.'cfg(unix)'.dependencies.libc]`, + // with only its own keys in the body. Recognizing the sub-table by + // its parent is what keeps both spellings in the check — matching + // the `dependencies.` prefix alone would take the first and let the + // target-scoped one fall through undetected. The trim handles the + // quoted-key spelling, `[dependencies."log"]`, which is also legal. + if !in_runtime_dependencies + && let Some((parent, name)) = header.rsplit_once('.') + && is_runtime_table(parent) + { + dependencies.push(name.trim_matches('"')); + } + continue; + } + // A dependency key sits at column 0. Skipping indented and commented + // lines is what keeps a multi-line inline table's `features = [..]` + // continuation, or a commented-out `# tokio = ..`, from being read as a + // crate name and failing this test under a name that is not a crate. + if !in_runtime_dependencies || line.starts_with([' ', '\t', '#']) { + continue; + } + if let Some((name, _)) = line.split_once('=') { + let name = name.trim(); + if !name.is_empty() { + dependencies.push(name); + } + } + } + // A renamed section or a reordered manifest would otherwise leave this test + // asserting over an empty list and passing vacuously. + assert!( + !dependencies.is_empty(), + "parsed no runtime dependencies from Cargo.toml; the section header or layout moved \ + and this test would silently stop checking anything" + ); + + // Odd-index pieces of a backtick split are the code spans. Matching spans + // rather than raw substrings is what keeps `log` from being "documented" by + // the word `logging`, and what lets `` `noodles::sam` `` document `noodles`. + let code_spans: Vec<&str> = crate_docs + .lines() + .filter(|line| line.starts_with("//!")) + .flat_map(|line| line.split('`').skip(1).step_by(2)) + .collect(); + let undocumented: Vec<&str> = dependencies + .iter() + .filter(|dependency| { + !code_spans.iter().any(|span| { + span == *dependency + || span.strip_prefix(*dependency).is_some_and(|rest| rest.starts_with("::")) + }) + }) + .copied() + .collect(); + + assert!( + undocumented.is_empty(), + "crate docs in lib.rs enumerate the dependency graph but omit {undocumented:?}; \ + add them to the list or drop the claim" + ); +} diff --git a/crates/fgumi-pipeline-core/src/topology.rs b/crates/fgumi-pipeline-core/src/topology.rs new file mode 100644 index 000000000..99d445ea2 --- /dev/null +++ b/crates/fgumi-pipeline-core/src/topology.rs @@ -0,0 +1,380 @@ +//! `ChainGraph`: per-step queue + branch-consumer tracking. Used by +//! `PipelineBuilder::build()` to assert all-outputs-wired and by the +//! runtime to construct queue topology. + +/// Static branch-index display names covering every branch up to [`MAX_ARITY`]. +/// Output branch counts are bounded by `MAX_ARITY` at registration, so a valid +/// branch index always maps to a name and the build-error message never prints a +/// placeholder. Out-of-range indices fall back to `"?"` (never hit in practice, +/// but keeps the function total). +fn branch_name(branch: usize) -> &'static str { + const BRANCH_NAMES: [&str; crate::outputs::MAX_ARITY] = ["0", "1", "2", "3"]; + BRANCH_NAMES.get(branch).copied().unwrap_or("?") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StepIdx(pub usize); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct BranchIdx(pub usize); + +#[derive(Debug, Default)] +pub struct ChainGraph { + /// `consumers[(producer, branch)] = Some(consumer)` if wired. + consumers: Vec>, + /// `consumer_input_slots[(producer, branch)] = Some(slot)` — + /// which input branch of the consumer this edge feeds. + /// Defaults to `0` for single-input consumers; multi-input + /// consumers (`Step2` / future `StepN`) record their per-input + /// slot explicitly so [`crate::runtime::contexts`] + /// can build the right [`TwoInputHandles`] wrapper. + consumer_input_slots: Vec>, + /// `branch_count[step]` — number of output branches for that step. + branch_counts: Vec, + /// `branch_offsets[step]` — running prefix sum of `branch_counts` *before* + /// `step` (i.e. `sum(branch_counts[..step])`). This is the base index into + /// the flat `consumers` / `consumer_input_slots` arrays for `step`'s first + /// output branch, so `consumer_slot_index` is O(1) instead of re-summing the + /// prefix on every call. Pushed once per step in + /// [`Self::register_step_with_input_arity`]; static thereafter. + branch_offsets: Vec, + /// `input_arities[step]` — number of input branches for that + /// step (1 for single-input [`Step`](crate::step::Step) impls, 2 for [`Step2`](crate::step::Step2), + /// N for future `StepN`). Defaults to 1 via [`Self::register_step`]; + /// multi-input steps register via + /// [`Self::register_step_with_input_arity`]. + input_arities: Vec, + /// Step names for error messages. + step_names: Vec<&'static str>, +} + +impl ChainGraph { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Register a step with a single input branch (`input_arity = 1`). + /// Convenience shorthand for [`Self::register_step_with_input_arity`]. + pub fn register_step(&mut self, name: &'static str, branch_count: usize) -> StepIdx { + self.register_step_with_input_arity(name, branch_count, 1) + } + + /// Register a step with an explicit input arity. Multi-input + /// steps (`Step2` and future `StepN`) pass `input_arity > 1`; + /// sources pass `input_arity = 0` (their input is implicit). + pub fn register_step_with_input_arity( + &mut self, + name: &'static str, + branch_count: usize, + input_arity: usize, + ) -> StepIdx { + let idx = StepIdx(self.branch_counts.len()); + // The offset for this step is the total branch count of all prior steps, + // i.e. the current length of the flat `consumers` array before we grow it. + self.branch_offsets.push(self.consumers.len()); + self.branch_counts.push(branch_count); + self.input_arities.push(input_arity); + self.step_names.push(name); + self.consumers.resize(self.consumers.len() + branch_count, None); + self.consumer_input_slots.resize(self.consumer_input_slots.len() + branch_count, None); + idx + } + + /// Wire a (producer, branch) → consumer link, into consumer's + /// input branch 0. Convenience shorthand for + /// [`Self::wire_to_slot`] used by single-input consumers. + /// + /// # Panics + /// + /// Panics if the (producer, branch) is already wired (the type system + + /// `Chain` move semantics ensure single-consumer in practice; this is a + /// defensive check). + pub fn wire(&mut self, producer: StepIdx, branch: BranchIdx, consumer: StepIdx) { + self.wire_to_slot(producer, branch, consumer, 0); + } + + /// Wire a (producer, branch) → (consumer, consumer's input slot) + /// link. Multi-input consumers (`Step2` / future `StepN`) record + /// the consumer's input-slot index so + /// [`crate::runtime::contexts`] can + /// route each edge to the right per-branch input handle. + /// + /// # Panics + /// + /// Panics if the (producer, branch) is already wired (defensive), or if + /// `producer`, `branch`, `consumer`, or `consumer_input_slot` are out of + /// range. + pub fn wire_to_slot( + &mut self, + producer: StepIdx, + branch: BranchIdx, + consumer: StepIdx, + consumer_input_slot: usize, + ) { + assert!( + consumer.0 < self.input_arities.len(), + "consumer StepIdx({}) out of range (graph has {} steps)", + consumer.0, + self.input_arities.len() + ); + let consumer_arity = self.input_arities[consumer.0]; + assert!( + consumer_input_slot < consumer_arity, + "consumer_input_slot {consumer_input_slot} out of range for step '{}' \ + with input_arity {consumer_arity}", + self.step_names[consumer.0] + ); + let slot = self.consumer_slot_index(producer, branch); + assert!( + self.consumers[slot].is_none(), + "branch already wired: {:?} branch {:?} → {:?}", + producer, + branch, + self.consumers[slot] + ); + self.consumers[slot] = Some(consumer); + self.consumer_input_slots[slot] = Some(consumer_input_slot); + } + + /// Returns the consumer-input-slot this (producer, branch) edge + /// feeds, if wired. Single-input consumers always return + /// `Some(0)`; multi-input consumers return `Some(0)` or + /// `Some(1)` depending on which input branch the edge feeds. + #[must_use] + pub fn consumer_input_slot(&self, producer: StepIdx, branch: BranchIdx) -> Option { + let slot = self.consumer_slot_index(producer, branch); + self.consumer_input_slots[slot] + } + + /// Returns the input arity of a step (1 for single-input + /// [`Step`](crate::step::Step) impls, 2 for [`Step2`](crate::step::Step2), etc.). Sources have arity 0. + #[must_use] + pub fn input_arity(&self, step: StepIdx) -> usize { + self.input_arities[step.0] + } + + /// Returns the first unwired branch as `(producer, branch_idx, branch_name)`, + /// or `None` if every output branch is wired. + #[must_use] + pub fn first_unwired(&self) -> Option<(StepIdx, BranchIdx, &'static str)> { + for (producer_usize, &branch_count) in self.branch_counts.iter().enumerate() { + for branch in 0..branch_count { + let producer = StepIdx(producer_usize); + let slot = self.consumer_slot_index(producer, BranchIdx(branch)); + if self.consumers[slot].is_none() { + return Some((producer, BranchIdx(branch), branch_name(branch))); + } + } + } + None + } + + #[must_use] + pub fn consumer(&self, producer: StepIdx, branch: BranchIdx) -> Option { + let slot = self.consumer_slot_index(producer, branch); + self.consumers[slot] + } + + /// Returns the first step wired INTO `consumer`, or `None` if nothing feeds + /// it. Used by `PipelineBuilder::build` to reject an edge into a source, + /// whose implicit input means the edge would never be consumed. + #[must_use] + pub fn first_producer_into(&self, consumer: StepIdx) -> Option { + for (producer_usize, &branch_count) in self.branch_counts.iter().enumerate() { + let producer = StepIdx(producer_usize); + for branch in 0..branch_count { + if self.consumer(producer, BranchIdx(branch)) == Some(consumer) { + return Some(producer); + } + } + } + None + } + + #[must_use] + pub fn step_name(&self, step: StepIdx) -> &'static str { + self.step_names[step.0] + } + + #[must_use] + pub fn n_steps(&self) -> usize { + self.branch_counts.len() + } + + #[must_use] + pub fn branch_count(&self, step: StepIdx) -> usize { + self.branch_counts[step.0] + } + + fn consumer_slot_index(&self, producer: StepIdx, branch: BranchIdx) -> usize { + assert!( + producer.0 < self.branch_counts.len(), + "producer StepIdx({}) out of range (graph has {} steps)", + producer.0, + self.branch_counts.len() + ); + let branch_count = self.branch_counts[producer.0]; + assert!( + branch.0 < branch_count, + "branch BranchIdx({}) out of range for producer '{}' with {} branches", + branch.0, + self.step_names[producer.0], + branch_count + ); + // `branch_offsets[producer]` is the precomputed prefix sum of all prior + // steps' branch counts (maintained in `register_step_with_input_arity`), + // so this is O(1) rather than re-summing `branch_counts[..producer.0]`. + self.branch_offsets[producer.0] + branch.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fresh_graph_is_empty() { + let g = ChainGraph::new(); + assert_eq!(g.n_steps(), 0); + assert!(g.first_unwired().is_none()); + } + + #[test] + fn sink_with_no_branches_is_wired() { + let mut g = ChainGraph::new(); + g.register_step("Sink", 0); + assert!(g.first_unwired().is_none()); + } + + #[test] + fn unwired_single_output_step_is_detected() { + let mut g = ChainGraph::new(); + g.register_step("Source", 1); + let (producer, branch, name) = g.first_unwired().unwrap(); + assert_eq!(producer, StepIdx(0)); + assert_eq!(branch, BranchIdx(0)); + assert_eq!(name, "0"); + } + + #[test] + fn wired_chain_is_clean() { + let mut g = ChainGraph::new(); + let src = g.register_step("Source", 1); + let sink = g.register_step("Sink", 0); + g.wire(src, BranchIdx(0), sink); + assert!(g.first_unwired().is_none()); + assert_eq!(g.consumer(src, BranchIdx(0)), Some(sink)); + } + + #[test] + fn unwired_fanout_branch_is_detected() { + let mut g = ChainGraph::new(); + let src = g.register_step("Source", 1); + let mid = g.register_step("FanOut", 2); + let sink = g.register_step("Sink", 0); + g.wire(src, BranchIdx(0), mid); + g.wire(mid, BranchIdx(0), sink); + let (producer, branch, _) = g.first_unwired().unwrap(); + assert_eq!(producer, StepIdx(1)); + assert_eq!(branch, BranchIdx(1)); + } + + #[test] + fn branch_offsets_route_multi_branch_consumers_correctly() { + // Several multi-branch producers in a row: the precomputed branch + // offsets must keep each (producer, branch) edge mapped to a distinct + // flat slot so `consumer`/`consumer_input_slot` return the right links. + let mut g = ChainGraph::new(); + let src = g.register_step("Source", 1); // offset 0, slot 0 + let fan3 = g.register_step("Fan3", 3); // offset 1, slots 1..4 + let fan2 = g.register_step("Fan2", 2); // offset 4, slots 4..6 + let s0 = g.register_step("S0", 0); + let s1 = g.register_step("S1", 0); + let s2 = g.register_step("S2", 0); + let s3 = g.register_step("S3", 0); + + g.wire(src, BranchIdx(0), fan3); + g.wire(fan3, BranchIdx(0), s0); + g.wire(fan3, BranchIdx(1), fan2); + g.wire(fan3, BranchIdx(2), s1); + g.wire(fan2, BranchIdx(0), s2); + g.wire(fan2, BranchIdx(1), s3); + + assert!(g.first_unwired().is_none()); + assert_eq!(g.consumer(src, BranchIdx(0)), Some(fan3)); + assert_eq!(g.consumer(fan3, BranchIdx(0)), Some(s0)); + assert_eq!(g.consumer(fan3, BranchIdx(1)), Some(fan2)); + assert_eq!(g.consumer(fan3, BranchIdx(2)), Some(s1)); + assert_eq!(g.consumer(fan2, BranchIdx(0)), Some(s2)); + assert_eq!(g.consumer(fan2, BranchIdx(1)), Some(s3)); + } + + #[test] + #[should_panic(expected = "already wired")] + fn double_wire_panics() { + let mut g = ChainGraph::new(); + let src = g.register_step("Source", 1); + let sink1 = g.register_step("Sink1", 0); + let sink2 = g.register_step("Sink2", 0); + g.wire(src, BranchIdx(0), sink1); + g.wire(src, BranchIdx(0), sink2); // panics + } + + /// `wire_to_slot` must record which of a multi-input consumer's slots each + /// edge feeds. `contexts::find_all_producers` orders the consumer's input + /// handles by that slot, so a mix-up silently swaps `StepCtx2::a` and + /// `StepCtx2::b` and mis-wires a merge step — with no type error, since both + /// inputs are commonly the same type. + #[test] + fn wire_to_slot_records_each_consumer_input_slot() { + let mut g = ChainGraph::new(); + let a = g.register_step("A", 1); + let b = g.register_step("B", 1); + let join = g.register_step_with_input_arity("Join", 0, 2); + g.wire_to_slot(a, BranchIdx(0), join, 0); + g.wire_to_slot(b, BranchIdx(0), join, 1); + assert_eq!(g.consumer_input_slot(a, BranchIdx(0)), Some(0)); + assert_eq!(g.consumer_input_slot(b, BranchIdx(0)), Some(1)); + assert_eq!(g.input_arity(join), 2); + } + + /// `first_producer_into` backs `PipelineBuilder::build`'s wired-into-a-source + /// check, which is the only guard covering a source appended through + /// `Chain::chain` (registered with a consumer's arity 1, so `wire_to_slot` + /// accepts the edge). + #[test] + fn first_producer_into_finds_the_incoming_edge() { + let mut g = ChainGraph::new(); + let src = g.register_step("Source", 1); + let mid = g.register_step("Mid", 1); + let sink = g.register_step("Sink", 0); + g.wire(src, BranchIdx(0), mid); + g.wire(mid, BranchIdx(0), sink); + + assert_eq!(g.first_producer_into(src), None, "nothing feeds the source"); + assert_eq!(g.first_producer_into(mid), Some(src)); + assert_eq!(g.first_producer_into(sink), Some(mid)); + } + + #[test] + #[should_panic(expected = "out of range")] + fn wire_into_zero_arity_source_panics() { + // Sources register with `input_arity = 0` (their input is implicit), so + // even slot 0 must be rejected — there is no valid input branch to wire. + let mut g = ChainGraph::new(); + let producer = g.register_step("Producer", 1); + let source = g.register_step_with_input_arity("Source", 1, 0); + g.wire(producer, BranchIdx(0), source); // slot 0, but arity 0 → panics + } + + #[test] + #[should_panic(expected = "consumer StepIdx(5) out of range")] + fn wire_to_out_of_range_consumer_panics() { + // A consumer index past the registered steps must produce a + // deterministic range error rather than an opaque out-of-bounds panic. + let mut g = ChainGraph::new(); + let producer = g.register_step("Producer", 1); + g.wire(producer, BranchIdx(0), StepIdx(5)); // no step 5 registered + } +} diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.rs b/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.rs new file mode 100644 index 000000000..0e693e648 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.rs @@ -0,0 +1,54 @@ +//! `Chain>::chain` requires `S::Input = u32`. Chaining a +//! step whose `Input = u64` must fail to compile, not at runtime. + +use std::io; + +use fgumi_pipeline_core::PipelineBuilder; +use fgumi_pipeline_core::outputs::Single; +use fgumi_pipeline_core::queues::QueueSpec; +use fgumi_pipeline_core::reorder::BranchOrdering; +use fgumi_pipeline_core::step::{Step, StepCtx, StepKind, StepOutcome, StepProfile}; + +#[derive(Clone)] +struct U32Source; +impl Step for U32Source { + type Input = (); + type Outputs = Single; + fn profile(&self) -> StepProfile { + StepProfile { + name: "U32Source", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![QueueSpec::CountBounded { capacity: 1 }], + branch_ordering: vec![BranchOrdering::None], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::Finished) + } +} + +#[derive(Clone)] +struct U64Sink; +impl Step for U64Sink { + type Input = u64; // mismatch with U32Source's Single output + type Outputs = (); + fn profile(&self) -> StepProfile { + StepProfile { + name: "U64Sink", + kind: StepKind::Exclusive, + sticky: false, + output_queues: vec![], + branch_ordering: vec![], + } + } + fn try_run(&mut self, _ctx: &mut StepCtx<'_, Self>) -> io::Result { + Ok(StepOutcome::NoProgress) + } +} + +fn main() { + let builder = PipelineBuilder::new(); + // Type error: cannot chain U64Sink (Input = u64) onto Chain>. + let _ = builder.chain(U32Source).chain(U64Sink); +} diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.stderr b/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.stderr new file mode 100644 index 000000000..e96b7ff33 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/chain_input_type_mismatch.stderr @@ -0,0 +1,21 @@ +error[E0271]: type mismatch resolving `::Input == u32` + --> tests/compile-fail/chain_input_type_mismatch.rs:53:44 + | +53 | let _ = builder.chain(U32Source).chain(U64Sink); + | ----- ^^^^^^^ type mismatch resolving `::Input == u32` + | | + | required by a bound introduced by this call + | +note: expected this to be `u32` + --> tests/compile-fail/chain_input_type_mismatch.rs:34:18 + | +34 | type Input = u64; // mismatch with U32Source's Single output + | ^^^ +note: required by a bound in `fgumi_pipeline_core::Chain::<'b, Single>::chain` + --> src/builder.rs + | + | pub fn chain(self, step: S) -> Chain<'b, S::Outputs> + | ----- required by a bound in this associated function + | where + | S: Step, + | ^^^^^^^^^ required by this bound in `Chain::<'b, Single>::chain` diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.rs b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.rs new file mode 100644 index 000000000..d09b209d8 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.rs @@ -0,0 +1,22 @@ +//! `OrderedBytesSingle` requires `T: HeapSize + Ordered`. A type that +//! impls `Ordered` but NOT `HeapSize` must fail to compile. + +use fgumi_pipeline_core::item::Ordered; +use fgumi_pipeline_core::outputs::OrderedBytesSingle; + +struct OrderedNoHeap { + serial: u64, +} + +impl Ordered for OrderedNoHeap { + fn ordinal(&self) -> u64 { + self.serial + } +} + +fn _instantiate() -> std::marker::PhantomData> { + // Bound `T: HeapSize` is unsatisfied for `OrderedNoHeap`. + std::marker::PhantomData +} + +fn main() {} diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.stderr b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.stderr new file mode 100644 index 000000000..7b97cc491 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_heapsize.stderr @@ -0,0 +1,53 @@ +error[E0277]: the trait bound `OrderedNoHeap: HeapSize` is not satisfied + --> tests/compile-fail/ordered_bytes_single_requires_heapsize.rs:17:47 + | +17 | fn _instantiate() -> std::marker::PhantomData> { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `HeapSize` is not implemented for `OrderedNoHeap` + --> tests/compile-fail/ordered_bytes_single_requires_heapsize.rs:7:1 + | + 7 | struct OrderedNoHeap { + | ^^^^^^^^^^^^^^^^^^^^ + = help: the following other types implement trait `HeapSize`: + () + Option + Sequenced + String + Vec + bool + char + f32 + and $N others +note: required by a bound in `OrderedBytesSingle` + --> src/outputs.rs + | + | pub struct OrderedBytesSingle(PhantomData T>); + | ^^^^^^^^ required by this bound in `OrderedBytesSingle` + +error[E0277]: the trait bound `OrderedNoHeap: HeapSize` is not satisfied + --> tests/compile-fail/ordered_bytes_single_requires_heapsize.rs:19:5 + | +19 | std::marker::PhantomData + | ^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `HeapSize` is not implemented for `OrderedNoHeap` + --> tests/compile-fail/ordered_bytes_single_requires_heapsize.rs:7:1 + | + 7 | struct OrderedNoHeap { + | ^^^^^^^^^^^^^^^^^^^^ + = help: the following other types implement trait `HeapSize`: + () + Option + Sequenced + String + Vec + bool + char + f32 + and $N others +note: required by a bound in `OrderedBytesSingle` + --> src/outputs.rs + | + | pub struct OrderedBytesSingle(PhantomData T>); + | ^^^^^^^^ required by this bound in `OrderedBytesSingle` diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.rs b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.rs new file mode 100644 index 000000000..2c1d6dfc6 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.rs @@ -0,0 +1,22 @@ +//! `OrderedBytesSingle` requires `T: HeapSize + Ordered`. A type that +//! impls `HeapSize` but NOT `Ordered` must fail to compile. + +use fgumi_pipeline_core::item::HeapSize; +use fgumi_pipeline_core::outputs::OrderedBytesSingle; + +struct HeapNoOrdered { + bytes: Vec, +} + +impl HeapSize for HeapNoOrdered { + fn heap_size(&self) -> usize { + self.bytes.len() + } +} + +fn _instantiate() -> std::marker::PhantomData> { + // Bound `T: Ordered` is unsatisfied for `HeapNoOrdered`. + std::marker::PhantomData +} + +fn main() {} diff --git a/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.stderr b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.stderr new file mode 100644 index 000000000..6da30361f --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile-fail/ordered_bytes_single_requires_ordered.stderr @@ -0,0 +1,43 @@ +error[E0277]: the trait bound `HeapNoOrdered: Ordered` is not satisfied + --> tests/compile-fail/ordered_bytes_single_requires_ordered.rs:17:47 + | +17 | fn _instantiate() -> std::marker::PhantomData> { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `Ordered` is not implemented for `HeapNoOrdered` + --> tests/compile-fail/ordered_bytes_single_requires_ordered.rs:7:1 + | + 7 | struct HeapNoOrdered { + | ^^^^^^^^^^^^^^^^^^^^ +help: the trait `Ordered` is implemented for `Sequenced` + --> src/reorder.rs + | + | impl Ordered for Sequenced { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +note: required by a bound in `OrderedBytesSingle` + --> src/outputs.rs + | + | pub struct OrderedBytesSingle(PhantomData T>); + | ^^^^^^^ required by this bound in `OrderedBytesSingle` + +error[E0277]: the trait bound `HeapNoOrdered: Ordered` is not satisfied + --> tests/compile-fail/ordered_bytes_single_requires_ordered.rs:19:5 + | +19 | std::marker::PhantomData + | ^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `Ordered` is not implemented for `HeapNoOrdered` + --> tests/compile-fail/ordered_bytes_single_requires_ordered.rs:7:1 + | + 7 | struct HeapNoOrdered { + | ^^^^^^^^^^^^^^^^^^^^ +help: the trait `Ordered` is implemented for `Sequenced` + --> src/reorder.rs + | + | impl Ordered for Sequenced { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +note: required by a bound in `OrderedBytesSingle` + --> src/outputs.rs + | + | pub struct OrderedBytesSingle(PhantomData T>); + | ^^^^^^^ required by this bound in `OrderedBytesSingle` diff --git a/crates/fgumi-pipeline-core/tests/compile_fail.rs b/crates/fgumi-pipeline-core/tests/compile_fail.rs new file mode 100644 index 000000000..615eec272 --- /dev/null +++ b/crates/fgumi-pipeline-core/tests/compile_fail.rs @@ -0,0 +1,56 @@ +//! Compile-fail tests for the unified pipeline core. Each `.rs` file under +//! `tests/compile-fail/` is expected to fail to compile with the trait +//! bounds and type checks the framework promises. +//! +//! trybuild covers the type-mismatch compile-fail invariants: that a chain +//! link's `Step::Input` must match its producer's item type +//! (`tests/compile-fail/chain_input_type_mismatch.rs`), and that +//! `OrderedBytesSingle` requires both `HeapSize` and `Ordered` +//! (`ordered_bytes_single_requires_{heapsize,ordered}.rs`). +//! +//! ## The `.stderr` fixtures are whole-diagnostic, not assertions +//! +//! trybuild compares each fixture to rustc's normalized stderr for **equality**; +//! it has no wildcard or subset matching, so a `.stderr` file records the entire +//! diagnostic — including rustc's `the following other types implement trait +//! HeapSize` / `the trait Ordered is implemented for Sequenced` help blocks. +//! Those blocks are an inventory of the crate's impls, not part of the bound +//! under test, so adding a `HeapSize` or `Ordered` impl anywhere rewrites them +//! and fails this test for a reason unrelated to `OrderedBytesSingle`. +//! +//! They cannot simply be deleted from the fixtures — that is a mismatch like any +//! other and fails immediately. So when a diff here is confined to those help +//! blocks (or to rustc's wording), it is a **re-bless**, not a contract +//! regression: check that the `error[E0277]` lines and the `required by a bound +//! in OrderedBytesSingle` notes still say what they should, then regenerate with +//! `TRYBUILD=overwrite cargo nextest run -p fgumi-pipeline-core --test compile_fail`. +//! A diff that touches an `E0277` line or a `required by a bound` note is the +//! real signal and must not be blessed away. +//! +//! That re-bless runs through a bare `cargo nextest run`, not the repo's +//! `cargo ci-test` alias: the alias hardcodes `--workspace`, which wins over the +//! `-p` filter and also selects `fgumi-cli-macros`' own `compile_fail` binary, so +//! blessing through it would rewrite that crate's fixtures in the same pass. + +/// The number of fixtures `tests/compile-fail/` is expected to hold. Asserted +/// before handing the glob to trybuild because `compile_fail` **passes when the +/// glob matches nothing** — so renaming or moving the fixture directory would +/// silently retire every compile-time contract above while CI stayed green. +/// Raise this when adding a fixture. +const EXPECTED_FIXTURES: usize = 3; + +#[test] +fn pipeline_core_compile_fail() { + let fixtures = std::fs::read_dir("tests/compile-fail") + .expect("tests/compile-fail must exist") + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "rs")) + .count(); + assert!( + fixtures >= EXPECTED_FIXTURES, + "expected at least {EXPECTED_FIXTURES} compile-fail fixtures, found {fixtures}; \ + a zero-match glob makes `compile_fail` a silent no-op" + ); + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compile-fail/*.rs"); +}