From 924270b3ba41c19a468f36b13345edff89e6db2a Mon Sep 17 00:00:00 2001 From: Nils Homer Date: Tue, 11 Aug 2026 23:02:46 -0700 Subject: [PATCH 1/8] feat(metrics): add reason-keyed template filter accounting Introduce TemplateFilterReason and TemplateFilterCounts: one variant per rejection site in the pre-grouping template filter, and a collector that records every decision in both template and primary-read units. This replaces the practice of reusing group's fgbio output schema as the measurement primitive, which capped the reason vocabulary at fgbio's four discard columns. Named template_filter rather than filter because fgumi filter is a real command with its own metrics, and every module in this crate is named for the command it measures. --- crates/fgumi-metrics/src/lib.rs | 2 + crates/fgumi-metrics/src/template_filter.rs | 298 ++++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 crates/fgumi-metrics/src/template_filter.rs diff --git a/crates/fgumi-metrics/src/lib.rs b/crates/fgumi-metrics/src/lib.rs index d26520e8d..9c2b134ea 100644 --- a/crates/fgumi-metrics/src/lib.rs +++ b/crates/fgumi-metrics/src/lib.rs @@ -18,6 +18,7 @@ pub mod group; pub mod rejection; pub mod shared; pub mod simplex; +pub mod template_filter; pub mod writer; use serde::{Deserialize, Serialize}; @@ -106,6 +107,7 @@ pub use group::{FamilySizeMetrics, PositionGroupSizeMetrics, UmiGroupingMetrics} pub use rejection::{RejectionReason, format_count}; pub use shared::UmiMetric; pub use simplex::{SimplexFamilySizeMetric, SimplexMetricsCollector, SimplexYieldMetric}; +pub use template_filter::{TemplateFilterCounts, TemplateFilterReason}; pub use writer::{read_metrics, read_metrics_auto, write_metrics}; #[cfg(test)] diff --git a/crates/fgumi-metrics/src/template_filter.rs b/crates/fgumi-metrics/src/template_filter.rs new file mode 100644 index 000000000..18ddde326 --- /dev/null +++ b/crates/fgumi-metrics/src/template_filter.rs @@ -0,0 +1,298 @@ +//! Template-level filter accounting shared by the `group` and `dedup` commands. +//! +//! Both commands run the same pre-grouping template filter but report it +//! differently: `group` emits fgbio's five-column `UmiGroupingMetric` in +//! **primary-record** units, `dedup` emits fgumi-native `filtered_*` columns in +//! **template** units. [`TemplateFilterCounts`] is the single measurement those +//! two views render from, so it records every decision in both units and keys +//! rejections by [`TemplateFilterReason`] — one variant per rejection site in the +//! filter, independent of any output file's column vocabulary. + +/// Why the pre-grouping template filter rejected a template. +/// +/// One variant per rejection site, not per output column: fgbio's +/// `UmiGroupingMetric` collapses six of these into a single +/// `discarded_poor_alignment` column, which is a property of that file format +/// rather than of the filter. Adding a rejection site requires adding a variant +/// here, which breaks the exhaustive matches in every renderer until each decides +/// what to do with it. +/// +/// `#[repr(usize)]` is load-bearing: [`TemplateFilterCounts`] indexes its arrays +/// by `reason as usize`, and [`Self::ALL`] is ordered to match. +// `EnumIter` (test-only) auto-generates `iter()` so coverage assertions enumerate +// every variant without a hand-maintained list that can drift. +#[cfg_attr(test, derive(strum::EnumIter))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(usize)] +pub enum TemplateFilterReason { + /// A record was shorter than the minimum BAM record length (corrupt input). + MalformedRecord, + /// The template had neither an R1 nor an R2 primary record. + NoPrimaryReads, + /// Every primary read in the template was unmapped. + Unmapped, + /// A primary read carried the QC-fail (`0x200`) flag. + NotPassingFilter, + /// A mapped primary read's mapping quality was below the threshold. + LowMappingQuality, + /// A primary read's mate mapping quality (`MQ` tag) was below the threshold. + LowMateMappingQuality, + /// A primary read had no UMI tag. + MissingUmi, + /// The UMI contained at least one `N` base. + NsInUmi, + /// The UMI was shorter than the configured minimum length. + UmiTooShort, +} + +impl TemplateFilterReason { + /// Number of variants. + pub const COUNT: usize = 9; + + /// Every variant, in discriminant order. + pub const ALL: [Self; Self::COUNT] = [ + Self::MalformedRecord, + Self::NoPrimaryReads, + Self::Unmapped, + Self::NotPassingFilter, + Self::LowMappingQuality, + Self::LowMateMappingQuality, + Self::MissingUmi, + Self::NsInUmi, + Self::UmiTooShort, + ]; + + /// Snake-case identifier for this reason, used as the suffix of the + /// `filtered_*` column in `dedup`'s metrics file and in log summaries. + #[must_use] + pub fn column_suffix(self) -> &'static str { + match self { + Self::MalformedRecord => "malformed_record", + Self::NoPrimaryReads => "no_primary_reads", + Self::Unmapped => "unmapped", + Self::NotPassingFilter => "not_passing_filter", + Self::LowMappingQuality => "low_mapping_quality", + Self::LowMateMappingQuality => "low_mate_mapping_quality", + Self::MissingUmi => "missing_umi", + Self::NsInUmi => "ns_in_umi", + Self::UmiTooShort => "umi_too_short", + } + } +} + +/// Per-reason accounting for the pre-grouping template filter. +/// +/// The filter is the only writer: each template that enters is recorded exactly +/// once, via [`Self::record_accepted`] or [`Self::record_rejected`]. That makes +/// `total == accepted + sum(rejected)` true by construction in both units. +/// +/// Counts are tracked in two units simultaneously: +/// - **templates**: one per template, regardless of how many records it holds +/// - **primary reads**: the number of primary (non-secondary, non-supplementary) +/// records in the template, which is fgbio's unit for `UmiGroupingMetric` +/// +/// A template with neither an R1 nor an R2 contributes `1` template and `0` +/// primary reads, so the units are not proportional and neither is derivable +/// from the other. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TemplateFilterCounts { + accepted_templates: u64, + accepted_primary_reads: u64, + rejected_templates: [u64; TemplateFilterReason::COUNT], + rejected_primary_reads: [u64; TemplateFilterReason::COUNT], +} + +impl TemplateFilterCounts { + /// Creates an empty set of counts. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Records a template that passed the filter, carrying `primary_reads` + /// primary records. + pub fn record_accepted(&mut self, primary_reads: u64) { + self.accepted_templates += 1; + self.accepted_primary_reads += primary_reads; + } + + /// Records a template rejected for `reason`, carrying `primary_reads` primary + /// records. + pub fn record_rejected(&mut self, reason: TemplateFilterReason, primary_reads: u64) { + self.rejected_templates[reason as usize] += 1; + self.rejected_primary_reads[reason as usize] += primary_reads; + } + + /// Adds every count in `other` into `self`. + pub fn merge(&mut self, other: &Self) { + self.accepted_templates += other.accepted_templates; + self.accepted_primary_reads += other.accepted_primary_reads; + for index in 0..TemplateFilterReason::COUNT { + self.rejected_templates[index] += other.rejected_templates[index]; + self.rejected_primary_reads[index] += other.rejected_primary_reads[index]; + } + } + + /// Templates that passed the filter. + #[must_use] + pub fn accepted_templates(&self) -> u64 { + self.accepted_templates + } + + /// Primary reads in templates that passed the filter. + #[must_use] + pub fn accepted_primary_reads(&self) -> u64 { + self.accepted_primary_reads + } + + /// Templates rejected for `reason`. + #[must_use] + pub fn rejected_templates(&self, reason: TemplateFilterReason) -> u64 { + self.rejected_templates[reason as usize] + } + + /// Primary reads in templates rejected for `reason`. + #[must_use] + pub fn rejected_primary_reads(&self, reason: TemplateFilterReason) -> u64 { + self.rejected_primary_reads[reason as usize] + } + + /// Templates rejected for any reason. + #[must_use] + pub fn total_rejected_templates(&self) -> u64 { + self.rejected_templates.iter().sum() + } + + /// Primary reads in templates rejected for any reason. + #[must_use] + pub fn total_rejected_primary_reads(&self) -> u64 { + self.rejected_primary_reads.iter().sum() + } + + /// Templates that entered the filter (accepted plus rejected). + #[must_use] + pub fn total_templates(&self) -> u64 { + self.accepted_templates + self.total_rejected_templates() + } + + /// Primary reads that entered the filter (accepted plus rejected). + #[must_use] + pub fn total_primary_reads(&self) -> u64 { + self.accepted_primary_reads + self.total_rejected_primary_reads() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use strum::IntoEnumIterator; + + /// `TemplateFilterCounts` indexes its arrays by `reason as usize`, so `ALL` + /// must list every variant in discriminant order or counts land in the wrong + /// bucket. + #[test] + fn all_covers_every_variant_in_discriminant_order() { + assert_eq!(TemplateFilterReason::ALL.len(), TemplateFilterReason::COUNT); + assert_eq!(TemplateFilterReason::iter().count(), TemplateFilterReason::COUNT); + for (position, &reason) in TemplateFilterReason::ALL.iter().enumerate() { + assert_eq!(reason as usize, position, "{reason:?} is out of order in ALL"); + } + } + + /// Column suffixes become metric column names: unique and `snake_case`. + #[test] + fn column_suffixes_are_unique_snake_case() { + let mut seen = std::collections::HashSet::new(); + for reason in TemplateFilterReason::ALL { + let suffix = reason.column_suffix(); + assert!(seen.insert(suffix), "duplicate column suffix {suffix}"); + assert!( + suffix.bytes().all(|b| b.is_ascii_lowercase() || b == b'_'), + "{suffix} is not snake_case" + ); + } + } + + /// The central invariant: every template entering the filter is recorded + /// exactly once, so totals reconcile in both units. + #[rstest::rstest] + #[case::only_accepts(&[], 3, 2)] + #[case::only_rejects(&[(TemplateFilterReason::NsInUmi, 2)], 0, 0)] + #[case::mixed(&[(TemplateFilterReason::NotPassingFilter, 2), (TemplateFilterReason::MalformedRecord, 1)], 4, 2)] + #[case::zero_primary_reads(&[(TemplateFilterReason::NoPrimaryReads, 0)], 1, 2)] + fn totals_reconcile( + #[case] rejects: &[(TemplateFilterReason, u64)], + #[case] accepts: u64, + #[case] primary_reads_per_accept: u64, + ) { + let mut counts = TemplateFilterCounts::new(); + for _ in 0..accepts { + counts.record_accepted(primary_reads_per_accept); + } + for &(reason, primary_reads) in rejects { + counts.record_rejected(reason, primary_reads); + } + + assert_eq!( + counts.total_templates(), + counts.accepted_templates() + counts.total_rejected_templates() + ); + assert_eq!( + counts.total_primary_reads(), + counts.accepted_primary_reads() + counts.total_rejected_primary_reads() + ); + assert_eq!(counts.accepted_templates(), accepts); + assert_eq!(counts.accepted_primary_reads(), accepts * primary_reads_per_accept); + assert_eq!(counts.total_rejected_templates(), rejects.len() as u64); + assert_eq!( + counts.total_rejected_primary_reads(), + rejects.iter().map(|&(_, n)| n).sum::() + ); + } + + /// A rejection lands in its own bucket and nowhere else. + #[rstest::rstest] + #[case::malformed(TemplateFilterReason::MalformedRecord)] + #[case::no_primary(TemplateFilterReason::NoPrimaryReads)] + #[case::unmapped(TemplateFilterReason::Unmapped)] + #[case::non_pf(TemplateFilterReason::NotPassingFilter)] + #[case::low_mapq(TemplateFilterReason::LowMappingQuality)] + #[case::low_mate_mapq(TemplateFilterReason::LowMateMappingQuality)] + #[case::missing_umi(TemplateFilterReason::MissingUmi)] + #[case::ns_in_umi(TemplateFilterReason::NsInUmi)] + #[case::umi_too_short(TemplateFilterReason::UmiTooShort)] + fn rejection_lands_in_exactly_one_bucket(#[case] recorded: TemplateFilterReason) { + let mut counts = TemplateFilterCounts::new(); + counts.record_rejected(recorded, 2); + + for reason in TemplateFilterReason::ALL { + let expected_templates = u64::from(reason == recorded); + let expected_reads = if reason == recorded { 2 } else { 0 }; + assert_eq!(counts.rejected_templates(reason), expected_templates, "{reason:?}"); + assert_eq!(counts.rejected_primary_reads(reason), expected_reads, "{reason:?}"); + } + } + + /// Merge aggregates per-worker counters; it must be additive on every field. + #[test] + fn merge_is_additive() { + let mut left = TemplateFilterCounts::new(); + left.record_accepted(2); + left.record_rejected(TemplateFilterReason::NsInUmi, 2); + + let mut right = TemplateFilterCounts::new(); + right.record_accepted(1); + right.record_rejected(TemplateFilterReason::NsInUmi, 2); + right.record_rejected(TemplateFilterReason::Unmapped, 2); + + left.merge(&right); + + assert_eq!(left.accepted_templates(), 2); + assert_eq!(left.accepted_primary_reads(), 3); + assert_eq!(left.rejected_templates(TemplateFilterReason::NsInUmi), 2); + assert_eq!(left.rejected_primary_reads(TemplateFilterReason::NsInUmi), 4); + assert_eq!(left.rejected_templates(TemplateFilterReason::Unmapped), 1); + assert_eq!(left.total_templates(), 5); + assert_eq!(left.total_primary_reads(), 9); + } +} From 33ed4183bc8180dd5c0bfc107355acfe229d3908 Mon Sep 17 00:00:00 2001 From: Nils Homer Date: Tue, 11 Aug 2026 23:11:36 -0700 Subject: [PATCH 2/8] test(group): pin the fgbio five-column grouping metrics header The struct-level test pins UmiGroupingMetrics' serialization; this pins what the command actually writes, so an internal refactor of the filter accounting cannot change the file fgbio has to be able to read. --- tests/integration/test_group_command.rs | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/integration/test_group_command.rs b/tests/integration/test_group_command.rs index e488152cb..f32f8c2e4 100644 --- a/tests/integration/test_group_command.rs +++ b/tests/integration/test_group_command.rs @@ -110,6 +110,48 @@ fn test_group_command_rejects_n_bases_in_umi() { assert!(metric.discarded_ns_in_umi > 0, "Should have discarded reads with N in UMI"); } +/// `group`'s metrics file is fgbio's `UmiGroupingMetric` and must stay exactly +/// five columns with fgbio's spellings, whatever fgumi tracks internally. The +/// struct-level test in `fgumi-metrics` pins the serialization; this pins what the +/// command actually writes to disk, so an internal refactor of the filter +/// accounting cannot change the file fgbio has to be able to read. +#[test] +fn test_grouping_metrics_header_is_fgbio_five_columns() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let input_bam = temp_dir.path().join("input.bam"); + let output_bam = temp_dir.path().join("output.bam"); + let metrics_file = temp_dir.path().join("metrics.txt"); + + create_test_input_bam(&input_bam); + + let cmd = GroupReadsByUmi::try_parse_from([ + "group", + "--input", + input_bam.to_str().unwrap(), + "--output", + output_bam.to_str().unwrap(), + "--strategy", + "identity", + "--edits", + "0", + "--grouping-metrics", + metrics_file.to_str().unwrap(), + "--compression-level", + "1", + ]) + .expect("failed to parse group args"); + cmd.execute("fgumi group").expect("Failed to run group command"); + + let content = fs::read_to_string(&metrics_file).expect("Failed to read metrics file"); + assert_eq!( + content.lines().next(), + Some( + "accepted_sam_records\tdiscarded_non_pf\tdiscarded_poor_alignment\t\ + discarded_ns_in_umi\tdiscarded_umis_to_short" + ) + ); +} + /// Helper function to create a test BAM file with multiple UMI families. fn create_test_input_bam(path: &PathBuf) { let header = create_minimal_header("chr1", 10000); From ee66dd522e3bef811d01cc8e2a54fa91859cf2db Mon Sep 17 00:00:00 2001 From: Nils Homer Date: Tue, 11 Aug 2026 23:24:51 -0700 Subject: [PATCH 3/8] refactor: record filter rejections by reason in group and dedup Replace FilterMetrics with TemplateFilterCounts in both filters. FilterMetrics was group's fgbio output schema used as a measurement primitive: four discard buckets named after fgbio's columns, which dedup borrowed while counting a different unit into them (its `accepted_templates` field held a primary-read count in group and a template count in dedup). group's .grouping_metrics.txt is unchanged -- verified byte-identical across three filter regimes -- because UmiGroupingMetrics::set_filter_counts projects the nine reasons back onto fgbio's four columns in primary-record units. dedup's metrics file is also unchanged here; wiring the new counts into its output is a separate change. Both filters are converted together because deleting FilterMetrics breaks dedup, and every commit must build. The existing filter tests now assert the specific reason rather than the bucket, which immediately caught a mislabelled case: the dedup truncated-aux-record test was reported as a poor-alignment rejection when the record actually clears the length check and is rejected for a missing UMI. --- crates/fgumi-metrics/src/group.rs | 94 ++++++++++++++ src/lib/commands/dedup.rs | 195 ++++++++++++++++-------------- src/lib/commands/group.rs | 134 ++++++++++---------- src/lib/grouper.rs | 42 +------ src/lib/metrics/mod.rs | 2 + 5 files changed, 269 insertions(+), 198 deletions(-) diff --git a/crates/fgumi-metrics/src/group.rs b/crates/fgumi-metrics/src/group.rs index 8179977d9..b10dfbd96 100644 --- a/crates/fgumi-metrics/src/group.rs +++ b/crates/fgumi-metrics/src/group.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::Metric; +use crate::template_filter::{TemplateFilterCounts, TemplateFilterReason}; /// Build a size distribution from (size, count) pairs. /// @@ -107,6 +108,43 @@ impl UmiGroupingMetrics { pub fn new() -> Self { Self::default() } + + /// Fills the fgbio filter columns from [`TemplateFilterCounts`]. + /// + /// fgbio's `UmiGroupingMetric` counts **primary records**, not templates, so + /// this reads the primary-read side of `counts`. The exhaustive match makes + /// adding a new [`TemplateFilterReason`] a compile error until it is assigned + /// a column, so a rejection can never be silently dropped from the output. + /// + /// Six reasons share `discarded_poor_alignment` because fgbio's schema has no + /// finer bucket for them. That collapse is lossy and is retained only for + /// fgbio parity; `dedup` reports each reason separately. + /// + /// Leaves the family-size and summary fields untouched; the caller fills those. + pub fn set_filter_counts(&mut self, counts: &TemplateFilterCounts) { + self.total_records = counts.total_primary_reads(); + self.accepted_records = counts.accepted_primary_reads(); + self.discarded_non_pf = 0; + self.discarded_poor_alignment = 0; + self.discarded_ns_in_umi = 0; + self.discarded_umi_too_short = 0; + + for reason in TemplateFilterReason::ALL { + let reads = counts.rejected_primary_reads(reason); + let column = match reason { + TemplateFilterReason::NotPassingFilter => &mut self.discarded_non_pf, + TemplateFilterReason::NsInUmi => &mut self.discarded_ns_in_umi, + TemplateFilterReason::UmiTooShort => &mut self.discarded_umi_too_short, + TemplateFilterReason::MalformedRecord + | TemplateFilterReason::NoPrimaryReads + | TemplateFilterReason::Unmapped + | TemplateFilterReason::LowMappingQuality + | TemplateFilterReason::LowMateMappingQuality + | TemplateFilterReason::MissingUmi => &mut self.discarded_poor_alignment, + }; + *column += reads; + } + } } impl Metric for UmiGroupingMetrics { @@ -258,6 +296,62 @@ mod tests { assert_eq!(metrics.unique_molecule_ids, 0); } + /// fgbio columns are in **primary-read** units, and the six reasons fgbio has + /// no bucket for must all land in `discarded_poor_alignment`. This mapping is + /// what keeps group's output byte-identical across the `TemplateFilterCounts` + /// refactor. + #[rstest::rstest] + #[case::malformed(TemplateFilterReason::MalformedRecord, 0, 2, 0, 0)] + #[case::no_primary(TemplateFilterReason::NoPrimaryReads, 0, 2, 0, 0)] + #[case::unmapped(TemplateFilterReason::Unmapped, 0, 2, 0, 0)] + #[case::non_pf(TemplateFilterReason::NotPassingFilter, 2, 0, 0, 0)] + #[case::low_mapq(TemplateFilterReason::LowMappingQuality, 0, 2, 0, 0)] + #[case::low_mate_mapq(TemplateFilterReason::LowMateMappingQuality, 0, 2, 0, 0)] + #[case::missing_umi(TemplateFilterReason::MissingUmi, 0, 2, 0, 0)] + #[case::ns_in_umi(TemplateFilterReason::NsInUmi, 0, 0, 2, 0)] + #[case::umi_too_short(TemplateFilterReason::UmiTooShort, 0, 0, 0, 2)] + fn set_filter_counts_maps_reasons_to_fgbio_columns( + #[case] reason: TemplateFilterReason, + #[case] expected_non_pf: u64, + #[case] expected_poor_alignment: u64, + #[case] expected_ns_in_umi: u64, + #[case] expected_umi_too_short: u64, + ) { + let mut counts = TemplateFilterCounts::new(); + counts.record_accepted(2); + counts.record_rejected(reason, 2); + + let mut metrics = UmiGroupingMetrics::new(); + metrics.set_filter_counts(&counts); + + assert_eq!(metrics.accepted_records, 2, "accepted is in primary-read units"); + assert_eq!(metrics.total_records, 4); + assert_eq!(metrics.discarded_non_pf, expected_non_pf); + assert_eq!(metrics.discarded_poor_alignment, expected_poor_alignment); + assert_eq!(metrics.discarded_ns_in_umi, expected_ns_in_umi); + assert_eq!(metrics.discarded_umi_too_short, expected_umi_too_short); + } + + /// No rejection may be dropped: the four fgbio columns must sum to the total. + #[test] + fn set_filter_counts_conserves_every_rejection() { + let mut counts = TemplateFilterCounts::new(); + counts.record_accepted(2); + for reason in TemplateFilterReason::ALL { + counts.record_rejected(reason, 2); + } + + let mut metrics = UmiGroupingMetrics::new(); + metrics.set_filter_counts(&counts); + + let discarded = metrics.discarded_non_pf + + metrics.discarded_poor_alignment + + metrics.discarded_ns_in_umi + + metrics.discarded_umi_too_short; + assert_eq!(discarded, counts.total_rejected_primary_reads()); + assert_eq!(metrics.total_records, metrics.accepted_records + discarded); + } + #[test] fn test_umi_grouping_metrics_serializes_fgbio_five_columns() { use fgoxide::io::DelimFile; diff --git a/src/lib/commands/dedup.rs b/src/lib/commands/dedup.rs index 24266b763..60fc12006 100644 --- a/src/lib/commands/dedup.rs +++ b/src/lib/commands/dedup.rs @@ -23,11 +23,10 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use crate::assigner::{PairedUmiAssigner, Strategy, UmiAssigner}; -use crate::grouper::{ - FilterMetrics, RawPositionGroup, RecordPositionGrouper, build_templates_from_records, -}; +use crate::grouper::{RawPositionGroup, RecordPositionGrouper, build_templates_from_records}; use crate::logging::OperationTimer; use crate::metrics::group::FamilySizeMetrics; +use crate::metrics::{TemplateFilterCounts, TemplateFilterReason}; use crate::read_info::LibraryIndex; use crate::sam::SamTag; use crate::sam::is_template_coordinate_sorted; @@ -87,8 +86,8 @@ pub struct DedupMetrics { pub supplementary_reads: u64, /// Secondary/supplementary without `tc` tag pub missing_tc_tag: u64, - /// Filter metrics from position grouping - pub filter_metrics: FilterMetrics, + /// Per-reason counts from the template filter. + pub filter_counts: TemplateFilterCounts, } impl DedupMetrics { @@ -103,7 +102,7 @@ impl DedupMetrics { self.secondary_reads += other.secondary_reads; self.supplementary_reads += other.supplementary_reads; self.missing_tc_tag += other.missing_tc_tag; - self.filter_metrics.merge(&other.filter_metrics); + self.filter_counts.merge(&other.filter_counts); } /// Calculate duplicate rate. @@ -298,9 +297,11 @@ fn score_template(template: &Template) -> i64 { fn filter_template( template: &Template, config: &DedupFilterConfig, - metrics: &mut FilterMetrics, + counts: &mut TemplateFilterCounts, ) -> bool { - metrics.total_templates += 1; + // Primary-record count, recorded alongside the template count so `group` and + // `dedup` can render the same measurement in their respective units. + let primary_reads = u64::from(template.r1().is_some()) + u64::from(template.r2().is_some()); // Fail closed when *any* record (primary, secondary, or supplementary) is // shorter than the minimum BAM record length: a truncated record indicates @@ -308,7 +309,7 @@ fn filter_template( // malformed record silently dropped (and later panic in RawRecordView::new // on the dedup/serialize path). if template.records().iter().any(|r| r.len() < fgumi_raw_bam::MIN_BAM_RECORD_LEN) { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::MalformedRecord, primary_reads); return false; } @@ -316,7 +317,7 @@ fn filter_template( let raw_r2 = template.r2(); if raw_r1.is_none() && raw_r2.is_none() { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::NoPrimaryReads, primary_reads); return false; } @@ -325,7 +326,7 @@ fn filter_template( && raw_r2 .is_none_or(|r| (RawRecordView::new(r).flags() & fgumi_raw_bam::flags::UNMAPPED) != 0); if both_unmapped { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::Unmapped, primary_reads); return false; } @@ -334,14 +335,14 @@ fn filter_template( let flg = RawRecordView::new(raw).flags(); if !config.include_non_pf && (flg & fgumi_raw_bam::flags::QC_FAIL) != 0 { - metrics.discarded_non_pf += 1; + counts.record_rejected(TemplateFilterReason::NotPassingFilter, primary_reads); return false; } if (flg & fgumi_raw_bam::flags::UNMAPPED) == 0 { let mapq = fgumi_raw_bam::mapq(raw); if mapq < config.min_mapq { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::LowMappingQuality, primary_reads); return false; } } @@ -415,7 +416,7 @@ fn filter_template( // Compare as signed so a negative MQ (e.g., MQ:c:-1) fails the filter // rather than wrapping to 255 via `as u8`. if mq < i64::from(config.min_mapq) { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::LowMateMappingQuality, primary_reads); return false; } } @@ -428,25 +429,25 @@ fn filter_template( if let Some(umi_bytes) = found_umi { match validate_umi(umi_bytes) { UmiValidation::ContainsN => { - metrics.discarded_ns_in_umi += 1; + counts.record_rejected(TemplateFilterReason::NsInUmi, primary_reads); return false; } UmiValidation::Valid(base_count) => { if let Some(min_len) = config.min_umi_length && base_count < min_len { - metrics.discarded_umi_too_short += 1; + counts.record_rejected(TemplateFilterReason::UmiTooShort, primary_reads); return false; } } } } else { - metrics.discarded_poor_alignment += 1; + counts.record_rejected(TemplateFilterReason::MissingUmi, primary_reads); return false; } } - metrics.accepted_templates += 1; + counts.record_accepted(primary_reads); true } @@ -805,13 +806,13 @@ fn process_position_group( }; // Filter templates - let mut filter_metrics = FilterMetrics::new(); + let mut filter_counts = TemplateFilterCounts::new(); let filtered_templates: Vec