From 424d3d84f0c4c1f09118773bcc53399c466a68a6 Mon Sep 17 00:00:00 2001 From: Roy Date: Tue, 18 Aug 2026 00:26:14 +0000 Subject: [PATCH 1/5] perf(cpu-ep): stop the half GEMM forking the pool for work too small to repay it `gemm_impl` split every half GEMM across the pool with no work guard, so it spent 0.14 ms of fork/join overhead to multiply an 8x64 by a 64x64 -- work one core finishes in 0.046 ms. Splitting was 3.1x *slower* than not splitting. That is now the common case, not a corner: since the f16 prefill gate (#1140) this kernel only ever sees the small shapes that decline widening, so the unguarded fork was mis-sized for every shape it still serves. Add the same guard its siblings already apply (`half_gemv::PARALLEL_MIN_WORK`, `accelerate_gemm`'s `k*n` bound), with the crossover measured here rather than copied: this kernel forks per row-block instead of per stripe, so it needs a larger operand to repay the split. Measured `serial/parallel`, interleaved rep-by-rep, p50 of 9, pinned, two runs per thread count: m*k*n T=4 T=16 32_768 0.70x 0.32x 262_144 0.91x/0.92x 0.93x/0.74x 524_288 1.00x/1.00x 1.15x/0.96x 786_432 1.04x/1.04x 1.28x/1.22x 1_048_576 1.06x/1.06x 1.36x/1.37x 786_432 is the smallest size that wins at every thread count in every run. The guard changes scheduling only, so both routes stay bit-identical. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/kernels/half_gemm.rs | 296 +++++++++++++++++- 1 file changed, 288 insertions(+), 8 deletions(-) diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs index a76d140bd9..6b9da2b7c5 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs @@ -329,7 +329,14 @@ fn gemm_impl( c.fill(0.0); let threads = rayon::current_num_threads(); - let mc = if threads <= 1 { + // Splitting is not free: the fork costs, and the workers keep spinning + // after the region ends, which competes with whatever runs next. Below + // `PARALLEL_MIN_WORK` that costs more than the split saves, so stay serial + // -- the same guard `half_gemv` and `accelerate_gemm` already apply. + let parallel = forced_route().unwrap_or_else(|| { + threads > 1 && m.saturating_mul(k).saturating_mul(n) >= PARALLEL_MIN_WORK + }); + let mc = if !parallel { MAX_MC.min(m) } else { let rows = m.div_ceil(threads.saturating_mul(2)).clamp(1, MAX_MC); @@ -340,15 +347,106 @@ fn gemm_impl( } }; + let block = |block_index: usize, c_block: &mut [f32]| { + let first_row = block_index * mc; + let rows = c_block.len() / n; + gemm_block::( + a, a_layout, b, b_layout, c_block, first_row, rows, k, n, path, + ); + }; + + if !parallel { + count_serial_gemm(); + for (block_index, c_block) in c.chunks_mut(mc * n).enumerate() { + block(block_index, c_block); + } + return; + } + c.par_chunks_mut(mc * n) .enumerate() - .for_each(|(block_index, c_block)| { - let first_row = block_index * mc; - let rows = c_block.len() / n; - gemm_block::( - a, a_layout, b, b_layout, c_block, first_row, rows, k, n, path, - ); - }); + .for_each(|(block_index, c_block)| block(block_index, c_block)); +} + +/// Minimum multiply-accumulate count (`m*k*n`) before splitting a half GEMM +/// across the pool. +/// +/// `gemm_impl` used to fork unconditionally, so it spent 0.14 ms of pool +/// overhead to multiply an 8x64 by a 64x64 -- work a single core finishes in +/// 0.046 ms. That is the common case now: since the f16 prefill gate landed, +/// this kernel only ever sees the *small* shapes that decline widening, so the +/// unguarded fork was mis-sized for every shape it still serves. +/// +/// Measured `serial/parallel` (>1 = splitting wins), interleaved rep-by-rep, +/// `p50` of 9, pinned to 16 physical cores, two independent runs per thread +/// count (`bench_half_gemm_parallel_threshold`): +/// +/// | m*k*n | T=4 run1/run2 | T=16 run1/run2 | +/// |-----------|---------------|----------------| +/// | 32_768 | 0.70x | 0.32x | +/// | 131_072 | 0.73x | 0.70x | +/// | 262_144 | 0.91x/0.92x | 0.93x/0.74x | +/// | 393_216 | 0.96x/0.98x | 1.06x/0.92x | +/// | 524_288 | 1.00x/1.00x | 1.15x/0.96x | +/// | 786_432 | 1.04x/1.04x | 1.28x/1.22x | +/// | 1_048_576 | 1.06x/1.06x | 1.36x/1.37x | +/// +/// `786_432` is the smallest size that wins at *every* thread count in *every* +/// run; `524_288` only breaks even and dips to 0.96x. Sibling kernels apply the +/// same guard (`half_gemv::PARALLEL_MIN_WORK`, `accelerate_gemm`'s `k*n` +/// bound); this one is measured separately because it forks per row-block +/// rather than per stripe, so its fork count -- and therefore its overhead -- +/// is larger for the same operand. +const PARALLEL_MIN_WORK: usize = 786_432; + +#[cfg(test)] +thread_local! { + /// Test-only override letting a benchmark time both routes in one process. + static FORCED_ROUTE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +#[inline] +fn forced_route() -> Option { + #[cfg(test)] + { + FORCED_ROUTE.with(std::cell::Cell::get) + } + #[cfg(not(test))] + { + None + } +} + +#[cfg(test)] +fn with_forced_route(parallel: bool, op: impl FnOnce() -> R) -> R { + FORCED_ROUTE.with(|c| c.set(Some(parallel))); + let out = op(); + FORCED_ROUTE.with(|c| c.set(None)); + out +} + +#[cfg(test)] +thread_local! { + /// Counts half GEMMs that declined to split, so a test can prove the guard + /// fired rather than inferring it from a timing. + static SERIAL_GEMMS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[inline] +fn count_serial_gemm() { + #[cfg(test)] + SERIAL_GEMMS.with(|c| c.set(c.get() + 1)); +} + +/// Number of unsplit half GEMMs on this thread since the last reset. +#[cfg(test)] +fn serial_gemms() -> usize { + SERIAL_GEMMS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_serial_gemms() { + SERIAL_GEMMS.with(|c| c.set(0)); } #[allow(clippy::too_many_arguments)] @@ -1087,3 +1185,185 @@ mod tests { } } } + +#[cfg(test)] +mod half_gemm_par_bench { + use super::*; + use std::time::Instant; + + fn operands(m: usize, k: usize, n: usize) -> (Vec, Vec, Vec) { + let a = (0..m * k) + .map(|i| half::f16::from_f32(((i % 13) as f32 - 6.0) * 0.1).to_bits()) + .collect(); + let b = (0..k * n) + .map(|i| half::f16::from_f32(((i % 7) as f32 - 3.0) * 0.1).to_bits()) + .collect(); + (a, b, vec![0.0f32; m * n]) + } + + fn run(a: &[u16], b: &[u16], c: &mut [f32], m: usize, k: usize, n: usize, parallel: bool) { + with_forced_route(parallel, || { + gemm_impl::( + a, + MatrixLayout::row_major(k), + b, + MatrixLayout::row_major(n), + c, + m, + k, + n, + ExecutionPath::Scalar, + ) + }); + } + + /// Sites [`PARALLEL_MIN_WORK`]: serial vs pool-split half GEMM, interleaved + /// rep-by-rep so both routes see the same machine load. Pin the run + /// (`taskset -c 0-15`) and set `RAYON_NUM_THREADS`. + #[test] + #[ignore = "microbench: run explicitly with --ignored --nocapture"] + fn bench_half_gemm_parallel_threshold() { + println!("threads={}", rayon::current_num_threads()); + println!("m,k,n,work,serial_ms,par_ms,serial/par"); + for &(m, k, n) in &[ + (8usize, 256usize, 128usize), // 262_144 + (8, 256, 192), // 393_216 + (8, 256, 256), // 524_288 + (8, 384, 256), // 786_432 + (8, 512, 256), // 1_048_576 + (15, 256, 256), // 983_040 + (4, 512, 512), // 1_048_576 + ] { + let (a, b, mut c) = operands(m, k, n); + run(&a, &b, &mut c, m, k, n, false); + let expected = c.clone(); + run(&a, &b, &mut c, m, k, n, true); + assert_eq!(c, expected, "{m}x{k}x{n}: split changed the result"); + + let (mut sv, mut pv) = (Vec::new(), Vec::new()); + for _ in 0..9 { + let t = Instant::now(); + run(&a, &b, &mut c, m, k, n, false); + sv.push(t.elapsed().as_secs_f64() * 1e3); + let t = Instant::now(); + run(&a, &b, &mut c, m, k, n, true); + pv.push(t.elapsed().as_secs_f64() * 1e3); + } + sv.sort_by(f64::total_cmp); + pv.sort_by(f64::total_cmp); + println!( + "{m},{k},{n},{},{:.4},{:.4},{:.2}", + m * k * n, + sv[4], + pv[4], + sv[4] / pv[4] + ); + } + } +} + +#[cfg(test)] +mod half_gemm_guard_tests { + use super::*; + + fn operands(m: usize, k: usize, n: usize) -> (Vec, Vec, Vec) { + let a = (0..m * k) + .map(|i| half::f16::from_f32(((i % 23) as f32 - 11.0) * 0.07).to_bits()) + .collect(); + let b = (0..k * n) + .map(|i| half::f16::from_f32(((i % 17) as f32 - 8.0) * 0.05).to_bits()) + .collect(); + (a, b, vec![0.0f32; m * n]) + } + + fn gemm(a: &[u16], b: &[u16], c: &mut [f32], m: usize, k: usize, n: usize) { + gemm_impl::( + a, + MatrixLayout::row_major(k), + b, + MatrixLayout::row_major(n), + c, + m, + k, + n, + ExecutionPath::Scalar, + ); + } + + /// The threshold is a measured crossover, not a guess -- an earlier guess + /// sat 12x too low, inside the range the sweep shows losing 0.74x-0.93x. + /// Pin the value so a silent edit has to come back through the sweep. + #[test] + fn half_gemm_parallel_threshold_matches_the_measured_crossover() { + assert_eq!( + PARALLEL_MIN_WORK, 786_432, + "smallest m*k*n measured to win at every thread count; re-run \ + bench_half_gemm_parallel_threshold before changing it" + ); + } + + /// Falsifies the guard itself: work below the crossover must stay on one + /// thread, and work above it must still split. Without the first half, an + /// 8x64x64 GEMM forks and takes 3.1x longer than doing it serially. + #[test] + fn half_gemm_declines_to_split_work_below_the_crossover() { + if rayon::current_num_threads() < 2 { + eprintln!("skipping: needs a multi-thread pool"); + return; + } + // 8*64*64 = 32_768, the shape measured at 0.32x when split. + let (a, b, mut c) = operands(8, 64, 64); + reset_serial_gemms(); + gemm(&a, &b, &mut c, 8, 64, 64); + assert_eq!( + serial_gemms(), + 1, + "work below PARALLEL_MIN_WORK must not fork the pool" + ); + + // 8*512*256 = 1_048_576, measured 1.36x/1.37x in favour of splitting. + let (a, b, mut c) = operands(8, 512, 256); + reset_serial_gemms(); + gemm(&a, &b, &mut c, 8, 512, 256); + assert_eq!( + serial_gemms(), + 0, + "work above PARALLEL_MIN_WORK must still be split" + ); + } + + /// The guard changes *scheduling*, never arithmetic: both routes must agree + /// bit-for-bit, including on shapes whose row count is not a multiple of + /// the register-block height so the final block is a tail. + /// + /// Deliberately spans `m > MAX_MC`: the serial route blocks at + /// `MAX_MC.min(m)`, so every shape with `m <= 64` yields a *single* block + /// and never exercises the loop's block indexing. Without a `m > 64` shape + /// here, corrupting the serial route's `first_row` goes undetected. + #[test] + fn both_routes_agree_bit_for_bit() { + assert_eq!(MAX_MC, 64, "shape list below is chosen to straddle MAX_MC"); + for &(m, k, n) in &[ + (1usize, 64usize, 64usize), + (3, 65, 33), + (8, 64, 64), + (13, 129, 67), + (15, 256, 256), + (16, 257, 129), + (65, 33, 49), // just over MAX_MC: two serial blocks, second a tail + (130, 65, 67), // several serial blocks with a tail + (192, 17, 23), // exact multiple of MAX_MC: no tail + ] { + let (a, b, mut c) = operands(m, k, n); + with_forced_route(false, || gemm(&a, &b, &mut c, m, k, n)); + let serial = c.clone(); + with_forced_route(true, || gemm(&a, &b, &mut c, m, k, n)); + let sb: Vec = serial.iter().map(|v| v.to_bits()).collect(); + let pb: Vec = c.iter().map(|v| v.to_bits()).collect(); + assert!( + sb == pb, + "{m}x{k}x{n}: split and serial half GEMM disagree bit-for-bit" + ); + } + } +} From dbc3da2efe4c095457209fbca281ea45d435bb86 Mon Sep 17 00:00:00 2001 From: Roy Date: Tue, 18 Aug 2026 01:20:43 +0000 Subject: [PATCH 2/5] review: lower the threshold to the value that wins at every thread count, and never fork for one block Opus review found the fixed 786_432 threshold was tuned for T=16 and forced serial in a band where T=2/T=8 genuinely benefit from splitting -- a real regression against the pre-PR always-fork code at partial pool sizes. Re-swept m*k*n across T=2/4/8/16/32, two runs each. The crossover does move with the pool, but *not monotonically*: T=4 wants the highest threshold and T=8 the lowest, because for small m the block count is pinned by m rather than by the pool. A thread-scaled formula would fit that noise, so keep one constant and lower it to 524_288 -- the smallest size that wins at every measured thread count in every run (worst case 1.01x). 262_144 stays declined: it wins at T=2/T=8 but loses at T=4 (0.92x) and T=32 (0.80x). Also decline a "split" that yields a single block: it cannot use more than one thread, so it is pure fork overhead however large the operand. m == 1 always lands there -- a 1x1024 by 1024x1024 GEMV cleared the work bound and forked for one block. Tests for the three review findings: - exact-boundary test, so swapping >= for > is caught (it previously was not) - single-block test - bf16 parity, rather than assuming it inherits f16's guarantee Document that m*k*n is a proxy: the split is over rows of C, so speedup is bounded by block count, and the parallel route re-packs B per row-block. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/kernels/half_gemm.rs | 193 ++++++++++++++---- .../weight-offload-tests/qmoe-1179131-8.bin | Bin 0 -> 18436 bytes 2 files changed, 157 insertions(+), 36 deletions(-) create mode 100644 crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs index 6b9da2b7c5..58a258e0a6 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs @@ -333,12 +333,7 @@ fn gemm_impl( // after the region ends, which competes with whatever runs next. Below // `PARALLEL_MIN_WORK` that costs more than the split saves, so stay serial // -- the same guard `half_gemv` and `accelerate_gemm` already apply. - let parallel = forced_route().unwrap_or_else(|| { - threads > 1 && m.saturating_mul(k).saturating_mul(n) >= PARALLEL_MIN_WORK - }); - let mc = if !parallel { - MAX_MC.min(m) - } else { + let split_mc = { let rows = m.div_ceil(threads.saturating_mul(2)).clamp(1, MAX_MC); if rows == 1 { 1 @@ -346,6 +341,14 @@ fn gemm_impl( rows.div_ceil(MR).saturating_mul(MR).min(MAX_MC) } }; + let parallel = forced_route().unwrap_or_else(|| { + threads > 1 + && m.saturating_mul(k).saturating_mul(n) >= PARALLEL_MIN_WORK + // A split into a single block cannot use more than one thread, so + // it is pure fork overhead. `m == 1` always lands here. + && m.div_ceil(split_mc) > 1 + }); + let mc = if parallel { split_mc } else { MAX_MC.min(m) }; let block = |block_index: usize, c_block: &mut [f32]| { let first_row = block_index * mc; @@ -377,27 +380,40 @@ fn gemm_impl( /// this kernel only ever sees the *small* shapes that decline widening, so the /// unguarded fork was mis-sized for every shape it still serves. /// -/// Measured `serial/parallel` (>1 = splitting wins), interleaved rep-by-rep, -/// `p50` of 9, pinned to 16 physical cores, two independent runs per thread -/// count (`bench_half_gemm_parallel_threshold`): +/// Measured `serial/parallel` (>1 = splitting wins) at `m = 8`, interleaved +/// rep-by-rep, `p50` of 9, pinned to 16 physical cores, two independent runs +/// per thread count (`bench_half_gemm_parallel_threshold`): /// -/// | m*k*n | T=4 run1/run2 | T=16 run1/run2 | -/// |-----------|---------------|----------------| -/// | 32_768 | 0.70x | 0.32x | -/// | 131_072 | 0.73x | 0.70x | -/// | 262_144 | 0.91x/0.92x | 0.93x/0.74x | -/// | 393_216 | 0.96x/0.98x | 1.06x/0.92x | -/// | 524_288 | 1.00x/1.00x | 1.15x/0.96x | -/// | 786_432 | 1.04x/1.04x | 1.28x/1.22x | -/// | 1_048_576 | 1.06x/1.06x | 1.36x/1.37x | +/// | m*k*n | T=2 | T=4 | T=8 | T=16 | T=32 | +/// |-----------|-----------|-----------|-----------|-----------|-----------| +/// | 32_768 | 0.88/0.86 | 0.51/0.55 | 0.63/0.75 | 0.36/0.31 | 0.21/0.19 | +/// | 65_536 | 0.79/0.78 | 0.63/0.63 | 0.85/0.88 | 0.53/0.53 | 0.41/0.33 | +/// | 131_072 | 1.02/1.02 | 0.79/0.80 | 1.14/1.18 | 0.76/0.60 | 0.63/0.54 | +/// | 262_144 | 1.20/1.20 | 0.92/0.92 | 1.16/1.51 | 1.03/1.02 | 0.86/0.80 | +/// | 393_216 | 1.27/1.28 | 0.91/0.97 | 1.27/1.64 | 1.04/1.14 | 0.99/1.04 | +/// | 524_288 | 1.34/1.30 | 1.01/1.02 | 1.29/1.76 | 1.20/1.08 | 1.19/1.12 | +/// | 1_048_576 | 1.41/1.37 | 1.08/1.04 | 1.47/1.95 | 1.34/1.30 | 1.31/1.31 | /// -/// `786_432` is the smallest size that wins at *every* thread count in *every* -/// run; `524_288` only breaks even and dips to 0.96x. Sibling kernels apply the -/// same guard (`half_gemv::PARALLEL_MIN_WORK`, `accelerate_gemm`'s `k*n` -/// bound); this one is measured separately because it forks per row-block -/// rather than per stripe, so its fork count -- and therefore its overhead -- -/// is larger for the same operand. -const PARALLEL_MIN_WORK: usize = 786_432; +/// `524_288` is the smallest size that wins at *every* measured thread count in +/// *every* run (worst case 1.01x). Below it the picture is genuinely mixed -- +/// `262_144` wins at T=2/T=8 but loses at T=4 (0.92x) and T=32 (0.80x) -- so +/// declining is the right call there even though it leaves some partial-pool +/// wins on the table. +/// +/// Deliberately a single constant rather than a function of thread count: the +/// measured crossover is *not* monotone in threads (T=4 wants the highest +/// threshold, T=8 the lowest), because for small `m` the block count is pinned +/// by `m` rather than by the pool, so a thread-scaled formula would be fitting +/// noise. +/// +/// `m*k*n` is a proxy, not a cost model. The split is over *rows of C*, so the +/// achievable speedup is bounded by the block count, and the parallel route +/// re-packs `B` per row-block -- for small `m` (`mc == 1`) that is `m` +/// redundant packs. That is why the ratios plateau near 1.1x at T=4 rather than +/// scaling with the pool, and why some shapes above the threshold still gain +/// little. Sibling kernels apply the same kind of guard +/// (`half_gemv::PARALLEL_MIN_WORK`, `accelerate_gemm`'s `k*n` bound). +const PARALLEL_MIN_WORK: usize = 524_288; #[cfg(test)] thread_local! { @@ -1226,13 +1242,18 @@ mod half_gemm_par_bench { println!("threads={}", rayon::current_num_threads()); println!("m,k,n,work,serial_ms,par_ms,serial/par"); for &(m, k, n) in &[ - (8usize, 256usize, 128usize), // 262_144 - (8, 256, 192), // 393_216 - (8, 256, 256), // 524_288 - (8, 384, 256), // 786_432 - (8, 512, 256), // 1_048_576 - (15, 256, 256), // 983_040 - (4, 512, 512), // 1_048_576 + (8usize, 64usize, 64usize), // 32_768 + (8, 128, 64), // 65_536 + (8, 128, 128), // 131_072 + (8, 192, 128), // 196_608 + (8, 256, 128), // 262_144 + (8, 256, 192), // 393_216 + (8, 256, 256), // 524_288 + (8, 384, 256), // 786_432 + (8, 512, 256), // 1_048_576 + (8, 512, 384), // 1_572_864 + (8, 512, 512), // 2_097_152 + (8, 768, 512), // 3_145_728 ] { let (a, b, mut c) = operands(m, k, n); run(&a, &b, &mut c, m, k, n, false); @@ -1276,6 +1297,20 @@ mod half_gemm_guard_tests { (a, b, vec![0.0f32; m * n]) } + fn gemm_as(a: &[u16], b: &[u16], c: &mut [f32], m: usize, k: usize, n: usize) { + gemm_impl::( + a, + MatrixLayout::row_major(k), + b, + MatrixLayout::row_major(n), + c, + m, + k, + n, + ExecutionPath::Scalar, + ); + } + fn gemm(a: &[u16], b: &[u16], c: &mut [f32], m: usize, k: usize, n: usize) { gemm_impl::( a, @@ -1291,12 +1326,14 @@ mod half_gemm_guard_tests { } /// The threshold is a measured crossover, not a guess -- an earlier guess - /// sat 12x too low, inside the range the sweep shows losing 0.74x-0.93x. - /// Pin the value so a silent edit has to come back through the sweep. + /// sat 8x too low, inside the range the sweep shows losing 0.3x-0.9x. Pin + /// the value so a silent edit has to come back through the sweep, which + /// spans five thread counts precisely because the crossover moves with the + /// pool. #[test] fn half_gemm_parallel_threshold_matches_the_measured_crossover() { assert_eq!( - PARALLEL_MIN_WORK, 786_432, + PARALLEL_MIN_WORK, 524_288, "smallest m*k*n measured to win at every thread count; re-run \ bench_half_gemm_parallel_threshold before changing it" ); @@ -1321,7 +1358,7 @@ mod half_gemm_guard_tests { "work below PARALLEL_MIN_WORK must not fork the pool" ); - // 8*512*256 = 1_048_576, measured 1.36x/1.37x in favour of splitting. + // 8*512*256 = 1_048_576, measured 1.31x-1.95x in favour of splitting. let (a, b, mut c) = operands(8, 512, 256); reset_serial_gemms(); gemm(&a, &b, &mut c, 8, 512, 256); @@ -1332,6 +1369,90 @@ mod half_gemm_guard_tests { ); } + /// Pins the comparison at *exactly* the threshold. Without this, swapping + /// `>=` for `>` in the guard changes behaviour at the boundary and every + /// other test still passes. + #[test] + fn the_crossover_itself_splits_and_one_mac_below_it_does_not() { + if rayon::current_num_threads() < 2 { + eprintln!("skipping: needs a multi-thread pool"); + return; + } + // 8 * 256 * 256 == 524_288 == PARALLEL_MIN_WORK exactly. + let (m, k, n) = (8usize, 256usize, 256usize); + assert_eq!( + m * k * n, + PARALLEL_MIN_WORK, + "shape must sit on the boundary" + ); + let (a, b, mut c) = operands(m, k, n); + reset_serial_gemms(); + gemm(&a, &b, &mut c, m, k, n); + assert_eq!( + serial_gemms(), + 0, + "work equal to PARALLEL_MIN_WORK must split (the bound is inclusive)" + ); + + // One MAC below the threshold: k*n is one column short. + let n1 = n - 1; + assert!(m * k * n1 < PARALLEL_MIN_WORK); + let (a, b, mut c) = operands(m, k, n1); + reset_serial_gemms(); + gemm(&a, &b, &mut c, m, k, n1); + assert_eq!( + serial_gemms(), + 1, + "work below PARALLEL_MIN_WORK must stay serial" + ); + } + + /// A split into one block cannot use more than one thread, so forking for + /// it is pure overhead however large the operand is. `m == 1` is the case + /// that matters: a 1x2048 by 2048x2048 GEMV is 4.2M MACs, far above the + /// threshold, yet yields exactly one block. + #[test] + fn a_single_block_is_never_split_however_large_the_operand() { + if rayon::current_num_threads() < 2 { + eprintln!("skipping: needs a multi-thread pool"); + return; + } + let (m, k, n) = (1usize, 1024usize, 1024usize); + assert!(m * k * n > PARALLEL_MIN_WORK, "must clear the work bound"); + let (a, b, mut c) = operands(m, k, n); + reset_serial_gemms(); + gemm(&a, &b, &mut c, m, k, n); + assert_eq!( + serial_gemms(), + 1, + "a one-block split is pure fork overhead and must be declined" + ); + } + + /// The routing is element-type agnostic, so bf16 must get the same + /// bit-for-bit guarantee as f16 rather than inheriting it by assumption. + #[test] + fn bf16_routes_agree_bit_for_bit() { + for &(m, k, n) in &[(8usize, 64usize, 64usize), (13, 129, 67), (130, 65, 67)] { + let a: Vec = (0..m * k) + .map(|i| half::bf16::from_f32(((i % 23) as f32 - 11.0) * 0.07).to_bits()) + .collect(); + let b: Vec = (0..k * n) + .map(|i| half::bf16::from_f32(((i % 17) as f32 - 8.0) * 0.05).to_bits()) + .collect(); + let mut c = vec![0.0f32; m * n]; + with_forced_route(false, || gemm_as::(&a, &b, &mut c, m, k, n)); + let serial = c.clone(); + with_forced_route(true, || gemm_as::(&a, &b, &mut c, m, k, n)); + let sb: Vec = serial.iter().map(|v| v.to_bits()).collect(); + let pb: Vec = c.iter().map(|v| v.to_bits()).collect(); + assert!( + sb == pb, + "{m}x{k}x{n}: split and serial bf16 half GEMM disagree bit-for-bit" + ); + } + } + /// The guard changes *scheduling*, never arithmetic: both routes must agree /// bit-for-bit, including on shapes whose row count is not a multiple of /// the register-block height so the final block is a tail. diff --git a/crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin b/crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin new file mode 100644 index 0000000000000000000000000000000000000000..7069aeaab4213bd52038a636ced391231fcefab5 GIT binary patch literal 18436 zcmeI&-A&|B5JX`|wi^X0kOHs!ND8Dt3cL{7g4h*ZU)=2eyUO_M%y8)c%8TtWo;$S0?nwNWPe$Z6B0 zkBm;C>&vwPf%TAQEl%UCMw9iByCRwBBcoHOk9=~PSsP`dkDNA5`p6|jppQIVPUEy` z(noHiO!Se_Dbz&vHYW|q}xvL13*Bopf)Q;tG? zUx5&ix<*rC3) zM@FYmA9=c*#%a@}kDNA5`p6|jppSfVnpqoVqL180ndl?u+D)!6Q;tIGAmj#MCi=)FM4*p+a++BiWulLqHck4-=oIQBPnXj;ZJP9v+b9!#0@s&kEl%UCMw9iByCRub4;h_8edLqV%-SduedM%h(nl^K0)6D^ zavG;ilRk1AWulLaPN6>X$!TV7l!-oa+BE4Smk@zI@^m?k)28YBe?FhzH2#0v+VS^y u&Reg^`^SI1_ITgV@%C8Dx8CpTo4vlr`@ZMDo&WFqzGvsdoDc8oeE16-!Fa>~ literal 0 HcmV?d00001 From f0943870afb8b0fea64608e53581fffc40c11fde Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 18 Aug 2026 02:58:50 +0000 Subject: [PATCH 3/5] review: raise the half-GEMM fork threshold to the crossover the pinned build actually shows `codegen-units = 1` for `onnx-runtime-ep-cpu` (#1174) makes the *serial* route materially faster, so the point where splitting starts to repay the fork moves up. Re-ran `bench_half_gemm_parallel_threshold` at RAYON 2/4/8/16 with that pin in place, two runs per thread count: | m*k*n | T=2 | T=4 | T=8 | T=16 | |-----------|------|-----------|------|-----------| | 262_144 | 1.20 | 0.92/0.91 | 1.52 | 0.64/0.95 | | 393_216 | 1.28 | 0.99/0.96 | 1.66 | 0.80/1.10 | | 524_288 | 1.32 | 0.99/0.98 | 1.81 | 1.03/1.19 | | 786_432 | 1.33 | 1.03/0.92 | 1.35 | 0.97/1.25 | | 1_048_576 | 1.37 | 1.05/1.05 | 1.46 | 1.31/1.34 | The rule is unchanged -- the smallest size that wins at *every* measured thread count in *every* run -- but the answer is now `1_048_576`, not `524_288`: `524_288` is a wash at T=4 (0.99/0.98) and `786_432` regresses there (0.92) and at T=16 (0.97). Below the threshold the loss is still the 0.32-0.37x the guard exists to stop, so the guard itself is unaffected. Boundary tests are moved with the constant so they keep testing the boundary: the "must split" shape becomes 8x512x384, the exact-threshold shape becomes 8x512x256, and the one-block shape becomes 1x1024x2048. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/kernels/half_gemm.rs | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs index 58a258e0a6..d5572a408d 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/half_gemm.rs @@ -394,11 +394,26 @@ fn gemm_impl( /// | 524_288 | 1.34/1.30 | 1.01/1.02 | 1.29/1.76 | 1.20/1.08 | 1.19/1.12 | /// | 1_048_576 | 1.41/1.37 | 1.08/1.04 | 1.47/1.95 | 1.34/1.30 | 1.31/1.31 | /// -/// `524_288` is the smallest size that wins at *every* measured thread count in -/// *every* run (worst case 1.01x). Below it the picture is genuinely mixed -- -/// `262_144` wins at T=2/T=8 but loses at T=4 (0.92x) and T=32 (0.80x) -- so -/// declining is the right call there even though it leaves some partial-pool -/// wins on the table. +/// Re-measured after `codegen-units = 1` was pinned for this crate, which makes +/// the *serial* route materially faster and therefore moves the crossover up. +/// Same harness, two runs per thread count: +/// +/// | m*k*n | T=2 | T=4 | T=8 | T=16 | +/// |-----------|-----------|-----------|-----------|-----------| +/// | 262_144 | 1.20 | 0.92/0.91 | 1.52 | 0.64/0.95 | +/// | 393_216 | 1.28 | 0.99/0.96 | 1.66 | 0.80/1.10 | +/// | 524_288 | 1.32 | 0.99/0.98 | 1.81 | 1.03/1.19 | +/// | 786_432 | 1.33 | 1.03/0.92 | 1.35 | 0.97/1.25 | +/// | 1_048_576 | 1.37 | 1.05/1.05 | 1.46 | 1.31/1.34 | +/// +/// `1_048_576` is now the smallest size that wins at every measured thread count +/// in every run (worst case 1.05x). `524_288` no longer qualifies: it is a wash +/// at T=4 (0.99/0.98) and `786_432` dips to 0.92 there and 0.97 at T=16. The +/// rule is unchanged -- smallest size that wins everywhere -- only the build it +/// is measured against is faster, so the answer moved. +/// +/// Below the threshold the picture is not merely mixed but bad: 0.32-0.37 at +/// T=16 for the smallest shapes, which is the 3x the guard exists to stop. /// /// Deliberately a single constant rather than a function of thread count: the /// measured crossover is *not* monotone in threads (T=4 wants the highest @@ -413,7 +428,7 @@ fn gemm_impl( /// scaling with the pool, and why some shapes above the threshold still gain /// little. Sibling kernels apply the same kind of guard /// (`half_gemv::PARALLEL_MIN_WORK`, `accelerate_gemm`'s `k*n` bound). -const PARALLEL_MIN_WORK: usize = 524_288; +const PARALLEL_MIN_WORK: usize = 1_048_576; #[cfg(test)] thread_local! { @@ -1333,7 +1348,7 @@ mod half_gemm_guard_tests { #[test] fn half_gemm_parallel_threshold_matches_the_measured_crossover() { assert_eq!( - PARALLEL_MIN_WORK, 524_288, + PARALLEL_MIN_WORK, 1_048_576, "smallest m*k*n measured to win at every thread count; re-run \ bench_half_gemm_parallel_threshold before changing it" ); @@ -1358,10 +1373,10 @@ mod half_gemm_guard_tests { "work below PARALLEL_MIN_WORK must not fork the pool" ); - // 8*512*256 = 1_048_576, measured 1.31x-1.95x in favour of splitting. - let (a, b, mut c) = operands(8, 512, 256); + // 8*512*384 = 1_572_864, measured 1.33x-1.51x in favour of splitting. + let (a, b, mut c) = operands(8, 512, 384); reset_serial_gemms(); - gemm(&a, &b, &mut c, 8, 512, 256); + gemm(&a, &b, &mut c, 8, 512, 384); assert_eq!( serial_gemms(), 0, @@ -1378,8 +1393,8 @@ mod half_gemm_guard_tests { eprintln!("skipping: needs a multi-thread pool"); return; } - // 8 * 256 * 256 == 524_288 == PARALLEL_MIN_WORK exactly. - let (m, k, n) = (8usize, 256usize, 256usize); + // 8 * 512 * 256 == 1_048_576 == PARALLEL_MIN_WORK exactly. + let (m, k, n) = (8usize, 512usize, 256usize); assert_eq!( m * k * n, PARALLEL_MIN_WORK, @@ -1417,7 +1432,7 @@ mod half_gemm_guard_tests { eprintln!("skipping: needs a multi-thread pool"); return; } - let (m, k, n) = (1usize, 1024usize, 1024usize); + let (m, k, n) = (1usize, 1024usize, 2048usize); assert!(m * k * n > PARALLEL_MIN_WORK, "must clear the work bound"); let (a, b, mut c) = operands(m, k, n); reset_serial_gemms(); From 1a6d4c4fc925a2fedf6ac767e3c906bc469b93a7 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 18 Aug 2026 04:39:58 +0000 Subject: [PATCH 4/5] review: drop the stray weight-offload scratch file from this branch `target-leon/weight-offload-tests/qmoe-1179131-8.bin` is a generated test artifact that a local run dropped into the tree and a `git add -A` swept up. It has nothing to do with the half-GEMM fork threshold. Two siblings of it are already tracked on `main`; untracking those is a separate cleanup and does not belong in a perf PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../weight-offload-tests/qmoe-1179131-8.bin | Bin 18436 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin diff --git a/crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin b/crates/onnx-runtime-ep-cpu/target-leon/weight-offload-tests/qmoe-1179131-8.bin deleted file mode 100644 index 7069aeaab4213bd52038a636ced391231fcefab5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18436 zcmeI&-A&|B5JX`|wi^X0kOHs!ND8Dt3cL{7g4h*ZU)=2eyUO_M%y8)c%8TtWo;$S0?nwNWPe$Z6B0 zkBm;C>&vwPf%TAQEl%UCMw9iByCRwBBcoHOk9=~PSsP`dkDNA5`p6|jppQIVPUEy` z(noHiO!Se_Dbz&vHYW|q}xvL13*Bopf)Q;tG? zUx5&ix<*rC3) zM@FYmA9=c*#%a@}kDNA5`p6|jppSfVnpqoVqL180ndl?u+D)!6Q;tIGAmj#MCi=)FM4*p+a++BiWulLqHck4-=oIQBPnXj;ZJP9v+b9!#0@s&kEl%UCMw9iByCRub4;h_8edLqV%-SduedM%h(nl^K0)6D^ zavG;ilRk1AWulLaPN6>X$!TV7l!-oa+BE4Smk@zI@^m?k)28YBe?FhzH2#0v+VS^y u&Reg^`^SI1_ITgV@%C8Dx8CpTo4vlr`@ZMDo&WFqzGvsdoDc8oeE16-!Fa>~ From 9e82c18338b2546e0cf468ab8fdf481a4a616a74 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 18 Aug 2026 04:44:35 +0000 Subject: [PATCH 5/5] review: carry the rustfmt repair so this branch can go green This branch merged `main` after #1200 landed a trailing blank line in `pipeline/mod.rs`, which fails the required `Rust quality` check. #1204 fixes it on `main`; the identical one-line deletion here lets this PR go green without waiting, and merges as a no-op once #1204 lands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/onnx-genai-engine/src/pipeline/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/onnx-genai-engine/src/pipeline/mod.rs b/crates/onnx-genai-engine/src/pipeline/mod.rs index 814b2a5234..dbe88458ad 100644 --- a/crates/onnx-genai-engine/src/pipeline/mod.rs +++ b/crates/onnx-genai-engine/src/pipeline/mod.rs @@ -3681,4 +3681,3 @@ pipeline: } } } -