diff --git a/crates/mlas-sys/src/lib.rs b/crates/mlas-sys/src/lib.rs index d58564b8f5..c3f6612ae0 100644 --- a/crates/mlas-sys/src/lib.rs +++ b/crates/mlas-sys/src/lib.rs @@ -58,6 +58,33 @@ unsafe extern "C" { multithread: c_int, ); + fn mlas_qgemm_pack_b_size(n: usize, k: usize, a_is_signed: c_int, b_is_signed: c_int) -> usize; + fn mlas_qgemm_pack_b( + n: usize, + k: usize, + b: *const c_void, + ldb: usize, + a_is_signed: c_int, + b_is_signed: c_int, + packed_b: *mut c_void, + ); + fn mlas_qgemm_i32_packed( + m: usize, + n: usize, + k: usize, + a_is_signed: c_int, + b_is_signed: c_int, + a: *const c_void, + lda: usize, + zero_point_a: u8, + packed_b: *const c_void, + zero_point_b: *const u8, + per_column_zero_points: c_int, + c: *mut i32, + ldc: usize, + multithread: c_int, + ); + fn mlas_sgemm_pack_b_size(trans_a: c_int, trans_b: c_int, n: usize, k: usize) -> usize; fn mlas_sgemm_pack_b( trans_a: c_int, @@ -1186,6 +1213,148 @@ pub fn qgemm_i32( } } +/// A constant quantized `B` pre-packed into MLAS's kernel layout. +/// +/// [`qgemm_i32`] leaves `BIsPacked` unset, so MLAS re-packs the whole `k x n` +/// weight on every call. ORT pre-packs a constant weight once at session +/// initialisation instead; this is the same thing. +/// +/// The packed layout is chosen by the kernel MLAS dispatches to, so a pack is +/// only valid for the same `(n, k, a_signed, b_signed)` **on the same machine**. +/// It must never be serialised, cached to disk, or shared across processes. +pub struct QgemmPackedB { + ptr: *mut u8, + layout: std::alloc::Layout, + n: usize, + k: usize, + a_signed: bool, + b_signed: bool, +} + +// SAFETY: the buffer is written once during `new` and only ever read afterwards +// (MLAS takes it as `const void*`), so sharing it across threads is sound. This +// mirrors `PackedB` above. +unsafe impl Send for QgemmPackedB {} +unsafe impl Sync for QgemmPackedB {} + +impl QgemmPackedB { + /// Pack a row-major `k x n` quantized B (`ldb == n`). + /// + /// Returns `None` when MLAS reports no packed layout for this shape and + /// signedness combination, which is its documented way of saying "call the + /// unpacked path"; callers must then keep using [`qgemm_i32`]. + pub fn new(n: usize, k: usize, b: &[u8], a_signed: bool, b_signed: bool) -> Option { + assert_eq!(b.len(), k * n, "B must be k*n bytes"); + if n == 0 || k == 0 { + return None; + } + // SAFETY: a pure size query with no pointer arguments. + let size = + unsafe { mlas_qgemm_pack_b_size(n, k, c_int::from(a_signed), c_int::from(b_signed)) }; + if size == 0 { + return None; + } + let layout = std::alloc::Layout::from_size_align(size, 64).ok()?; + // SAFETY: `size` is non-zero, so the layout is non-zero-sized. + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null(), "packed quantized-B allocation failed"); + // SAFETY: `b` is `k * n` bytes as asserted, `ldb = n` matches the + // row-major layout, and `ptr` addresses `size` writable bytes -- the + // exact size MLAS just asked for. + unsafe { + mlas_qgemm_pack_b( + n, + k, + b.as_ptr().cast::(), + n, + c_int::from(a_signed), + c_int::from(b_signed), + ptr.cast::(), + ); + } + Some(Self { + ptr, + layout, + n, + k, + a_signed, + b_signed, + }) + } + + /// The logical `(k, n)` dimensions and `(a_signed, b_signed)` this pack was + /// built for. A caller must not use it for any other combination. + pub fn identity(&self) -> (usize, usize, bool, bool) { + (self.k, self.n, self.a_signed, self.b_signed) + } +} + +impl Drop for QgemmPackedB { + fn drop(&mut self) { + // SAFETY: `ptr`/`layout` are the pair returned by `alloc_zeroed` in + // `new` and this runs at most once. + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } +} + +/// [`qgemm_i32`] against a B that was pre-packed by [`QgemmPackedB::new`]. +/// +/// Panics if `packed` was built for a different shape or signedness, which +/// would otherwise make MLAS read the wrong layout. +#[allow(clippy::too_many_arguments)] +pub fn qgemm_i32_packed( + m: usize, + n: usize, + k: usize, + a: &[u8], + a_signed: bool, + zero_point_a: u8, + packed: &QgemmPackedB, + zero_point_b: QgemmZeroPoints<'_>, + c: &mut [i32], +) { + assert_eq!(a.len(), m * k, "A must be m*k bytes"); + assert_eq!(c.len(), m * n, "C must be m*n i32"); + assert_eq!( + packed.identity(), + (k, n, a_signed, packed.b_signed), + "the packed B was built for a different shape or signedness" + ); + if m == 0 || n == 0 { + return; + } + let (zero_point_b_ptr, per_column) = match zero_point_b { + QgemmZeroPoints::PerTensor(ref value) => (std::ptr::from_ref(value), 0), + QgemmZeroPoints::PerColumn(values) => { + assert_eq!(values.len(), n, "per-column zero points must be n long"); + (values.as_ptr(), 1) + } + }; + ensure_threading(); + // SAFETY: `a` and `c` are exactly the sizes asserted above, `packed` owns a + // buffer MLAS itself sized and filled for this `(n, k, signedness)`, the + // zero point pointer is either an `n`-long slice or a borrow of a local that + // outlives the call, and the shim writes only through `c`. + unsafe { + mlas_qgemm_i32_packed( + m, + n, + k, + c_int::from(a_signed), + c_int::from(packed.b_signed), + a.as_ptr().cast::(), + k, + zero_point_a, + packed.ptr.cast::(), + zero_point_b_ptr, + per_column, + c.as_mut_ptr(), + n, + c_int::from(mlas_threading_degree() > 1), + ); + } +} + /// Whether this machine's `u8 x i8` integer GEMM kernel is exact. /// /// On AVX2 without VNNI, MLAS's `u8 x i8` kernel multiplies raw bytes with @@ -3682,6 +3851,83 @@ mod tests { ); } + /// A pre-packed B must give bit-identical `i32` accumulators to the + /// unpacked path, for every signedness MLAS supports here and for both + /// zero-point layouts. + /// + /// Bit-identical is the right bar: the pack only changes B's memory layout, + /// not the arithmetic, so any difference is a packing bug rather than a + /// tolerable reordering. + #[test] + fn qgemm_packed_b_is_bit_identical_to_the_unpacked_path() { + let (m, n, k) = (5usize, 19usize, 37usize); + let a: Vec = (0..m * k).map(|i| (i * 37 % 251) as u8).collect(); + let b: Vec = (0..k * n).map(|i| (i * 53 % 241) as u8).collect(); + let mut checked = 0usize; + for (a_signed, b_signed) in [(false, false), (false, true), (true, true)] { + if !a_signed && b_signed && !qgemm_u8s8_is_exact() { + continue; + } + let Some(packed) = QgemmPackedB::new(n, k, &b, a_signed, b_signed) else { + continue; + }; + assert_eq!(packed.identity(), (k, n, a_signed, b_signed)); + let zero_point_a = 7u8; + let per_column: Vec = (0..n).map(|i| (i * 11 % 97) as u8).collect(); + for zero_points in [ + QgemmZeroPoints::PerTensor(13), + QgemmZeroPoints::PerColumn(&per_column), + ] { + let mut want = vec![0i32; m * n]; + qgemm_i32( + m, + n, + k, + &a, + a_signed, + zero_point_a, + &b, + b_signed, + zero_points, + &mut want, + ); + let mut got = vec![0i32; m * n]; + qgemm_i32_packed( + m, + n, + k, + &a, + a_signed, + zero_point_a, + &packed, + zero_points, + &mut got, + ); + assert_eq!( + got, want, + "packed and unpacked qgemm disagreed for a_signed={a_signed} \ + b_signed={b_signed}" + ); + checked += 1; + } + } + assert!( + checked > 0, + "no signedness/zero-point combination was exercised, so this test proved nothing" + ); + } + + #[test] + fn qgemm_packed_b_declines_empty_shapes() { + assert!(QgemmPackedB::new(0, 4, &[], false, false).is_none()); + assert!(QgemmPackedB::new(4, 0, &[], false, false).is_none()); + } + + #[test] + fn qgemm_packed_b_is_send_sync() { + assert_send_sync::(); + } + #[test] fn qgemm_i32_tolerates_empty_shapes() { let mut c: Vec = Vec::new(); diff --git a/crates/mlas-sys/vendor/shim.cpp b/crates/mlas-sys/vendor/shim.cpp index 968ebac1ea..64426e0d3d 100644 --- a/crates/mlas-sys/vendor/shim.cpp +++ b/crates/mlas-sys/vendor/shim.cpp @@ -764,3 +764,84 @@ extern "C" void mlas_qgemm_i32( 1, multithread != 0 ? kMlasParallelSentinel : nullptr); } + +// ---- Pre-packed B for the quantized GEMM ------------------------------------ +// +// `mlas_qgemm_i32` above leaves `BIsPacked = false`, so MLAS re-packs the whole +// K-by-N weight into its kernel layout on every call. ORT instead pre-packs a +// constant B once at session initialisation and sets `BIsPacked`. The packed +// layout is chosen per kernel, so the pack is only valid for the same +// `(N, K, AIsSigned, BIsSigned)` on the same machine -- it must never be +// serialised or shared across processes. + +extern "C" size_t mlas_qgemm_pack_b_size( + size_t N, + size_t K, + int a_is_signed, + int b_is_signed) +{ + return MlasGemmPackBSize(N, K, a_is_signed != 0, b_is_signed != 0, nullptr); +} + +extern "C" void mlas_qgemm_pack_b( + size_t N, + size_t K, + const void* B, + size_t ldb, + int a_is_signed, + int b_is_signed, + void* packed_b) +{ + MlasGemmPackB( + N, K, + static_cast(B), + ldb, + a_is_signed != 0, + b_is_signed != 0, + packed_b); +} + +extern "C" void mlas_qgemm_i32_packed( + size_t M, + size_t N, + size_t K, + int a_is_signed, + int b_is_signed, + const void* A, + size_t lda, + uint8_t zero_point_a, + const void* packed_b, + const uint8_t* zero_point_b, + int per_column_zero_points, + int32_t* C, + size_t ldc, + int multithread) +{ + MLAS_GEMM_QUANT_SHAPE_PARAMS shape; + shape.M = M; + shape.N = N; + shape.K = K; + shape.AIsSigned = a_is_signed != 0; + shape.BIsSigned = b_is_signed != 0; + shape.IsAccumulateMode = false; + + MLAS_GEMM_QUANT_DATA_PARAMS data; + data.A = static_cast(A); + data.lda = lda; + data.ZeroPointA = zero_point_a; + data.B = packed_b; + // Ignored when `BIsPacked`, but leave it consistent with the logical shape. + data.ldb = N; + data.ZeroPointB = zero_point_b; + data.BIsPacked = true; + data.PerColumnZeroPoints = per_column_zero_points != 0; + data.C = C; + data.ldc = ldc; + data.OutputProcessor = nullptr; + + MlasGemmBatch( + shape, + &data, + 1, + multithread != 0 ? kMlasParallelSentinel : nullptr); +} diff --git a/crates/onnx-runtime-ep-cpu/src/assignment_policy.rs b/crates/onnx-runtime-ep-cpu/src/assignment_policy.rs index c99d9637f3..487c12fdab 100644 --- a/crates/onnx-runtime-ep-cpu/src/assignment_policy.rs +++ b/crates/onnx-runtime-ep-cpu/src/assignment_policy.rs @@ -180,11 +180,32 @@ const MEASURED_MIN_WEIGHTS: usize = 1 << 20; /// | `MatMulNBits` int4 M=1 | 1.00 | 1.52 | 1.74 | 2.23 | 2.21 | 4.28 | /// /// So these are not slow kernels — they are kernels that realise roughly half -/// of ORT's parallel speedup. Fixing that is a threadpool/partitioning problem -/// (#1054 removed one cause, the 8-worker cap on the standalone MLAS pool), not -/// a kernel-tuning one, and until it is fixed the honest answer at capability -/// time is to let ORT run these nodes. The thread count is not visible here, so -/// a claim would have to hold at *every* count; none of these do. +/// of ORT's parallel speedup. +/// +/// It is **not the threadpool**, which is what this note used to claim. #1054 +/// removed one real cause (the 8-worker cap on the standalone MLAS pool), but +/// driving an MLAS GEMM directly through this crate's work-stealing pool at +/// `K = N = 3584`, `M = 512` gives 99.9 ms at one thread, 14.3 ms at eight and +/// 13.6 ms at sixteen — a 7.0-7.4x speedup, about what ORT achieves — with +/// `serial_fallback_calls == 0` and `scheduled_iterations >= pool_threads * +/// parallel_for_calls`. The pool can drive an MLAS GEMM to 7x on this host, so +/// it is not the ceiling here. +/// +/// Scope, because it matters: that measurement is taken on the **integer +/// `QLinearMatMul`** path, the only kernel here with a committed phase-splitting +/// instrument (`qlinear_phase_report`). It is a proxy. All three kernels share +/// the pool, so "the pool is not the ceiling" transfers — but `QLinearMatMul`'s +/// serial phases (requantize, `i32` accumulator staging) do not exist in +/// `matmul.rs` or `matmul_nbits.rs`, so no millisecond figure transfers. Those +/// kernels' own per-call serial work has **not** been decomposed; that is open. +/// +/// On the proxy, non-GEMM work is ~2-4% of the call at one thread and 22-40% at +/// eight, of which only a ~2 ms requantize-plus-alloc slice is repeatable and +/// thread-stable; the rest is unattributed and varied 3x across identical runs +/// on this contended host. The Amdahl *direction* is clear, the decomposition is +/// not. Until it is, the honest answer at capability time is to let ORT run +/// these nodes. The thread count is not visible here, so a claim would have to +/// hold at *every* count; none of these do. const DECODE_PARALLEL_NOTE: &str = "measured slower than ORT's CPU kernel at every thread count \ from 2 to 16 on x86-64 AVX2 (this EP realises about half of ORT's parallel speedup; at one \ thread it is at parity)"; @@ -417,9 +438,9 @@ fn matmul_weight_elements(op: &Node, shapes: &[Shape]) -> Option { .ok()? .checked_mul(usize::try_from(n).ok()?); } - // `MatMul`/`Gemm` take B as input 1. `QLinearMatMul` never reaches here: - // it loses at every size measured, so it is decided before the size gate. - let b = shapes.get(1)?; + // `MatMul`/`Gemm` take B as input 1; `QLinearMatMul` takes it as input 3, + // after `a_scale` and `a_zero_point`. + let b = shapes.get(if op.op_type == "QLinearMatMul" { 3 } else { 1 })?; b.iter() .map(|dim| dim.as_static()) .try_fold(1usize, |acc, dim| acc.checked_mul(dim?)) @@ -445,30 +466,61 @@ fn matmul_family_preference( return None; } - // `QLinearMatMul` is the one case that does not need a size gate: it loses - // by more than an order of magnitude at the smallest shape measured, and it - // loses for a *structural* reason rather than a tuning one. #1058 bound - // MLAS's integer QGEMM and took u8 x u8 from 27-119x down to 3-4x, but ORT - // pre-packs the constant B once at session init while this kernel packs - // inside every call -- at M=1 a 12.8 MB pack dominates a 1.7 ms call, which - // is the whole of the remaining 22x. Signed x signed is worse still (3.0x) - // because MLAS documents `AIsSigned` as unsupported off ARM, so on x86 only - // u8 x u8 reaches the fast path at all. + // `QLinearMatMul` splits by activation dtype, because the two dtypes now + // land on opposite sides of the line. + // + // #1058 bound MLAS's integer QGEMM (27-119x down to 3-4x). This round added + // a session-lifetime pre-pack for a constant B, dropped the per-call dense + // copy of the whole weight, parallelized requantization, and translated the + // signedness combinations MLAS has no kernel for into `u8 x u8` by moving + // the operand and its zero point together (`XOR 0x80` / `+128`), which is + // exact over the integers. + // + // Ratios are ours/ORT p50, K = N = 3584, steady state, lower is better: + // + // | threads | u8 M=1 | u8 M=128 | u8 M=512 | i8 M=1 | i8 M=128 | i8 M=512 | + // |---------|-------:|---------:|---------:|-------:|---------:|---------:| + // | 1 | 1.13 | 1.20 | 1.20 | 0.03 | 0.25 | 0.26 | + // | 4 | 1.80 | 2.18 | 2.07 | 0.05 | 0.39 | 0.38 | + // | 8 | 2.33 | 2.43 | 2.12 | 0.09 | 0.47 | 0.42 | + // | 16 | 2.34 | 2.65 | 2.08 | 0.10 | 0.60 | 0.42 | + // + // Both rules are measured on x86-64 only and applied on every architecture, + // which is the same convention the rest of this module uses. aarch64 has + // native `i8 x i8` kernels (SDOT/SMMLA) that need no translation at all, so + // the claim there is if anything conservative; correctness on that lane is + // covered unconditionally by + // `qgemm_i32_matches_the_integer_oracle_for_every_signedness`, but the + // *speed* is unmeasured and is not claimed to be measured. // - // | shape (K=N=3584, 8 threads) | u8 x u8 | i8 x i8 | - // |-----------------------------|--------:|--------:| - // | M=1 | 22.04 | 2.23 | - // | M=128 | 4.19 | 3.02 | - // | M=512 | 4.05 | 3.07 | + // `i8 x i8` is claimed: it wins at every measured point by 1.7x-33x, and it + // wins because ORT's own signed path is far slower than its unsigned one + // (ORT i8 M=128 T=8 is 21.7 ms against its own 4.0 ms for u8), not because + // this kernel does anything different for it -- after the translation both + // dtypes run the identical `u8 x u8` MLAS kernel at the same speed. // - // Closing it needs a prepack hook for constant initializers, with its own - // correctness surface (weight identity, address reuse, dynamic weights). - // Until that exists ORT should run these nodes. + // `u8 x u8` still loses and is still deferred. The residue is thread + // scaling, not packing: at one thread it is 1.13-1.20x, and the gap widens + // with threads exactly as [`DECODE_PARALLEL_NOTE`] describes for every + // other dense path here. Cold cost is repaid immediately -- the first call + // at K=N=3584 M=1 costs 6.35 ms including the pack against 5.90 ms for the + // never-packed path, and every later call is 0.108 ms. if op.op_type == "QLinearMatMul" { + let signed = |index: usize| input_dtypes.get(index) == Some(&DataType::Int8); + if signed(0) && signed(3) { + return Some(ClaimPreference::Claim); + } + let weights = matmul_weight_elements(op, shapes); + if !weights.is_some_and(|elements| elements >= MEASURED_MIN_WEIGHTS) { + // Unmeasured and small: a partition boundary would cost more than + // the ratio above, so keep the historical claim. + return Some(ClaimPreference::Claim); + } return Some(ClaimPreference::defer( - "measured 2.2-22x slower than ORT's CPU QLinearMatMul on x86-64 AVX2: ORT pre-packs \ - the constant B once at session init while this kernel packs inside every call, and \ - MLAS's integer GEMM does not accept signed activations off ARM", + "measured 1.13-2.65x slower than ORT's CPU QLinearMatMul for unsigned activations on \ + x86-64 AVX2 (K=N=3584, 1-16 threads); the constant weight is now pre-packed once and \ + the residual gap is thread scaling, which widens from 1.13x at one thread to 2.65x \ + at sixteen", )); } @@ -674,29 +726,104 @@ mod tests { } } - /// `QLinearMatMul` loses at every shape measured, including the smallest, - /// and for a structural reason (ORT pre-packs B once; this kernel packs per - /// call). It defers regardless of size. + fn qlinear_shapes(b: Shape) -> Vec { + vec![ + shape(&[1, 8]), + shape(&[]), + shape(&[]), + b, + shape(&[]), + shape(&[]), + shape(&[]), + shape(&[]), + ] + } + + fn qlinear_dtypes(a: DataType, b: DataType) -> Vec { + vec![a, DataType::Float32, a, b, DataType::Float32, b] + } + + /// Unsigned activations still lose in the measured region, so they defer; + /// below it the ratio is unmeasured and a partition boundary would cost + /// more than it saves. + /// + /// The weight for `QLinearMatMul` is input **3**, not input 1 -- reading + /// the wrong index here would size every node by an empty scale tensor and + /// silently claim the whole operator back. #[test] - fn qlinear_matmul_defers_at_every_size() { + fn qlinear_matmul_defers_unsigned_activations_in_the_measured_region() { + let big = claim_preference( + &dense_node("QLinearMatMul"), + 22, + &qlinear_shapes(shape(&[3584, 3584])), + &qlinear_dtypes(DataType::Uint8, DataType::Uint8), + ); + assert!( + !big.is_claim(), + "u8 activations are 1.13-2.65x slower than ORT at K=N=3584" + ); + + let small = claim_preference( + &dense_node("QLinearMatMul"), + 22, + &qlinear_shapes(shape(&[8, 8])), + &qlinear_dtypes(DataType::Uint8, DataType::Uint8), + ); + assert!( + small.is_claim(), + "a 64-weight QLinearMatMul is outside every measurement here" + ); + } + + /// Signed activations against signed weights are now faster than ORT at + /// every measured point (1.7x-33x), so they must be claimed. + /// + /// ORT's own signed path is the slow one -- after the `XOR 0x80` + /// translation both dtypes run the identical MLAS `u8 x u8` kernel here at + /// the same speed -- so this is a real assignment win, not a measurement + /// artefact of our own two paths differing. + #[test] + fn qlinear_matmul_claims_signed_activations() { for b in [shape(&[8, 8]), shape(&[3584, 3584])] { - let shapes = vec![ - shape(&[1, 8]), - shape(&[]), - shape(&[]), - b, - shape(&[]), - shape(&[]), - shape(&[]), - shape(&[]), - ]; let pref = claim_preference( &dense_node("QLinearMatMul"), 22, - &shapes, - &[DataType::Uint8], + &qlinear_shapes(b), + &qlinear_dtypes(DataType::Int8, DataType::Int8), + ); + assert!( + pref.is_claim(), + "i8 x i8 QLinearMatMul beats ORT at every measured shape" + ); + } + // Mixed signedness was not measured, so it must not borrow the signed + // win: it follows the same size-gated rule as u8 x u8 -- deferred where + // that was measured to lose, claimed below it like every other + // unmeasured shape in this module. + for (a, b) in [ + (DataType::Uint8, DataType::Int8), + (DataType::Int8, DataType::Uint8), + ] { + assert!( + !claim_preference( + &dense_node("QLinearMatMul"), + 22, + &qlinear_shapes(shape(&[3584, 3584])), + &qlinear_dtypes(a, b), + ) + .is_claim(), + "{a:?} x {b:?} is unmeasured; it must not inherit the signed claim" + ); + assert!( + claim_preference( + &dense_node("QLinearMatMul"), + 22, + &qlinear_shapes(shape(&[8, 8])), + &qlinear_dtypes(a, b), + ) + .is_claim(), + "{a:?} x {b:?} below the measured region keeps the historical claim" ); - assert!(!pref.is_claim(), "QLinearMatMul is 2.2-22x slower than ORT"); } } diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/qlinear_matmul.rs b/crates/onnx-runtime-ep-cpu/src/kernels/qlinear_matmul.rs index 9979d5e725..6e126293a9 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/qlinear_matmul.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/qlinear_matmul.rs @@ -13,12 +13,104 @@ use crate::strided::numel; /// never trades measurable throughput for it. const PARALLEL_MIN_WORK: usize = 1 << 16; -pub struct QLinearMatMulKernel; +/// Identity a pre-packed constant `B` is only valid for. +/// +/// MLAS chooses the packed layout from the shape and signedness, and the pack +/// is built from one specific weight buffer, so all five have to match before a +/// cached pack may be reused. `addr` is the weight's base address, which is +/// stable for a graph initializer over the executor's lifetime. +#[cfg(feature = "mlas")] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +struct QgemmPackKey { + addr: usize, + k: usize, + n: usize, + a_signed: bool, + b_signed: bool, + /// Whether the packed bytes are the weight's own bytes or the sign-flipped + /// ones. A pack built from flipped bytes answers a different question, so + /// it must never be served to a call that did not flip. + flip_b: bool, +} + +#[derive(Default)] +pub struct QLinearMatMulKernel { + /// Which operands the graph guarantees are constant initializers. Only + /// index 3 (`B`) is consulted; the rest are carried so the array lines up + /// with the operand list. + constant_inputs: [bool; 8], + /// `B` pre-packed into MLAS's quantized kernel layout, built at most once. + /// + /// `None` inside the `OnceLock` records that MLAS declined to pack this + /// shape, so the unpacked path is used and no further attempt is made. + #[cfg(feature = "mlas")] + packed_b: std::sync::OnceLock>, +} + pub struct QLinearMatMulFactory; impl KernelFactory for QLinearMatMulFactory { fn create(&self, _node: &Node, _shapes: &[Vec]) -> Result> { - Ok(Box::new(QLinearMatMulKernel)) + Ok(Box::new(QLinearMatMulKernel::default())) + } +} + +impl QLinearMatMulKernel { + /// The identity a pack for this call would have, or `None` when `B` must + /// not be packed at all. + /// + /// Declines for a non-constant `B`, and for a batched `B` where each batch + /// is a different weight and one pack could not serve them. + #[cfg(feature = "mlas")] + fn pack_key( + &self, + b: &TensorView<'_>, + geometry: &Geometry, + plan: &QgemmPlan, + ) -> Option { + if !self.constant_inputs[3] { + return None; + } + if geometry.b_batch.iter().any(|&dimension| dimension != 1) { + return None; + } + Some(QgemmPackKey { + addr: b.data_ptr::() as usize, + k: geometry.k, + n: geometry.n, + a_signed: plan.a_signed, + b_signed: plan.b_signed, + flip_b: plan.flip_b, + }) + } + + /// An already-built pack for `key`, without building one. + /// + /// Separate from [`pack_build`](Self::pack_build) so a call that already + /// has a pack can skip densifying `B` entirely -- the dense bytes exist + /// only to feed the packer. + #[cfg(feature = "mlas")] + fn pack_lookup(&self, key: QgemmPackKey) -> Option<&mlas_sys::QgemmPackedB> { + self.packed_b + .get()? + .as_ref() + .and_then(|(cached, packed)| (*cached == key).then_some(packed)) + } + + /// Build the pack for `key` from `b_bytes`, at most once per kernel. + /// + /// A stored `None` records that MLAS declined this shape, so the unpacked + /// path serves every later call without another attempt. + #[cfg(feature = "mlas")] + fn pack_build(&self, key: QgemmPackKey, b_bytes: &[u8]) -> Option<&mlas_sys::QgemmPackedB> { + if b_bytes.len() != key.k.checked_mul(key.n)? { + return None; + } + let packed = mlas_sys::QgemmPackedB::new(key.n, key.k, b_bytes, key.a_signed, key.b_signed); + self.packed_b + .get_or_init(|| packed.map(|packed| (key, packed))) + .as_ref() + .and_then(|(cached, packed)| (*cached == key).then_some(packed)) } } @@ -195,6 +287,12 @@ fn dims_broadcastable(left: Dim, right: Dim) -> bool { } impl Kernel for QLinearMatMulKernel { + fn set_constant_inputs(&mut self, constant_inputs: &[bool]) { + for (index, is_constant) in self.constant_inputs.iter_mut().enumerate() { + *is_constant = constant_inputs.get(index).copied().unwrap_or(false); + } + } + fn execute(&self, inputs: &[TensorView], outputs: &mut [TensorMut]) -> Result<()> { check_arity("QLinearMatMul", inputs, outputs, 8, 8, 1)?; let a = &inputs[0]; @@ -239,10 +337,26 @@ impl Kernel for QLinearMatMulKernel { // The fallback's widened copies are four bytes per element; the MLAS // route hands the raw bytes to the kernel. Materialize only what the // chosen path reads, so neither pays for the other's buffers. - let (a_bytes, b_bytes, a, b) = if qgemm.is_some() { + // A constant B is packed once. After that its dense bytes are dead -- + // they exist only to feed the packer -- so skip the copy, which is + // `k * n` bytes on every call and dominates decode at large K and N. + #[cfg(feature = "mlas")] + let pack_key = qgemm + .as_ref() + .and_then(|plan| self.pack_key(&inputs[3], &geometry, plan)); + #[cfg(feature = "mlas")] + let pack_ready = pack_key.is_some_and(|key| self.pack_lookup(key).is_some()); + #[cfg(not(feature = "mlas"))] + let pack_ready = false; + #[cfg_attr(not(feature = "mlas"), allow(unused_mut))] + let (mut a_bytes, mut b_bytes, a, b) = if qgemm.is_some() { ( to_dense_bytes(&inputs[0])?, - to_dense_bytes(&inputs[3])?, + if pack_ready { + Vec::new() + } else { + to_dense_bytes(&inputs[3])? + }, Vec::new(), Vec::new(), ) @@ -254,6 +368,25 @@ impl Kernel for QLinearMatMulKernel { read_quantized(&inputs[3])?, ) }; + // Move any operand MLAS has no kernel for into the sign domain it does, + // before it is either packed or handed to the unpacked entry point. + #[cfg(feature = "mlas")] + if let Some(plan) = &qgemm { + if plan.flip_a { + flip_sign_domain(&mut a_bytes); + } + if plan.flip_b { + flip_sign_domain(&mut b_bytes); + } + } + // Loop-invariant: the pack depends only on the weight, not the batch. + #[cfg(feature = "mlas")] + let packed = pack_key.and_then(|key| match self.pack_lookup(key) { + Some(packed) => Some(packed), + None => self.pack_build(key, &b_bytes), + }); + #[cfg(not(feature = "mlas"))] + let packed: Option<&()> = None; let (m, k, n) = (geometry.m, geometry.k, geometry.n); let mut output = Vec::with_capacity(geometry.result_len); let mut batch_index = vec![0; geometry.batch_shape.len()]; @@ -261,6 +394,7 @@ impl Kernel for QLinearMatMulKernel { // many-batch call allocates once rather than once per batch. let mut products: Vec = Vec::new(); let mut b_zero_points: Vec = Vec::new(); + let mut b_scales: Vec = Vec::new(); for batch in 0..geometry.batch_count { let a_batch = geometry.a_batch_offset(&batch_index); let b_batch = geometry.b_batch_offset(&batch_index); @@ -276,26 +410,38 @@ impl Kernel for QLinearMatMulKernel { } b_zero_points.clear(); b_zero_points.extend((0..n).map(|column| b_quant.at(b_batch, column).1)); + b_scales.clear(); + b_scales.extend((0..n).map(|column| b_quant.at(b_batch, column).0)); products.clear(); products.resize(m * n, 0); if let Some(plan) = &qgemm { - plan.run( - m, - n, - k, - &a_bytes[a_offset..a_offset + m * k], - &b_bytes[b_offset..b_offset + k * n], - &b_zero_points, - &mut products, - )?; + match packed { + Some(packed) => plan.run_packed( + m, + n, + k, + &a_bytes[a_offset..a_offset + m * k], + packed, + &b_zero_points, + &mut products, + )?, + None => plan.run( + m, + n, + k, + &a_bytes[a_offset..a_offset + m * k], + &b_bytes[b_offset..b_offset + k * n], + &b_zero_points, + &mut products, + )?, + } requantize_rows( &products, &a_quant, - &b_quant, a_batch, - b_batch, + &b_scales, n, y_scale, y_zero_point, @@ -362,9 +508,8 @@ impl Kernel for QLinearMatMulKernel { requantize_rows( &products, &a_quant, - &b_quant, a_batch, - b_batch, + &b_scales, n, y_scale, y_zero_point, @@ -529,8 +674,12 @@ fn read_quantized(view: &TensorView) -> Result> { /// that synthesizes quantization parameters cannot silently truncate one into /// a different kernel input; such a case declines to the exact loop instead. #[cfg(feature = "mlas")] -fn zero_point_byte(value: i32, signed: bool) -> Option { - if signed { +fn zero_point_byte(value: i32, signed: bool, flipped: bool) -> Option { + if flipped { + // The operand bytes were shifted by +128 in the unsigned domain, so the + // zero point shifts identically and `a - za` is unchanged. + i8::try_from(value).ok().map(|value| (value as u8) ^ 0x80) + } else if signed { i8::try_from(value).ok().map(|value| value as u8) } else { u8::try_from(value).ok() @@ -553,14 +702,38 @@ fn zero_point_byte(value: i32, signed: bool) -> Option { /// activations against `i8` weights can saturate. `qgemm_u8s8_is_exact()` /// probes the running machine for exactly that, so the fast path is taken /// only where it is exact, and VNNI/AMX hosts get it automatically. +/// +/// Operand combinations outside that set are not declined but *translated*: an +/// operand is reinterpreted in the unsigned domain by `XOR 0x80`, with `+128` +/// applied to its zero point. `sum_k (a_k - za)(b_k - zb)` is a difference of +/// integers, so shifting an operand and its zero point by the same constant +/// leaves every accumulator bit-identical, and the call lands on the `u8 x u8` +/// kernel this file already trusts as exact. Without it, `i8` activations sat +/// on the scalar loop and lost to ONNX Runtime by 5-6x. #[cfg(feature = "mlas")] struct QgemmPlan { + /// Signedness handed to MLAS, i.e. *after* any flip. a_signed: bool, b_signed: bool, + /// Whether the caller must `XOR 0x80` the operand's bytes first. + flip_a: bool, + flip_b: bool, zero_point_a: u8, b_zero_point_bytes: std::cell::RefCell>, } +/// Reinterpret quantized bytes in the opposite signedness domain. +/// +/// `XOR 0x80` is `+128` modulo 256, i.e. exactly the `i8 <-> u8` bijection that +/// preserves order. Applied to an operand *and* its zero point it cancels in +/// `a - za`, which is why it is free of any accuracy cost. +#[cfg(feature = "mlas")] +fn flip_sign_domain(bytes: &mut [u8]) { + for byte in bytes.iter_mut() { + *byte ^= 0x80; + } +} + #[cfg(feature = "mlas")] impl QgemmPlan { fn select( @@ -578,30 +751,58 @@ impl QgemmPlan { // MLAS documents (mlas.h, `MLAS_GEMM_QUANT_SHAPE_PARAMS`) that signed // activations are unsupported off ARM, and on ARM only alongside signed // weights. The generic kernel happens to answer correctly outside that - // envelope today, but relying on it would be relying on an accident, so - // stay inside the documented set and let the exact loop take the rest. + // envelope today, but relying on it would be relying on an accident. // // The ARM `i8 x i8` kernels accumulate through non-saturating `vmull` / // `vpadalq` / dot-product / SMMLA instructions, so unlike `u8 x i8` on // AVX2 they need no exactness probe. That is asserted unconditionally // by `qgemm_i32_matches_the_integer_oracle_for_every_signedness`, which // runs on every architecture including the aarch64 CI lanes. - let signed_activations_supported = cfg!(target_arch = "aarch64") && b_signed; - if a_signed && !signed_activations_supported { - return None; - } - if !a_signed && b_signed && !mlas_sys::qgemm_u8s8_is_exact() { - return None; - } - let zero_point_a = zero_point_byte(a_quant.at(0, 0).1, a_signed)?; + let native = if a_signed { + cfg!(target_arch = "aarch64") && b_signed + } else if b_signed { + mlas_sys::qgemm_u8s8_is_exact() + } else { + true + }; + // Anything outside the native envelope is translated into `u8 x u8` + // rather than declined. Only the operands actually out of domain move. + let (flip_a, flip_b) = if native { + (false, false) + } else { + (a_signed, b_signed) + }; + let zero_point_a = zero_point_byte(a_quant.at(0, 0).1, a_signed, flip_a)?; Some(Self { - a_signed, - b_signed, + a_signed: a_signed && !flip_a, + b_signed: b_signed && !flip_b, + flip_a, + flip_b, zero_point_a, b_zero_point_bytes: std::cell::RefCell::new(Vec::new()), }) } + /// The `b_zero_point` column vector as the bytes MLAS expects, in the same + /// sign domain the weight bytes were handed over in. + fn b_zero_point_bytes(&self, b_zero_points: &[i32], n: usize) -> Result<()> { + let mut bytes = self.b_zero_point_bytes.borrow_mut(); + bytes.clear(); + bytes.reserve(n); + for &value in b_zero_points { + bytes.push( + zero_point_byte(value, self.b_signed || self.flip_b, self.flip_b).ok_or_else( + || { + EpError::KernelFailed(format!( + "QLinearMatMul: b_zero_point {value} does not fit the operand dtype" + )) + }, + )?, + ); + } + Ok(()) + } + fn run( &self, m: usize, @@ -612,16 +813,8 @@ impl QgemmPlan { b_zero_points: &[i32], products: &mut [i32], ) -> Result<()> { - let mut bytes = self.b_zero_point_bytes.borrow_mut(); - bytes.clear(); - bytes.reserve(n); - for &value in b_zero_points { - bytes.push(zero_point_byte(value, self.b_signed).ok_or_else(|| { - EpError::KernelFailed(format!( - "QLinearMatMul: b_zero_point {value} does not fit the operand dtype" - )) - })?); - } + self.b_zero_point_bytes(b_zero_points, n)?; + let bytes = self.b_zero_point_bytes.borrow(); mlas_sys::qgemm_i32( m, n, @@ -636,6 +829,33 @@ impl QgemmPlan { ); Ok(()) } + + /// [`run`](Self::run) against a `B` that was pre-packed once. + fn run_packed( + &self, + m: usize, + n: usize, + k: usize, + a: &[u8], + packed: &mlas_sys::QgemmPackedB, + b_zero_points: &[i32], + products: &mut [i32], + ) -> Result<()> { + self.b_zero_point_bytes(b_zero_points, n)?; + let bytes = self.b_zero_point_bytes.borrow(); + mlas_sys::qgemm_i32_packed( + m, + n, + k, + a, + self.a_signed, + self.zero_point_a, + packed, + mlas_sys::QgemmZeroPoints::PerColumn(&bytes), + products, + ); + Ok(()) + } } #[cfg(not(feature = "mlas"))] @@ -659,48 +879,121 @@ impl QgemmPlan { ) -> Result<()> { unreachable!("QgemmPlan::select never yields a plan without the mlas feature") } + + /// Unreachable without `mlas`: `packed` is always `None` there, so this + /// exists only to keep the dispatch in `execute` type-checking. + #[allow(clippy::too_many_arguments)] + fn run_packed( + &self, + _: usize, + _: usize, + _: usize, + _: &[u8], + _: &(), + _: &[i32], + _: &mut [i32], + ) -> Result<()> { + unreachable!("no pack exists without the mlas feature") + } } #[allow(clippy::too_many_arguments)] +/// Requantize the `i32` accumulators into the output dtype. +/// +/// Rows are independent, so this appends into a pre-sized region and walks the +/// rows in parallel once there is enough work to pay for the fork. Two +/// properties are load-bearing and must not be traded for speed: +/// +/// * the per-element arithmetic is still `a_scale * b_scale / y_scale` in that +/// association, so results stay bit-identical to the serial version -- float +/// multiply and divide do not reassociate; +/// * each row writes only its own `n` bytes, so the output is identical +/// whether the rows run serially or in parallel. +/// +/// `b_scales` is the per-column scale gathered once per batch by the caller; +/// looking it up per element was a `QuantParams::at` call inside the innermost +/// loop. fn requantize_rows( products: &[i32], a_quant: &QuantParams, - b_quant: &QuantParams, a_batch: usize, - b_batch: usize, + b_scales: &[f32], n: usize, y_scale: f32, y_zero_point: i32, dtype: DataType, output: &mut Vec, ) -> Result<()> { - for (row, accumulators) in products.chunks_exact(n).enumerate() { + // Both supported dtypes are one byte wide, so the output length is known. + // Reject anything else once here rather than per element. + if !matches!(dtype, DataType::Int8 | DataType::Uint8) { + return Err(EpError::KernelFailed(format!( + "QLinearMatMul: unsupported output dtype {dtype:?}" + ))); + } + let base = output.len(); + output.resize(base + products.len(), 0); + let destination = &mut output[base..]; + + // `a_scale * b_scale / y_scale` is the same expression for every row when + // `a_scale` is per tensor, so evaluate it once per column instead of once + // per element. The association is untouched, so the products are the same + // `f32` bits -- this removes a division per output element, not a rounding + // step. A per-row `a_scale` genuinely varies by row and keeps the divide. + let shared_scales: Option> = (!a_quant.per_axis).then(|| { + let a_scale = a_quant.at(a_batch, 0).0; + b_scales + .iter() + .map(|&b_scale| a_scale * b_scale / y_scale) + .collect() + }); + + let requantize_row = |row: usize, accumulators: &[i32], bytes: &mut [u8]| { let a_scale = a_quant.at(a_batch, row).0; - for (column, &accumulated) in accumulators.iter().enumerate() { - let b_scale = b_quant.at(b_batch, column).0; - let scale = a_scale * b_scale / y_scale; + let quantize = |accumulated: i32, scale: f32, byte: &mut u8| { let value = (accumulated as f32 * scale).round_ties_even() as i64 + i64::from(y_zero_point); - push_quantized(output, dtype, value)?; - } - } - Ok(()) -} - -fn push_quantized(bytes: &mut Vec, dtype: DataType, value: i64) -> Result<()> { - match dtype { - DataType::Int8 => { - bytes.push(value.clamp(i8::MIN as i64, i8::MAX as i64) as i8 as u8); - Ok(()) + *byte = match dtype { + DataType::Int8 => value.clamp(i8::MIN as i64, i8::MAX as i64) as i8 as u8, + _ => value.clamp(u8::MIN as i64, u8::MAX as i64) as u8, + }; + }; + match &shared_scales { + Some(scales) => { + for ((&accumulated, &scale), byte) in + accumulators.iter().zip(scales).zip(bytes.iter_mut()) + { + quantize(accumulated, scale, byte); + } + } + None => { + for ((&accumulated, &b_scale), byte) in + accumulators.iter().zip(b_scales).zip(bytes.iter_mut()) + { + quantize(accumulated, a_scale * b_scale / y_scale, byte); + } + } } - DataType::Uint8 => { - bytes.push(value.clamp(0, u8::MAX as i64) as u8); - Ok(()) + }; + + if products.len() >= PARALLEL_MIN_WORK { + destination + .par_chunks_mut(n) + .zip(products.par_chunks_exact(n)) + .enumerate() + .for_each(|(row, (bytes, accumulators))| { + requantize_row(row, accumulators, bytes); + }); + } else { + for (row, (bytes, accumulators)) in destination + .chunks_mut(n) + .zip(products.chunks_exact(n)) + .enumerate() + { + requantize_row(row, accumulators, bytes); } - other => Err(EpError::KernelFailed(format!( - "QLinearMatMul: unsupported output dtype {other:?}" - ))), } + Ok(()) } struct Geometry { @@ -877,7 +1170,7 @@ mod tests { fn execute(inputs: [&Owned; 8], output_dtype: DataType, output_shape: &[usize]) -> Owned { let mut output = Owned::zeros(output_dtype, output_shape); - QLinearMatMulKernel + QLinearMatMulKernel::default() .execute(&inputs.map(|input| input.view()), &mut [output.view_mut()]) .unwrap(); output @@ -1089,7 +1382,7 @@ mod tests { let y_scale = Owned::f32(&[], &[1.0]); let y_zero = Owned::u8(&[], &[0]); let mut out = Owned::zeros(DataType::Uint8, &[2, 1]); - let error = QLinearMatMulKernel + let error = QLinearMatMulKernel::default() .execute( &[ a.view(), @@ -1254,8 +1547,8 @@ mod tests { } /// Every decline below is a correctness rule, not a tuning knob: MLAS takes - /// a single `ZeroPointA`, documents signed activations as unsupported off - /// ARM, and can saturate `u8 x i8` in an `i16` pair accumulator. + /// a single `ZeroPointA`, and a zero point that does not fit the operand + /// dtype cannot be truncated. #[cfg(feature = "mlas")] #[test] fn qgemm_plan_declines_every_case_it_cannot_reproduce_exactly() { @@ -1264,26 +1557,500 @@ mod tests { QgemmPlan::select(&quant_params(true, 128, 4), false, false, &geometry).is_none(), "per-row activation zero points have no MLAS equivalent" ); + assert!( + QgemmPlan::select(&quant_params(false, 300, 4), false, false, &geometry).is_none(), + "a zero point that does not fit the operand dtype must fall back \ + rather than be truncated" + ); + let empty = Geometry::new(&[0, 32], &[32, 8]).unwrap(); + assert!( + QgemmPlan::select(&quant_params(false, 128, 4), false, false, &empty).is_none(), + "an empty shape has nothing to hand MLAS" + ); + } + + /// Every signedness combination must produce a plan, and every combination + /// MLAS has no kernel for must be reached by translating the offending + /// operand into the unsigned domain rather than by declining. + /// + /// This is the non-vacuity guard for + /// `qlinear_matmul_reordered_accumulation_is_bit_identical`: without it + /// the `i8` half of that sweep could silently be proving only that the + /// scalar fallback equals itself. + #[cfg(feature = "mlas")] + #[test] + fn signed_operands_are_translated_rather_than_declined() { + let geometry = Geometry::new(&[4, 32], &[32, 8]).unwrap(); + let signed_activations_native = cfg!(target_arch = "aarch64"); + let cases = [ + // (a_signed, b_signed, expected flip_a, expected flip_b) + (false, false, false, false), + (false, true, false, !mlas_sys::qgemm_u8s8_is_exact()), + ( + true, + true, + !signed_activations_native, + !signed_activations_native, + ), + (true, false, true, false), + ]; + for (a_signed, b_signed, flip_a, flip_b) in cases { + // The zero point has to be legal in the operand's *own* dtype; + // that is what the flip then has to carry across. + let zero_point = if a_signed { -1 } else { 128 }; + let plan = QgemmPlan::select( + &quant_params(false, zero_point, 4), + a_signed, + b_signed, + &geometry, + ) + .unwrap_or_else(|| panic!("a_signed={a_signed} b_signed={b_signed} must yield a plan")); + assert_eq!( + (plan.flip_a, plan.flip_b), + (flip_a, flip_b), + "a_signed={a_signed} b_signed={b_signed}" + ); + assert_eq!( + (plan.a_signed, plan.b_signed), + (a_signed && !flip_a, b_signed && !flip_b), + "a flipped operand must be handed to MLAS as unsigned" + ); + } + } + + /// Returns whether this kernel is holding a built pack. + #[cfg(all(test, feature = "mlas"))] + fn qlinear_pack_state(kernel: &QLinearMatMulKernel) -> Option { + kernel.packed_b.get().map(|slot| slot.is_some()) + } + + #[cfg(feature = "mlas")] + struct PackFixture { + a: Owned, + b: Owned, + m: usize, + n: usize, + } + + /// Shapes big enough that MLAS actually accepts a pack, with values that + /// span the whole byte range so a wrong pack cannot coincidentally agree. + #[cfg(feature = "mlas")] + fn pack_fixture(m: usize, k: usize, n: usize, seed: usize) -> PackFixture { + PackFixture { + a: Owned::u8( + &[m, k], + &(0..m * k) + .map(|i| ((i * 37 + seed * 11) % 256) as u8) + .collect::>(), + ), + b: Owned::u8( + &[k, n], + &(0..k * n) + .map(|i| ((i * 91 + seed * 53 + 7) % 256) as u8) + .collect::>(), + ), + m, + n, + } + } + + #[cfg(feature = "mlas")] + fn run_with(kernel: &QLinearMatMulKernel, fixture: &PackFixture) -> Owned { + let mut output = Owned::zeros(DataType::Uint8, &[fixture.m, fixture.n]); + let (a_scale, a_zero) = (Owned::f32(&[], &[0.02]), Owned::u8(&[], &[128])); + let (b_scale, b_zero) = (Owned::f32(&[], &[0.01]), Owned::u8(&[], &[127])); + let (y_scale, y_zero) = (Owned::f32(&[], &[0.05]), Owned::u8(&[], &[120])); + let inputs = [ + fixture.a.view(), + a_scale.view(), + a_zero.view(), + fixture.b.view(), + b_scale.view(), + b_zero.view(), + y_scale.view(), + y_zero.view(), + ]; + kernel.execute(&inputs, &mut [output.view_mut()]).unwrap(); + output + } + + /// A constant weight must be packed exactly once and then reused, and the + /// packed answer must equal the unpacked one bit for bit. + /// + /// Without the reuse this kernel re-packed and re-copied the whole `k * n` + /// weight on every call, which is what made decode 20x slower than ONNX + /// Runtime. + #[cfg(feature = "mlas")] + #[test] + fn a_constant_weight_is_packed_once_and_reused() { + let fixture = pack_fixture(3, 64, 96, 1); + + let mut unpacked = QLinearMatMulKernel::default(); + unpacked.set_constant_inputs(&[false; 8]); + let reference = run_with(&unpacked, &fixture); + assert!( + qlinear_pack_state(&unpacked).is_none(), + "a non-constant weight must never be packed: it can change under us" + ); + + let mut packed = QLinearMatMulKernel::default(); + packed.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + let first = run_with(&packed, &fixture); assert_eq!( - QgemmPlan::select(&quant_params(false, 128, 4), false, true, &geometry).is_some(), - mlas_sys::qgemm_u8s8_is_exact(), - "u8 x i8 may only be routed to MLAS where that kernel is exact" + qlinear_pack_state(&packed), + Some(true), + "MLAS must accept a pack for this shape, or the rest of this test \ + proves nothing" ); - let signed_activations_ok = cfg!(target_arch = "aarch64"); + let second = run_with(&packed, &fixture); + + assert_eq!(first.bytes, reference.bytes, "first packed call"); + assert_eq!(second.bytes, reference.bytes, "reused packed call"); + } + + /// Splits a steady-state call into "inside MLAS" and "everything else", so + /// a thread-scaling gap can be attributed rather than guessed at. + /// + /// `#[ignore]`; run with + /// `ONNX_GENAI_MLAS_THREADPOOL_THREADS=8 cargo test --release --features mlas \ + /// qlinear_phase_report -- --ignored --nocapture`. + #[cfg(feature = "mlas")] + #[test] + #[ignore = "reports timings; not a pass/fail assertion"] + fn qlinear_phase_report() { + let (k, n) = (3584usize, 3584usize); + for m in [1usize, 128, 512] { + let fixture = pack_fixture(m, k, n, 1); + let mut kernel = QLinearMatMulKernel::default(); + kernel.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + let reps = if m == 1 { 100 } else { 10 }; + for _ in 0..3 { + let _ = run_with(&kernel, &fixture); + } + let start = std::time::Instant::now(); + for _ in 0..reps { + let _ = run_with(&kernel, &fixture); + } + let whole = start.elapsed() / reps; + + let a_bytes = fixture.a.bytes.clone(); + let packed = kernel + .pack_lookup( + kernel + .pack_key( + &fixture.b.view(), + &Geometry::new(&[m, k], &[k, n]).unwrap(), + &QgemmPlan::select( + &quant_params(false, 128, 1), + false, + false, + &Geometry::new(&[m, k], &[k, n]).unwrap(), + ) + .unwrap(), + ) + .unwrap(), + ) + .unwrap(); + let zeros = vec![127u8; n]; + let mut products = vec![0i32; m * n]; + for _ in 0..3 { + mlas_sys::qgemm_i32_packed( + m, + n, + k, + &a_bytes, + false, + 128, + packed, + mlas_sys::QgemmZeroPoints::PerColumn(&zeros), + &mut products, + ); + } + let start = std::time::Instant::now(); + for _ in 0..reps { + mlas_sys::qgemm_i32_packed( + m, + n, + k, + &a_bytes, + false, + 128, + packed, + mlas_sys::QgemmZeroPoints::PerColumn(&zeros), + &mut products, + ); + } + let gemm = start.elapsed() / reps; + + let a_quant = quant_params(false, 128, 1); + let b_scales = vec![0.01f32; n]; + let mut output = Vec::new(); + let start = std::time::Instant::now(); + for _ in 0..reps { + output.clear(); + requantize_rows( + &products, + &a_quant, + 0, + &b_scales, + n, + 0.05, + 120, + DataType::Uint8, + &mut output, + ) + .unwrap(); + } + let requant = start.elapsed() / reps; + + let start = std::time::Instant::now(); + for _ in 0..reps { + // Deliberately the same `clear` + `resize` shape the kernel + // uses, so this measures what the kernel pays. + let mut scratch: Vec = Vec::new(); + scratch.clear(); + scratch.resize(m * n, 0); + std::hint::black_box(&scratch); + } + let alloc = start.elapsed() / reps; + + // The partition counters are printed, not asserted, because the + // docs cite them as the reason the pool is exonerated. A reader + // reproducing this should see `serial_fallback=0` and + // `sched_per_call >= threads`. + let stats = mlas_sys::mlas_threading_stats(); + let sched_per_call = stats + .scheduled_iterations + .checked_div(stats.parallel_for_calls) + .unwrap_or(0); + println!( + "m={m} threads={}: whole={whole:?} gemm={gemm:?} requant={requant:?} \ + products-alloc={alloc:?} unattributed={:?} \ + sched_per_call={sched_per_call} serial_fallback={}", + stats.pool_threads, + whole + .saturating_sub(gemm) + .saturating_sub(requant) + .saturating_sub(alloc), + stats.serial_fallback_calls, + ); + } + } + + /// Reports the one-time cost of the pack against a steady-state call, at a + /// real decode shape, because the benchmark harness runs a parity check + /// before it starts timing and therefore never sees a first call. + /// + /// `#[ignore]` so CI does not pay for a 3584x3584 pack; run with + /// `cargo test --release --features mlas qlinear_pack_cost -- --ignored --nocapture`. + #[cfg(feature = "mlas")] + #[test] + #[ignore = "reports timings; not a pass/fail assertion"] + fn qlinear_pack_cost_report() { + let fixture = pack_fixture(1, 3584, 3584, 1); + let mut kernel = QLinearMatMulKernel::default(); + kernel.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + + let start = std::time::Instant::now(); + let _ = run_with(&kernel, &fixture); + let cold = start.elapsed(); + assert_eq!(qlinear_pack_state(&kernel), Some(true)); + + let start = std::time::Instant::now(); + for _ in 0..20 { + let _ = run_with(&kernel, &fixture); + } + let steady = start.elapsed() / 20; + + let mut unpacked = QLinearMatMulKernel::default(); + unpacked.set_constant_inputs(&[false; 8]); + let start = std::time::Instant::now(); + for _ in 0..20 { + let _ = run_with(&unpacked, &fixture); + } + let never_packed = start.elapsed() / 20; + + println!( + "k=n=3584 m=1: cold(first call incl. pack)={cold:?} steady={steady:?} \ + never-packed={never_packed:?}" + ); + } + + /// The pack is keyed on the weight's identity, so a second weight must + /// never be served the first one's pack. `addr` alone is not enough -- + /// hence the shape and signedness in the key -- and a stale pack would be + /// a silent wrong answer, not a slowdown. + #[cfg(feature = "mlas")] + #[test] + fn a_different_weight_is_never_served_the_cached_pack() { + let first = pack_fixture(3, 64, 96, 1); + let second = pack_fixture(3, 64, 96, 2); + assert_ne!( + first.b.bytes, second.b.bytes, + "the two weights must actually differ" + ); + + let mut kernel = QLinearMatMulKernel::default(); + kernel.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + let _ = run_with(&kernel, &first); + assert_eq!(qlinear_pack_state(&kernel), Some(true)); + let served = run_with(&kernel, &second); + + let mut fresh = QLinearMatMulKernel::default(); + fresh.set_constant_inputs(&[false; 8]); assert_eq!( - QgemmPlan::select(&quant_params(false, -1, 4), true, true, &geometry).is_some(), - signed_activations_ok, - "signed activations are documented as ARM-only" + served.bytes, + run_with(&fresh, &second).bytes, + "the second weight must be computed from itself, not from the \ + cached pack of the first" ); + } + + /// The two pack guards shadow each other through `execute` -- a batched `B` + /// is refused by `pack_key` before `pack_build` can refuse its byte count + /// -- so each is falsified directly here instead of through a compound + /// injection. + /// + /// The length guard is not decoration: `QgemmPackedB::new` *asserts* on a + /// buffer that is not `k * n`, because it hands the pointer to MLAS, so + /// dropping the guard turns a graceful decline into a panic. + #[cfg(feature = "mlas")] + #[test] + fn each_pack_guard_declines_on_its_own() { + let mut kernel = QLinearMatMulKernel::default(); + kernel.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + let plan_geometry = Geometry::new(&[3, 64], &[64, 96]).unwrap(); + let plan = + QgemmPlan::select(&quant_params(false, 128, 4), false, false, &plan_geometry).unwrap(); + + let batched = Owned::u8(&[2, 64, 96], &vec![7u8; 2 * 64 * 96]); + let batched_geometry = Geometry::new(&[2, 3, 64], &[2, 64, 96]).unwrap(); assert!( - QgemmPlan::select(&quant_params(false, -1, 4), true, false, &geometry).is_none(), - "signed activations against unsigned weights are outside the \ - documented envelope on every architecture" + kernel + .pack_key(&batched.view(), &batched_geometry, &plan) + .is_none(), + "a per-batch weight has no single pack" ); + + let flat = Owned::u8(&[64, 96], &vec![7u8; 64 * 96]); + let key = kernel + .pack_key(&flat.view(), &plan_geometry, &plan) + .expect("a constant 2-D weight is packable, or the rest is vacuous"); assert!( - QgemmPlan::select(&quant_params(false, 300, 4), false, false, &geometry).is_none(), - "a zero point that does not fit the operand dtype must fall back \ - rather than be truncated" + kernel.pack_build(key, &[7u8; 8]).is_none(), + "a byte count that is not k * n must be declined, not packed" + ); + assert!( + kernel.pack_build(key, &vec![7u8; 64 * 96]).is_some(), + "MLAS must accept this shape, or the decline above proves nothing" + ); + } + + /// A batched `B` is a different weight per batch, so one pack cannot serve + /// it: the kernel must decline to build one, and every batch must still be + /// computed from its own weight. + /// + /// Two independent guards refuse this -- the batch check in `pack_key` and + /// the `k * n` length check in `pack_build` -- so the assertion that + /// matters, and the one a single injection can falsify, is that batch 1 is + /// not answered with batch 0's weights. + #[cfg(feature = "mlas")] + #[test] + fn a_batched_weight_declines_the_pack() { + let (batches, m, k, n) = (2usize, 3usize, 64usize, 96usize); + let a = Owned::u8( + &[batches, m, k], + &(0..batches * m * k) + .map(|i| ((i * 37 + 11) % 256) as u8) + .collect::>(), + ); + let b = Owned::u8( + &[batches, k, n], + &(0..batches * k * n) + .map(|i| ((i * 91 + 7) % 256) as u8) + .collect::>(), + ); + let mut kernel = QLinearMatMulKernel::default(); + kernel.set_constant_inputs(&[false, false, false, true, false, false, false, false]); + let mut output = Owned::zeros(DataType::Uint8, &[batches, m, n]); + let (a_scale, a_zero) = (Owned::f32(&[], &[0.02]), Owned::u8(&[], &[128])); + let (b_scale, b_zero) = (Owned::f32(&[], &[0.01]), Owned::u8(&[], &[127])); + let (y_scale, y_zero) = (Owned::f32(&[], &[0.05]), Owned::u8(&[], &[120])); + kernel + .execute( + &[ + a.view(), + a_scale.view(), + a_zero.view(), + b.view(), + b_scale.view(), + b_zero.view(), + y_scale.view(), + y_zero.view(), + ], + &mut [output.view_mut()], + ) + .unwrap(); + assert!( + qlinear_pack_state(&kernel).is_none(), + "a per-batch weight has no single pack" + ); + + let mut unpacked = QLinearMatMulKernel::default(); + unpacked.set_constant_inputs(&[false; 8]); + let mut expected = Owned::zeros(DataType::Uint8, &[batches, m, n]); + unpacked + .execute( + &[ + a.view(), + a_scale.view(), + a_zero.view(), + b.view(), + b_scale.view(), + b_zero.view(), + y_scale.view(), + y_zero.view(), + ], + &mut [expected.view_mut()], + ) + .unwrap(); + assert_eq!( + output.bytes, expected.bytes, + "every batch must use its own weight" + ); + assert_ne!( + output.bytes[..m * n], + output.bytes[m * n..], + "the two batches must actually differ, or serving batch 0's pack \ + to batch 1 would be undetectable" + ); + } + + /// `XOR 0x80` must move the operand and its zero point by the same amount, + /// or the shift stops cancelling in `a - za` and every accumulator is off + /// by a multiple of `k`. + #[cfg(feature = "mlas")] + #[test] + fn the_sign_flip_moves_bytes_and_zero_points_together() { + for value in i8::MIN..=i8::MAX { + let mut byte = [value as u8]; + flip_sign_domain(&mut byte); + assert_eq!( + i32::from(byte[0]), + i32::from(value) + 128, + "flipping {value} must land on its unsigned image" + ); + assert_eq!( + zero_point_byte(i32::from(value), true, true), + Some(byte[0]), + "the zero point of {value} must move with it" + ); + } + assert_eq!( + zero_point_byte(128, true, true), + None, + "a zero point outside the signed dtype must still be rejected, not \ + wrapped by the flip" ); } @@ -1294,6 +2061,9 @@ mod tests { (1, 37, 65), (3, 64, 16), (5, 129, 33), + // Above `PARALLEL_MIN_WORK`, so `requantize_rows` forks. The + // parallel and serial row walks must agree bit for bit. + (96, 40, 900), ] { for per_axis in [false, true] { // --- Uint8 --- diff --git a/docs/performance/CPU_MATMUL_ASSIGNMENT.md b/docs/performance/CPU_MATMUL_ASSIGNMENT.md index 3d438c989e..21d91a1959 100644 --- a/docs/performance/CPU_MATMUL_ASSIGNMENT.md +++ b/docs/performance/CPU_MATMUL_ASSIGNMENT.md @@ -125,12 +125,24 @@ which is the decode case and stays claimed. | `MatMul` | f16 | 128 | 3584 | 8 | 7.77 | 8.77 | defer (noisy) | | `MatMul` | f16 | 128 | 3584 | 16 | 7.10 | 7.46 | defer | | `MatMul` | f16 | 128 | 3584 | 32 | 5.34 | 6.24 | defer | -| `QLinearMatMul` | u8 x u8 | 1 | 3584 | 8 | 22.04 | 27.30 | defer | -| `QLinearMatMul` | u8 x u8 | 128 | 3584 | 8 | 4.19 | 4.23 | defer | -| `QLinearMatMul` | u8 x u8 | 512 | 3584 | 8 | 4.05 | 4.16 | defer | -| `QLinearMatMul` | i8 x i8 | 1 | 3584 | 8 | 2.23 | 2.37 | defer | -| `QLinearMatMul` | i8 x i8 | 128 | 3584 | 8 | 3.02 | 3.05 | defer | -| `QLinearMatMul` | i8 x i8 | 512 | 3584 | 8 | 3.07 | 3.21 | defer | +| `QLinearMatMul` | u8 x u8 | 1 | 3584 | 1 | 1.13 | 1.18 | defer | +| `QLinearMatMul` | u8 x u8 | 1 | 3584 | 8 | 2.33 | 2.77 | defer | +| `QLinearMatMul` | u8 x u8 | 1 | 3584 | 16 | 2.34 | 2.58 | defer | +| `QLinearMatMul` | u8 x u8 | 128 | 3584 | 1 | 1.20 | 1.21 | defer | +| `QLinearMatMul` | u8 x u8 | 128 | 3584 | 8 | 2.43 | 2.80 | defer | +| `QLinearMatMul` | u8 x u8 | 128 | 3584 | 16 | 2.65 | 3.42 | defer | +| `QLinearMatMul` | u8 x u8 | 512 | 3584 | 1 | 1.20 | 1.21 | defer | +| `QLinearMatMul` | u8 x u8 | 512 | 3584 | 8 | 2.12 | 2.22 | defer | +| `QLinearMatMul` | u8 x u8 | 512 | 3584 | 16 | 2.08 | 2.18 | defer | +| `QLinearMatMul` | i8 x i8 | 1 | 3584 | 1 | **0.03** | 0.03 | **claim** | +| `QLinearMatMul` | i8 x i8 | 1 | 3584 | 8 | **0.09** | 0.09 | **claim** | +| `QLinearMatMul` | i8 x i8 | 1 | 3584 | 16 | **0.10** | 0.10 | **claim** | +| `QLinearMatMul` | i8 x i8 | 128 | 3584 | 1 | **0.25** | 0.25 | **claim** | +| `QLinearMatMul` | i8 x i8 | 128 | 3584 | 8 | **0.47** | 0.53 | **claim** | +| `QLinearMatMul` | i8 x i8 | 128 | 3584 | 16 | **0.60** | 0.69 | **claim** | +| `QLinearMatMul` | i8 x i8 | 512 | 3584 | 1 | **0.26** | 0.26 | **claim** | +| `QLinearMatMul` | i8 x i8 | 512 | 3584 | 8 | **0.42** | 0.43 | **claim** | +| `QLinearMatMul` | i8 x i8 | 512 | 3584 | 16 | **0.42** | 0.47 | **claim** | Ranges **outside** the measured region keep the historical claim, and the code says so explicitly: `K * N < 2^20`, symbolic/dynamic weight shapes, and dense dtypes other than f32/f16 (where a @@ -145,11 +157,57 @@ At **one thread this EP is at parity** (f32 1.00 / 0.97, int4 1.00 / 0.99). The are added and closes again only at 32, where ORT's own scaling saturates. We realise roughly **half** of ORT's parallel speedup. -That is a threadpool/partitioning problem, not a kernel problem. #1054 removed one cause — the -standalone MLAS pool was clamped to `min(available, 8)` workers and never saw the EP's requested -thread count, which cost 2.16x -> 1.61x at 32 vCPU — but the residual 1.4-2.4x at 2-16 threads is -still open. Because the thread count is **not visible at capability time**, a claim would have to -hold at every count, and none of these do. +#1054 removed one cause — the standalone MLAS pool was clamped to `min(available, 8)` workers and +never saw the EP's requested thread count, which cost 2.16x -> 1.61x at 32 vCPU. The residual +1.4-2.4x (ours/ORT, lower is better) at 2-16 threads is still open. + +**It is not the pool.** Driving an MLAS GEMM directly through this crate's work-stealing pool, +`K = N = 3584`, `M = 512`, warmed, on the same host: + +| pool threads | 1 | 2 | 4 | 8 | 16 | 32 | +|---|---:|---:|---:|---:|---:|---:| +| MLAS GEMM only | 99.9 ms | 60.3 ms | 26.9 ms | 14.3 ms | 13.6 ms | 11.0 ms | +| speedup vs 1 thread | 1.0x | 1.7x | 3.7x | **7.0x** | **7.4x** | 9.1x | + +MLAS requests at least `pool_threads` partitions per call and they are dispatched. The instrument +below prints them: on this host `sched_per_call` comes out at exactly the pool width (8 at +`--threads 8`) with `serial_fallback=0`. The *committed assertion* in `mlas-sys` is only the +inequality `scheduled_iterations >= pool_threads`, so the inequality is what is proven and the +equality is what is observed. Either way the primitive and the pool together scale about as well as +ORT does. The pool is therefore **not** where the 1.4-2.4x goes. What is left is the per-call work +*around* the primitive, which does not shrink with threads. + +**Read the scope of that evidence carefully.** The numbers above are measured on the **integer +`QLinearMatMul`** path (`qgemm_i32_packed`), which is the only kernel here with a committed +phase-splitting instrument. That is a *proxy*, and it licenses exactly one transferable conclusion: +this crate's work-stealing pool can drive an MLAS GEMM to 7x on this host, so the pool is not the +ceiling for f32 or int4 either — all three go through the same pool. It does **not** license the +specific phase names or milliseconds for f32/int4. `QLinearMatMul`'s serial phases (requantize, the +`i32` accumulator staging) **do not exist** in `matmul.rs` or `matmul_nbits.rs`; those kernels have +their own, different per-call work (dequant staging, activation densification, executor tensor +handling) which has **not** been decomposed. Doing that decomposition is open work. + +What the instrument actually shows, on `QLinearMatMul`, `K = N = 3584`, `M = 512`: + +| phase | 1 thread | 8 threads | +|---|---:|---:| +| MLAS GEMM | ~100 ms | 14.5-15.7 ms | +| requantize + accumulator alloc | 1.7-2.5 ms | ~1.5 ms | +| unattributed (densify, executor, measurement) | 0-2 ms | 2.5-8.5 ms | +| **non-GEMM share of the call** | **~2-4%** | **~22-40%** | + +Only the requantize + alloc slice is repeatable and genuinely thread-stable at ~2 ms. The +unattributed remainder is **not** constant in the thread count — it is larger at 8 threads than at +1, and it varied 2.5 / 4.3 / 8.5 ms across three consecutive runs of an identical configuration on +this shared, heavily contended host. So the honest statement is a bound: **non-GEMM work is 22-40% +of the call at 8 threads and ~2-4% at 1**, it is dominated by a term we have not attributed, and the +Amdahl direction is clear even though the decomposition is not. + +`qlinear_phase_report` (`#[ignore]`d and `#[cfg(feature = "mlas")]`, in `kernels/qlinear_matmul.rs`) +reproduces the split. + +Because the thread count is **not visible at capability time**, a claim would have to hold at every +count, and none of these do. ### 2. A kernel gap — f16 dense @@ -158,20 +216,50 @@ casts around MLAS's f32 kernels; this EP's f16 path does not reach the same prim largest single dense loss in the matrix and the most tractable: the fix is to route f16 `MatMul` through the same MLAS `sgemm` the f32 path already uses, with a cast, rather than a bespoke kernel. -### 3. Per-call packing — `QLinearMatMul` - -#1058 bound MLAS's integer QGEMM and took u8 x u8 from **27-119x** down to 3-4x. What remains is -structural: **ORT pre-packs the constant B once at session init; this kernel packs inside every -call.** At M=1 a 12.8 MB pack dominates a 1.7 ms call, which is the whole of the residual 22x. -`QLinearMatMulFactory::create` ignores its `Node`, so the kernel is stateless and prepacking would -need a constant-initializer hook with its own correctness surface (weight identity, **address -reuse**, dynamic weights). - -Signed x signed is separately capped: MLAS documents `AIsSigned` as unsupported off ARM -(`mlas.h:610-611`), so on x86 only u8 x u8 reaches the fast path at all. On AVX2 without VNNI, u8 x -i8 is additionally **not bit-exact** — `vpmaddubsw` sums adjacent products into a saturating i16, and -`255*(-128) + 255*(-128)` clamps — so it is excluded by a runtime exactness probe rather than an ISA -table, which lets VNNI/AMX hosts pick the fast path up automatically. +### 3. Per-call packing — `QLinearMatMul` (**fixed**) + +#1058 bound MLAS's integer QGEMM and took u8 x u8 from **27-119x** down to 3-4x. What remained was +structural: **ORT pre-packed the constant B once at session init; this kernel packed inside every +call**, and additionally copied the whole `K * N` weight into a dense buffer on every call. At M=1 a +12.8 MB pack and copy dominated a 1.7 ms call, which was the whole of the residual 22x. + +That is now fixed. The kernel takes `set_constant_inputs`, keys a session-lifetime pack on the +weight's full identity (address, `K`, `N`, both signedness flags, and whether the bytes were sign +translated), and skips the dense copy entirely once the pack exists. Cold cost is repaid on the +**first** call: + +| K = N = 3584, M = 1 | time | +|---|---| +| first call, including dense copy + MLAS pack | 6.35 ms | +| every later call | 0.108 ms | +| never-packed path (what every call used to cost) | 5.90 ms | + +Signedness is no longer a cap either. MLAS documents `AIsSigned` as unsupported off ARM +(`mlas.h:610-611`), and on AVX2 without VNNI u8 x i8 is **not bit-exact** — `vpmaddubsw` sums +adjacent products into a saturating i16, and `255*(-128) + 255*(-128)` clamps. Rather than decline +those combinations to a scalar loop, the offending operand is now *translated* into the unsigned +domain: `XOR 0x80` on its bytes and `+128` on its zero point. Since the kernel computes +`sum_k (a_k - za)(b_k - zb)`, shifting an operand and its zero point by the same constant leaves +every `i32` accumulator bit-identical, and the call lands on the `u8 x u8` kernel. i8 activations +went from 5.5x slower than ORT to **1.7x-33x faster**, and are now claimed. + +What is left for u8 x u8 is thread scaling, not packing: 1.13-1.20x at one thread, widening to +2.08-2.65x at sixteen — the same root cause as [parallel efficiency](#1-parallel-efficiency-not-kernel-quality--f32-dense-and-int4-matmulnbits). + +Both `QLinearMatMul` rules were measured on **x86-64 AVX2 only** and are applied on every +architecture, which is the same convention the rest of this table uses. aarch64 has native `i8 x i8` +kernels (SDOT/SMMLA) that need no translation at all, so the claim there is if anything +conservative — but its *speed* is unmeasured here and is not claimed to be measured. Correctness on +that lane is covered unconditionally by `qgemm_i32_matches_the_integer_oracle_for_every_signedness`. + +Mixed signedness (`u8 x i8`, `i8 x u8`) was not measured either way. It follows the u8 rule rather +than borrowing the signed win: deferred in the measured region, claimed below it like every other +unmeasured shape here. + +The `i8 x i8` "before" ratios above (5.20/4.97/5.22 at 8 threads) were re-measured against ONNX +Runtime 1.27.0 for this round on the same host. An earlier round recorded 2.23-3.07 for the same +declined scalar path; the kernel side did not change between the two, so the difference is the +baseline and the harness, not a regression. ## Precision