From 34dec16570f43b234e0d9e494f427ed2c8153690 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 18 Aug 2026 06:39:09 +0000 Subject: [PATCH 1/2] Put map_ps's doc comment and target_feature back on map_ps A merge left the 'Apply an 8-lane kernel across a slice' doc comment and both of its attributes attached to map_bias_ps, which was inserted between them and their function. map_ps was left with no #[inline] and, more importantly, no #[target_feature(enable = "avx2,fma")]. That matters because LLVM will not inline a callee whose feature set is a superset of its caller's: without avx2,fma on map_ps, the kernel closures cannot be inlined into its loop. It is harmless today only because every caller is itself avx2,fma, so map_ps inlines upwards into the caller and the closure folds in afterwards -- an inlining decision, not a guarantee. Grow map_ps past the threshold and every activation takes a real call per 8 elements. Codegen is byte-identical before and after (verified by diffing the disassembly of tanh_avx2), so this is a latent-hazard fix, not a perf change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/kernels/simd_activations.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs b/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs index 0a8a454db4..a9ec53c7b9 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs @@ -1767,13 +1767,10 @@ mod avx2 { } } - /// Apply an 8-lane kernel across a slice. The `< 8` remainder is processed - /// through the *same* kernel via a masked load/store, so every element of - /// the output — tail included — is computed identically. - #[inline] - #[target_feature(enable = "avx2,fma")] /// Like [`map_ps`], but adds a `width`-element bias row to each `width`-element /// slab of the input before applying `kernel`. + #[inline] + #[target_feature(enable = "avx2,fma")] pub(super) unsafe fn map_bias_ps( input: &[f32], bias: &[f32], @@ -1813,6 +1810,22 @@ mod avx2 { } } + /// Apply an 8-lane kernel across a slice. The `< 8` remainder is processed + /// through the *same* kernel via a masked load/store, so every element of + /// the output — tail included — is computed identically. + /// + /// The `target_feature` is load-bearing, not decoration. LLVM will not + /// inline a callee whose feature set is a superset of its caller's, so + /// without `avx2,fma` here the `kernel` closures — which are all + /// `avx2,fma` — cannot be inlined into this loop. Today that is masked: + /// every caller is itself `avx2,fma`, so `map_ps` gets inlined *upwards* + /// into the caller first and the closure folds in afterwards. That is a + /// property of the current inlining decision, not a guarantee. Grow this + /// function past the inline threshold and the closure becomes a real call + /// per 8 elements, which for a kernel like `erf_ps` would also force all + /// of its constants to be re-materialised on every call. + #[inline] + #[target_feature(enable = "avx2,fma")] pub(super) unsafe fn map_ps( input: &[f32], output: &mut [f32], From 2e393217b1e09d1665f6c8ff877d1e75eb2890dd Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 18 Aug 2026 06:47:30 +0000 Subject: [PATCH 2/2] Say 'strict superset': equal feature sets do inline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs b/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs index a9ec53c7b9..99ca47d91b 100644 --- a/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs +++ b/crates/onnx-runtime-ep-cpu/src/kernels/simd_activations.rs @@ -1815,9 +1815,10 @@ mod avx2 { /// the output — tail included — is computed identically. /// /// The `target_feature` is load-bearing, not decoration. LLVM will not - /// inline a callee whose feature set is a superset of its caller's, so - /// without `avx2,fma` here the `kernel` closures — which are all - /// `avx2,fma` — cannot be inlined into this loop. Today that is masked: + /// inline a callee that requires a feature its caller lacks — a *strict* + /// superset; equal feature sets inline fine, which is the whole point of + /// putting `avx2,fma` back here. Without it the `kernel` closures — which + /// are all `avx2,fma` — cannot be inlined into this loop. Today that is masked: /// every caller is itself `avx2,fma`, so `map_ps` gets inlined *upwards* /// into the caller first and the closure folds in afterwards. That is a /// property of the current inlining decision, not a guarantee. Grow this