diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index a6dee4fcc0a66..2e815961e3ad5 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -248,6 +248,7 @@ function(setup_mlas_source_for_windows) ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp ) set_source_files_properties(${mlas_platform_srcs_avx512} PROPERTIES COMPILE_FLAGS "/arch:AVX512") @@ -932,6 +933,7 @@ else() ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp ) set_source_files_properties(${mlas_platform_srcs_avx512f} PROPERTIES COMPILE_FLAGS "-mavx512f") diff --git a/onnxruntime/core/mlas/lib/intrinsics/avx512/gelu_avx512f.cpp b/onnxruntime/core/mlas/lib/intrinsics/avx512/gelu_avx512f.cpp index 4a9f3a100ed65..a7753bf7ae8a3 100644 --- a/onnxruntime/core/mlas/lib/intrinsics/avx512/gelu_avx512f.cpp +++ b/onnxruntime/core/mlas/lib/intrinsics/avx512/gelu_avx512f.cpp @@ -205,6 +205,34 @@ MlasGeluErfKernelAvx512FExactImpl( } } +void +MlasErfKernelAvx512FImpl( + const float* Input, + float* Output, + size_t N + ) +{ + const GeluAvx512BroadcastConstants Constants; + while (N >= 16) { + const __m512 X = _mm512_loadu_ps(Input); + const __m512 Result = MlasGeluErfAvx512(X, Constants); + + _mm512_storeu_ps(Output, Result); + + Input += 16; + Output += 16; + N -= 16; + } + + if (N > 0) { + const __mmask16 TailMask = __mmask16((1u << static_cast(N)) - 1u); + const __m512 X = _mm512_maskz_loadu_ps(TailMask, Input); + const __m512 Result = MlasGeluErfAvx512(X, Constants); + + _mm512_mask_storeu_ps(Output, TailMask, Result); + } +} + } // namespace void @@ -217,3 +245,14 @@ MlasGeluErfKernelAvx512F( { MlasGeluErfKernelAvx512FExactImpl(Input, Output, N); } + +void +MLASCALL +MlasErfKernelAvx512F( + const float* Input, + float* Output, + size_t N + ) +{ + MlasErfKernelAvx512FImpl(Input, Output, N); +} diff --git a/onnxruntime/core/mlas/lib/intrinsics/avx512/reorder_avx512f.cpp b/onnxruntime/core/mlas/lib/intrinsics/avx512/reorder_avx512f.cpp new file mode 100644 index 0000000000000..c0516a5ff4798 --- /dev/null +++ b/onnxruntime/core/mlas/lib/intrinsics/avx512/reorder_avx512f.cpp @@ -0,0 +1,144 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + reorder_avx512f.cpp + +Abstract: + + This module implements AVX-512 accelerated helpers for the NCHW<->NCHWc + reorder routines when the NCHWc block size is 16. The baseline reorder path + transposes 4x4 tiles with SSE2, requiring four transposes per 16-wide block; + these routines transpose full 16x16 tiles in one pass. + + The transposes are pure data-movement: for a tile, + Output[p*DstStride + c] = Input[c*SrcStride + p]. + +--*/ + +#include + +#include "mlasi.h" + +namespace { + +// +// Transpose a 16x16 float tile. +// +// Input row c (0..15): 16 contiguous floats at Input + c*SrcStride. +// Output row p (0..15): 16 contiguous floats at Output + p*DstStride, with +// Output[p*DstStride + c] = Input[c*SrcStride + p]. +// +MLAS_FORCEINLINE +void +MlasReorderTranspose16x16Avx512F( + const float* Input, + float* Output, + size_t SrcStride, + size_t DstStride + ) +{ + __m512 r[16]; + for (int i = 0; i < 16; i++) { + r[i] = _mm512_loadu_ps(Input + i * SrcStride); + } + + __m512 t[16]; + for (int i = 0; i < 8; i++) { + t[2 * i + 0] = _mm512_unpacklo_ps(r[2 * i], r[2 * i + 1]); + t[2 * i + 1] = _mm512_unpackhi_ps(r[2 * i], r[2 * i + 1]); + } + + __m512 u[16]; + for (int i = 0; i < 4; i++) { + u[4 * i + 0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(t[4 * i + 0]), _mm512_castps_pd(t[4 * i + 2]))); + u[4 * i + 1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(t[4 * i + 0]), _mm512_castps_pd(t[4 * i + 2]))); + u[4 * i + 2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(t[4 * i + 1]), _mm512_castps_pd(t[4 * i + 3]))); + u[4 * i + 3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(t[4 * i + 1]), _mm512_castps_pd(t[4 * i + 3]))); + } + + __m512 v[16]; + for (int i = 0; i < 4; i++) { + v[i + 0] = _mm512_shuffle_f32x4(u[i], u[i + 4], 0x88); + v[i + 4] = _mm512_shuffle_f32x4(u[i], u[i + 4], 0xDD); + v[i + 8] = _mm512_shuffle_f32x4(u[i + 8], u[i + 12], 0x88); + v[i + 12] = _mm512_shuffle_f32x4(u[i + 8], u[i + 12], 0xDD); + } + + for (int i = 0; i < 8; i++) { + _mm512_storeu_ps(Output + (i + 0) * DstStride, _mm512_shuffle_f32x4(v[i], v[i + 8], 0x88)); + _mm512_storeu_ps(Output + (i + 8) * DstStride, _mm512_shuffle_f32x4(v[i], v[i + 8], 0xDD)); + } +} + +} // namespace + +// +// Reorder one full 16-channel NCHW block into NCHWc (block size 16). +// +// Source S: 16 channels, each InputSize contiguous spatial floats (stride InputSize). +// Dest D: InputSize spatial rows, each 16 contiguous channel floats (stride 16). +// D[p*16 + c] = S[c*InputSize + p]. +// +// Equivalent to the InputChannelsThisIteration == BlockSize == 16 case of the +// scalar MlasReorderInputNchw inner loops. +// +void +MLASCALL +MlasReorderInputNchwBlock16Avx512F( + const float* S, + float* D, + size_t InputSize + ) +{ + size_t p = 0; + for (; p + 16 <= InputSize; p += 16) { + MlasReorderTranspose16x16Avx512F(S + p, D + p * 16, InputSize, 16); + } + for (; p < InputSize; p++) { + float* d = D + p * 16; + const float* s = S + p; + for (int c = 0; c < 16; c++) { + d[c] = s[c * InputSize]; + } + } +} + +// +// Reorder one full 16-channel NCHWc block into NCHW (block size 16), for a +// contiguous run of OutputSize spatial positions. +// +// Source S: OutputSize spatial rows, each 16 contiguous channel floats (stride 16). +// Dest D: 16 channels, each OutputSize contiguous spatial floats (stride OutputSize). +// D[c*OutputSize + p] = S[p*16 + c]. +// +// Equivalent to the OutputChannelsThisIteration == BlockSize == 16 case of the +// scalar MlasReorderOutputNchwThreaded inner loops. +// +void +MLASCALL +MlasReorderOutputNchwBlock16Avx512F( + const float* S, + float* D, + size_t OutputSize + ) +{ + size_t p = 0; + for (; p + 16 <= OutputSize; p += 16) { + // Source rows are spatial positions (stride 16 channels); dest rows are + // channels (stride OutputSize). Transpose with SrcStride=16, + // DstStride=OutputSize maps Output[c*OutputSize + p] = Input[p*16 + c]. + MlasReorderTranspose16x16Avx512F(S + p * 16, D + p, 16, OutputSize); + } + for (; p < OutputSize; p++) { + const float* s = S + p * 16; + float* d = D + p; + for (int c = 0; c < 16; c++) { + d[c * OutputSize] = s[c]; + } + } +} diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 2c2b5f282db16..425ccd00de7ed 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -1360,9 +1360,32 @@ extern "C" { MLAS_QUANTIZE_LINEAR_S8_KERNEL MlasQuantizeLinearS8KernelAvx512F; MLAS_QUANTIZE_LINEAR_U8_KERNEL MlasQuantizeLinearU8KernelAvx512F; MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasGeluErfKernelAvx512F; + MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasErfKernelAvx512F; MLAS_COMPUTE_UNARY_FLOAT_KERNEL MlasSiluKernelAvx512F; #endif +#if defined(MLAS_TARGET_AMD64) +// +// AVX-512 accelerated NCHW<->NCHWc reorder block helpers (block size 16). +// Declared unconditionally for AMD64; only invoked when NchwcBlockSize == 16. +// +void +MLASCALL +MlasReorderInputNchwBlock16Avx512F( + const float* S, + float* D, + size_t InputSize + ); + +void +MLASCALL +MlasReorderOutputNchwBlock16Avx512F( + const float* S, + float* D, + size_t OutputSize + ); +#endif + MLAS_REDUCE_MAXIMUM_FLOAT_KERNEL MlasReduceMaximumF32Kernel; MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL MlasReduceMinimumMaximumF32Kernel; #if defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV) diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 50c3764595378..d8a73c1c8e8e5 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -555,6 +555,7 @@ Return Value: if (((Cpuid7[1] & 0x10000) != 0) && ((xcr0 & 0xE0) == 0xE0)) { this->GeluErfKernelRoutine = MlasGeluErfKernelAvx512F; + this->ErfKernelRoutine = MlasErfKernelAvx512F; this->SiluKernelRoutine = MlasSiluKernelAvx512F; this->GemmFloatKernel = MlasGemmFloatKernelAvx512F; this->GemmDoubleKernel = MlasGemmDoubleKernelAvx512F; diff --git a/onnxruntime/core/mlas/lib/reorder.cpp b/onnxruntime/core/mlas/lib/reorder.cpp index b329ea2ffb149..4979f55f7bb78 100644 --- a/onnxruntime/core/mlas/lib/reorder.cpp +++ b/onnxruntime/core/mlas/lib/reorder.cpp @@ -260,6 +260,19 @@ Return Value: const size_t InputChannelsThisIteration = std::min(i, BlockSize); i -= InputChannelsThisIteration; +#if defined(MLAS_TARGET_AMD64) + // + // Fast path: on AVX-512 (BlockSize == 16) a full 16-channel block is a + // 16-wide transpose. Bit-exact with the SSE2 4x4 path below. + // + if (BlockSize == 16 && InputChannelsThisIteration == 16) { + MlasReorderInputNchwBlock16Avx512F(S, D, InputSize); + S += BlockSize * InputSize; + D += BlockSize * InputSize; + continue; + } +#endif + const float* s = S; float* d = D; size_t InputSizeRemaining = InputSize; @@ -502,6 +515,19 @@ Return Value: BlockSize : OutputChannels - BlockSize * LastTaskInBatchIndex; const size_t AlignedOutputChannelsThisIteration = OutputChannelsThisIteration & (~3); +#if defined(MLAS_TARGET_AMD64) + // + // Fast path: on AVX-512 (BlockSize == 16) a full 16-channel block is a + // 16-wide transpose. Bit-exact with the SSE2 4x4 path below. + // + if (BlockSize == 16 && OutputChannelsThisIteration == 16) { + MlasReorderOutputNchwBlock16Avx512F(S, D, OutputSize); + S += BlockSize * OutputSize; + D += OutputChannelsThisIteration * OutputSize; + continue; + } +#endif + const float* s = S; float* d = D; size_t OutputSizeRemaining = OutputSize; diff --git a/onnxruntime/test/mlas/unittest/test_erf.cpp b/onnxruntime/test/mlas/unittest/test_erf.cpp new file mode 100644 index 0000000000000..23bd6abd10923 --- /dev/null +++ b/onnxruntime/test/mlas/unittest/test_erf.cpp @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include "test_util.h" +#include "core/mlas/lib/mlasi.h" + +// Exercises MlasComputeErf. On AVX-512 this dispatches to the 16-wide +// MlasErfKernelAvx512F (added for the MobileClip GELU path); on other targets it +// uses the FMA3/scalar kernel. +// +// Two levels of verification: +// 1. Accuracy-neutrality vs base: on AVX-512 hardware, compare the new +// MlasErfKernelAvx512F directly against the base MlasErfKernelFma3 kernel +// (the pointer this change replaced). This proves "matches base", which is +// the actual guarantee -- not merely "matches the math". +// 2. Mathematical correctness: MlasComputeErf vs std::erf within the +// polynomial's accuracy tolerance, sweeping lengths that straddle the +// 16-lane boundary so the vector main loop and masked tail are both hit. +class MlasErfTest : public MlasTestBase { + private: + MatrixGuardBuffer BufferInput; + MatrixGuardBuffer BufferOutput; + MatrixGuardBuffer BufferOutputBase; + + // MLAS erf is a minimax polynomial approximation; empirically its absolute + // error vs std::erf stays well under 1e-5. Conservative bound guards the + // kernel (width/dispatch bugs) without being flaky on the approximation. + static constexpr float AbsTolerance = 2e-5f; + +#if defined(MLAS_TARGET_AMD64) + static bool Avx512Available() { + return GetMlasPlatform().ErfKernelRoutine == MlasErfKernelAvx512F; + } +#endif // MLAS_TARGET_AMD64 + + static uint32_t Bits(float f) { + uint32_t u; + std::memcpy(&u, &f, sizeof(u)); + return u; + } + + // ULP distance between two finite floats of the same sign convention. + static uint32_t UlpDiff(float a, float b) { + if (a == b) return 0; + int32_t ia = static_cast(Bits(a)); + int32_t ib = static_cast(Bits(b)); + // Map to a monotonic ordering across the sign boundary. + if (ia < 0) ia = static_cast(0x80000000u) - ia; + if (ib < 0) ib = static_cast(0x80000000u) - ib; + int64_t d = int64_t(ia) - int64_t(ib); + return static_cast(d < 0 ? -d : d); + } + +#if defined(MLAS_TARGET_AMD64) + // Level 1: AVX-512 kernel must match the base FMA3 kernel to <= 1 ULP. + // (The two implementations share the erf polynomial coefficients but use + // different instruction sequences -- hand-written AVX2 asm vs AVX-512 + // intrinsics -- so FMA reassociation could in principle differ by the last + // bit. Measured divergence on current hardware is 0 ULP, i.e. bit-exact; the + // 1 ULP bound is retained as the safe contract across microarchitectures.) + void TestMatchesBase(size_t N) { + if (!Avx512Available()) { + return; // Base FMA3 kernel is what runs anyway; nothing to compare. + } + + float* Input = BufferInput.GetBuffer(N); + float* Opt = BufferOutput.GetBuffer(N); + float* Base = BufferOutputBase.GetBuffer(N); + + for (size_t i = 0; i < N; i++) { + Input[i] = -6.0f + 12.0f * (static_cast(i % 257) / 256.0f); + } + + MlasErfKernelAvx512F(Input, Opt, N); + MlasErfKernelFma3(Input, Base, N); + + uint32_t max_ulp = 0; + for (size_t i = 0; i < N; i++) { + max_ulp = std::max(max_ulp, UlpDiff(Opt[i], Base[i])); + } + ASSERT_LE(max_ulp, 1u) << " AVX-512 erf diverges from base FMA3 by " << max_ulp + << " ULP at N=" << N; + } + + // Level 1 for edge inputs: NaN, +/-inf, denormals, large saturating magnitudes. + // erf saturates to +/-1 outside ~[-4, 4]; NaN must propagate. + void TestSpecialValuesMatchBase() { + if (!Avx512Available()) { + return; + } + + const float nan = std::numeric_limits::quiet_NaN(); + const float inf = std::numeric_limits::infinity(); + const float denorm = std::numeric_limits::denorm_min(); + const std::vector specials = { + nan, -nan, inf, -inf, denorm, -denorm, + 0.0f, -0.0f, 1e-30f, -1e-30f, 10.0f, -10.0f, 1e30f, -1e30f, + 3.9f, -3.9f, 4.1f, -4.1f}; + const size_t N = specials.size(); + + float* Input = BufferInput.GetBuffer(N); + float* Opt = BufferOutput.GetBuffer(N); + float* Base = BufferOutputBase.GetBuffer(N); + std::copy(specials.begin(), specials.end(), Input); + + MlasErfKernelAvx512F(Input, Opt, N); + MlasErfKernelFma3(Input, Base, N); + + for (size_t i = 0; i < N; i++) { + if (std::isnan(Base[i])) { + ASSERT_TRUE(std::isnan(Opt[i])) << " expected NaN at i=" << i << ", input=" << Input[i]; + } else { + ASSERT_LE(UlpDiff(Opt[i], Base[i]), 1u) + << " special value mismatch at i=" << i << ", input=" << Input[i] + << ", opt=" << Opt[i] << ", base=" << Base[i]; + } + } + } +#endif // MLAS_TARGET_AMD64 + + // Level 2: mathematical correctness vs std::erf, in place (matches the + // MobileClip GELU usage where the activation updates the tensor in place). + void TestMathInPlace(size_t N) { + float* Buffer = BufferInput.GetBuffer(N); + std::vector Reference(N); + + for (size_t i = 0; i < N; i++) { + Buffer[i] = -4.0f + 8.0f * (static_cast(i % 101) / 100.0f); + Reference[i] = std::erf(Buffer[i]); + } + + MlasComputeErf(Buffer, Buffer, N); + + for (size_t i = 0; i < N; i++) { + ASSERT_NEAR(Buffer[i], Reference[i], AbsTolerance) << " N=" << N << ", i=" << i; + } + } + + public: + static const char* GetTestSuiteName() { + static const std::string suite_name("Erf"); + return suite_name.c_str(); + } + + void ExecuteShort(void) override { + // Lengths crossing the 16-lane boundary: exact multiples, one-past, and odd + // tails exercise both the vector main loop and the masked remainder. + for (size_t n : {size_t(1), size_t(3), size_t(7), size_t(15), size_t(16), + size_t(17), size_t(31), size_t(32), size_t(33), size_t(48), + size_t(63), size_t(64), size_t(255), size_t(1000)}) { +#if defined(MLAS_TARGET_AMD64) + TestMatchesBase(n); +#endif + TestMathInPlace(n); + } +#if defined(MLAS_TARGET_AMD64) + TestSpecialValuesMatchBase(); +#endif + } +}; + +static UNUSED_VARIABLE bool added_to_main = AddTestRegister([](bool is_short_execute) { + return is_short_execute ? MlasDirectShortExecuteTests::RegisterShortExecute() : 0; +}); diff --git a/onnxruntime/test/mlas/unittest/test_reorder_input.cpp b/onnxruntime/test/mlas/unittest/test_reorder_input.cpp new file mode 100644 index 0000000000000..e525daa977bff --- /dev/null +++ b/onnxruntime/test/mlas/unittest/test_reorder_input.cpp @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test_util.h" + +// Exercises MlasReorderInputNchw (NCHW -> NCHWc). On AVX-512 the block size is +// 16 and a full 16-channel block takes the MlasReorderInputNchwBlock16Avx512F +// fast path; this test asserts that path (and the scalar tail path for partial +// blocks) is bit-exact with a plain scalar reference. +class MlasReorderInputTest : public MlasTestBase { + private: + const size_t BlockSize = MlasNchwcGetBlockSize(); + + MatrixGuardBuffer BufferInput; + MatrixGuardBuffer BufferOutput; + MatrixGuardBuffer BufferOutputReference; + + void Test(size_t Channels, size_t Height, size_t Width) { + const size_t InputSize = Height * Width; + const size_t NchwcChannels = (Channels + BlockSize - 1) & ~(BlockSize - 1); + + // MlasReorderInputNchw gathers input channels four at a time, so it reads + // up to the next multiple of four channels; allocate the source rounded up + // accordingly (the extra channels are zero and land in NCHWc padding lanes). + const size_t PaddedInputChannels = (Channels + 3) & ~size_t(3); + const size_t InputBufferElements = PaddedInputChannels * InputSize; + const size_t OutputBufferElements = NchwcChannels * InputSize; + + float* Input = BufferInput.GetBuffer(InputBufferElements); + float* Output = BufferOutput.GetBuffer(OutputBufferElements); + float* OutputReference = BufferOutputReference.GetBuffer(OutputBufferElements); + + // Zero the padding channels [Channels, PaddedInputChannels) so the gather of + // a partial group reads defined values and the reference agrees. + for (size_t c = Channels; c < PaddedInputChannels; c++) { + std::fill_n(Input + c * InputSize, InputSize, 0.0f); + } + + // Padding lanes of a partial trailing block must be written as zero by the + // routine; seed both buffers with a sentinel so a missed zero-fill fails. + std::fill_n(Output, OutputBufferElements, -0.5f); + std::fill_n(OutputReference, OutputBufferElements, -0.5f); + + MlasReorderInputNchw(Input, Output, Channels, InputSize); + ReferenceReorderInput(Channels, InputSize, Input, OutputReference); + + ASSERT_EQ(memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)), 0) + << " channels=" << Channels << ", height=" << Height << ", width=" << Width; + } + + // NCHW source [Channels][InputSize] -> NCHWc dest, laid out as blocks of + // BlockSize channels, each block storing [InputSize][BlockSize]. Channels + // beyond the real count in a trailing block are zero-padded. + void ReferenceReorderInput(size_t Channels, + size_t InputSize, + const float* Input, + float* Output) { + const size_t NumBlocks = (Channels + BlockSize - 1) / BlockSize; + + for (size_t b = 0; b < NumBlocks; b++) { + float* block = Output + b * BlockSize * InputSize; + for (size_t hw = 0; hw < InputSize; hw++) { + for (size_t c = 0; c < BlockSize; c++) { + const size_t channel = b * BlockSize + c; + block[hw * BlockSize + c] = + (channel < Channels) ? Input[channel * InputSize + hw] : 0.0f; + } + } + } + } + + public: + static const char* GetTestSuiteName() { + static const std::string suite_name("ReorderInput"); + return suite_name.c_str(); + } + + void ExecuteShort(void) override { + // Channel counts span exact-block multiples (16, 32, 48 -> AVX-512 fast + // path only) and partial trailing blocks (scalar tail path), including 1. + for (size_t c = 1; c < 48; c++) { + Test(c, 112, 112); // large spatial, InputSize multiple of 16 + Test(c, 15, 21); // InputSize = 315, not a multiple of 16 (tail spatial) + Test(c, 11, 11); // small odd spatial + } + } +}; + +static UNUSED_VARIABLE bool added_to_main = AddTestRegister([](bool is_short_execute) { + return (MlasNchwcGetBlockSize() > 1 && is_short_execute) + ? MlasDirectShortExecuteTests::RegisterShortExecute() + : 0; +});