From c11667c144b597557a93cfdb7fc222a8b5fc8c6f Mon Sep 17 00:00:00 2001 From: BalintCsala Date: Thu, 10 Sep 2026 12:02:45 +0000 Subject: [PATCH 1/2] Create ProblemDescription structs --- .../plans/RMSnorm/RMSnormBwdPlan.cpp | 8 +- .../plans/RMSnorm/RMSnormCommon.hpp | 49 +++ .../plans/RMSnorm/RMSnormFwdPlan.cpp | 8 +- .../plans/batchnorm/BatchnormBwdPlan.cpp | 160 ++++------ .../plans/batchnorm/BatchnormCommon.hpp | 301 ++++++++++-------- .../batchnorm/BatchnormFwdTrainingPlan.cpp | 57 ++-- .../plans/layernorm/LayernormBwdPlan.cpp | 44 +-- .../plans/layernorm/LayernormFwdPlan.cpp | 38 +-- .../plans/layernorm/LayernormPlanBuilder.cpp | 38 +-- .../plans/layernorm/LayernormUtilities.cpp | 42 +++ .../plans/layernorm/LayernormUtilities.hpp | 44 +++ .../plans/Batchnorm/TestBatchnormCommon.cpp | 152 ++++++--- .../plans/Layernorm/TestLayernormFwdPlan.cpp | 18 ++ .../plans/RMSnorm/TestRMSnormFwdPlan.cpp | 18 ++ 14 files changed, 545 insertions(+), 432 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormBwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormBwdPlan.cpp index 9ed52b2ff489..54a9a15a3966 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormBwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormBwdPlan.cpp @@ -95,10 +95,10 @@ void RMSnormBwdPlan::compile([[maybe_unused]] const IKernelCompiler& kernelCompi } // Get problem dimensions - const unsigned normalizeDim = getNormalizeDim(_params.x()->dims(), _params.scale()->dims()); - const int64_t stride = getStride(_params.x(), normalizeDim); - const int64_t outerSize = getOuterSize(_params.x()->dims(), normalizeDim, stride); - const int64_t innerSize = getInnerSize(_params.x()->dims(), normalizeDim); + const ProblemDescription problem(_params.x(), _params.scale(), Direction::BACKWARD); + const int64_t stride = problem.stride(); + const int64_t outerSize = problem.outerSize(); + const int64_t innerSize = problem.innerSize(); if(outerSize * stride >= UINT32_MAX) { throw hipdnn_plugin_sdk::HipdnnPluginException(HIPDNN_PLUGIN_STATUS_BAD_PARAM, diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp index d6d7c55ef9cb..8dcc137d0005 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp @@ -9,6 +9,12 @@ namespace hip_kernel_provider::rmsnorm { +enum class Direction +{ + FORWARD, + BACKWARD +}; + inline int64_t getOuterSize(const flatbuffers::Vector* xDims, unsigned normalizeDim, int64_t stride) { @@ -66,4 +72,47 @@ inline unsigned getNormalizeDim(const flatbuffers::Vector* xDims, return static_cast(normalizeDim); } +class ProblemDescription +{ +public: + ProblemDescription(const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* x, + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* scale, + Direction direction) + : _direction(direction) + , _normalizeDim(getNormalizeDim(x->dims(), scale->dims())) + , _stride(getStride(x, _normalizeDim)) + , _outerSize(getOuterSize(x->dims(), _normalizeDim, _stride)) + , _innerSize(getInnerSize(x->dims(), _normalizeDim)) + { + } + + Direction direction() const + { + return _direction; + } + unsigned normalizeDim() const + { + return _normalizeDim; + } + int64_t outerSize() const + { + return _outerSize; + } + int64_t innerSize() const + { + return _innerSize; + } + int64_t stride() const + { + return _stride; + } + +private: + Direction _direction; + unsigned _normalizeDim; + int64_t _stride; + int64_t _outerSize; + int64_t _innerSize; +}; + } diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp index 00bcab05cc25..47cabed04599 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp @@ -113,10 +113,10 @@ void RMSnormFwdPlan::compile(const IKernelCompiler& kernelCompiler, // dimension of 2, outerSize of N, stride of C, and innerSize of HxW. // The kernel will therefore consist of NxC workgroups with each workgroup normalizing // over HxW elements using a fixed number of threads. - const unsigned normalizeDim = getNormalizeDim(_params.x()->dims(), _params.scale()->dims()); - const int64_t stride = getStride(_params.x(), normalizeDim); - const int64_t outerSize = getOuterSize(_params.x()->dims(), normalizeDim, stride); - const int64_t innerSize = getInnerSize(_params.x()->dims(), normalizeDim); + const ProblemDescription problem(_params.x(), _params.scale(), Direction::FORWARD); + const int64_t stride = problem.stride(); + const int64_t outerSize = problem.outerSize(); + const int64_t innerSize = problem.innerSize(); if(outerSize * stride >= UINT32_MAX) { throw hipdnn_plugin_sdk::HipdnnPluginException(HIPDNN_PLUGIN_STATUS_BAD_PARAM, diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp index b8cf5c34d63e..010a79bfcd78 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp @@ -17,49 +17,36 @@ namespace hip_kernel_provider::batchnorm { -struct ProblemDims +static ProblemDescription extractProblemDescription(const BatchnormBwdParams& params) { - size_t n = 0; - size_t c = 0; - size_t h = 0; - size_t w = 0; - unsigned int inCstride = 0; - unsigned int inNhw = 0; - unsigned int inChw = 0; - unsigned int inNchw = 0; - bool isLayoutNHWC = false; - bool useFp16Mix = false; - bool useBfp16Mix = false; - bool useFp32 = true; -}; - -static ProblemDims extractProblemDims(const BatchnormBwdParams& params) -{ - ProblemDims dims{}; - const auto xDataType = params.x()->data_type(); const auto scaleDataType = params.scale()->data_type(); - dims.useFp16Mix = (xDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::HALF - && scaleDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT); - dims.useBfp16Mix = (xDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::BFLOAT16 - && scaleDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT); - dims.useFp32 = !dims.useFp16Mix && !dims.useBfp16Mix; + const bool useFp16Mix + = (xDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::HALF + && scaleDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT); + const bool useBfp16Mix + = (xDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::BFLOAT16 + && scaleDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT); const auto* xDims = params.x()->dims(); + size_t n = 0; + size_t c = 0; + size_t h = 0; + size_t w = 0; if(xDims->size() == 4) { - dims.n = static_cast(xDims->Get(0)); - dims.c = static_cast(xDims->Get(1)); - dims.h = static_cast(xDims->Get(2)); - dims.w = static_cast(xDims->Get(3)); + n = static_cast(xDims->Get(0)); + c = static_cast(xDims->Get(1)); + h = static_cast(xDims->Get(2)); + w = static_cast(xDims->Get(3)); } else if(xDims->size() == 5) { - dims.n = static_cast(xDims->Get(0)); - dims.c = static_cast(xDims->Get(1)); + n = static_cast(xDims->Get(0)); + c = static_cast(xDims->Get(1)); const auto d = static_cast(xDims->Get(2)); - dims.h = d * static_cast(xDims->Get(3)); - dims.w = static_cast(xDims->Get(4)); + h = d * static_cast(xDims->Get(3)); + w = static_cast(xDims->Get(4)); } else { @@ -68,12 +55,15 @@ static ProblemDims extractProblemDims(const BatchnormBwdParams& params) + std::to_string(xDims->size())); } - dims.inCstride = static_cast(dims.h * dims.w); - dims.inNhw = static_cast(dims.n) * dims.inCstride; - dims.inChw = static_cast(dims.c) * dims.inCstride; - dims.inNchw = static_cast(dims.n) * dims.inChw; - dims.isLayoutNHWC = isChannelLastLayout(params.x()); - return dims; + return ProblemDescription(n, + c, + h, + w, + isChannelLastLayout(params.x()), + useFp16Mix, + useBfp16Mix, + Direction::BACKWARD, + 1); } BatchnormBwdParams::BatchnormBwdParams( @@ -190,7 +180,7 @@ size_t BatchnormBwdPlan::getWorkspaceSize([[maybe_unused]] const Handle& handle) void BatchnormBwdPlan::compile(const IKernelCompiler& kernelCompiler, const hipDeviceProp_t& deviceProperties) { - const auto dims = extractProblemDims(_params); + const auto problem = extractProblemDescription(_params); if(_params.optActivation().has_value()) { @@ -207,49 +197,21 @@ void BatchnormBwdPlan::compile(const IKernelCompiler& kernelCompiler, // in MIOpen backward_spatial.cpp. const unsigned int stashValuesBwd = !_usesSavedStats ? 4u : 2u; KernelConfig config; - if(useMultiple(dims.n, - dims.h, - dims.w, - dims.useFp16Mix || dims.useBfp16Mix, - dims.isLayoutNHWC, - Direction::BACKWARD)) + if(useMultiple(problem)) { - const size_t minWorkgroups = 1; - defaultConfigSpatialMultiple(dims.n, - dims.c, - dims.h, - dims.w, - dims.isLayoutNHWC, - dims.useFp32, - minWorkgroups, - stashValuesBwd, - config); + defaultConfigSpatialMultiple(problem, stashValuesBwd, config); if(config.variant == -1) { - defaultConfigSpatialSingle(dims.n, - dims.h, - dims.w, - dims.useFp16Mix, - dims.useBfp16Mix, - dims.isLayoutNHWC, - Direction::BACKWARD, - config); + defaultConfigSpatialSingle(problem, config); } } else { - defaultConfigSpatialSingle(dims.n, - dims.h, - dims.w, - dims.useFp16Mix, - dims.useBfp16Mix, - dims.isLayoutNHWC, - Direction::BACKWARD, - config); + defaultConfigSpatialSingle(problem, config); } _kernelVariant = config.variant; - _invInNhw = 1.0f / static_cast(dims.inNhw); + _invInNhw = 1.0f / static_cast(problem.inNhw()); size_t xlocalsize = config.xlocalsize; const size_t ylocalsize = config.ylocalsize; @@ -276,29 +238,29 @@ void BatchnormBwdPlan::compile(const IKernelCompiler& kernelCompiler, _params.scale(), deviceProperties, activationMode); - options.update("HIP_PLUGIN_USE_FPMIX", dims.useFp16Mix); - options.update("HIP_PLUGIN_USE_BFPMIX", dims.useBfp16Mix); + options.update("HIP_PLUGIN_USE_FPMIX", problem.useFp16Mix()); + options.update("HIP_PLUGIN_USE_BFPMIX", problem.useBfp16Mix()); // Not using FP16 and BFP16 paths due to affine data type requirements options.update("HIP_PLUGIN_USE_FP16", 0); options.update("HIP_PLUGIN_USE_BFP16", 0); options.update("HIP_PLUGIN_BN_USESAVED", _usesSavedStats); - options.update("HIP_PLUGIN_BN_N", dims.n); - options.update("HIP_PLUGIN_BN_C", dims.c); - options.update("HIP_PLUGIN_BN_HW", dims.inCstride); - options.update("HIP_PLUGIN_BN_NHW", dims.inNhw); - options.update("HIP_PLUGIN_BN_CHW", dims.inChw); - options.update("HIP_PLUGIN_BN_NCHW", dims.inNchw); + options.update("HIP_PLUGIN_BN_N", problem.n()); + options.update("HIP_PLUGIN_BN_C", problem.c()); + options.update("HIP_PLUGIN_BN_HW", problem.inCstride()); + options.update("HIP_PLUGIN_BN_NHW", problem.inNhw()); + options.update("HIP_PLUGIN_BN_CHW", problem.inChw()); + options.update("HIP_PLUGIN_BN_NCHW", problem.inNchw()); options.update("HIP_PLUGIN_BN_VARIANT", _kernelVariant); if(_kernelVariant != 2) { xlocalsize = 1024; - if(((dims.inCstride < 256) && (dims.n < 256)) - || ((dims.inCstride < 100) && (dims.n <= 256))) + if(((problem.inCstride() < 256) && (problem.n() < 256)) + || ((problem.inCstride() < 100) && (problem.n() <= 256))) { xlocalsize = 256; } - xgridsize = dims.c * xlocalsize; + xgridsize = problem.c() * xlocalsize; ldsSize = static_cast(xlocalsize); options.update("HIP_PLUGIN_BN_GRP0", xlocalsize); @@ -314,30 +276,24 @@ void BatchnormBwdPlan::compile(const IKernelCompiler& kernelCompiler, } else { - if(dims.isLayoutNHWC) + if(problem.isLayoutNHWC()) { - xgridsize = xlocalsize * ((dims.c / config.vectorsize + xlocalsize - 1) / xlocalsize); - ygridsize = ylocalsize * ((dims.inCstride + ylocalsize - 1) / ylocalsize); + xgridsize + = xlocalsize * ((problem.c() / config.vectorsize + xlocalsize - 1) / xlocalsize); + ygridsize = ylocalsize * ((problem.inCstride() + ylocalsize - 1) / ylocalsize); } else { - xgridsize = xlocalsize * ((dims.c + xlocalsize - 1) / xlocalsize); - ygridsize - = ylocalsize * ((dims.inCstride / config.vectorsize + ylocalsize - 1) / ylocalsize); + xgridsize = xlocalsize * ((problem.c() + xlocalsize - 1) / xlocalsize); + ygridsize = ylocalsize + * ((problem.inCstride() / config.vectorsize + ylocalsize - 1) / ylocalsize); } - zgridsize = zlocalsize * ((dims.n / config.nelements + zlocalsize - 1) / zlocalsize); - - stashMethod = getStashMethod(dims.isLayoutNHWC, - dims.useFp32, - stashValuesBwd, - dims.c, - dims.n, - dims.inCstride, - ylocalsize, - zlocalsize, - config.nelements); - - if(dims.isLayoutNHWC && dims.c % 2 == 0 && xlocalsize % 2 == 0) + zgridsize = zlocalsize * ((problem.n() / config.nelements + zlocalsize - 1) / zlocalsize); + + stashMethod + = getStashMethod(problem, stashValuesBwd, ylocalsize, zlocalsize, config.nelements); + + if(problem.isLayoutNHWC() && problem.c() % 2 == 0 && xlocalsize % 2 == 0) { xlocalsizeFinal = 2; zlocalsizeFinal = zgridsize / zlocalsize * zlocalsize; diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormCommon.hpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormCommon.hpp index a0a39ebf009c..023b9763983f 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormCommon.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormCommon.hpp @@ -18,6 +18,100 @@ enum class Direction : int BACKWARD = 2 }; +class ProblemDescription +{ +public: + ProblemDescription(size_t n, + size_t c, + size_t h, + size_t w, + bool isLayoutNHWC, + bool useFp16Mix, + bool useBfp16Mix, + Direction direction, + size_t minWorkgroups) + : _n(n) + , _c(c) + , _h(h) + , _w(w) + , _isLayoutNHWC(isLayoutNHWC) + , _useFp16Mix(useFp16Mix) + , _useBfp16Mix(useBfp16Mix) + , _direction(direction) + , _minWorkgroups(minWorkgroups) + { + } + + size_t n() const + { + return _n; + } + size_t c() const + { + return _c; + } + size_t h() const + { + return _h; + } + size_t w() const + { + return _w; + } + bool isLayoutNHWC() const + { + return _isLayoutNHWC; + } + bool useFp16Mix() const + { + return _useFp16Mix; + } + bool useBfp16Mix() const + { + return _useBfp16Mix; + } + bool useFp32() const + { + return !_useFp16Mix && !_useBfp16Mix; + } + Direction direction() const + { + return _direction; + } + size_t minWorkgroups() const + { + return _minWorkgroups; + } + + unsigned int inCstride() const + { + return static_cast(_h * _w); + } + unsigned int inNhw() const + { + return static_cast(_n * _h * _w); + } + unsigned int inChw() const + { + return static_cast(_c * _h * _w); + } + unsigned int inNchw() const + { + return static_cast(_n * _c * _h * _w); + } + +private: + size_t _n; + size_t _c; + size_t _h; + size_t _w; + bool _isLayoutNHWC; + bool _useFp16Mix; + bool _useBfp16Mix; + Direction _direction; + size_t _minWorkgroups; +}; + struct KernelConfig { int variant = -1; @@ -30,11 +124,7 @@ struct KernelConfig // Compute workgroup size configuration given a problem (NHWC) and a vectorsize // It supports only 2D workgroups -inline void getLocalConfigNHWC(size_t c, - size_t h, - size_t w, - bool isFp32, - size_t minWorkgroups, +inline void getLocalConfigNHWC(const ProblemDescription& problem, size_t vectorsize, size_t& xlocalsize, size_t& ylocalsize) @@ -43,31 +133,32 @@ inline void getLocalConfigNHWC(size_t c, unsigned int xlocalsizeLimit = 64; if(vectorsize > 1) { - xlocalsizeLimit = isFp32 ? 16 : 32; + xlocalsizeLimit = problem.useFp32() ? 16 : 32; } // shared memory size per workgroup is fixed unsigned int maxLocalsize = 1024 / vectorsize; // default local config in case the while loop is not entered - xlocalsize = std::min( - size_t{1} << static_cast(std::ceil(std::log2(std::max(c / vectorsize, size_t{1})))), - static_cast(xlocalsizeLimit)); + xlocalsize = std::min(size_t{1} << static_cast( + std::ceil(std::log2(std::max(problem.c() / vectorsize, size_t{1})))), + static_cast(xlocalsizeLimit)); ylocalsize = maxLocalsize / xlocalsize; size_t nworkgroups = 0; // decrease maxLocalsize until the number of workgroups is greater than 80% // of the available CUs - while(nworkgroups < minWorkgroups && maxLocalsize >= xlocalsizeLimit && maxLocalsize > 64) + while(nworkgroups < problem.minWorkgroups() && maxLocalsize >= xlocalsizeLimit + && maxLocalsize > 64) { // xlocalsize must be power of 2 as reductions in the kernels rely on it, here c is rounded // up to next power of 2. - xlocalsize = std::min(size_t{1} << static_cast( - std::ceil(std::log2(std::max(c / vectorsize, size_t{1})))), + xlocalsize = std::min(size_t{1} << static_cast(std::ceil( + std::log2(std::max(problem.c() / vectorsize, size_t{1})))), static_cast(xlocalsizeLimit)); ylocalsize = maxLocalsize / xlocalsize; - nworkgroups = ((c / vectorsize + xlocalsize - 1) / xlocalsize) - * ((h * w + ylocalsize - 1) / ylocalsize); + nworkgroups = ((problem.c() / vectorsize + xlocalsize - 1) / xlocalsize) + * ((problem.inCstride() + ylocalsize - 1) / ylocalsize); maxLocalsize >>= 1; } } @@ -75,12 +166,7 @@ inline void getLocalConfigNHWC(size_t c, // Provide workgroup sizes for spatial multiple configuration. // It returns the preferred spatial multiple configuration, which is used without tuning. // If tuning is enabled, this configuration is also added to the group of instances. -inline void getSpatialMultipleConfig(size_t c, - size_t h, - size_t w, - bool isLayoutNHWC, - bool isFp32, - size_t minWorkgroups, +inline void getSpatialMultipleConfig(const ProblemDescription& problem, size_t vectorsize, size_t& xlocalsize, size_t& ylocalsize) @@ -89,16 +175,16 @@ inline void getSpatialMultipleConfig(size_t c, xlocalsize = 1; ylocalsize = 1; - const size_t inCstride = h * w; + const size_t inCstride = problem.inCstride(); - if(isLayoutNHWC) + if(problem.isLayoutNHWC()) { - if(c % vectorsize != 0) + if(problem.c() % vectorsize != 0) { // xlocalsize and ylocalsize already initialized to 1 return; } - getLocalConfigNHWC(c, h, w, isFp32, minWorkgroups, vectorsize, xlocalsize, ylocalsize); + getLocalConfigNHWC(problem, vectorsize, xlocalsize, ylocalsize); } else { @@ -121,37 +207,32 @@ inline void getSpatialMultipleConfig(size_t c, // Check if spatial multiple implementation can be used for a given problem // and workgroup configuration. -inline bool isSpatialMultipleApplicable(size_t n, - size_t c, - size_t h, - size_t w, - bool isLayoutNHWC, - bool isFp32, +inline bool isSpatialMultipleApplicable(const ProblemDescription& problem, size_t vectorsize, unsigned int stashValues, size_t ylocalsize, size_t zlocalsize, size_t nelements) { - const auto inCstride = static_cast(h * w); + const auto inCstride = problem.inCstride(); - if(isLayoutNHWC) + if(problem.isLayoutNHWC()) { // check if the provided vectorsize can be used - if(c % vectorsize != 0) + if(problem.c() % vectorsize != 0) { return false; } - stashValues *= (isFp32 ? 1 : 2); + stashValues *= (problem.useFp32() ? 1 : 2); const unsigned int lastYlocalsize = inCstride % ylocalsize == 0 ? static_cast(ylocalsize) : inCstride % ylocalsize; const unsigned int lastZocalsize - = n % (zlocalsize * nelements) == 0 + = problem.n() % (zlocalsize * nelements) == 0 ? static_cast(zlocalsize * nelements) - : n % static_cast(zlocalsize * nelements); + : problem.n() % static_cast(zlocalsize * nelements); // FP32: // - last block must have enough space to stash intermediate results in HW dimension @@ -163,7 +244,7 @@ inline bool isSpatialMultipleApplicable(size_t n, // be large enough // - if C is not multiple of 2, intermediate results are stored in N dimension splitting // float values in group of 2 bytes. N must be large enough - if((!isFp32 && (c % 2 != 0 && lastZocalsize < stashValues)) + if((!problem.useFp32() && (problem.c() % 2 != 0 && lastZocalsize < stashValues)) || ((lastYlocalsize < stashValues) && (lastZocalsize < stashValues))) { return false; @@ -182,14 +263,14 @@ inline bool isSpatialMultipleApplicable(size_t n, : inCstride % ylocalsize; const unsigned int lastZocalsize - = n % (zlocalsize * nelements) == 0 + = problem.n() % (zlocalsize * nelements) == 0 ? static_cast(zlocalsize * nelements) - : n % static_cast(zlocalsize * nelements); + : problem.n() % static_cast(zlocalsize * nelements); // Restrictions: // - last block must have enough space to stash intermediate results in HW dimension // - if last block doesn't fit, intermediate results are stored in N dimension which must // be large enough - stashValues *= (isFp32 ? 1 : 2); + stashValues *= (problem.useFp32() ? 1 : 2); if(lastYlocalsize < stashValues && lastZocalsize < stashValues) { return false; @@ -198,14 +279,13 @@ inline bool isSpatialMultipleApplicable(size_t n, return true; } -inline bool useMultiple( - size_t n, size_t h, size_t w, bool isFp16OrBfp16Mix, bool isLayoutNHWC, Direction direction) +inline bool useMultiple(const ProblemDescription& problem) { - const auto inCstride = static_cast(h * w); - const auto inNhw = static_cast(n) * inCstride; + const auto inCstride = problem.inCstride(); + const auto inNhw = problem.inNhw(); const auto thrInNhw = static_cast(32 * 1024 * 1024); - if(!isLayoutNHWC && direction == Direction::BACKWARD) + if(!problem.isLayoutNHWC() && problem.direction() == Direction::BACKWARD) { if((inNhw < thrInNhw && inCstride > 1024) || (inNhw < thrInNhw && inCstride > 512) || inCstride <= 512) @@ -214,14 +294,15 @@ inline bool useMultiple( } } - if(!isLayoutNHWC && direction == Direction::FORWARD_TRAINING) + if(!problem.isLayoutNHWC() && problem.direction() == Direction::FORWARD_TRAINING) { - const bool condition1 = (n < 3) || (inCstride <= 512) - || (inNhw < 33554432 && inCstride > 1024) - || (n >= 256 && inCstride > 60 && isFp16OrBfp16Mix) - || (isFp16OrBfp16Mix && inCstride > 512); + const bool condition1 + = (problem.n() < 3) || (inCstride <= 512) || (inNhw < 33554432 && inCstride > 1024) + || (problem.n() >= 256 && inCstride > 60 + && (problem.useFp16Mix() || problem.useBfp16Mix())) + || ((problem.useFp16Mix() || problem.useBfp16Mix()) && inCstride > 512); - const bool condition2 = (n <= 768) || (inCstride <= 150); + const bool condition2 = (problem.n() <= 768) || (inCstride <= 150); if(condition1 && condition2) { @@ -233,47 +314,39 @@ inline bool useMultiple( } // Provide the stash method to use for spatial multiple implementation -inline int getStashMethod(bool isLayoutNHWC, - bool isFp32, +inline int getStashMethod(const ProblemDescription& problem, unsigned int stashValues, - size_t c, - size_t n, - size_t inCstride, size_t ylocalsize, size_t zlocalsize, size_t nelements) { // See `batchnorm_functions.hpp` for stash implementation of different methods int stashMethod = 0; - stashValues *= (isFp32 ? 1 : 2); - const unsigned int lastYlocalsize = (inCstride) % ylocalsize == 0 + stashValues *= (problem.useFp32() ? 1 : 2); + const unsigned int lastYlocalsize = problem.inCstride() % ylocalsize == 0 ? static_cast(ylocalsize) - : static_cast((inCstride) % ylocalsize); - const unsigned int lastZocalsize = n % (zlocalsize * nelements) == 0 - ? static_cast(zlocalsize * nelements) - : n % static_cast(zlocalsize * nelements); + : problem.inCstride() % ylocalsize; + const unsigned int lastZocalsize + = problem.n() % (zlocalsize * nelements) == 0 + ? static_cast(zlocalsize * nelements) + : problem.n() % static_cast(zlocalsize * nelements); if(lastYlocalsize < stashValues && lastZocalsize >= stashValues) { stashMethod = 1; } - if(isLayoutNHWC && !isFp32 && (c % 2 != 0) && (lastZocalsize >= stashValues)) + if(problem.isLayoutNHWC() && !problem.useFp32() && (problem.c() % 2 != 0) + && (lastZocalsize >= stashValues)) { stashMethod = 2; } return stashMethod; } -inline void defaultConfigSpatialSingle(size_t n, - size_t h, - size_t w, - bool isFp16Mix, - bool isBfp16Mix, - bool isLayoutNHWC, - Direction direction, - KernelConfig& config) +inline void defaultConfigSpatialSingle(const ProblemDescription& problem, KernelConfig& config) { - const auto inCstride = static_cast(h * w); - const auto inNhw = static_cast(n * inCstride); + const auto n = problem.n(); + const auto inCstride = problem.inCstride(); + const auto inNhw = problem.inNhw(); // NCHW supports also variants 0 and 3 which can be much faster than // variant 1 but have more restrictions. Here we decide if we use variant @@ -283,11 +356,11 @@ inline void defaultConfigSpatialSingle(size_t n, // we add the latter for tuning to be sure and because it is cheap to run. // NOTE: Currently we don't have the tuning infrastructure in place, so we // are only selecting one variant to run based on heuristics. - if(!isLayoutNHWC) + if(!problem.isLayoutNHWC()) { - if(direction == Direction::BACKWARD) + if(problem.direction() == Direction::BACKWARD) { - if((inCstride < 200) && (inCstride > 60) && isFp16Mix) + if((inCstride < 200) && (inCstride > 60) && problem.useFp16Mix()) { config.variant = 1; config.vectorsize = 1; @@ -348,8 +421,9 @@ inline void defaultConfigSpatialSingle(size_t n, } if((inNhw < 33554432 && inCstride > 1024) - || ((n >= 256) && (inCstride > 60) && (isFp16Mix || isBfp16Mix)) - || ((inCstride > 512) && (isFp16Mix || isBfp16Mix))) + || ((n >= 256) && (inCstride > 60) + && (problem.useFp16Mix() || problem.useBfp16Mix())) + || ((inCstride > 512) && (problem.useFp16Mix() || problem.useBfp16Mix()))) { config.variant = 1; config.vectorsize = 1; @@ -379,13 +453,7 @@ inline void defaultConfigSpatialSingle(size_t n, // - for NHWC an hybrid approach is used, xlocalsize and vectorsize are set using heuristics, // while ylocalsize, zlocalsize and nelements are added to the tuning with some // additional restrictions based on heuristics to keep the number of instances low -inline void defaultConfigSpatialMultiple(size_t n, - size_t c, - size_t h, - size_t w, - bool isLayoutNHWC, - bool isFp32, - size_t minWorkgroups, +inline void defaultConfigSpatialMultiple(const ProblemDescription& problem, unsigned int stashValues, KernelConfig& config) { @@ -393,28 +461,16 @@ inline void defaultConfigSpatialMultiple(size_t n, size_t ylocalsizeDefault = 1; const size_t zlocalsizeDefault = 1; size_t vectorsizeDefault = 4; - const size_t nelementsDefault = n; + const size_t nelementsDefault = problem.n(); - if(isLayoutNHWC) + if(problem.isLayoutNHWC()) { // First add the default instance, which should work well for a large range of problems { - getSpatialMultipleConfig(c, - h, - w, - isLayoutNHWC, - isFp32, - minWorkgroups, - vectorsizeDefault, - xlocalsizeDefault, - ylocalsizeDefault); - - if(isSpatialMultipleApplicable(n, - c, - h, - w, - isLayoutNHWC, - isFp32, + getSpatialMultipleConfig( + problem, vectorsizeDefault, xlocalsizeDefault, ylocalsizeDefault); + + if(isSpatialMultipleApplicable(problem, vectorsizeDefault, stashValues, ylocalsizeDefault, @@ -433,22 +489,10 @@ inline void defaultConfigSpatialMultiple(size_t n, if(vectorsizeDefault > 1) { vectorsizeDefault = 1; - getSpatialMultipleConfig(c, - h, - w, - isLayoutNHWC, - isFp32, - minWorkgroups, - vectorsizeDefault, - xlocalsizeDefault, - ylocalsizeDefault); - - if(isSpatialMultipleApplicable(n, - c, - h, - w, - isLayoutNHWC, - isFp32, + getSpatialMultipleConfig( + problem, vectorsizeDefault, xlocalsizeDefault, ylocalsizeDefault); + + if(isSpatialMultipleApplicable(problem, vectorsizeDefault, stashValues, ylocalsizeDefault, @@ -476,22 +520,9 @@ inline void defaultConfigSpatialMultiple(size_t n, // applicable) while(vectorsizeDefault > 0) { - getSpatialMultipleConfig(c, - h, - w, - isLayoutNHWC, - isFp32, - minWorkgroups, - vectorsizeDefault, - xlocalsizeDefault, - ylocalsizeDefault); - - if(isSpatialMultipleApplicable(n, - c, - h, - w, - isLayoutNHWC, - isFp32, + getSpatialMultipleConfig(problem, vectorsizeDefault, xlocalsizeDefault, ylocalsizeDefault); + + if(isSpatialMultipleApplicable(problem, vectorsizeDefault, stashValues, ylocalsizeDefault, diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormFwdTrainingPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormFwdTrainingPlan.cpp index f0a86f5a2fdc..e4f3b99cd0ed 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormFwdTrainingPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormFwdTrainingPlan.cpp @@ -211,7 +211,6 @@ void BatchnormFwdTrainingPlan::compile(const IKernelCompiler& kernelCompiler, const bool useBfp16Mix = (xDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::BFLOAT16 && scaleDataType == hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT); - const bool useFp32 = !useFp16Mix && !useBfp16Mix; // Extract dimensions from x tensor const auto* xDims = _trainingParams.x()->dims(); @@ -277,47 +276,34 @@ void BatchnormFwdTrainingPlan::compile(const IKernelCompiler& kernelCompiler, // Get the kernel launch configuration based on heuristics hip_kernel_provider::batchnorm::KernelConfig config; + const size_t minWorkgroups = std::max( + size_t(1), size_t(0.6f * static_cast(deviceProperties.multiProcessorCount))); + const hip_kernel_provider::batchnorm::ProblemDescription problem( + n, + c, + h, + w, + isLayoutNHWC, + useFp16Mix, + useBfp16Mix, + hip_kernel_provider::batchnorm::Direction::FORWARD_TRAINING, + minWorkgroups); // Define default configuration based on heuristics and // add all other valid configurations for the given problem - if(hip_kernel_provider::batchnorm::useMultiple( - n, - h, - w, - useFp16Mix || useBfp16Mix, - isLayoutNHWC, - hip_kernel_provider::batchnorm::Direction::FORWARD_TRAINING)) + if(hip_kernel_provider::batchnorm::useMultiple(problem)) { - // Determine the minimum number of workgroups - const size_t minWorkgroups = std::max( - size_t(1), size_t(0.6f * static_cast(deviceProperties.multiProcessorCount))); hip_kernel_provider::batchnorm::defaultConfigSpatialMultiple( - n, c, h, w, isLayoutNHWC, useFp32, minWorkgroups, stashValuesFwd, config); + problem, stashValuesFwd, config); if(config.variant == -1) { // If the default spatial multiple function failed to select a valid configuration, // get a default spatial single configuration as fallback - hip_kernel_provider::batchnorm::defaultConfigSpatialSingle( - n, - h, - w, - useFp16Mix, - useBfp16Mix, - isLayoutNHWC, - hip_kernel_provider::batchnorm::Direction::FORWARD_TRAINING, - config); + hip_kernel_provider::batchnorm::defaultConfigSpatialSingle(problem, config); } } else { - hip_kernel_provider::batchnorm::defaultConfigSpatialSingle( - n, - h, - w, - useFp16Mix, - useBfp16Mix, - isLayoutNHWC, - hip_kernel_provider::batchnorm::Direction::FORWARD_TRAINING, - config); + hip_kernel_provider::batchnorm::defaultConfigSpatialSingle(problem, config); } variant = config.variant; @@ -357,15 +343,8 @@ void BatchnormFwdTrainingPlan::compile(const IKernelCompiler& kernelCompiler, zgridsize = zlocalsize * ((n / nelements + zlocalsize - 1) / zlocalsize); // Get the stash method based on problem size and WG size - stashMethod = hip_kernel_provider::batchnorm::getStashMethod(isLayoutNHWC, - useFp32, - stashValuesFwd, - c, - n, - inCstride, - ylocalsize, - zlocalsize, - nelements); + stashMethod = hip_kernel_provider::batchnorm::getStashMethod( + problem, stashValuesFwd, ylocalsize, zlocalsize, nelements); // WG size for Final kernels (NHWC) if(isLayoutNHWC && c % 2 == 0 && xlocalsize % 2 == 0) diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormBwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormBwdPlan.cpp index c384f0878bfe..e0dbf00f2cf2 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormBwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormBwdPlan.cpp @@ -104,44 +104,12 @@ const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* LayernormBwdParams LayernormBwdPlan::LayernormBwdPlan(LayernormBwdParams&& params) : _params(std::move(params)) { - // Extract dimensions from x tensor - const auto* xDims = _params.x()->dims(); - const auto* xStrides = _params.x()->strides(); - const auto strideOrder = hipdnn_data_sdk::utilities::extractStrideOrder( - std::vector(xStrides->begin(), xStrides->end())); - - const size_t normalizedDim - = layernorm::guessNormalizedDim(_params.x(), _params.scale(), _params.mean()); - int64_t outerSize = 1; - int64_t innerSize = 1; - int64_t stride = 1; - const auto layoutNHWC = hipdnn_data_sdk::utilities::TensorLayout::NHWC; - const auto layoutNDHWC = hipdnn_data_sdk::utilities::TensorLayout::NDHWC; - - if(normalizedDim > 1 - && (strideOrder == layoutNHWC.strideOrder || strideOrder == layoutNDHWC.strideOrder)) - { - stride = static_cast(xDims->Get(1)); - } - - for(unsigned int i = 0; i < xDims->size(); ++i) - { - if(i < normalizedDim) - { - if(stride == 1 || i != 1) // Don't add C to outerSize if there is a stride - { - outerSize *= static_cast(xDims->Get(i)); - } - } - else - { - innerSize *= static_cast(xDims->Get(i)); - } - } - - _outerSize = outerSize; - _innerSize = innerSize; - _stride = stride; + const auto statAttr + = _params.mean() == nullptr ? std::nullopt : std::make_optional(_params.mean()); + const ProblemDescription problem(_params.x(), _params.scale(), statAttr, Direction::BACKWARD); + _outerSize = problem.outerSize(); + _innerSize = problem.innerSize(); + _stride = problem.stride(); _localSize = LayernormBwdParams::MAX_LOCAL_SIZE; } diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormFwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormFwdPlan.cpp index 9ac95a8cabe6..9f1d81ff826c 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormFwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormFwdPlan.cpp @@ -97,10 +97,6 @@ void LayernormFwdPlan::compile(const IKernelCompiler& kernelCompiler, { // Extract dimensions from x tensor const auto* xDims = _params.x()->dims(); - const auto* xStrides = _params.x()->strides(); - const auto strideOrder = hipdnn_data_sdk::utilities::extractStrideOrder( - std::vector(xStrides->begin(), xStrides->end())); - // Ensure that the input tensor is either 4D or 5D if(xDims->size() != 4 && xDims->size() != 5) { @@ -109,34 +105,12 @@ void LayernormFwdPlan::compile(const IKernelCompiler& kernelCompiler, + std::to_string(xDims->size())); } - const size_t normalizedDim - = layernorm::guessNormalizedDim(_params.x(), _params.scale(), _params.mean()); - int64_t outerSize = 1; - int64_t innerSize = 1; - int64_t stride = 1; - const auto layoutNHWC = hipdnn_data_sdk::utilities::TensorLayout::NHWC; - const auto layoutNDHWC = hipdnn_data_sdk::utilities::TensorLayout::NDHWC; - - if(normalizedDim > 1 - && (strideOrder == layoutNHWC.strideOrder || strideOrder == layoutNDHWC.strideOrder)) - { - stride = static_cast(xDims->Get(1)); - } - - for(unsigned int i = 0; i < xDims->size(); ++i) - { - if(i < normalizedDim) - { - if(stride == 1 || i != 1) // Don't add C to outerSize if there is a stride - { - outerSize *= static_cast(xDims->Get(i)); - } - } - else - { - innerSize *= static_cast(xDims->Get(i)); - } - } + const auto statAttr + = _params.mean() == nullptr ? std::nullopt : std::make_optional(_params.mean()); + const ProblemDescription problem(_params.x(), _params.scale(), statAttr, Direction::FORWARD); + const int64_t outerSize = problem.outerSize(); + const int64_t innerSize = problem.innerSize(); + const int64_t stride = problem.stride(); const int64_t xlocalsize = 1024; const int64_t xgridsize = outerSize * stride; diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormPlanBuilder.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormPlanBuilder.cpp index 8b05c43fb0df..556e07c02e9c 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormPlanBuilder.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormPlanBuilder.cpp @@ -38,39 +38,11 @@ size_t getMaxBwdWorkspaceSize(const Handle& handle, = attr->mean_tensor_uid().has_value() ? std::make_optional(opGraph.getTensorMap().at(attr->mean_tensor_uid().value())) : std::nullopt; - const auto* xDims = xTensorAttr->dims(); - const auto* xStrides = xTensorAttr->strides(); - const auto strideOrder = hipdnn_data_sdk::utilities::extractStrideOrder( - std::vector(xStrides->begin(), xStrides->end())); - - const size_t normalizedDim - = layernorm::guessNormalizedDim(xTensorAttr, scaleTensorAttr, meanTensorAttr); - int64_t outerSize = 1; - int64_t innerSize = 1; - int64_t stride = 1; - const auto layoutNHWC = hipdnn_data_sdk::utilities::TensorLayout::NHWC; - const auto layoutNDHWC = hipdnn_data_sdk::utilities::TensorLayout::NDHWC; - - if(normalizedDim > 1 - && (strideOrder == layoutNHWC.strideOrder || strideOrder == layoutNDHWC.strideOrder)) - { - stride = static_cast(xDims->Get(1)); - } - - for(unsigned int i = 0; i < xDims->size(); ++i) - { - if(i < normalizedDim) - { - if(stride == 1 || i != 1) // Don't add C to outerSize if there is a stride - { - outerSize *= static_cast(xDims->Get(i)); - } - } - else - { - innerSize *= static_cast(xDims->Get(i)); - } - } + const ProblemDescription problem( + xTensorAttr, scaleTensorAttr.value(), meanTensorAttr, Direction::BACKWARD); + const auto outerSize = problem.outerSize(); + const auto innerSize = problem.innerSize(); + const auto stride = problem.stride(); const bool parallel = LayernormBwdPlan::isParallel(deviceProperties, static_cast(LayernormBwdParams::MAX_LOCAL_SIZE), diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp index 95017506813e..6ba039c7db7d 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include "core/Utils.hpp" @@ -220,4 +222,44 @@ size_t guessNormalizedDim( return normalizedDimMin > 0 ? normalizedDimMin : normalizedDimMax; } +ProblemDescription::ProblemDescription( + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* ioAttr, + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* affineAttr, + std::optional statAttr, + Direction direction) + : _direction(direction) + , _normalizedDim(guessNormalizedDim(ioAttr, affineAttr, statAttr)) + , _outerSize(1) + , _innerSize(1) + , _stride(1) +{ + const auto* dims = ioAttr->dims(); + const auto* strides = ioAttr->strides(); + const auto strideOrder = hipdnn_data_sdk::utilities::extractStrideOrder( + std::vector(strides->begin(), strides->end())); + const auto layoutNHWC = hipdnn_data_sdk::utilities::TensorLayout::NHWC; + const auto layoutNDHWC = hipdnn_data_sdk::utilities::TensorLayout::NDHWC; + + if(_normalizedDim > 1 + && (strideOrder == layoutNHWC.strideOrder || strideOrder == layoutNDHWC.strideOrder)) + { + _stride = dims->Get(1); + } + + for(unsigned int i = 0; i < dims->size(); ++i) + { + if(i < _normalizedDim) + { + if(_stride == 1 || i != 1) + { + _outerSize *= dims->Get(i); + } + } + else + { + _innerSize *= dims->Get(i); + } + } +} + } // namespace hip_kernel_provider::layernorm diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp index 0c87e92e2c74..a5d3dfa1c39f 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp @@ -12,6 +12,50 @@ namespace hip_kernel_provider::layernorm { +enum class Direction +{ + FORWARD, + BACKWARD +}; + +class ProblemDescription +{ +public: + ProblemDescription( + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* ioAttr, + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* affineAttr, + std::optional statAttr, + Direction direction); + + Direction direction() const + { + return _direction; + } + size_t normalizedDim() const + { + return _normalizedDim; + } + int64_t outerSize() const + { + return _outerSize; + } + int64_t innerSize() const + { + return _innerSize; + } + int64_t stride() const + { + return _stride; + } + +private: + Direction _direction; + size_t _normalizedDim; + int64_t _outerSize; + int64_t _innerSize; + int64_t _stride; +}; + size_t getMinNormalizedDimFromAffine( const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* ioAttr, const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* affineAttr); diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Batchnorm/TestBatchnormCommon.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Batchnorm/TestBatchnormCommon.cpp index ae36f3b41039..8de6d9ff8a91 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Batchnorm/TestBatchnormCommon.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Batchnorm/TestBatchnormCommon.cpp @@ -9,6 +9,51 @@ using namespace hip_kernel_provider::batchnorm; +namespace +{ + +ProblemDescription makeProblem(size_t n, + size_t c, + size_t h, + size_t w, + bool isLayoutNHWC, + bool isFp32, + size_t minWorkgroups = 1, + Direction direction = Direction::FORWARD_TRAINING) +{ + return {n, c, h, w, isLayoutNHWC, !isFp32, false, direction, minWorkgroups}; +} + +} // namespace + +TEST(TestProblemDescription, ExposesProblemProperties) +{ + const ProblemDescription problem(32, 64, 8, 16, true, false, true, Direction::BACKWARD, 42); + + EXPECT_EQ(problem.n(), 32); + EXPECT_EQ(problem.c(), 64); + EXPECT_EQ(problem.h(), 8); + EXPECT_EQ(problem.w(), 16); + EXPECT_TRUE(problem.isLayoutNHWC()); + EXPECT_FALSE(problem.useFp16Mix()); + EXPECT_TRUE(problem.useBfp16Mix()); + EXPECT_FALSE(problem.useFp32()); + EXPECT_EQ(problem.direction(), Direction::BACKWARD); + EXPECT_EQ(problem.minWorkgroups(), 42); +} + +TEST(TestProblemDescription, ComputesDerivedDimensions) +{ + const ProblemDescription problem( + 32, 64, 8, 16, false, false, false, Direction::FORWARD_TRAINING, 1); + + EXPECT_TRUE(problem.useFp32()); + EXPECT_EQ(problem.inCstride(), 128); + EXPECT_EQ(problem.inNhw(), 4096); + EXPECT_EQ(problem.inChw(), 8192); + EXPECT_EQ(problem.inNchw(), 262144); +} + // ============================================================================ // getLocalConfigNHWC - computes local workgroup size for NHWC layout // ============================================================================ @@ -18,7 +63,7 @@ TEST(TestGetLocalConfigNhwcFp32, ProducesSquareWorkgroup) size_t x = 0; size_t y = 0; - getLocalConfigNHWC(128, 16, 16, true, 1, 4, x, y); + getLocalConfigNHWC(makeProblem(1, 128, 16, 16, true, true), 4, x, y); EXPECT_EQ(x, 16); EXPECT_EQ(y, 16); } @@ -28,7 +73,7 @@ TEST(TestGetLocalConfigNhwc, MixedPrecisionUsesWiderX) size_t x = 0; size_t y = 0; - getLocalConfigNHWC(128, 16, 16, false, 1, 4, x, y); + getLocalConfigNHWC(makeProblem(1, 128, 16, 16, true, false), 4, x, y); EXPECT_EQ(x, 32); EXPECT_EQ(y, 8); } @@ -39,7 +84,7 @@ TEST(TestGetLocalConfigNhwc, HandlesVectorsizeScaling) size_t y = 0; // With vectorsize of 8, the maxlocalsize should be reduced - getLocalConfigNHWC(1024, 16, 16, true, 1, 8, x, y); + getLocalConfigNHWC(makeProblem(1, 1024, 16, 16, true, true), 8, x, y); EXPECT_EQ(x, 16); EXPECT_EQ(y, 8); } @@ -48,12 +93,12 @@ TEST(TestGetLocalConfigNhwc, HandlesIncreasedMinWorkgroups) { size_t xSmallWG = 0; size_t ySmallWG = 0; - getLocalConfigNHWC(64, 64, 64, true, 10, 1, xSmallWG, ySmallWG); + getLocalConfigNHWC(makeProblem(1, 64, 64, 64, true, true, 10), 1, xSmallWG, ySmallWG); // With high minWorkgroups requirement, the workgroup size should be reduced size_t xLargeWG = 0; size_t yLargeWG = 0; - getLocalConfigNHWC(64, 64, 64, true, 10000, 1, xLargeWG, yLargeWG); + getLocalConfigNHWC(makeProblem(1, 64, 64, 64, true, true, 10000), 1, xLargeWG, yLargeWG); EXPECT_EQ(xSmallWG, xLargeWG); EXPECT_LT(yLargeWG, ySmallWG); @@ -64,7 +109,7 @@ TEST(TestGetLocalConfigNhwc, HandlesSmallChannels) size_t x = 0; size_t y = 0; - getLocalConfigNHWC(2, 16, 16, true, 1, 4, x, y); + getLocalConfigNHWC(makeProblem(1, 2, 16, 16, true, true), 4, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 128); } @@ -74,7 +119,7 @@ TEST(TestGetLocalConfigNhwc, HandlesSmallInitialMaxLocalsize) size_t x = 0; size_t y = 0; - getLocalConfigNHWC(128, 16, 16, true, 1, 4, x, y); + getLocalConfigNHWC(makeProblem(1, 128, 16, 16, true, true), 4, x, y); EXPECT_EQ(x, 16); EXPECT_EQ(y, 16); } @@ -88,7 +133,7 @@ TEST(TestGetSpatialMultipleConfig, NhwcReturnsDefaultOnMisalignment) size_t x = 0; size_t y = 0; - getSpatialMultipleConfig(3, 16, 16, true, true, 80, 4, x, y); + getSpatialMultipleConfig(makeProblem(1, 3, 16, 16, true, true, 80), 4, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 1); } @@ -98,7 +143,7 @@ TEST(TestGetSpatialMultipleConfig, NchwReturnsDefaultOnMisalignment) size_t x = 0; size_t y = 0; - getSpatialMultipleConfig(64, 3, 5, false, true, 80, 4, x, y); + getSpatialMultipleConfig(makeProblem(1, 64, 3, 5, false, true, 80), 4, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 1); } @@ -108,7 +153,7 @@ TEST(TestGetSpatialMultipleConfig, NhwcCalculatesWorkgroupSize) size_t x = 0; size_t y = 0; - getSpatialMultipleConfig(64, 16, 16, true, true, 80, 4, x, y); + getSpatialMultipleConfig(makeProblem(1, 64, 16, 16, true, true, 80), 4, x, y); EXPECT_EQ(x, 16); EXPECT_EQ(y, 8); } @@ -118,7 +163,7 @@ TEST(TestGetSpatialMultipleConfig, NchwHandlesLargeSpatial) size_t x = 0; size_t y = 0; - getSpatialMultipleConfig(64, 64, 64, false, true, 80, 1, x, y); + getSpatialMultipleConfig(makeProblem(1, 64, 64, 64, false, true, 80), 1, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 1024); } @@ -128,11 +173,11 @@ TEST(TestGetSpatialMultipleConfig, NchwScalesDownForSmallSpatial) size_t x = 0; size_t y = 0; - getSpatialMultipleConfig(64, 8, 16, false, true, 80, 1, x, y); + getSpatialMultipleConfig(makeProblem(1, 64, 8, 16, false, true, 80), 1, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 128); - getSpatialMultipleConfig(64, 4, 8, false, true, 80, 1, x, y); + getSpatialMultipleConfig(makeProblem(1, 64, 4, 8, false, true, 80), 1, x, y); EXPECT_EQ(x, 1); EXPECT_EQ(y, 64); } @@ -144,37 +189,43 @@ TEST(TestGetSpatialMultipleConfig, NchwScalesDownForSmallSpatial) TEST(TestIsSpatialMultipleApplicable, NhwcVectorAlignmentFail) { // If C is not divisible by vectorsize with NHWC layout, it should fail immediately - EXPECT_FALSE(isSpatialMultipleApplicable(64, 63, 16, 16, true, true, 4, 32, 64, 1, 64)); + EXPECT_FALSE( + isSpatialMultipleApplicable(makeProblem(64, 63, 16, 16, true, true), 4, 32, 64, 1, 64)); } TEST(TestIsSpatialMultipleApplicable, NchwSpatialAlignmentFail) { // If H*W is not divisible by vectorsize with NCHW layout, it should fail immediately - EXPECT_FALSE(isSpatialMultipleApplicable(64, 64, 3, 5, false, true, 4, 32, 64, 1, 64)); + EXPECT_FALSE( + isSpatialMultipleApplicable(makeProblem(64, 64, 3, 5, false, true), 4, 32, 64, 1, 64)); } TEST(TestIsSpatialMultipleApplicable, StashFitsInSpatialDimension) { // The last block of spatial dimension is large enough to hold stashed values - EXPECT_TRUE(isSpatialMultipleApplicable(64, 64, 16, 16, true, true, 4, 32, 64, 1, 64)); + EXPECT_TRUE( + isSpatialMultipleApplicable(makeProblem(64, 64, 16, 16, true, true), 4, 32, 64, 1, 64)); } TEST(TestIsSpatialMultipleApplicable, StashFitsInBatchDimension) { // Even if the spatial remainder is small, it works if the batch remainder is large enough - EXPECT_TRUE(isSpatialMultipleApplicable(128, 64, 2, 5, true, true, 1, 32, 64, 1, 64)); + EXPECT_TRUE( + isSpatialMultipleApplicable(makeProblem(128, 64, 2, 5, true, true), 1, 32, 64, 1, 64)); } TEST(TestIsSpatialMultipleApplicable, StashDoesNotFitInSpatialOrBatchDimension) { // Both spatial and batch remainders are smaller than the required stash values - EXPECT_FALSE(isSpatialMultipleApplicable(10, 64, 2, 5, true, true, 1, 32, 64, 1, 64)); + EXPECT_FALSE( + isSpatialMultipleApplicable(makeProblem(10, 64, 2, 5, true, true), 1, 32, 64, 1, 64)); } TEST(TestIsSpatialMultipleApplicable, MixedPrecisionOddCFailsOnSmallZ) { // For FP16/BF16, if C is odd, the intermediate results MUST fit in the batch dimension - EXPECT_FALSE(isSpatialMultipleApplicable(10, 65, 16, 16, true, false, 1, 16, 64, 1, 64)); + EXPECT_FALSE( + isSpatialMultipleApplicable(makeProblem(10, 65, 16, 16, true, false), 1, 16, 64, 1, 64)); } // ==================================================================================================== @@ -183,32 +234,34 @@ TEST(TestIsSpatialMultipleApplicable, MixedPrecisionOddCFailsOnSmallZ) TEST(TestUseMultiple, BackwardSmallSpatialReturnsFalse) { - EXPECT_FALSE(useMultiple(64, 16, 16, false, false, Direction::BACKWARD)); + EXPECT_FALSE(useMultiple(makeProblem(64, 1, 16, 16, false, true, 1, Direction::BACKWARD))); } TEST(TestUseMultiple, BackwardLargeProblemReturnsTrue) { - EXPECT_TRUE(useMultiple(64, 1024, 1024, false, false, Direction::BACKWARD)); + EXPECT_TRUE(useMultiple(makeProblem(64, 1, 1024, 1024, false, true, 1, Direction::BACKWARD))); } TEST(TestUseMultiple, ForwardTrainingSmallProblemReturnsFalse) { - EXPECT_FALSE(useMultiple(2, 8, 8, false, false, Direction::FORWARD_TRAINING)); + EXPECT_FALSE(useMultiple(makeProblem(2, 1, 8, 8, false, true, 1, Direction::FORWARD_TRAINING))); } TEST(TestUseMultiple, ForwardTrainingLargeBatchReturnsTrue) { - EXPECT_TRUE(useMultiple(1024, 32, 32, false, false, Direction::FORWARD_TRAINING)); + EXPECT_TRUE( + useMultiple(makeProblem(1024, 1, 32, 32, false, true, 1, Direction::FORWARD_TRAINING))); } TEST(TestUseMultiple, ForwardTrainingMixedPrecisionHeuristic) { - EXPECT_FALSE(useMultiple(128, 32, 32, true, false, Direction::FORWARD_TRAINING)); + EXPECT_FALSE( + useMultiple(makeProblem(128, 1, 32, 32, false, false, 1, Direction::FORWARD_TRAINING))); } TEST(TestUseMultiple, NhwcAlwaysReturnsTrue) { - EXPECT_TRUE(useMultiple(2, 8, 8, false, true, Direction::FORWARD_TRAINING)); + EXPECT_TRUE(useMultiple(makeProblem(2, 1, 8, 8, true, true))); } // ==================================================================================================== @@ -217,22 +270,22 @@ TEST(TestUseMultiple, NhwcAlwaysReturnsTrue) TEST(TestGetStashMethod, ReturnsMethodZeroWhenSpatialFits) { - EXPECT_EQ(getStashMethod(false, true, 32, 64, 64, 1024, 1024, 1, 64), 0); + EXPECT_EQ(getStashMethod(makeProblem(64, 64, 32, 32, false, true), 32, 1024, 1, 64), 0); } TEST(TestGetStashMethod, ReturnsMethodOneWhenBatchStashRequired) { - EXPECT_EQ(getStashMethod(false, true, 32, 64, 64, 10, 64, 1, 64), 1); + EXPECT_EQ(getStashMethod(makeProblem(64, 64, 2, 5, false, true), 32, 64, 1, 64), 1); } TEST(TestGetStashMethod, ReturnsMethodTwoForNhwcOddCMixedPrecision) { - EXPECT_EQ(getStashMethod(true, false, 16, 65, 64, 1024, 1024, 1, 64), 2); + EXPECT_EQ(getStashMethod(makeProblem(64, 65, 32, 32, true, false), 16, 1024, 1, 64), 2); } TEST(TestGetStashMethod, MixedPrecisionScalesStashValues) { - EXPECT_EQ(getStashMethod(false, false, 16, 64, 64, 20, 64, 1, 64), 1); + EXPECT_EQ(getStashMethod(makeProblem(64, 64, 4, 5, false, false), 16, 64, 1, 64), 1); } // ==================================================================================================== @@ -243,7 +296,8 @@ TEST(TestDefaultConfigSpatialSingle, NhwcDefaultVariantOne) { KernelConfig config; - defaultConfigSpatialSingle(64, 16, 16, false, false, true, Direction::BACKWARD, config); + defaultConfigSpatialSingle( + ProblemDescription(64, 1, 16, 16, true, false, false, Direction::BACKWARD, 1), config); EXPECT_EQ(config.variant, 1); EXPECT_EQ(config.vectorsize, 1); } @@ -252,7 +306,8 @@ TEST(TestDefaultConfigSpatialSingle, NchwBackwardSmallSpatialSmallBatch) { KernelConfig config; - defaultConfigSpatialSingle(32, 16, 16, false, false, false, Direction::BACKWARD, config); + defaultConfigSpatialSingle( + ProblemDescription(32, 1, 16, 16, false, false, false, Direction::BACKWARD, 1), config); EXPECT_EQ(config.variant, 0); EXPECT_EQ(config.vectorsize, 1); } @@ -261,7 +316,8 @@ TEST(TestDefaultConfigSpatialSingle, NchwBackwardSmallSpatialLargeBatch) { KernelConfig config; - defaultConfigSpatialSingle(128, 20, 10, false, false, false, Direction::BACKWARD, config); + defaultConfigSpatialSingle( + ProblemDescription(128, 1, 20, 10, false, false, false, Direction::BACKWARD, 1), config); EXPECT_EQ(config.variant, 3); EXPECT_EQ(config.vectorsize, 1); } @@ -270,7 +326,8 @@ TEST(TestDefaultConfigSpatialSingle, NchwBackwardMidSpatialSmallBatch) { KernelConfig config; - defaultConfigSpatialSingle(16, 20, 30, false, false, false, Direction::BACKWARD, config); + defaultConfigSpatialSingle( + ProblemDescription(16, 1, 20, 30, false, false, false, Direction::BACKWARD, 1), config); EXPECT_EQ(config.variant, 3); EXPECT_EQ(config.vectorsize, 1); } @@ -279,7 +336,9 @@ TEST(TestDefaultConfigSpatialSingle, NchwForwardLargeSpatialOrMixed) { KernelConfig config; - defaultConfigSpatialSingle(64, 32, 32, true, false, false, Direction::FORWARD_TRAINING, config); + defaultConfigSpatialSingle( + ProblemDescription(64, 1, 32, 32, false, true, false, Direction::FORWARD_TRAINING, 1), + config); EXPECT_EQ(config.variant, 1); EXPECT_EQ(config.vectorsize, 1); } @@ -288,7 +347,9 @@ TEST(TestDefaultConfigSpatialSingle, NchwForwardSmallDefault) { KernelConfig config; - defaultConfigSpatialSingle(10, 8, 8, false, false, false, Direction::FORWARD_TRAINING, config); + defaultConfigSpatialSingle( + ProblemDescription(10, 1, 8, 8, false, false, false, Direction::FORWARD_TRAINING, 1), + config); EXPECT_EQ(config.variant, 0); EXPECT_EQ(config.vectorsize, 1); } @@ -301,7 +362,7 @@ TEST(TestDefaultConfigSpatialMultiple, NhwcFullConfigCheck) { KernelConfig config; - defaultConfigSpatialMultiple(128, 64, 16, 16, true, true, 80, 32, config); + defaultConfigSpatialMultiple(makeProblem(128, 64, 16, 16, true, true, 80), 32, config); EXPECT_EQ(config.variant, 2); EXPECT_EQ(config.vectorsize, 4); EXPECT_EQ(config.nelements, 128); @@ -314,7 +375,7 @@ TEST(TestDefaultConfigSpatialMultiple, NhwcFallbackConfigCheck) { KernelConfig config; - defaultConfigSpatialMultiple(64, 25, 16, 16, true, true, 80, 32, config); + defaultConfigSpatialMultiple(makeProblem(64, 25, 16, 16, true, true, 80), 32, config); EXPECT_EQ(config.variant, 2); EXPECT_EQ(config.vectorsize, 1); EXPECT_EQ(config.nelements, 64); @@ -327,7 +388,7 @@ TEST(TestDefaultConfigSpatialMultiple, NchwFullConfigCheck) { KernelConfig config; - defaultConfigSpatialMultiple(128, 64, 16, 16, false, true, 80, 32, config); + defaultConfigSpatialMultiple(makeProblem(128, 64, 16, 16, false, true, 80), 32, config); EXPECT_EQ(config.variant, 2); EXPECT_EQ(config.vectorsize, 1); EXPECT_EQ(config.nelements, 128); @@ -340,7 +401,7 @@ TEST(TestDefaultConfigSpatialMultiple, NchwSingleVectorConfig) { KernelConfig config; - defaultConfigSpatialMultiple(16, 32, 1, 2, false, true, 80, 1, config); + defaultConfigSpatialMultiple(makeProblem(16, 32, 1, 2, false, true, 80), 1, config); EXPECT_EQ(config.variant, 2); EXPECT_EQ(config.vectorsize, 1); EXPECT_EQ(config.nelements, 16); @@ -353,7 +414,7 @@ TEST(TestDefaultConfigSpatialMultiple, NchwOddSpatialFallback) { KernelConfig config; - defaultConfigSpatialMultiple(64, 64, 3, 5, false, true, 80, 32, config); + defaultConfigSpatialMultiple(makeProblem(64, 64, 3, 5, false, true, 80), 32, config); EXPECT_EQ(config.variant, 2); EXPECT_EQ(config.vectorsize, 1); EXPECT_EQ(config.nelements, 64); @@ -367,7 +428,7 @@ TEST(TestDefaultConfigSpatialMultiple, NoConfigAssignedOnFailure) KernelConfig config; // StashValues requirement 12345 is impossible to satisfy - defaultConfigSpatialMultiple(1, 1, 1, 1, false, true, 80, 12345, config); + defaultConfigSpatialMultiple(makeProblem(1, 1, 1, 1, false, true, 80), 12345, config); EXPECT_EQ(config.variant, -1); } @@ -387,7 +448,7 @@ TEST(TestBatchnormBwdStashFix, NotSavedValueRejectsAsanShapeFromVariant2) // dx at {2,3,1,1} holds 2 stash fields, not 4, so a 4-field request (what a // stats-computing backward pass needs) makes the gate reject the reduction path. - defaultConfigSpatialMultiple(2, 3, 1, 1, true, true, 1, 4, config); + defaultConfigSpatialMultiple(makeProblem(2, 3, 1, 1, true, true), 4, config); EXPECT_NE(config.variant, 2); } @@ -397,7 +458,7 @@ TEST(TestBatchnormBwdStashFix, OldValueAcceptedAsanShapeIntoVariant2) // A 2-field request fits dx at this shape, so the gate selects the reduction path. // This is the boundary the field count must clear: 2 fits where 4 does not. - defaultConfigSpatialMultiple(2, 3, 1, 1, true, true, 1, 2, config); + defaultConfigSpatialMultiple(makeProblem(2, 3, 1, 1, true, true), 2, config); EXPECT_EQ(config.variant, 2); } @@ -407,6 +468,7 @@ TEST(TestBatchnormBwdStashFix, ApplicabilityGateFlipsOnStashValues) // resolves to ylocalsize=256, zlocalsize=1, nelements=2, whose last block holds 2 // stash fields but not 4, so the gate accepts a 2-field request and rejects a // 4-field one. - EXPECT_TRUE(isSpatialMultipleApplicable(2, 3, 1, 1, true, true, 1, 2, 256, 1, 2)); - EXPECT_FALSE(isSpatialMultipleApplicable(2, 3, 1, 1, true, true, 1, 4, 256, 1, 2)); + const auto problem = makeProblem(2, 3, 1, 1, true, true); + EXPECT_TRUE(isSpatialMultipleApplicable(problem, 1, 2, 256, 1, 2)); + EXPECT_FALSE(isSpatialMultipleApplicable(problem, 1, 4, 256, 1, 2)); } diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Layernorm/TestLayernormFwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Layernorm/TestLayernormFwdPlan.cpp index fad999f03426..e4220bd84372 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Layernorm/TestLayernormFwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/Layernorm/TestLayernormFwdPlan.cpp @@ -3,6 +3,7 @@ #include "../TestPlanCommon.hpp" #include "engines/hip_mlops_engine/plans/layernorm/LayernormFwdPlan.hpp" +#include "engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp" #include "mocks/MockCompiledProgram.hpp" #include "mocks/MockKernelCompiler.hpp" #include "mocks/MockRunnableKernel.hpp" @@ -92,6 +93,23 @@ TEST(TestLayernormFwdParams, IsNotCopyConstructible) EXPECT_FALSE(std::is_copy_constructible_v); } +TEST(TestLayernormProblemDescription, ComputesExecutionGeometry) +{ + auto builder = hipdnn_test_sdk::utilities::createValidLayernormFpropGraph(); + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper graph( + builder.GetBufferPointer(), builder.GetSize()); + const auto& attr = *graph.getNode(0).attributes_as_LayernormAttributes(); + const LayernormFwdParams params(attr, graph.getTensorMap()); + + const ProblemDescription problem(params.x(), params.scale(), params.mean(), Direction::FORWARD); + + EXPECT_EQ(problem.direction(), Direction::FORWARD); + EXPECT_EQ(problem.normalizedDim(), 1); + EXPECT_EQ(problem.outerSize(), 1); + EXPECT_EQ(problem.innerSize(), 588); + EXPECT_EQ(problem.stride(), 1); +} + namespace { diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/RMSnorm/TestRMSnormFwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/RMSnorm/TestRMSnormFwdPlan.cpp index 4ab3bcd7d48b..c657d6fbcc62 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/RMSnorm/TestRMSnormFwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/hip_mlops_engine/plans/RMSnorm/TestRMSnormFwdPlan.cpp @@ -7,6 +7,7 @@ #include +#include "engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp" #include "engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.hpp" #include "mocks/MockCompiledProgram.hpp" #include "mocks/MockKernelCompiler.hpp" @@ -90,6 +91,23 @@ TEST(TestRMSnormFwdParams, IsNotCopyConstructible) { EXPECT_FALSE(std::is_copy_constructible_v); } + +TEST(TestRMSnormProblemDescription, ComputesExecutionGeometry) +{ + auto builder = hipdnn_test_sdk::utilities::createValidRMSNormGraph(); + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper graph( + builder.GetBufferPointer(), builder.GetSize()); + const auto& attr = *graph.getNode(0).attributes_as_RMSNormAttributes(); + const RMSnormFwdParams params(attr, graph.getTensorMap()); + + const ProblemDescription problem(params.x(), params.scale(), Direction::FORWARD); + + EXPECT_EQ(problem.direction(), Direction::FORWARD); + EXPECT_EQ(problem.normalizeDim(), 1); + EXPECT_EQ(problem.outerSize(), 1); + EXPECT_EQ(problem.innerSize(), 150528); + EXPECT_EQ(problem.stride(), 1); +} // ============================================================================ // RMSnormFwdPlan - helpers // ============================================================================ From 49011c7e58529b8fbd96a769ae1da33db8927b15 Mon Sep 17 00:00:00 2001 From: BalintCsala Date: Wed, 16 Sep 2026 09:49:31 +0000 Subject: [PATCH 2/2] Apply suggestions --- .../plans/RMSnorm/RMSnormCommon.hpp | 112 ++++++++++-------- .../plans/RMSnorm/RMSnormFwdPlan.cpp | 17 --- .../plans/batchnorm/BatchnormBwdPlan.cpp | 18 +-- .../plans/layernorm/LayernormUtilities.cpp | 3 - .../plans/layernorm/LayernormUtilities.hpp | 6 +- 5 files changed, 77 insertions(+), 79 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp index 8dcc137d0005..df625c4abf71 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormCommon.hpp @@ -15,65 +15,83 @@ enum class Direction BACKWARD }; -inline int64_t - getOuterSize(const flatbuffers::Vector* xDims, unsigned normalizeDim, int64_t stride) +// Infer the outer and inner normalization sizes. +// 1) Work out the normalization dimension, as the index of the first dimension in +// scale tensor that is not 1. +// 2) Work out the outerSize as the size of input dimensions for which scale is 1. +// We will have a work-group for each of these dimensions. +// When stride is not 1, we are in a channel-last layout and we ignore the +// channel dimension when calculating the outer size. +// 3) Work out the innerSize as the size of the input dimensions for which scale is not 1. +// +// For an input of [N, C, H, W] with scale [1, C, H, W] this will give: a normalization +// dimension of 1, outerSize of N, and innerSize of CxHxW. +// The kernel will therefore consist of N workgroups with each workgroup normalizing over +// CxHxW elements using a fixed number of threads. +// For an input of [N, H, W, C] with scale [1, H, W, 1] this will give: a normalization +// dimension of 2, outerSize of N, stride of C, and innerSize of HxW. +// The kernel will therefore consist of NxC workgroups with each workgroup normalizing +// over HxW elements using a fixed number of threads. +class ProblemDescription { - int64_t outerSize = 1; - for(unsigned i = 0; i < normalizeDim; ++i) + static unsigned getNormalizeDim(const flatbuffers::Vector* xDims, + const flatbuffers::Vector* scaleDims) + { + const std::vector xDimsVec(xDims->begin(), xDims->end()); + const std::vector scaleDimsVec(scaleDims->begin(), scaleDims->end()); + + // Find number of trailing dims where scaleDims[i] == inputDims[i] + const auto [scaleMismatch, _] = std::mismatch( + scaleDimsVec.rbegin(), scaleDimsVec.rend(), xDimsVec.rbegin(), xDimsVec.rend()); + const auto matchCount + = static_cast(std::distance(scaleDimsVec.rbegin(), scaleMismatch)); + + // Scale must have at least one normalization axis, so account for the case where + // input has a single batch and scale matches exactly. + const auto normalizeDim + = (matchCount == scaleDimsVec.size()) ? 1 : scaleDimsVec.size() - matchCount; + return static_cast(normalizeDim); + } + + static int64_t getStride(const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* x, + unsigned normalizeDim) { - // Add channel size only if there is no stride - if(i == 1 && stride != 1) + int64_t stride = 1; + const auto isLayoutNHWC = core::utils::isChannelLastLayout(x); + if(normalizeDim > 1 && isLayoutNHWC) { - continue; + stride = static_cast(x->dims()->Get(1)); } - outerSize *= static_cast(xDims->Get(i)); + return stride; } - return outerSize; -} -inline int64_t getInnerSize(const flatbuffers::Vector* xDims, unsigned normalizeDim) -{ - int64_t innerSize = 1; - for(unsigned i = normalizeDim; i < xDims->size(); ++i) + static int64_t getOuterSize(const flatbuffers::Vector* xDims, + unsigned normalizeDim, + int64_t stride) { - innerSize *= xDims->Get(i); + int64_t outerSize = 1; + for(unsigned i = 0; i < normalizeDim; ++i) + { + // Add channel size only if there is no stride + if(i == 1 && stride != 1) + { + continue; + } + outerSize *= static_cast(xDims->Get(i)); + } + return outerSize; } - return innerSize; -} -inline int64_t getStride(const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* x, - unsigned normalizeDim) -{ - int64_t stride = 1; - auto isLayoutNHWC = core::utils::isChannelLastLayout(x); - if(normalizeDim > 1 && isLayoutNHWC) + static int64_t getInnerSize(const flatbuffers::Vector* xDims, unsigned normalizeDim) { - stride = static_cast(x->dims()->Get(1)); + int64_t innerSize = 1; + for(unsigned i = normalizeDim; i < xDims->size(); ++i) + { + innerSize *= xDims->Get(i); + } + return innerSize; } - return stride; -} - -inline unsigned getNormalizeDim(const flatbuffers::Vector* xDims, - const flatbuffers::Vector* scaleDims) -{ - const std::vector xDimsVec(xDims->begin(), xDims->end()); - const std::vector scaleDimsVec(scaleDims->begin(), scaleDims->end()); - // Find number of trailing dims where scaleDims[i] == inputDims[i] - const auto [scaleMismatch, _] = std::mismatch( - scaleDimsVec.rbegin(), scaleDimsVec.rend(), xDimsVec.rbegin(), xDimsVec.rend()); - const auto matchCount - = static_cast(std::distance(scaleDimsVec.rbegin(), scaleMismatch)); - - // Scale must have at least one normalization axis, so account for the case where input has a single batch and scale - // matches exactly. - const auto normalizeDim - = (matchCount == scaleDimsVec.size()) ? 1 : scaleDimsVec.size() - matchCount; - return static_cast(normalizeDim); -} - -class ProblemDescription -{ public: ProblemDescription(const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* x, const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* scale, diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp index 47cabed04599..5bdc74ecf6b4 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/RMSnorm/RMSnormFwdPlan.cpp @@ -96,23 +96,6 @@ void RMSnormFwdPlan::compile(const IKernelCompiler& kernelCompiler, + std::to_string(xRank)); } - // Infer the outer and inner normalization sizes. - // 1) Work out the normalization dimension, as the index of the first dimension in - // scale tensor that is not 1. - // 2) Work out the outerSize as the size of input dimensions for which scale is 1. - // We will have a work-group for each of these dimensions. - // When stride is not 1, we are in a channel-last layout and we ignore the - // channel dimension when calculating the outer size. - // 3) Work out the innerSize as the size of the intput dimensions for which scale nor 1. - // - // For an input of [N, C, H, W] with scale [1, C, H, W] this will give: a normalization - // dimension of 1, outerSize of N, and innerSize of CxHxW. - // The kernel will therefore consist of N workgroups with each workgroup normalizing over - // CxHxW elements using a fixed number of threads. - // For an input of [N, H, W, C] with scale [1, H, W, 1] this will give: a normalization - // dimension of 2, outerSize of N, stride of C, and innerSize of HxW. - // The kernel will therefore consist of NxC workgroups with each workgroup normalizing - // over HxW elements using a fixed number of threads. const ProblemDescription problem(_params.x(), _params.scale(), Direction::FORWARD); const int64_t stride = problem.stride(); const int64_t outerSize = problem.outerSize(); diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp index 010a79bfcd78..c31d3f484cc7 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/batchnorm/BatchnormBwdPlan.cpp @@ -55,15 +55,15 @@ static ProblemDescription extractProblemDescription(const BatchnormBwdParams& pa + std::to_string(xDims->size())); } - return ProblemDescription(n, - c, - h, - w, - isChannelLastLayout(params.x()), - useFp16Mix, - useBfp16Mix, - Direction::BACKWARD, - 1); + return {n, + c, + h, + w, + isChannelLastLayout(params.x()), + useFp16Mix, + useBfp16Mix, + Direction::BACKWARD, + 1}; } BatchnormBwdParams::BatchnormBwdParams( diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp index 6ba039c7db7d..7c7e9252965a 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.cpp @@ -229,9 +229,6 @@ ProblemDescription::ProblemDescription( Direction direction) : _direction(direction) , _normalizedDim(guessNormalizedDim(ioAttr, affineAttr, statAttr)) - , _outerSize(1) - , _innerSize(1) - , _stride(1) { const auto* dims = ioAttr->dims(); const auto* strides = ioAttr->strides(); diff --git a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp index a5d3dfa1c39f..602d1dd89907 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/layernorm/LayernormUtilities.hpp @@ -51,9 +51,9 @@ class ProblemDescription private: Direction _direction; size_t _normalizedDim; - int64_t _outerSize; - int64_t _innerSize; - int64_t _stride; + int64_t _outerSize{1}; + int64_t _innerSize{1}; + int64_t _stride{1}; }; size_t getMinNormalizedDimFromAffine(