From 78f07059611f3028cd75b5e1819e6ff22dbc985b Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Mon, 15 Dec 2025 22:04:35 +0100 Subject: [PATCH 01/33] Porting OpTensorLeadingOnes kernels to HIP --- .../miopen/src/kernels/MIOpenTensorKernels.cl | 16 - .../src/kernels/MIOpenTensorKernelsHip.cpp | 228 ++++++++- .../test/gtest/tensor_4d_generic_ocl_hip.cpp | 227 ++++----- .../gtest/tensor_fwd_bias_generic_ocl_hip.cpp | 39 +- .../test/gtest/tensor_fwd_bias_ocl_hip.cpp | 51 +- .../tensor_leading_ones_generic_ocl_hip.cpp | 454 ++++++++++++++++++ .../gtest/tensor_leading_ones_ocl_hip.cpp | 449 +++++++++++++++++ 7 files changed, 1277 insertions(+), 187 deletions(-) create mode 100644 projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp create mode 100644 projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp diff --git a/projects/miopen/src/kernels/MIOpenTensorKernels.cl b/projects/miopen/src/kernels/MIOpenTensorKernels.cl index b885e1e7b1b9..cebbe16d9081 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernels.cl +++ b/projects/miopen/src/kernels/MIOpenTensorKernels.cl @@ -50,7 +50,6 @@ MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); } #ifdef USE_FWD_BIAS - __kernel void OpTensorFwdBias(global MIOPEN_TYPE* a, global MIOPEN_TYPE* b, const int b_c, @@ -125,11 +124,9 @@ __kernel void OpTensorFwdBias(global MIOPEN_TYPE* a, } } } - #endif #ifdef USE_FWD_BIAS_GENERIC - __kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -211,12 +208,10 @@ __kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a, } } } - #endif // DLOWELL : cutting out this section #ifdef USE_LEADING_ONES - __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, global MIOPEN_TYPE* b, global MIOPEN_TYPE* c, @@ -235,11 +230,9 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, const int num_wg, const unsigned int bitmap) { - /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value */ - global MIOPEN_TYPE* a_off = a + Aoffset; global MIOPEN_TYPE* b_off = b + Boffset; global MIOPEN_TYPE* c_off = c + Coffset; @@ -303,12 +296,9 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, } } } - #endif -// DLOWELL : cutting out this section #ifdef USE_LEADING_ONES_GENERIC - __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -334,7 +324,6 @@ __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, const int num_wg, const unsigned int bitmap) { - /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value */ @@ -426,11 +415,9 @@ __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, } } } - #endif #ifdef USE_4D_TENSOR_GENERIC - __kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -538,7 +525,6 @@ __kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a, } } } - #endif #ifdef USE_5D_TENSOR_GENERIC @@ -671,7 +657,6 @@ __kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a, } } } - #endif #ifdef USE_2D_TENSOR_LITE @@ -781,5 +766,4 @@ __kernel void Op2dTensorLite(const global MIOPEN_TYPE* a, } } } - #endif diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 13333827068e..a475c05257f1 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -97,7 +97,6 @@ extern "C" __global__ void Op1dTensorGeneric(const MIOPEN_TYPE* a, c_ptr += c_step; } } - #endif #ifdef USE_2D_TENSOR_GENERIC @@ -154,7 +153,6 @@ extern "C" __global__ void Op2dTensorGeneric(const MIOPEN_TYPE* a, gid += step; } } - #endif #ifdef USE_2D_TENSOR_SQUASH @@ -420,7 +418,6 @@ extern "C" __global__ void Op3dTensorGeneric(const MIOPEN_TYPE* a, gid += step; } } - #endif #ifdef USE_4D_TENSOR_GENERIC @@ -643,9 +640,9 @@ extern "C" __global__ void OpTensorFwdBias(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, + const long Aoffset, + const long Boffset, + const long Coffset, const int num_wg, const int incr_wg) { @@ -727,9 +724,9 @@ extern "C" __global__ void OpTensorFwdBiasGeneric(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, const int work_per_wg, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, + const long Aoffset, + const long Boffset, + const long Coffset, const int num_wg, const int incr_wg) { @@ -796,3 +793,216 @@ extern "C" __global__ void OpTensorFwdBiasGeneric(MIOPEN_TYPE* a, } } #endif + +#ifdef USE_LEADING_ONES +extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, + MIOPEN_TYPE* b, + MIOPEN_TYPE* c, + const int c_c, + const int c_h, + const int c_w, + const int c_nstride, + const int c_cstride, + const int work_per_wg, + const MIOPEN_TYPE alpha0, + const MIOPEN_TYPE alpha1, + const MIOPEN_TYPE beta, + const long Aoffset, + const long Boffset, + const long Coffset, + const int num_wg, + const unsigned int bitmap) +{ + /* Special case for leading ones where the total no. of threads is the + * inner_product of the tensor dims. Each thread just updates one value + */ + + MIOPEN_TYPE* a_off = a + Aoffset; + MIOPEN_TYPE* b_off = b + Boffset; + MIOPEN_TYPE* c_off = c + Coffset; + + int gid = (bitmap == 0xF) ? (blockIdx.x * blockDim.x + threadIdx.x) : blockIdx.x; + + // num_wg: the number of workgroups should be launched + // MAX_NUM_WG: the maximum number of workgroups actually launched +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" + if(beta == static_cast(0)) +#pragma clang diagnostic pop + { + for(; gid < num_wg; gid += MAX_NUM_WG) + { + int lid = (bitmap == 0xF) ? 0 : threadIdx.x; + int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; + MIOPEN_TYPE operand = b_off[gid] * alpha1; + + int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; + int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; + int o_c = + (bitmap & (1 << 2)) + ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % + c_c) + : 0; + int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * + ((bitmap & (1 << 2)) ? c_c : 1)); + + while(lid < work_per_wg) + { + int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; + c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand); + lid += lcl_sz; + } + } + } + else + { + for(; gid < num_wg; gid += MAX_NUM_WG) + { + int lid = (bitmap == 0xF) ? 0 : threadIdx.x; + int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; + MIOPEN_TYPE operand = b_off[gid] * alpha1; + + int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; + int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; + int o_c = + (bitmap & (1 << 2)) + ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % + c_c) + : 0; + int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * + ((bitmap & (1 << 2)) ? c_c : 1)); + + while(lid < work_per_wg) + { + int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; + c_off[index] = + MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index]; + lid += lcl_sz; + } + } + } +} +#endif + +#ifdef USE_LEADING_ONES_GENERIC +extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, + const int a_nstride, + const int a_cstride, + const int a_hstride, + MIOPEN_TYPE* b, + const int b_nstride, + const int b_cstride, + const int b_hstride, + MIOPEN_TYPE* c, + const int c_c, + const int c_h, + const int c_w, + const int c_nstride, + const int c_cstride, + const int c_hstride, + const MIOPEN_TYPE alpha0, + const MIOPEN_TYPE alpha1, + const MIOPEN_TYPE beta, + const int work_per_wg, + const long Aoffset, + const long Boffset, + const long Coffset, + const int num_wg, + const unsigned int bitmap) +{ + /* Special case for leading ones where the total no. of threads is the + * inner_product of the tensor dims. Each thread just updates one value + */ + + MIOPEN_TYPE* a_off = a + Aoffset; + MIOPEN_TYPE* b_off = b + Boffset; + MIOPEN_TYPE* c_off = c + Coffset; + + int gid = (bitmap == 0xF) ? blockIdx.x * blockDim.x + threadIdx.x : blockIdx.x; + + // num_wg: the number of workgroups should be launched + // MAX_NUM_WG: the maximum number of workgroups actually launched +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" + if(beta == static_cast(0)) +#pragma clang diagnostic pop + { + for(; gid < num_wg; gid += MAX_NUM_WG) + { + int lid = (bitmap == 0xF) ? 0 : threadIdx.x; + int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; + + int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; + int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; + int o_c = + (bitmap & (1 << 2)) + ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % + c_c) + : 0; + int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * + ((bitmap & (1 << 2)) ? c_c : 1)); + + int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w; + MIOPEN_TYPE operand = b_off[bindex] * alpha1; + + while(lid < work_per_wg) + { + o_c = (bitmap & (1 << 2)) ? o_c : (lid % c_c); + o_h = (bitmap & (1 << 1)) + ? o_h + : ((bitmap & (1 << 2)) ? (lid / c_w) : ((lid / c_c) % c_h)); + o_w = (bitmap & (1 << 0)) + ? o_w + : ((bitmap & (1 << 1)) + ? lid + : ((bitmap & (1 << 2)) ? (lid % c_w) : ((lid / c_c) / c_h))); + int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w; + int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w; + c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand); + + lid += lcl_sz; + } + } + } + else + { + for(; gid < num_wg; gid += MAX_NUM_WG) + { + int lid = (bitmap == 0xF) ? 0 : threadIdx.x; + int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; + + int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; + int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; + int o_c = + (bitmap & (1 << 2)) + ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % + c_c) + : 0; + int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * + ((bitmap & (1 << 2)) ? c_c : 1)); + + int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w; + MIOPEN_TYPE operand = b_off[bindex] * alpha1; + + while(lid < work_per_wg) + { + o_c = (bitmap & (1 << 2)) ? o_c : (lid % c_c); + o_h = (bitmap & (1 << 1)) + ? o_h + : ((bitmap & (1 << 2)) ? (lid / c_w) : ((lid / c_c) % c_h)); + o_w = (bitmap & (1 << 0)) + ? o_w + : ((bitmap & (1 << 1)) + ? lid + : ((bitmap & (1 << 2)) ? (lid % c_w) : ((lid / c_c) / c_h))); + int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w; + int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w; + c_off[cindex] = + MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex]; + + lid += lcl_sz; + } + } + } +} +#endif diff --git a/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp index 4a4770c50669..fe0267dce0fb 100644 --- a/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (c) 2024 Advanced Micro Devices, Inc. + * Copyright (c) 2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,18 +23,12 @@ * SOFTWARE. * *******************************************************************************/ -#include "get_handle.hpp" -#include "random.hpp" -#include -#include #include -#include #include -#include - +#include "get_handle.hpp" +#include "verify.hpp" #include "perf_helper.hpp" -#include #define MAX_TENSOR_ELEM 17 #define PERF_ENABLE 0 @@ -52,7 +46,7 @@ template std::vector TensorsConfigs() { std::vector configs; - auto insertTestCase = [&configs](size_t& N, size_t& C, size_t& H, size_t& W) { + auto insertTestCase = [&configs](size_t N, size_t C, size_t H, size_t W) { configs.push_back( {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, W}, {C * H * W, H * W, W, 1}}); configs.push_back( @@ -89,41 +83,34 @@ std::vector TensorsConfigs() if constexpr(PERF_ENABLE) { - - const auto& handle = get_handle(); - size_t maxTotalSize; - - // Generate all NCHW tensors that are limited by L3 cache size - // or 2xL2 cache size when L3 is not available - if(miopen::StartsWith(handle.GetDeviceName(), "gfx90a") || - miopen::StartsWith(handle.GetDeviceName(), "gfx908")) - { - maxTotalSize = 16; // twice the 8MB L2 - } - else if(miopen::StartsWith(handle.GetDeviceName(), "gfx803")) - { - maxTotalSize = 4; // twice the 2MB L2 - } - else if(miopen::StartsWith(handle.GetDeviceName(), "gfx900") || - miopen::StartsWith(handle.GetDeviceName(), "gfx906")) - { - maxTotalSize = 8; // twice the 4MB L2 - } - else if(miopen::StartsWith(handle.GetDeviceName(), "gfx942")) + // Determine a cache-aware cap on total tensor elements for HIP/AMD: + // 1) Query L2 size via HIP and use 2x L2 as working set + // 2) Fallback to per-architecture table if L2 is not reported + size_t maxTotalSize = 0; + + // 1) HIP L2 cache query + int dev = -1; + if(hipSuccess == hipGetDevice(&dev)) { - maxTotalSize = 256; // 256MB L3 - } - else if(miopen::StartsWith(handle.GetDeviceName(), "gfx103")) - { - maxTotalSize = 128; // 128MB L3 + int L2_bytes = 0; + if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && + L2_bytes > 0) + { + // Use 2x L2 as a working-set heuristic + maxTotalSize = 2ul * static_cast(L2_bytes); + // Convert bytes -> elements of type T + maxTotalSize /= sizeof(T); + } } - else + + // 2) Fallback table by architecture family + if(maxTotalSize == 0) { - maxTotalSize = 4; // twice the 2MB L2, default case. + maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); } - maxTotalSize = maxTotalSize * 1024ull * 1024ull / sizeof(T); - + // Generate all NCHW tensors that are limited by L3 cache size + // or 2xL2 cache size when L3 is not available if constexpr(POW_2) { for(size_t N = 1; N <= maxTotalSize; N *= 2) @@ -262,14 +249,6 @@ struct Op4DTensorGenericTest } } - // if(std::accumulate(tensorsConfig.blens.begin(), - // tensorsConfig.blens.end(), - // 4, - // std::multiplies()) == 1) - // { - // bitmap = 4; - // } - num_wg_orig = num_wg; max_num_wg = 4096; num_wg = num_wg > max_num_wg ? max_num_wg : num_wg; @@ -289,7 +268,6 @@ struct Op4DTensorGenericTest void runCPU() { - std::vector A = tensA.data; std::vector B = tensB.data; std::vector C = tensC.data; @@ -342,48 +320,43 @@ struct Op4DTensorGenericTest std::string program_name = "MIOpenTensorKernels.cl"; std::string network_config_ocl = network_config + "-ocl"; - handle.AddKernel("Op4dTensorGeneric", - network_config_ocl, - program_name, - "Op4dTensorGeneric", - vld, - vgd, - params)(tensA_dev.get(), - static_cast(tensorsConfig.acstrides[0]), - static_cast(tensorsConfig.acstrides[1]), - static_cast(tensorsConfig.acstrides[2]), - tensB_dev.get(), - static_cast(tensorsConfig.blens[1]), - static_cast(tensorsConfig.blens[2]), - static_cast(tensorsConfig.blens[3]), - static_cast(tensorsConfig.bstrides[0]), - static_cast(tensorsConfig.bstrides[1]), - static_cast(tensorsConfig.bstrides[2]), - tensC_dev.get(), - static_cast(tensorsConfig.aclens[1]), - static_cast(tensorsConfig.aclens[2]), - static_cast(tensorsConfig.aclens[3]), - static_cast(tensorsConfig.acstrides[0]), - static_cast(tensorsConfig.acstrides[1]), - static_cast(tensorsConfig.acstrides[2]), - alpha0, - alpha1, - beta, - bitmap, - work_per_wg, - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(num_wg_orig)); + handle.AddKernel( + kernel_name, network_config_ocl, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + static_cast(tensorsConfig.acstrides[0]), + static_cast(tensorsConfig.acstrides[1]), + static_cast(tensorsConfig.acstrides[2]), + tensB_dev.get(), + static_cast(tensorsConfig.blens[1]), + static_cast(tensorsConfig.blens[2]), + static_cast(tensorsConfig.blens[3]), + static_cast(tensorsConfig.bstrides[0]), + static_cast(tensorsConfig.bstrides[1]), + static_cast(tensorsConfig.bstrides[2]), + tensC_dev.get(), + static_cast(tensorsConfig.aclens[1]), + static_cast(tensorsConfig.aclens[2]), + static_cast(tensorsConfig.aclens[3]), + static_cast(tensorsConfig.acstrides[0]), + static_cast(tensorsConfig.acstrides[1]), + static_cast(tensorsConfig.acstrides[2]), + alpha0, + alpha1, + beta, + bitmap, + work_per_wg, + 0L, + 0L, + 0L, + num_wg_orig); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); if constexpr(PERF_ENABLE) { ph.perfTest(handle, - "Op4dTensorGeneric", + kernel_name, network_config_ocl, - false, tensA_dev.get(), static_cast(tensorsConfig.acstrides[0]), static_cast(tensorsConfig.acstrides[1]), @@ -407,10 +380,10 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(num_wg_orig)); + 0L, + 0L, + 0L, + num_wg_orig); } } @@ -429,48 +402,43 @@ struct Op4DTensorGenericTest std::string program_name = "MIOpenTensorKernelsHip.cpp"; std::string network_config_hip = network_config + "-hip"; - handle.AddKernel("Op4dTensorGeneric", - network_config_hip, - program_name, - "Op4dTensorGeneric", - vld, - vgd, - params)(tensA_dev.get(), - static_cast(tensorsConfig.acstrides[0]), - static_cast(tensorsConfig.acstrides[1]), - static_cast(tensorsConfig.acstrides[2]), - tensB_dev.get(), - static_cast(tensorsConfig.blens[1]), - static_cast(tensorsConfig.blens[2]), - static_cast(tensorsConfig.blens[3]), - static_cast(tensorsConfig.bstrides[0]), - static_cast(tensorsConfig.bstrides[1]), - static_cast(tensorsConfig.bstrides[2]), - tensC_dev.get(), - static_cast(tensorsConfig.aclens[1]), - static_cast(tensorsConfig.aclens[2]), - static_cast(tensorsConfig.aclens[3]), - static_cast(tensorsConfig.acstrides[0]), - static_cast(tensorsConfig.acstrides[1]), - static_cast(tensorsConfig.acstrides[2]), - alpha0, - alpha1, - beta, - bitmap, - work_per_wg, - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(num_wg_orig)); + handle.AddKernel( + kernel_name, network_config_hip, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + static_cast(tensorsConfig.acstrides[0]), + static_cast(tensorsConfig.acstrides[1]), + static_cast(tensorsConfig.acstrides[2]), + tensB_dev.get(), + static_cast(tensorsConfig.blens[1]), + static_cast(tensorsConfig.blens[2]), + static_cast(tensorsConfig.blens[3]), + static_cast(tensorsConfig.bstrides[0]), + static_cast(tensorsConfig.bstrides[1]), + static_cast(tensorsConfig.bstrides[2]), + tensC_dev.get(), + static_cast(tensorsConfig.aclens[1]), + static_cast(tensorsConfig.aclens[2]), + static_cast(tensorsConfig.aclens[3]), + static_cast(tensorsConfig.acstrides[0]), + static_cast(tensorsConfig.acstrides[1]), + static_cast(tensorsConfig.acstrides[2]), + alpha0, + alpha1, + beta, + bitmap, + work_per_wg, + 0L, + 0L, + 0L, + num_wg_orig); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); if constexpr(PERF_ENABLE) { ph.perfTest(handle, - "Op4dTensorGeneric", + kernel_name, network_config_hip, - false, tensA_dev.get(), static_cast(tensorsConfig.acstrides[0]), static_cast(tensorsConfig.acstrides[1]), @@ -494,10 +462,10 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(num_wg_orig)); + 0L, + 0L, + 0L, + num_wg_orig); } } @@ -510,7 +478,7 @@ struct Op4DTensorGenericTest void verifyCPU() { auto error = miopen::rms_range(tensC_hip, tensC_cpu); - EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; + EXPECT_TRUE(error == 0) << "GPU outputs do not match CPU results. Error: " << error; } void TearDown() override @@ -537,10 +505,11 @@ struct Op4DTensorGenericTest stats += "_alpha0_" + std::to_string(alpha0) + "_alpha1_" + std::to_string(alpha1) + "_beta_" + std::to_string(beta) + "_" + miopen::GetDataType(data_type); - ph.writeStatsToCSV("tensor_4d.csv", stats); + ph.writeStatsToCSV("tensor_4d_generic.csv", stats); } } + const std::string kernel_name{"Op4dTensorGeneric"}; std::string network_config{}; std::string params{}; std::vector vld, vgd; diff --git a/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp index 65030faea638..8b1661a2c456 100644 --- a/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp @@ -85,7 +85,7 @@ std::vector TensorsConfigs() #if PERF_ENABLE // Determine a cache-aware cap on total tensor elements for HIP/AMD: // 1) Query L2 size via HIP and use 2x L2 as working set - // 2) Fallback to per-architecture table (MiB) if L2 is not reported + // 2) Fallback to per-architecture table if L2 is not reported size_t maxTotalSize = 0; // 1) HIP L2 cache query @@ -98,15 +98,16 @@ std::vector TensorsConfigs() { // Use 2x L2 as a working-set heuristic maxTotalSize = 2ul * static_cast(L2_bytes); + // Convert bytes -> elements of type T + maxTotalSize /= sizeof(T); } } - // 2) Fallback table by architecture family (MiB) + // 2) Fallback table by architecture family if(maxTotalSize == 0) - maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); - - // Convert bytes -> elements of type T - maxTotalSize = maxTotalSize / sizeof(T); + { + maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); + } for(int N = 1; N < maxTotalSize; N *= 4) { @@ -289,9 +290,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - uint64_t(0), // Aoffset - uint64_t(0), // Boffset - uint64_t(0), // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, incr_wg); @@ -316,9 +317,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - uint64_t(0), - uint64_t(0), - uint64_t(0), + 0L, + 0L, + 0L, num_wg, incr_wg); #endif @@ -358,9 +359,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - uint64_t(0), // Aoffset - uint64_t(0), // Boffset - uint64_t(0), // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, incr_wg); @@ -385,9 +386,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - uint64_t(0), - uint64_t(0), - uint64_t(0), + 0L, + 0L, + 0L, num_wg, incr_wg); #endif @@ -431,7 +432,7 @@ struct OpTensorFwdBiasGenericTest std::string network_config{}; std::string params{}; std::vector vld, vgd; - const int max_num_wg = 4096; + const int max_num_wg{4096}; int work_per_wg, num_wg, incr_wg{0}; tensor tensA; diff --git a/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp index f932bf71e381..e099b0c4a267 100644 --- a/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp @@ -86,7 +86,30 @@ std::vector TensorsConfigs() }; #if PERF_ENABLE - size_t maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()) / sizeof(T); + // Determine a cache-aware cap on total tensor elements for HIP/AMD: + // 1) Query L2 size via HIP and use 2x L2 as working set + // 2) Fallback to per-architecture table if L2 is not reported + size_t maxTotalSize = 0; + + // 1) HIP L2 cache query + int dev = -1; + if(hipSuccess == hipGetDevice(&dev)) + { + int L2_bytes = 0; + if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && + L2_bytes > 0) + { + // Use 2x L2 as a working-set heuristic + maxTotalSize = 2ul * static_cast(L2_bytes); + // Convert bytes -> elements of type T + maxTotalSize /= sizeof(T); + } + } + // 2) Fallback table by architecture family + if(maxTotalSize == 0) + { + maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); + } // Generate all NCHW tensors that are limited by L3 cache size // or 2xL2 cache size when L3 is not available @@ -255,9 +278,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - static_cast(0), // Aoffset - static_cast(0), // Boffset - static_cast(0), // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, incr_wg); @@ -279,9 +302,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - static_cast(0), - static_cast(0), - static_cast(0), + 0L, + 0L, + 0L, num_wg, incr_wg); #endif @@ -313,9 +336,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - static_cast(0), // Aoffset - static_cast(0), // Boffset - static_cast(0), // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, incr_wg); @@ -337,9 +360,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - static_cast(0), - static_cast(0), - static_cast(0), + 0L, + 0L, + 0L, num_wg, incr_wg); #endif @@ -382,7 +405,7 @@ struct OpTensorFwdBiasTest std::string network_config{}; std::string params{}; std::vector vld, vgd; - const int max_num_wg = 4096; + const int max_num_wg{4096}; int work_per_wg, num_wg, incr_wg{0}; tensor tensA; diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp new file mode 100644 index 000000000000..fa70564045d2 --- /dev/null +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -0,0 +1,454 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include +#include + +#include "get_handle.hpp" +#include "tensor_util.hpp" +#include "verify.hpp" + +#define PERF_ENABLE 0 +#if PERF_ENABLE +#include "perf_helper.hpp" +#endif + +struct TensorsConfig +{ + std::vector aclens; + std::vector acstrides; + std::vector blens; + std::vector bstrides; +}; + +template +std::vector TensorsConfigs() +{ + std::vector configs; + auto insertTestCase = [&configs](int N, int C, int H, int W) { + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, W}, {C * H * W, H * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, 1}, {C * H * 1, H * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, 1, W}, {C * 1 * W, 1 * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, 1, 1}, {C * 1 * 1, 1 * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, H, W}, {1 * H * W, H * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, H, 1}, {1 * H * 1, H * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, 1, W}, {1 * 1 * W, 1 * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, 1, 1}, {1 * 1 * 1, 1 * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, H, W}, {C * H * W, H * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, H, 1}, {C * H * 1, H * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, 1, W}, {C * 1 * W, 1 * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, 1, 1}, {C * 1 * 1, 1 * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, H, W}, {1 * H * W, H * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, H, 1}, {1 * H * 1, H * 1, 1, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, 1, W}, {1 * 1 * W, 1 * W, W, 1}}); + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, 1, 1}, {1 * 1 * 1, 1 * 1, 1, 1}}); + }; + +#if PERF_ENABLE + // Determine a cache-aware cap on total tensor elements for HIP/AMD: + // 1) Query L2 size via HIP and use 2x L2 as working set + // 2) Fallback to per-architecture table if L2 is not reported + size_t maxTotalSize = 0; + + // 1) HIP L2 cache query + int dev = -1; + if(hipSuccess == hipGetDevice(&dev)) + { + int L2_bytes = 0; + if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && + L2_bytes > 0) + { + // Use 2x L2 as a working-set heuristic + maxTotalSize = 2ul * static_cast(L2_bytes); + // Convert bytes -> elements of type T + maxTotalSize /= sizeof(T); + } + } + + // 2) Fallback table by architecture family + if(maxTotalSize == 0) + { + maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); + } + + for(int N = 1; N < maxTotalSize; N *= 2) + { + for(int C : {1, 4}) + { + for(int H = 1; H < maxTotalSize / (N * C); H *= 2) + { + for(int W = 1; W < maxTotalSize / (N * C * H); W *= 2) + { + size_t totalSize = N * C * H * W; + // Ensure the total size does not exceed the maximum limit + if(totalSize <= maxTotalSize) + { + insertTestCase(N, C, H, W); + } + } + } + } + } + + return configs; +#else + int N = 1; + int C = 1; + int H = 1; + int W = 1; + insertTestCase(N, C, H, W); + C = 20; + H = 16; + W = 8; + insertTestCase(N, C, H, W); + C = 32; + H = 64; + W = 16; + insertTestCase(N, C, H, W); + + return configs; +#endif +} + +template +struct OpTensorLeadingOnesGenericTest + : public ::testing::TestWithParam> +{ +protected: + void SetUp() override + { + auto&& handle = get_handle(); + std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + + data_type = miopen_type{}; + + // Generate elements in tensors + tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( + tensor_elem_gen_integer{17}); + tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( + tensor_elem_gen_integer{17}); + tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + + // Write the device tensors + tensA_dev = handle.Write(tensA.data); + tensB_dev = handle.Write(tensB.data); + + // Allocate output tensors for OCL and HIP + tensC_ocl = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + tensC_hip = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + + // Prepare all parameters needed for kernel + auto first_not_one = std::find_if( + tensorsConfig.blens.rbegin(), tensorsConfig.blens.rend(), [](int i) { return i != 1; }); + auto d = std::distance(tensorsConfig.blens.begin(), first_not_one.base()); + + num_wg = first_not_one != tensorsConfig.blens.rend() + ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) + : 1; + work_per_wg = std::accumulate(tensorsConfig.aclens.begin() + d, + tensorsConfig.aclens.end(), + 1, + std::multiplies()); + + bitmap = 0; + bitmap |= (1 << (tensorsConfig.blens.size() - d)); + + for(int i = (d - 2); i >= 0; i--) + { + if(tensorsConfig.blens[i] != 1) + { + bitmap |= (1 << (tensorsConfig.blens.size() - (i + 1))); + num_wg *= tensorsConfig.blens[i]; + } + else + { + work_per_wg *= tensorsConfig.aclens[i]; + } + } + + if(num_wg > max_num_wg) + num_wg = max_num_wg; + + size_t local_threads = 256; + size_t global_threads = num_wg * local_threads; + global_threads = (global_threads < local_threads) ? local_threads : global_threads; + + vld = {local_threads, 1, 1}; + vgd = {global_threads, 1, 1}; + + network_config += std::to_string(data_type) + "-miopenTensorOpAdd-" + + std::to_string(global_threads) + "-" + std::to_string(local_threads); + } + + void runOCL() // run OCL kernel + { + auto&& handle = get_handle(); + // Write data to device tensor + tensC_dev = handle.Write(tensC.data); + + params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); + params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::OpenCL{}); + params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES_GENERIC"; + + std::string program_name = "MIOpenTensorKernels.cl"; + std::string network_config_ocl = network_config + "-ocl"; + + handle.AddKernel( + kernel_name, network_config_ocl, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], // c_c + tensorsConfig.aclens[2], // c_h + tensorsConfig.aclens[3], // c_w + tensorsConfig.acstrides[0], // c_nstride + tensorsConfig.acstrides[1], // c_cstride + tensorsConfig.acstrides[2], // c_hstride + work_per_wg, + alpha0, + alpha1, + beta, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, + bitmap); + + tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); + +#if PERF_ENABLE + ph.perfTest(handle, + kernel_name, + network_config_ocl, + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], + tensorsConfig.aclens[2], + tensorsConfig.aclens[3], + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], + work_per_wg, + alpha0, + alpha1, + beta, + 0L, + 0L, + 0L, + num_wg, + bitmap); +#endif + } + + void runHIP() // run HIP kernel + { + auto&& handle = get_handle(); + tensC_dev = handle.Write(tensC.data); + + params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); + params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::HIP{}); + params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES_GENERIC"; + + std::string program_name = "MIOpenTensorKernelsHip.cpp"; + std::string network_config_hip = network_config + "-hip"; + + handle.AddKernel( + kernel_name, network_config_hip, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], // c_c + tensorsConfig.aclens[2], // c_h + tensorsConfig.aclens[3], // c_w + tensorsConfig.acstrides[0], // c_nstride + tensorsConfig.acstrides[1], // c_cstride + tensorsConfig.acstrides[2], // c_hstride + work_per_wg, + alpha0, + alpha1, + beta, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, + bitmap); + + tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); + +#if PERF_ENABLE + ph.perfTest(handle, + kernel_name, + network_config_hip, + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], + tensorsConfig.aclens[2], + tensorsConfig.aclens[3], + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], + work_per_wg, + alpha0, + alpha1, + beta, + 0L, + 0L, + 0L, + num_wg, + bitmap); +#endif + } + + void verify() // verify if the output tensors are same + { + auto error = miopen::rms_range(tensC_ocl, tensC_hip); + EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; + } + + void TearDown() override + { +#if PERF_ENABLE + std::string stats{}; + stats += "_aclens_" + std::to_string(tensorsConfig.aclens[0]) + "_" + + std::to_string(tensorsConfig.aclens[1]) + "_" + + std::to_string(tensorsConfig.aclens[2]) + "_" + + std::to_string(tensorsConfig.aclens[3]) + "_acstrides_" + + std::to_string(tensorsConfig.acstrides[0]) + "_" + + std::to_string(tensorsConfig.acstrides[1]) + "_" + + std::to_string(tensorsConfig.acstrides[2]) + "_" + + std::to_string(tensorsConfig.acstrides[3]); + stats += "_blens_" + std::to_string(tensorsConfig.blens[0]) + "_" + + std::to_string(tensorsConfig.blens[1]) + "_" + + std::to_string(tensorsConfig.blens[2]) + "_" + + std::to_string(tensorsConfig.blens[3]) + "_bstrides_" + + std::to_string(tensorsConfig.bstrides[0]) + "_" + + std::to_string(tensorsConfig.bstrides[1]) + "_" + + std::to_string(tensorsConfig.bstrides[2]) + "_" + + std::to_string(tensorsConfig.bstrides[3]); + stats += "_alpha0_" + std::to_string(alpha0) + "_alpha1_" + std::to_string(alpha1) + + "_beta_" + std::to_string(beta); + + std::string filename = + "tensor_leading_ones_generic-" + miopen::GetDataType(data_type) + ".csv"; + ph.writeStatsToCSV(filename, stats); +#endif + } + + const std::string kernel_name{"OpTensorLeadingOnesGeneric"}; + std::string network_config{}; + std::string params{}; + std::vector vld, vgd; + unsigned int bitmap; + int work_per_wg, num_wg; + int max_num_wg{4096}; + + tensor tensA; + tensor tensB; + tensor tensC; + tensor tensC_ocl; + tensor tensC_hip; + + miopenDataType_t data_type; + + miopen::Allocator::ManageDataPtr tensA_dev; + miopen::Allocator::ManageDataPtr tensB_dev; + miopen::Allocator::ManageDataPtr tensC_dev; + + TensorsConfig tensorsConfig; + T alpha0, alpha1, beta; + +#if PERF_ENABLE + PerfHelper ph; +#endif +}; + +using GPU_OpTensorLeadingOnesGenericTest_FP16 = OpTensorLeadingOnesGenericTest; + +TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP16, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesGenericTest_FP16, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); + +using GPU_OpTensorLeadingOnesGenericTest_FP32 = OpTensorLeadingOnesGenericTest; + +TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP32, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesGenericTest_FP32, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); + +using GPU_OpTensorLeadingOnesGenericTest_FP64 = OpTensorLeadingOnesGenericTest; + +TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP64, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesGenericTest_FP64, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp new file mode 100644 index 000000000000..ac5057843dfd --- /dev/null +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -0,0 +1,449 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include +#include + +#include "get_handle.hpp" +#include "tensor_util.hpp" +#include "verify.hpp" + +#define PERF_ENABLE 0 +#if PERF_ENABLE +#include "perf_helper.hpp" +#endif + +struct TensorsConfig +{ + std::vector aclens; + std::vector acstrides; + std::vector blens; + std::vector bstrides; +}; + +template +std::vector TensorsConfigs() +{ + std::vector configs; + auto insertTestCase = [&configs](int N, int C, int H, int W) { + configs.push_back( + {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, W}, {C * H * W, H * W, W, 1}}); + }; + +#if PERF_ENABLE + // Determine a cache-aware cap on total tensor elements for HIP/AMD: + // 1) Query L2 size via HIP and use 2x L2 as working set + // 2) Fallback to per-architecture table if L2 is not reported + size_t maxTotalSize = 0; + + // 1) HIP L2 cache query + int dev = -1; + if(hipSuccess == hipGetDevice(&dev)) + { + int L2_bytes = 0; + if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && + L2_bytes > 0) + { + // Use 2x L2 as a working-set heuristic + maxTotalSize = 2ul * static_cast(L2_bytes); + // Convert bytes -> elements of type T + maxTotalSize /= sizeof(T); + } + } + + // 2) Fallback table by architecture family + if(maxTotalSize == 0) + { + maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); + } + + int C = 4; + int H = 8; + int W = 8; + for(int N = 32; N <= maxTotalSize / (C * H * W); N *= 2) + { + insertTestCase(N, C, H, W); + } + + return configs; +#else + int N = 1; + int C = 1; + int H = 1; + int W = 1; + insertTestCase(N, C, H, W); + N = 1024; + C = 4; + H = 8; + W = 8; + insertTestCase(N, C, H, W); + N = 2048; + C = 4; + H = 16; + W = 16; + insertTestCase(N, C, H, W); + + return configs; +#endif +} + +template +struct OpTensorLeadingOnesTest + : public ::testing::TestWithParam> +{ +protected: + void SetUp() override + { + auto&& handle = get_handle(); + std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + + data_type = miopen_type{}; + + // Generate elements in tensors + tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( + tensor_elem_gen_integer{17}); + tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( + tensor_elem_gen_integer{17}); + tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + + // Write the device tensors + tensA_dev = handle.Write(tensA.data); + tensB_dev = handle.Write(tensB.data); + + // Allocate output tensors for OCL and HIP + tensC_ocl = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + tensC_hip = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + + // first_not_one is incorrect if btensor size equal to 1 + auto first_not_one = std::find_if( + tensorsConfig.blens.rbegin(), tensorsConfig.blens.rend(), [](int i) { return i != 1; }); + auto d = std::distance(tensorsConfig.blens.begin(), first_not_one.base()); + + num_wg = first_not_one != tensorsConfig.blens.rend() + ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) + : 1; + work_per_wg = std::accumulate(tensorsConfig.aclens.begin() + d, + tensorsConfig.aclens.end(), + 1, + std::multiplies()); + + bitmap = 0; + // update bitmap for first_not_one + bitmap |= (1 << (tensorsConfig.blens.size() - d)); + + for(int i = (d - 2); i >= 0; i--) + { + if(tensorsConfig.blens[i] != 1) + { + bitmap |= (1 << (tensorsConfig.blens.size() - (i + 1))); + num_wg *= tensorsConfig.blens[i]; + } + else + { + work_per_wg *= tensorsConfig.aclens[i]; + } + } + + // quick fix for btensor = <1,1,1,1> + if(1 == + std::accumulate( + tensorsConfig.blens.begin(), tensorsConfig.blens.end(), 1, std::multiplies())) + { + bitmap = 4; + } + + // Forward Convolution Bias specialization + // for fwd-bias, bitmap looks like <0,1,0,0> + // Is the no. of work-groups and the work for each wg balanced? + auto fwd_conv_bias = bitmap == (1 << 2) ? 1 : 0; + auto dims = tensorsConfig.aclens.size(); + auto c_n = tensorsConfig.aclens[0]; + // This block gives off indexing for 5d tensors, skipping + if(fwd_conv_bias == 1 && dims < 5 && num_wg < 640 && work_per_wg > 256 && c_n > 0) + { // 640 workgroups of size 256 needed to completely fill the GPU + work_per_wg /= c_n; + num_wg *= c_n; + } + + if(num_wg > max_num_wg) + num_wg = max_num_wg; + + size_t local_threads = 256; + + bool leading_ones = true; + for(int i = (d - 2); i >= 0; i--) + { + bool is_one = (bitmap & (1 << (dims - 1 - i))) != 0u; + leading_ones &= is_one; + } + if(leading_ones && work_per_wg < 64) + { + local_threads = 64; + } + + // Special case for adding tensors in place + size_t global_threads = + (static_cast(leading_ones) == 1 && (d - 1) == 3) ? num_wg : num_wg * local_threads; + global_threads = (global_threads < local_threads) ? local_threads : global_threads; + + vld = {local_threads, 1, 1}; + vgd = {global_threads, 1, 1}; + + network_config += std::to_string(data_type) + "-miopenTensorOpAdd-" + + std::to_string(global_threads) + "-" + std::to_string(local_threads); + } + + void runOCL() // run OCL kernel + { + auto&& handle = get_handle(); + // Write data to device tensor + tensC_dev = handle.Write(tensC.data); + + params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); + params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::OpenCL{}); + params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES"; + + std::string program_name = "MIOpenTensorKernels.cl"; + std::string network_config_ocl = network_config + "-ocl"; + + handle.AddKernel( + kernel_name, network_config_ocl, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], // c_c + tensorsConfig.aclens[2], // c_h + tensorsConfig.aclens[3], // c_w + tensorsConfig.acstrides[0], // c_nstride + tensorsConfig.acstrides[1], // c_cstride + tensorsConfig.acstrides[2], // c_hstride + work_per_wg, + alpha0, + alpha1, + beta, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, + bitmap); + + tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); + +#if PERF_ENABLE + ph.perfTest(handle, + kernel_name, + network_config_ocl, + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], + tensorsConfig.aclens[2], + tensorsConfig.aclens[3], + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], + work_per_wg, + alpha0, + alpha1, + beta, + 0L, + 0l, + 0L, + num_wg, + bitmap); +#endif + } + + void runHIP() // run HIP kernel + { + auto&& handle = get_handle(); + tensC_dev = handle.Write(tensC.data); + + params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); + params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::HIP{}); + params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES"; + + std::string program_name = "MIOpenTensorKernelsHip.cpp"; + std::string network_config_hip = network_config + "-hip"; + + handle.AddKernel( + kernel_name, network_config_hip, program_name, kernel_name, vld, vgd, params)( + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], // c_c + tensorsConfig.aclens[2], // c_h + tensorsConfig.aclens[3], // c_w + tensorsConfig.acstrides[0], // c_nstride + tensorsConfig.acstrides[1], // c_cstride + tensorsConfig.acstrides[2], // c_hstride + work_per_wg, + alpha0, + alpha1, + beta, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, + bitmap); + + tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); + +#if PERF_ENABLE + ph.perfTest(handle, + kernel_name, + network_config_hip, + tensA_dev.get(), + tensB_dev.get(), + tensC_dev.get(), + tensorsConfig.aclens[1], + tensorsConfig.aclens[2], + tensorsConfig.aclens[3], + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], + work_per_wg, + alpha0, + alpha1, + beta, + 0L, + 0L, + 0L, + num_wg, + bitmap); +#endif + } + + void verify() // verify if the output tensors are same + { + auto error = miopen::rms_range(tensC_ocl, tensC_hip); + EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; + } + + void TearDown() override + { +#if PERF_ENABLE + std::string stats{}; + stats += "_aclens_" + std::to_string(tensorsConfig.aclens[0]) + "_" + + std::to_string(tensorsConfig.aclens[1]) + "_" + + std::to_string(tensorsConfig.aclens[2]) + "_" + + std::to_string(tensorsConfig.aclens[3]) + "_acstrides_" + + std::to_string(tensorsConfig.acstrides[0]) + "_" + + std::to_string(tensorsConfig.acstrides[1]) + "_" + + std::to_string(tensorsConfig.acstrides[2]) + "_" + + std::to_string(tensorsConfig.acstrides[3]); + stats += "_blens_" + std::to_string(tensorsConfig.blens[0]) + "_" + + std::to_string(tensorsConfig.blens[1]) + "_" + + std::to_string(tensorsConfig.blens[2]) + "_" + + std::to_string(tensorsConfig.blens[3]) + "_bstrides_" + + std::to_string(tensorsConfig.bstrides[0]) + "_" + + std::to_string(tensorsConfig.bstrides[1]) + "_" + + std::to_string(tensorsConfig.bstrides[2]) + "_" + + std::to_string(tensorsConfig.bstrides[3]); + stats += "_alpha0_" + std::to_string(alpha0) + "_alpha1_" + std::to_string(alpha1) + + "_beta_" + std::to_string(beta); + + std::string filename = "tensor_leading_ones-" + miopen::GetDataType(data_type) + ".csv"; + ph.writeStatsToCSV(filename, stats); +#endif + } + + const std::string kernel_name{"OpTensorLeadingOnes"}; + std::string network_config{}; + std::string params{}; + std::vector vld, vgd; + unsigned int bitmap; + int work_per_wg, num_wg; + int max_num_wg{4096}; + + tensor tensA; + tensor tensB; + tensor tensC; + tensor tensC_ocl; + tensor tensC_hip; + + miopenDataType_t data_type; + + miopen::Allocator::ManageDataPtr tensA_dev; + miopen::Allocator::ManageDataPtr tensB_dev; + miopen::Allocator::ManageDataPtr tensC_dev; + + TensorsConfig tensorsConfig; + T alpha0, alpha1, beta; + +#if PERF_ENABLE + PerfHelper ph; +#endif +}; + +using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; + +TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesTest_FP16, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); + +using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; + +TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesTest_FP32, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); + +using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; + +TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) +{ + runOCL(); + runHIP(); + verify(); +} + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_OpTensorLeadingOnesTest_FP64, + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(1.0), + testing::Values(1.0), + testing::Values(0.0, 1.0))); From 669400639f811929b21d2c8c66815aee341920e7 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Wed, 17 Dec 2025 01:13:52 +0100 Subject: [PATCH 02/33] Check results of OpenCL kernel --- .../test/gtest/tensor_leading_ones_ocl_hip.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index ac5057843dfd..23d6f1bf0f50 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -93,7 +93,7 @@ std::vector TensorsConfigs() int C = 1; int H = 1; int W = 1; - insertTestCase(N, C, H, W); + //insertTestCase(N, C, H, W); N = 1024; C = 4; H = 8; @@ -103,7 +103,7 @@ std::vector TensorsConfigs() C = 4; H = 16; W = 16; - insertTestCase(N, C, H, W); + //insertTestCase(N, C, H, W); return configs; #endif @@ -251,6 +251,9 @@ struct OpTensorLeadingOnesTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); +std::cout << "OCL:"; +for (size_t i=0; i<10; ++i) std::cout << ' ' << tensC_ocl.data[i]; +std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -312,6 +315,9 @@ struct OpTensorLeadingOnesTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); +std::cout << "HIP:"; +for (size_t i=0; i<10; ++i) std::cout << ' ' << tensC_hip.data[i]; +std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -399,7 +405,7 @@ struct OpTensorLeadingOnesTest PerfHelper ph; #endif }; - +/* using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -431,7 +437,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +*/ using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) From fd07aed09f06bc82deecd2a95130375209aabe1a Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Wed, 17 Dec 2025 02:02:38 +0100 Subject: [PATCH 03/33] Fixed clang-format violations --- .../gtest/tensor_leading_ones_ocl_hip.cpp | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 23d6f1bf0f50..0209e9df6812 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -93,7 +93,7 @@ std::vector TensorsConfigs() int C = 1; int H = 1; int W = 1; - //insertTestCase(N, C, H, W); + // insertTestCase(N, C, H, W); N = 1024; C = 4; H = 8; @@ -103,7 +103,7 @@ std::vector TensorsConfigs() C = 4; H = 16; W = 16; - //insertTestCase(N, C, H, W); + // insertTestCase(N, C, H, W); return configs; #endif @@ -251,9 +251,10 @@ struct OpTensorLeadingOnesTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); -std::cout << "OCL:"; -for (size_t i=0; i<10; ++i) std::cout << ' ' << tensC_ocl.data[i]; -std::cout << '\n'; + std::cout << "OCL:"; + for(size_t i = 0; i < 10; ++i) + std::cout << ' ' << tensC_ocl.data[i]; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -315,9 +316,10 @@ std::cout << '\n'; bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); -std::cout << "HIP:"; -for (size_t i=0; i<10; ++i) std::cout << ' ' << tensC_hip.data[i]; -std::cout << '\n'; + std::cout << "HIP:"; + for(size_t i = 0; i < 10; ++i) + std::cout << ' ' << tensC_hip.data[i]; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -405,7 +407,7 @@ std::cout << '\n'; PerfHelper ph; #endif }; -/* + using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -437,7 +439,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) From e7af4ba475c5bad89779df4c39f06efe6ba946a3 Mon Sep 17 00:00:00 2001 From: stulebae Date: Wed, 17 Dec 2025 11:05:55 +0100 Subject: [PATCH 04/33] Update tensor_leading_ones_ocl_hip.cpp --- .../miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 0209e9df6812..a692107c4342 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -93,7 +93,7 @@ std::vector TensorsConfigs() int C = 1; int H = 1; int W = 1; - // insertTestCase(N, C, H, W); + insertTestCase(N, C, H, W); N = 1024; C = 4; H = 8; @@ -103,7 +103,7 @@ std::vector TensorsConfigs() C = 4; H = 16; W = 16; - // insertTestCase(N, C, H, W); + insertTestCase(N, C, H, W); return configs; #endif @@ -407,7 +407,7 @@ struct OpTensorLeadingOnesTest PerfHelper ph; #endif }; - +/* using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -423,7 +423,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +*/ using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) @@ -439,7 +439,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +/* using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) @@ -455,3 +455,4 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); +*/ From b2cab05d1451570fba0e1825048387a30cb0c853 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Thu, 18 Dec 2025 01:18:26 +0100 Subject: [PATCH 05/33] Fixing typo in parameters of kernel call --- .../test/gtest/tensor_leading_ones_ocl_hip.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index a692107c4342..2c102704c30a 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -89,7 +89,7 @@ std::vector TensorsConfigs() return configs; #else - int N = 1; + int N = 16; int C = 1; int H = 1; int W = 1; @@ -123,9 +123,12 @@ struct OpTensorLeadingOnesTest // Generate elements in tensors tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( - tensor_elem_gen_integer{17}); + tensor_elem_gen_integer{}); + for(size_t i = 0; i < 16; ++i) + std::cout << ' ' << tensA.data[i]; + std::cout << '\n'; tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( - tensor_elem_gen_integer{17}); + tensor_elem_gen_integer{}); tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; // Write the device tensors @@ -239,7 +242,6 @@ struct OpTensorLeadingOnesTest tensorsConfig.aclens[3], // c_w tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride - tensorsConfig.acstrides[2], // c_hstride work_per_wg, alpha0, alpha1, @@ -252,7 +254,7 @@ struct OpTensorLeadingOnesTest tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); std::cout << "OCL:"; - for(size_t i = 0; i < 10; ++i) + for(size_t i = 0; i < 16; ++i) std::cout << ' ' << tensC_ocl.data[i]; std::cout << '\n'; @@ -268,7 +270,6 @@ struct OpTensorLeadingOnesTest tensorsConfig.aclens[3], tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], - tensorsConfig.acstrides[2], work_per_wg, alpha0, alpha1, @@ -304,7 +305,6 @@ struct OpTensorLeadingOnesTest tensorsConfig.aclens[3], // c_w tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride - tensorsConfig.acstrides[2], // c_hstride work_per_wg, alpha0, alpha1, @@ -317,7 +317,7 @@ struct OpTensorLeadingOnesTest tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); std::cout << "HIP:"; - for(size_t i = 0; i < 10; ++i) + for(size_t i = 0; i < 16; ++i) std::cout << ' ' << tensC_hip.data[i]; std::cout << '\n'; @@ -333,7 +333,6 @@ struct OpTensorLeadingOnesTest tensorsConfig.aclens[3], tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], - tensorsConfig.acstrides[2], work_per_wg, alpha0, alpha1, From e1b03df693d8eaf920934681c54b919f08d87d0f Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Thu, 18 Dec 2025 16:46:13 +0100 Subject: [PATCH 06/33] Removing boost::small_vector from OpKernelArg --- .../src/include/miopen/op_kernel_args.hpp | 7 +- .../src/kernels/MIOpenTensorKernelsHip.cpp | 71 +++++-------------- .../gtest/tensor_leading_ones_ocl_hip.cpp | 49 +++++++------ 3 files changed, 45 insertions(+), 82 deletions(-) diff --git a/projects/miopen/src/include/miopen/op_kernel_args.hpp b/projects/miopen/src/include/miopen/op_kernel_args.hpp index 82a5ca9b0059..546d66ded533 100644 --- a/projects/miopen/src/include/miopen/op_kernel_args.hpp +++ b/projects/miopen/src/include/miopen/op_kernel_args.hpp @@ -7,11 +7,9 @@ #include #include #include -#include struct OpKernelArg { - OpKernelArg(char val, size_t sz) : buffer(sz) { std::fill(buffer.begin(), buffer.end(), val); } template @@ -23,15 +21,14 @@ struct OpKernelArg } template - OpKernelArg(T* arg) // NOLINT - : buffer(sizeof(T*)) + OpKernelArg(T* arg) : buffer(sizeof(T*)) { *(reinterpret_cast(buffer.data())) = arg; is_ptr = true; } std::size_t size() const { return buffer.size(); }; - boost::container::small_vector buffer; + std::vector buffer; bool is_ptr = false; }; diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index a475c05257f1..084d4ccdeee3 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -825,60 +825,27 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, // num_wg: the number of workgroups should be launched // MAX_NUM_WG: the maximum number of workgroups actually launched -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" - if(beta == static_cast(0)) -#pragma clang diagnostic pop - { - for(; gid < num_wg; gid += MAX_NUM_WG) - { - int lid = (bitmap == 0xF) ? 0 : threadIdx.x; - int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; - MIOPEN_TYPE operand = b_off[gid] * alpha1; - - int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; - int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; - int o_c = - (bitmap & (1 << 2)) - ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % - c_c) - : 0; - int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * - ((bitmap & (1 << 2)) ? c_c : 1)); - - while(lid < work_per_wg) - { - int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; - c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand); - lid += lcl_sz; - } - } - } - else + for(; gid < num_wg; gid += MAX_NUM_WG) { - for(; gid < num_wg; gid += MAX_NUM_WG) + int lid = (bitmap == 0xF) ? 0 : threadIdx.x; + int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; + + int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; + int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; + int o_c = + (bitmap & (1 << 2)) + ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % + c_c) + : 0; + int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * + ((bitmap & (1 << 2)) ? c_c : 1)); + + MIOPEN_TYPE operand = b_off[gid] * alpha1; + while(lid < work_per_wg) { - int lid = (bitmap == 0xF) ? 0 : threadIdx.x; - int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; - MIOPEN_TYPE operand = b_off[gid] * alpha1; - - int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; - int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; - int o_c = - (bitmap & (1 << 2)) - ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % - c_c) - : 0; - int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * - ((bitmap & (1 << 2)) ? c_c : 1)); - - while(lid < work_per_wg) - { - int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; - c_off[index] = - MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index]; - lid += lcl_sz; - } + int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; + c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index]; + lid += lcl_sz; } } } diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 2c102704c30a..e2ad8c49649e 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -116,21 +116,19 @@ struct OpTensorLeadingOnesTest protected: void SetUp() override { - auto&& handle = get_handle(); - std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); - data_type = miopen_type{}; + std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + // Generate elements in tensors tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( tensor_elem_gen_integer{}); - for(size_t i = 0; i < 16; ++i) - std::cout << ' ' << tensA.data[i]; - std::cout << '\n'; tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( tensor_elem_gen_integer{}); - tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( + tensor_elem_gen_integer{2}); + auto&& handle = get_handle(); // Write the device tensors tensA_dev = handle.Write(tensA.data); tensB_dev = handle.Write(tensB.data); @@ -143,10 +141,10 @@ struct OpTensorLeadingOnesTest auto first_not_one = std::find_if( tensorsConfig.blens.rbegin(), tensorsConfig.blens.rend(), [](int i) { return i != 1; }); auto d = std::distance(tensorsConfig.blens.begin(), first_not_one.base()); + num_wg = first_not_one != tensorsConfig.blens.rend() + ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) + : 1; - num_wg = first_not_one != tensorsConfig.blens.rend() - ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) - : 1; work_per_wg = std::accumulate(tensorsConfig.aclens.begin() + d, tensorsConfig.aclens.end(), 1, @@ -229,7 +227,7 @@ struct OpTensorLeadingOnesTest params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::OpenCL{}); params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES"; - std::string program_name = "MIOpenTensorKernels.cl"; + std::string program_name{"MIOpenTensorKernels.cl"}; std::string network_config_ocl = network_config + "-ocl"; handle.AddKernel( @@ -246,9 +244,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + 0LL, // Aoffset + 0LL, // Boffset + 0LL, // Coffset num_wg, bitmap); @@ -274,9 +272,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, - 0l, - 0L, + 0LL, + 0LL, + 0LL, num_wg, bitmap); #endif @@ -285,14 +283,15 @@ struct OpTensorLeadingOnesTest void runHIP() // run HIP kernel { auto&& handle = get_handle(); - tensC_dev = handle.Write(tensC.data); + // Write data to device tensor + tensC_dev = handle.Write(tensC.data); params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::HIP{}); params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES"; - std::string program_name = "MIOpenTensorKernelsHip.cpp"; + std::string program_name{"MIOpenTensorKernelsHip.cpp"}; std::string network_config_hip = network_config + "-hip"; handle.AddKernel( @@ -309,9 +308,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + 0LL, // Aoffset + 0LL, // Boffset + 0LL, // Coffset num_wg, bitmap); @@ -337,9 +336,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + 0LL, + 0LL, + 0LL, num_wg, bitmap); #endif From 7915d92c98d830cf603d12ace1bb4beb10d65bc1 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Fri, 19 Dec 2025 02:18:02 +0100 Subject: [PATCH 07/33] Fixed the order of arguments in kernel call --- .../src/kernels/MIOpenTensorKernelsHip.cpp | 14 +-- .../tensor_leading_ones_generic_ocl_hip.cpp | 88 ++++++++++++------- .../gtest/tensor_leading_ones_ocl_hip.cpp | 38 +++----- 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 084d4ccdeee3..c76e62021a53 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -807,9 +807,9 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const long Aoffset, - const long Boffset, - const long Coffset, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, const int num_wg, const unsigned int bitmap) { @@ -853,14 +853,17 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, #ifdef USE_LEADING_ONES_GENERIC extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, + const uint64_t Aoffset, const int a_nstride, const int a_cstride, const int a_hstride, MIOPEN_TYPE* b, + const uint64_t Boffset, const int b_nstride, const int b_cstride, const int b_hstride, MIOPEN_TYPE* c, + const uint64_t Coffset, const int c_c, const int c_h, const int c_w, @@ -870,11 +873,8 @@ extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const int work_per_wg, - const long Aoffset, - const long Boffset, - const long Coffset, const int num_wg, + const int work_per_wg, const unsigned int bitmap) { /* Special case for leading ones where the total no. of threads is the diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index fa70564045d2..42e820c8a83f 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -127,8 +127,6 @@ std::vector TensorsConfigs() } } } - - return configs; #else int N = 1; int C = 1; @@ -143,9 +141,8 @@ std::vector TensorsConfigs() H = 64; W = 16; insertTestCase(N, C, H, W); - - return configs; #endif + return configs; } template @@ -155,18 +152,19 @@ struct OpTensorLeadingOnesGenericTest protected: void SetUp() override { - auto&& handle = get_handle(); - std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); - data_type = miopen_type{}; + std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + // Generate elements in tensors tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( - tensor_elem_gen_integer{17}); + tensor_elem_gen_integer{}); tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( - tensor_elem_gen_integer{17}); - tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}; + tensor_elem_gen_integer{}); + tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( + [](auto...) { return 1; }); + auto&& handle = get_handle(); // Write the device tensors tensA_dev = handle.Write(tensA.data); tensB_dev = handle.Write(tensB.data); @@ -179,10 +177,10 @@ struct OpTensorLeadingOnesGenericTest auto first_not_one = std::find_if( tensorsConfig.blens.rbegin(), tensorsConfig.blens.rend(), [](int i) { return i != 1; }); auto d = std::distance(tensorsConfig.blens.begin(), first_not_one.base()); + num_wg = first_not_one != tensorsConfig.blens.rend() + ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) + : 1; - num_wg = first_not_one != tensorsConfig.blens.rend() - ? static_cast(*first_not_one == 0 ? 1 : *first_not_one) - : 1; work_per_wg = std::accumulate(tensorsConfig.aclens.begin() + d, tensorsConfig.aclens.end(), 1, @@ -229,13 +227,19 @@ struct OpTensorLeadingOnesGenericTest params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::OpenCL{}); params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES_GENERIC"; - std::string program_name = "MIOpenTensorKernels.cl"; + std::string program_name{"MIOpenTensorKernels.cl"}; std::string network_config_ocl = network_config + "-ocl"; handle.AddKernel( kernel_name, network_config_ocl, program_name, kernel_name, vld, vgd, params)( tensA_dev.get(), + tensorsConfig.acstrides[0], // a_nstride + tensorsConfig.acstrides[1], // a_cstride + tensorsConfig.acstrides[2], // a_hstride tensB_dev.get(), + tensorsConfig.bstrides[0], // b_nstride + tensorsConfig.bstrides[1], // b_cstride + tensorsConfig.bstrides[2], // b_hstride tensC_dev.get(), tensorsConfig.aclens[1], // c_c tensorsConfig.aclens[2], // c_h @@ -243,13 +247,13 @@ struct OpTensorLeadingOnesGenericTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride tensorsConfig.acstrides[2], // c_hstride - work_per_wg, alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + work_per_wg, + 0LL, // Aoffset + 0LL, // Boffset + 0LL, // Coffset num_wg, bitmap); @@ -260,7 +264,13 @@ struct OpTensorLeadingOnesGenericTest kernel_name, network_config_ocl, tensA_dev.get(), + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], tensB_dev.get(), + tensorsConfig.bstrides[0], + tensorsConfig.bstrides[1], + tensorsConfig.bstrides[2], tensC_dev.get(), tensorsConfig.aclens[1], tensorsConfig.aclens[2], @@ -268,13 +278,13 @@ struct OpTensorLeadingOnesGenericTest tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], tensorsConfig.acstrides[2], - work_per_wg, alpha0, alpha1, beta, - 0L, - 0L, - 0L, + work_per_wg, + 0LL, + 0LL, + 0LL, num_wg, bitmap); #endif @@ -283,36 +293,42 @@ struct OpTensorLeadingOnesGenericTest void runHIP() // run HIP kernel { auto&& handle = get_handle(); - tensC_dev = handle.Write(tensC.data); + // Write data to device tensor + tensC_dev = handle.Write(tensC.data); params = " -DMIOPEN_TYPE=" + miopen::GetDataType(data_type) + " -DMAX_NUM_WG=" + std::to_string(max_num_wg); params += " " + miopen::GetDataTypeKBP(data_type).GenerateFor(miopen::kbp::HIP{}); params += " -DMIOPEN_TENSOR_OP=miopenAdd -DUSE_LEADING_ONES_GENERIC"; - std::string program_name = "MIOpenTensorKernelsHip.cpp"; + std::string program_name{"MIOpenTensorKernelsHip.cpp"}; std::string network_config_hip = network_config + "-hip"; handle.AddKernel( kernel_name, network_config_hip, program_name, kernel_name, vld, vgd, params)( tensA_dev.get(), + uint64_t(0), // Aoffset + tensorsConfig.acstrides[0], // a_nstride + tensorsConfig.acstrides[1], // a_cstride + tensorsConfig.acstrides[2], // a_hstride tensB_dev.get(), + uint64_t(0), // Boffset + tensorsConfig.bstrides[0], // b_nstride + tensorsConfig.bstrides[1], // b_cstride + tensorsConfig.bstrides[2], // b_hstride tensC_dev.get(), - tensC_dev.get(), + uint64_t(0), // Coffset tensorsConfig.aclens[1], // c_c tensorsConfig.aclens[2], // c_h tensorsConfig.aclens[3], // c_w tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride tensorsConfig.acstrides[2], // c_hstride - work_per_wg, alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset num_wg, + work_per_wg, bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); @@ -322,22 +338,28 @@ struct OpTensorLeadingOnesGenericTest kernel_name, network_config_hip, tensA_dev.get(), + uint64_t(0), + tensorsConfig.acstrides[0], + tensorsConfig.acstrides[1], + tensorsConfig.acstrides[2], tensB_dev.get(), + uint64_t(0), + tensorsConfig.bstrides[0], + tensorsConfig.bstrides[1], + tensorsConfig.bstrides[2], tensC_dev.get(), + uint64_t(0), tensorsConfig.aclens[1], tensorsConfig.aclens[2], tensorsConfig.aclens[3], tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], tensorsConfig.acstrides[2], - work_per_wg, alpha0, alpha1, beta, - 0L, - 0L, - 0L, num_wg, + work_per_wg, bitmap); #endif } diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index e2ad8c49649e..843cc3271654 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -75,19 +75,13 @@ std::vector TensorsConfigs() // 2) Fallback table by architecture family if(maxTotalSize == 0) - { maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); - } int C = 4; int H = 8; int W = 8; for(int N = 32; N <= maxTotalSize / (C * H * W); N *= 2) - { insertTestCase(N, C, H, W); - } - - return configs; #else int N = 16; int C = 1; @@ -104,9 +98,8 @@ std::vector TensorsConfigs() H = 16; W = 16; insertTestCase(N, C, H, W); - - return configs; #endif + return configs; } template @@ -126,7 +119,7 @@ struct OpTensorLeadingOnesTest tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( tensor_elem_gen_integer{}); tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( - tensor_elem_gen_integer{2}); + [](auto...) { return 1; }); auto&& handle = get_handle(); // Write the device tensors @@ -251,10 +244,6 @@ struct OpTensorLeadingOnesTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); - std::cout << "OCL:"; - for(size_t i = 0; i < 16; ++i) - std::cout << ' ' << tensC_ocl.data[i]; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -308,17 +297,13 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0LL, // Aoffset - 0LL, // Boffset - 0LL, // Coffset + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); - std::cout << "HIP:"; - for(size_t i = 0; i < 16; ++i) - std::cout << ' ' << tensC_hip.data[i]; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -336,9 +321,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0LL, - 0LL, - 0LL, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, bitmap); #endif @@ -405,7 +390,7 @@ struct OpTensorLeadingOnesTest PerfHelper ph; #endif }; -/* + using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -421,7 +406,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) @@ -437,7 +422,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -/* + using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) @@ -453,4 +438,3 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ From aeba9b6b5ae5097a19bbd492deec55a26d508afe Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 23 Dec 2025 00:30:23 +0100 Subject: [PATCH 08/33] Changed arrange of TensorsConfig --- .../tensor_leading_ones_generic_ocl_hip.cpp | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 42e820c8a83f..12c159e6fbab 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -50,36 +50,6 @@ std::vector TensorsConfigs() auto insertTestCase = [&configs](int N, int C, int H, int W) { configs.push_back( {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, W}, {C * H * W, H * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, H, 1}, {C * H * 1, H * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, 1, W}, {C * 1 * W, 1 * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, C, 1, 1}, {C * 1 * 1, 1 * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, H, W}, {1 * H * W, H * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, H, 1}, {1 * H * 1, H * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, 1, W}, {1 * 1 * W, 1 * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {N, 1, 1, 1}, {1 * 1 * 1, 1 * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, H, W}, {C * H * W, H * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, H, 1}, {C * H * 1, H * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, 1, W}, {C * 1 * W, 1 * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, C, 1, 1}, {C * 1 * 1, 1 * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, H, W}, {1 * H * W, H * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, H, 1}, {1 * H * 1, H * 1, 1, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, 1, W}, {1 * 1 * W, 1 * W, W, 1}}); - configs.push_back( - {{N, C, H, W}, {C * H * W, H * W, W, 1}, {1, 1, 1, 1}, {1 * 1 * 1, 1 * 1, 1, 1}}); }; #if PERF_ENABLE From 73ef79592c9bc1215948ec7a2ae42d3225a80e75 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 23 Dec 2025 03:27:56 +0100 Subject: [PATCH 09/33] Added debug output --- .../miopen/src/kernels/MIOpenTensorKernels.cl | 6 +++--- .../gtest/tensor_leading_ones_ocl_hip.cpp | 20 ++++++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernels.cl b/projects/miopen/src/kernels/MIOpenTensorKernels.cl index cebbe16d9081..779ea0dd524b 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernels.cl +++ b/projects/miopen/src/kernels/MIOpenTensorKernels.cl @@ -221,14 +221,14 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, const int c_nstride, const int c_cstride, const int work_per_wg, + const int num_wg, + const unsigned int bitmap, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, const long Aoffset, const long Boffset, - const long Coffset, - const int num_wg, - const unsigned int bitmap) + const long Coffset) { /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 843cc3271654..0abcee57a427 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -207,6 +207,9 @@ struct OpTensorLeadingOnesTest network_config += std::to_string(data_type) + "-miopenTensorOpAdd-" + std::to_string(global_threads) + "-" + std::to_string(local_threads); + std::cout << "MIOPEN_TYPE=" << miopen::GetDataType(data_type) << '\n'; + std::cout << "work_per_wg=" << work_per_wg << " num_wg=" << num_wg << " bitmap=" << bitmap + << '\n'; } void runOCL() // run OCL kernel @@ -234,17 +237,21 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, + num_wg, + bitmap, alpha0, alpha1, beta, 0LL, // Aoffset 0LL, // Boffset - 0LL, // Coffset - num_wg, - bitmap); + 0LL // Coffset + ); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); - + std::cout << "OCL:"; + for(size_t i = 0; i < 16; ++i) + std::cout << ' ' << tensC_ocl.data[i]; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, kernel_name, @@ -304,7 +311,10 @@ struct OpTensorLeadingOnesTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); - + std::cout << "HIP:"; + for(size_t i = 0; i < 16; ++i) + std::cout << ' ' << tensC_hip.data[i]; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, kernel_name, From 19fa395f58a606488ba47b6e9c62b63ba095fc08 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 23 Dec 2025 16:57:13 +0100 Subject: [PATCH 10/33] Restored order of arguments in OpTensorLeadingOnes call --- .../miopen/src/kernels/MIOpenTensorKernels.cl | 6 ++--- .../gtest/tensor_leading_ones_ocl_hip.cpp | 24 ++++++++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernels.cl b/projects/miopen/src/kernels/MIOpenTensorKernels.cl index 779ea0dd524b..cebbe16d9081 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernels.cl +++ b/projects/miopen/src/kernels/MIOpenTensorKernels.cl @@ -221,14 +221,14 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, const int c_nstride, const int c_cstride, const int work_per_wg, - const int num_wg, - const unsigned int bitmap, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, const long Aoffset, const long Boffset, - const long Coffset) + const long Coffset, + const int num_wg, + const unsigned int bitmap) { /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 0abcee57a427..86f0656bd1a5 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -109,8 +109,7 @@ struct OpTensorLeadingOnesTest protected: void SetUp() override { - data_type = miopen_type{}; - + prng::reset_seed(); std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); // Generate elements in tensors @@ -205,11 +204,9 @@ struct OpTensorLeadingOnesTest vld = {local_threads, 1, 1}; vgd = {global_threads, 1, 1}; + data_type = miopen_type{}; network_config += std::to_string(data_type) + "-miopenTensorOpAdd-" + std::to_string(global_threads) + "-" + std::to_string(local_threads); - std::cout << "MIOPEN_TYPE=" << miopen::GetDataType(data_type) << '\n'; - std::cout << "work_per_wg=" << work_per_wg << " num_wg=" << num_wg << " bitmap=" << bitmap - << '\n'; } void runOCL() // run OCL kernel @@ -237,15 +234,14 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, - num_wg, - bitmap, alpha0, alpha1, beta, - 0LL, // Aoffset - 0LL, // Boffset - 0LL // Coffset - ); + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, + bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); std::cout << "OCL:"; @@ -268,9 +264,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0LL, - 0LL, - 0LL, + 0L, + 0L, + 0L, num_wg, bitmap); #endif From a630a1bd20ff3b8779c57ea23c7ae5da473e304c Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 23 Dec 2025 17:07:30 +0100 Subject: [PATCH 11/33] Update op_kernel_args.hpp --- projects/miopen/src/include/miopen/op_kernel_args.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/miopen/src/include/miopen/op_kernel_args.hpp b/projects/miopen/src/include/miopen/op_kernel_args.hpp index 546d66ded533..ea5fd532344a 100644 --- a/projects/miopen/src/include/miopen/op_kernel_args.hpp +++ b/projects/miopen/src/include/miopen/op_kernel_args.hpp @@ -7,6 +7,7 @@ #include #include #include +#include struct OpKernelArg { @@ -28,7 +29,7 @@ struct OpKernelArg } std::size_t size() const { return buffer.size(); }; - std::vector buffer; + boost::container::small_vector buffer; bool is_ptr = false; }; From aa751e14fa5659f9dacceeb33b49a0149ab8b06d Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 23 Dec 2025 17:10:02 +0100 Subject: [PATCH 12/33] Update tensor_leading_ones_ocl_hip.cpp --- .../gtest/tensor_leading_ones_ocl_hip.cpp | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 86f0656bd1a5..53fec520dfc1 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -237,17 +237,14 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + 0LL, // Aoffset + 0LL, // Boffset + 0LL, // Coffset num_wg, bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); - std::cout << "OCL:"; - for(size_t i = 0; i < 16; ++i) - std::cout << ' ' << tensC_ocl.data[i]; - std::cout << '\n'; + #if PERF_ENABLE ph.perfTest(handle, kernel_name, @@ -264,9 +261,9 @@ struct OpTensorLeadingOnesTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + 0LL, + 0LL, + 0LL, num_wg, bitmap); #endif @@ -307,10 +304,7 @@ struct OpTensorLeadingOnesTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); - std::cout << "HIP:"; - for(size_t i = 0; i < 16; ++i) - std::cout << ' ' << tensC_hip.data[i]; - std::cout << '\n'; + #if PERF_ENABLE ph.perfTest(handle, kernel_name, From 4913cebce596795c3be440ee0aac5e1cd657e684 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Wed, 24 Dec 2025 02:19:01 +0100 Subject: [PATCH 13/33] Fixed the type of 'alpha, beta' --- .../gtest/tensor_leading_ones_ocl_hip.cpp | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 53fec520dfc1..ce30fb11e7e8 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -234,12 +234,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, - alpha0, - alpha1, - beta, - 0LL, // Aoffset - 0LL, // Boffset - 0LL, // Coffset + (T)alpha0, + (T)alpha1, + (T)beta, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, bitmap); @@ -258,12 +258,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], work_per_wg, - alpha0, - alpha1, - beta, - 0LL, - 0LL, - 0LL, + (T)alpha0, + (T)alpha1, + (T)beta, + 0L, + 0L, + 0L, num_wg, bitmap); #endif @@ -294,9 +294,9 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, - alpha0, - alpha1, - beta, + (T)alpha0, + (T)alpha1, + (T)beta, uint64_t(0), // Aoffset uint64_t(0), // Boffset uint64_t(0), // Coffset @@ -318,9 +318,9 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], work_per_wg, - alpha0, - alpha1, - beta, + (T)alpha0, + (T)alpha1, + (T)beta, uint64_t(0), uint64_t(0), uint64_t(0), @@ -384,7 +384,7 @@ struct OpTensorLeadingOnesTest miopen::Allocator::ManageDataPtr tensC_dev; TensorsConfig tensorsConfig; - T alpha0, alpha1, beta; + double alpha0, alpha1, beta; #if PERF_ENABLE PerfHelper ph; From 07e7a81fbb8effec040cd472d58a9f7b02c296ed Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 20 Jan 2026 01:47:30 +0100 Subject: [PATCH 14/33] Fixing clang-format-violations in MIOpenTensorKernelsHip.cpp --- projects/miopen/CMakeLists.txt | 23 +- projects/miopen/src/CMakeLists.txt | 48 ++-- .../src/include/miopen/target_properties.hpp | 6 - .../src/kernels/MIOpenTensorKernelsHip.cpp | 218 +++++++++--------- projects/miopen/src/reducetensor.cpp | 4 +- projects/miopen/test/gtest/CMakeLists.txt | 2 +- 6 files changed, 150 insertions(+), 151 deletions(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 01960b235150..cbe7bdd5d224 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -77,8 +77,6 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -include(CTest) - find_package(Threads REQUIRED) find_package(ROCmCMakeBuildTools) @@ -341,21 +339,22 @@ if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_B if(MIOPEN_BUILD_CK) include(cmake/Dependencies.cmake) - execute_process ( - COMMAND bash -c "grep -P -o '^[ \t]*(? in requirements.txt") endif() - miopen_add_dependency(composable_kernel options NO_LOCAL) + miopen_add_dependency(composable_kernel HASH ${COMPOSABLE_KERNEL_HASH} options NO_LOCAL) else() - find_package(composable_kernel 1.0.0 COMPONENTS device_conv_operations) + find_package(composable_kernel COMPONENTS device_conv_operations) endif() endif() - if( MIOPEN_BACKEND STREQUAL "HIPNOGPU") + if(MIOPEN_BACKEND STREQUAL "HIPNOGPU") set(MIOPEN_MODE_NOGPU 1) endif() set(MIOPEN_BACKEND_HIP 1) @@ -500,7 +499,7 @@ find_package(Boost REQUIRED COMPONENTS filesystem CONFIG) find_path(HALF_INCLUDE_DIR half/half.hpp) message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") -option( MIOPEN_DEBUG_FIND_DB_CACHING "Use system find-db caching" ON) +option(MIOPEN_DEBUG_FIND_DB_CACHING "Use system find-db caching" ON) if(WIN32) set( DATA_INSTALL_DIR bin ) @@ -552,8 +551,8 @@ if(NOT WIN32 AND MIOPEN_USE_ROCTRACER) find_path(ROCTRACER_INCLUDE_DIR "roctracer/roctx.h") find_library(rocTracer roctx64) if(rocTracer AND ROCTRACER_INCLUDE_DIR) - MESSAGE(STATUS "Build with rocTracer: " ${rocTracer}) - MESSAGE(STATUS "rocTracer include directory: " ${ROCTRACER_INCLUDE_DIR}) + message(STATUS "Build with rocTracer: " ${rocTracer}) + message(STATUS "rocTracer include directory: " ${ROCTRACER_INCLUDE_DIR}) set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, roctracer") else() message(WARNING "rocTracer cannot be found! Build without rocTracer") diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index 05452988daa9..3d632f540d46 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -38,11 +38,9 @@ endif() # 2. How BF16 kernels (which are kind of mixed-precision now and expected to # remain in the future) perform final conversion (and rounding) of FP32 # to BF16 results. This affects the main functionality of the library. -option( MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) -option( MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) -option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) - -option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) +option(MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) +option(MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) +option(MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) configure_file("${PROJECT_SOURCE_DIR}/include/miopen/config.h.in" "${PROJECT_BINARY_DIR}/include/miopen/config.h") @@ -301,6 +299,8 @@ set( MIOpen_Source solver/layernorm/backward_t5layernorm.cpp solver/layernorm/forward_addlayernorm.cpp solver/layernorm/forward_layernorm.cpp + solver/layernorm/forward_layernorm2d_ck.cpp + solver/layernorm/forward_layernorm4d_ck.cpp solver/layernorm/forward_t5layernorm.cpp solver/mha/mha_solver_backward.cpp solver/mha/mha_solver_forward.cpp @@ -364,7 +364,14 @@ if(MIOPEN_ENABLE_SQLITE AND MIOPEN_ENABLE_SQLITE_KERN_CACHE) list(APPEND MIOpen_Source kern_db.cpp bz2.cpp) endif() -if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") +if(MIOPEN_USE_COMPOSABLEKERNEL) + list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f16_instance.cpp) + list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f32_instance.cpp) + list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f16_instance.cpp) + list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f32_instance.cpp) +endif() + +if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_INCLUDE "kernels/static_composable_kernel/include/*/*.hpp") file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_SOURCE "kernels/static_composable_kernel/src/*/*.cpp") file(GLOB_RECURSE COMPOSABLE_KERNEL_INCLUDE "legacy_composable_kernel/composable_kernel/include/*.hpp") @@ -378,7 +385,6 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_INCLUDE "kernels/gpu_general_tensor_reorder_kernel/*.hpp") file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_SOURCE "kernels/gpu_general_tensor_reorder_kernel/*.cpp") - set(MIOPEN_KERNEL_INCLUDES ${STATIC_COMPOSABLE_KERNEL_INCLUDE} ${COMPOSABLE_KERNEL_INCLUDE} @@ -596,7 +602,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/MIOpenT5LayerNorm.cpp kernels/MIOpenUpSample.cpp kernels/MIOpenUtilKernels4.cl - kernels/MIOpenUtilKernels5.cpp + kernels/MIOpenUtilKernels5.cl kernels/MIOpenVecAdd.cpp kernels/MIOpenVecAddOCL.cl kernels/MIOpenIm2d2Col.cpp @@ -611,7 +617,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/MIOpenConvBwdWrW1x1_PAD_read4.cl kernels/MIOpenConvFwd_LxL_11.cl kernels/MIOpenConvFFT.cpp - kernels/MIOpenRNNHiddenStateUpdate.cpp + kernels/MIOpenRNNHiddenStateUpdate.cl kernels/bugzilla_34765_detect.s kernels/dummy_kernel.s kernels/conv3x3.s @@ -693,7 +699,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/xform_bidirect_winograd_data.s kernels/xform_bidirect_winograd_filter.s kernels/xform_bidirect_winograd_out.s - kernels/UniversalTranspose.cpp) + kernels/UniversalTranspose.cl) # Kernels in development lists. # Should be ALWAYS empty in develop branch (at the time of PR merge) @@ -742,6 +748,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN ocl/utilocl.cpp ocl/dropoutocl.cpp ocl/gcn_asm_utils.cpp + ocl/rnn_util_ocl.cpp hip/hip_build_utils.cpp hip/batched_transpose_sol.cpp hip/general_tensor_reorder_sol.cpp @@ -926,9 +933,6 @@ if(MIOPEN_USE_COMPOSABLEKERNEL) # Use the aliased targets when we pull CK from /opt/rocm or other place on disc. set(MIOPEN_CK_LINK_FLAGS composable_kernel::device_conv_operations hip::host) endif() - if(GPU_TARGETS MATCHES "gfx942" OR GPU_TARGETS MATCHES "gfx950") - target_compile_definitions(MIOpen PRIVATE CK_ENABLE_TF32) - endif() endif() if(WIN32) @@ -957,10 +961,6 @@ if(BUILD_TESTING) target_compile_definitions(MIOpen PUBLIC $) endif() -if(MIOPEN_CK_BUILDER_EXPERIMENTAL) - target_compile_definitions(MIOpen PRIVATE CK_EXPERIMENTAL_BUILDER) -endif() - if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) target_link_libraries(MIOpen PRIVATE frugally-deep::fdeep Eigen3::Eigen) if(NOT TARGET nlohmann_json) @@ -1064,13 +1064,7 @@ local: # set_target_properties(MIOpen PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(MIOpen PROPERTIES VISIBILITY_INLINES_HIDDEN 1) endif() -####################################### -if(MIOPEN_ENABLE_SQLITE) - # MIOpen depends on SQLite - target_link_libraries(MIOpen PRIVATE SQLite::SQLite3) - target_link_libraries(MIOpen INTERFACE $) -endif() -############################################################ + # MIOpen depends on librt for Boost.Interprocess if(NOT WIN32 AND NOT APPLE) find_library(LIBRT rt) @@ -1080,6 +1074,12 @@ if(NOT WIN32 AND NOT APPLE) endif() endif() +if(MIOPEN_ENABLE_SQLITE) + # MIOpen depends on SQLite + target_link_libraries(MIOpen PRIVATE SQLite::SQLite3) + target_link_libraries(MIOpen INTERFACE $) +endif() + if(MIOPEN_USE_ROCTRACER) target_include_directories(MIOpen PRIVATE ${ROCTRACER_INCLUDE_DIR}) target_include_directories(MIOpen INTERFACE $) diff --git a/projects/miopen/src/include/miopen/target_properties.hpp b/projects/miopen/src/include/miopen/target_properties.hpp index cab04861a8e3..9f2962c7cfb0 100644 --- a/projects/miopen/src/include/miopen/target_properties.hpp +++ b/projects/miopen/src/include/miopen/target_properties.hpp @@ -29,18 +29,12 @@ #include #include #include -#include #define WORKAROUND_ISSUE_1204 1 // ROCm may incorrectly report "sramecc-" for gfx900. #define WORKAROUND_ISSUE_3001 1 namespace miopen { -static inline bool IsTF32Supported(const std::string& device_name) -{ - return device_name == "gfx942" || StartsWith(device_name, "gfx95"); -} - struct Handle; class TargetProperties diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 238fdae9a68e..4300b401d1e3 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -794,6 +794,118 @@ extern "C" __global__ void OpTensorFwdBiasGeneric(MIOPEN_TYPE* a, } #endif +#ifdef USE_2D_TENSOR_LITE +extern "C" __global__ void Op2dTensorLite(const MIOPEN_TYPE* a, + const int a_nstride, + const MIOPEN_TYPE* b, + const int b_nstride, + MIOPEN_TYPE* c, + const int c_nstride, + const MIOPEN_TYPE alpha0, + const MIOPEN_TYPE alpha1, + const MIOPEN_TYPE beta, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, + const size_t total_work, + const size_t total_work2, + const int use_beta, + const int use_bias) +{ + int gid0 = blockIdx.x * blockDim.x + threadIdx.x; + int gid1 = blockIdx.y * blockDim.y + threadIdx.y; + + MIOPEN_TYPE a_dat[RD_BLCK]; + MIOPEN_TYPE b_dat[RD_BLCK]; + MIOPEN_TYPE c_dat[RD_BLCK]; + + if(gid0 < total_work) + { + if(use_bias == 1) + { + int b_index = gid0 * RD_BLCK; + *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); + } + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" + if(beta == static_cast(0)) +#pragma clang diagnostic pop + { + for(; gid1 < total_work2; gid1 += gridDim.y * blockDim.y) + { + for(int i = 0; i < RD_BLCK; ++i) + { + c_dat[i] = (MIOPEN_TYPE)0; + } + + int a_index = gid1 * a_nstride + gid0 * RD_BLCK; + int c_index = gid1 * c_nstride + gid0 * RD_BLCK; + + *((READ_TYPE*)a_dat) = *((const READ_TYPE*)(a + Aoffset + a_index)); + if(use_beta == 1) + { + *((READ_TYPE*)c_dat) = *((const READ_TYPE*)(c + Coffset + c_index)); + } + + if(use_bias == 0) + { + int b_index = gid1 * b_nstride + gid0 * RD_BLCK; + *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); + } + + for(int i = 0; i < RD_BLCK; ++i) + { + if(use_beta == 1) + { + c_dat[i] = (MIOPEN_TYPE)0; + } + c_dat[i] += MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1); + } + + *((READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat); + } + } + else + { + for(; gid1 < total_work2; gid1 += gridDim.y * blockDim.y) + { + for(int i = 0; i < RD_BLCK; ++i) + { + c_dat[i] = (MIOPEN_TYPE)0; + } + + int a_index = gid1 * a_nstride + gid0 * RD_BLCK; + int c_index = gid1 * c_nstride + gid0 * RD_BLCK; + + *((READ_TYPE*)a_dat) = *((const READ_TYPE*)(a + Aoffset + a_index)); + if(use_beta == 1) + { + *((READ_TYPE*)c_dat) = *((const READ_TYPE*)(c + Coffset + c_index)); + } + + if(use_bias == 0) + { + int b_index = gid1 * b_nstride + gid0 * RD_BLCK; + *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); + } + + for(int i = 0; i < RD_BLCK; ++i) + { + if(use_beta == 1) + { + c_dat[i] *= beta; + } + c_dat[i] += MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1); + } + + *((READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat); + } + } + } +} +#endif + #ifdef USE_LEADING_ONES extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, MIOPEN_TYPE* b, @@ -968,112 +1080,6 @@ extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex]; lid += lcl_sz; -#ifdef USE_2D_TENSOR_LITE -extern "C" __global__ void Op2dTensorLite(const MIOPEN_TYPE* a, - const int a_nstride, - const MIOPEN_TYPE* b, - const int b_nstride, - MIOPEN_TYPE* c, - const int c_nstride, - const MIOPEN_TYPE alpha0, - const MIOPEN_TYPE alpha1, - const MIOPEN_TYPE beta, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, - const size_t total_work, - const size_t total_work2, - const int use_beta, - const int use_bias) -{ - int gid0 = blockIdx.x * blockDim.x + threadIdx.x; - int gid1 = blockIdx.y * blockDim.y + threadIdx.y; - - MIOPEN_TYPE a_dat[RD_BLCK]; - MIOPEN_TYPE b_dat[RD_BLCK]; - MIOPEN_TYPE c_dat[RD_BLCK]; - - if(gid0 < total_work) - { - if(use_bias == 1) - { - int b_index = gid0 * RD_BLCK; - *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); - } - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" - if(beta == static_cast(0)) -#pragma clang diagnostic pop - { - for(; gid1 < total_work2; gid1 += gridDim.y * blockDim.y) - { - for(int i = 0; i < RD_BLCK; ++i) - { - c_dat[i] = (MIOPEN_TYPE)0; - } - - int a_index = gid1 * a_nstride + gid0 * RD_BLCK; - int c_index = gid1 * c_nstride + gid0 * RD_BLCK; - - *((READ_TYPE*)a_dat) = *((const READ_TYPE*)(a + Aoffset + a_index)); - if(use_beta == 1) - { - *((READ_TYPE*)c_dat) = *((const READ_TYPE*)(c + Coffset + c_index)); - } - - if(use_bias == 0) - { - int b_index = gid1 * b_nstride + gid0 * RD_BLCK; - *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); - } - - for(int i = 0; i < RD_BLCK; ++i) - { - if(use_beta == 1) - { - c_dat[i] = (MIOPEN_TYPE)0; - } - c_dat[i] += MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1); - } - - *((READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat); - } - } - else - { - for(; gid1 < total_work2; gid1 += gridDim.y * blockDim.y) - { - for(int i = 0; i < RD_BLCK; ++i) - { - c_dat[i] = (MIOPEN_TYPE)0; - } - - int a_index = gid1 * a_nstride + gid0 * RD_BLCK; - int c_index = gid1 * c_nstride + gid0 * RD_BLCK; - - *((READ_TYPE*)a_dat) = *((const READ_TYPE*)(a + Aoffset + a_index)); - if(use_beta == 1) - { - *((READ_TYPE*)c_dat) = *((const READ_TYPE*)(c + Coffset + c_index)); - } - - if(use_bias == 0) - { - int b_index = gid1 * b_nstride + gid0 * RD_BLCK; - *((READ_TYPE*)b_dat) = *((const READ_TYPE*)(b + Boffset + b_index)); - } - - for(int i = 0; i < RD_BLCK; ++i) - { - if(use_beta == 1) - { - c_dat[i] *= beta; - } - c_dat[i] += MIOPEN_TENSOR_OP(a_dat[i] * alpha0, b_dat[i] * alpha1); - } - - *((READ_TYPE*)(c + Coffset + c_index)) = *((READ_TYPE*)c_dat); } } } diff --git a/projects/miopen/src/reducetensor.cpp b/projects/miopen/src/reducetensor.cpp index 8becd7fa005a..eddee1eb0506 100644 --- a/projects/miopen/src/reducetensor.cpp +++ b/projects/miopen/src/reducetensor.cpp @@ -37,8 +37,8 @@ #include // headers from composable kernel, to get consistent ID mapping -#include <../legacy_composable_kernel/composable_kernel/include/utility/data_type_enum.hpp> -#include <../legacy_composable_kernel/composable_kernel/include/utility/reduction_enums.hpp> +#include "legacy_composable_kernel/composable_kernel/include/utility/data_type_enum.hpp" +#include "legacy_composable_kernel/composable_kernel/include/utility/reduction_enums.hpp" namespace miopen { diff --git a/projects/miopen/test/gtest/CMakeLists.txt b/projects/miopen/test/gtest/CMakeLists.txt index cff6e6756f18..646fa959718c 100644 --- a/projects/miopen/test/gtest/CMakeLists.txt +++ b/projects/miopen/test/gtest/CMakeLists.txt @@ -229,7 +229,7 @@ else() add_gtest(miopen_gtest "${TESTS_CPP}") if(MIOPEN_USE_COMPOSABLEKERNEL) - if(MIOPEN_BUILD_CK ) + if(MIOPEN_BUILD_CK) # The include directories also dont propagate correctly when using the non aliased targets. # Manually including them fixes this problem for inline builds. target_include_directories(miopen_gtest SYSTEM PRIVATE From 1399dbd73f4c4940660e1a576290004f20a7d37c Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 20 Jan 2026 02:37:27 +0100 Subject: [PATCH 15/33] Removed mention of non existent source files from src/CMakeLists.txt --- projects/miopen/src/CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index 3d632f540d46..40b633ca674d 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -299,8 +299,8 @@ set( MIOpen_Source solver/layernorm/backward_t5layernorm.cpp solver/layernorm/forward_addlayernorm.cpp solver/layernorm/forward_layernorm.cpp - solver/layernorm/forward_layernorm2d_ck.cpp - solver/layernorm/forward_layernorm4d_ck.cpp +# solver/layernorm/forward_layernorm2d_ck.cpp +# solver/layernorm/forward_layernorm4d_ck.cpp solver/layernorm/forward_t5layernorm.cpp solver/mha/mha_solver_backward.cpp solver/mha/mha_solver_forward.cpp @@ -364,12 +364,12 @@ if(MIOPEN_ENABLE_SQLITE AND MIOPEN_ENABLE_SQLITE_KERN_CACHE) list(APPEND MIOpen_Source kern_db.cpp bz2.cpp) endif() -if(MIOPEN_USE_COMPOSABLEKERNEL) - list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f16_instance.cpp) - list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f32_instance.cpp) - list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f16_instance.cpp) - list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f32_instance.cpp) -endif() +#if(MIOPEN_USE_COMPOSABLEKERNEL) +# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f16_instance.cpp) +# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f32_instance.cpp) +# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f16_instance.cpp) +# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f32_instance.cpp) +#endif() if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_INCLUDE "kernels/static_composable_kernel/include/*/*.hpp") @@ -748,7 +748,7 @@ if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_ ocl/utilocl.cpp ocl/dropoutocl.cpp ocl/gcn_asm_utils.cpp - ocl/rnn_util_ocl.cpp +# ocl/rnn_util_ocl.cpp hip/hip_build_utils.cpp hip/batched_transpose_sol.cpp hip/general_tensor_reorder_sol.cpp From 55ba2b8e6c7331dfc5801eca9f65d4ade829479d Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 20 Jan 2026 02:52:45 +0100 Subject: [PATCH 16/33] Update CMakeLists.txt for MIOpenUtilKernels5 --- projects/miopen/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index 40b633ca674d..acaa0b4aae08 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -602,7 +602,7 @@ if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_ kernels/MIOpenT5LayerNorm.cpp kernels/MIOpenUpSample.cpp kernels/MIOpenUtilKernels4.cl - kernels/MIOpenUtilKernels5.cl + kernels/MIOpenUtilKernels5.cpp kernels/MIOpenVecAdd.cpp kernels/MIOpenVecAddOCL.cl kernels/MIOpenIm2d2Col.cpp From 78376e47afbae8c88edb57cfef04a3fd02079083 Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 20 Jan 2026 03:09:38 +0100 Subject: [PATCH 17/33] Update CMakeLists.txt for MIOpenRNNHiddenStateUpdate --- projects/miopen/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index acaa0b4aae08..79691e71b9bb 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -617,7 +617,7 @@ if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_ kernels/MIOpenConvBwdWrW1x1_PAD_read4.cl kernels/MIOpenConvFwd_LxL_11.cl kernels/MIOpenConvFFT.cpp - kernels/MIOpenRNNHiddenStateUpdate.cl + kernels/MIOpenRNNHiddenStateUpdate.cpp kernels/bugzilla_34765_detect.s kernels/dummy_kernel.s kernels/conv3x3.s From e923d42a48dd8b5d810a8e75013004f4c0629233 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Tue, 20 Jan 2026 15:44:05 +0100 Subject: [PATCH 18/33] Revert IsTF32Supported() in target_properties.hpp --- projects/miopen/CMakeLists.txt | 4 ++-- projects/miopen/src/CMakeLists.txt | 16 +++++----------- .../src/include/miopen/target_properties.hpp | 6 ++++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index cbe7bdd5d224..4d3ae83985f7 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -493,7 +493,7 @@ if(MIOPEN_USE_HIPRTC) endif() option(Boost_USE_STATIC_LIBS "Use boost static libraries" ON) -add_definitions(-DBOOST_ALL_NO_LIB=1) +#add_definitions(-DBOOST_ALL_NO_LIB=1) find_package(Boost REQUIRED COMPONENTS filesystem CONFIG) find_path(HALF_INCLUDE_DIR half/half.hpp) @@ -683,7 +683,7 @@ endfunction() # install for all targets; this was the behaviour before MIOPEN_INSTALL_GPU_DATABASES # was added. set(ALL_GPU_DATABASES gfx803 gfx900 gfx906 gfx908 gfx90a gfx942 gfx1030) -set(MIOPEN_INSTALL_GPU_DATABASES "${ALL_GPU_DATABASES}" CACHE LIST "List of AMDGPU targets for which to install databases") +set(MIOPEN_INSTALL_GPU_DATABASES "${ALL_GPU_DATABASES}" CACHE STRING "List of AMDGPU targets for which to install databases") # This list should be updated when a new KDB file is added to the project set(KDB_BZ2_FILES gfx942.kdb.bz2 gfx90a.kdb.bz2 gfx1030.kdb.bz2 gfx908.kdb.bz2 gfx906.kdb.bz2 gfx900.kdb.bz2) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index 79691e71b9bb..4c93e997d74e 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -299,8 +299,6 @@ set( MIOpen_Source solver/layernorm/backward_t5layernorm.cpp solver/layernorm/forward_addlayernorm.cpp solver/layernorm/forward_layernorm.cpp -# solver/layernorm/forward_layernorm2d_ck.cpp -# solver/layernorm/forward_layernorm4d_ck.cpp solver/layernorm/forward_t5layernorm.cpp solver/mha/mha_solver_backward.cpp solver/mha/mha_solver_forward.cpp @@ -364,13 +362,6 @@ if(MIOPEN_ENABLE_SQLITE AND MIOPEN_ENABLE_SQLITE_KERN_CACHE) list(APPEND MIOpen_Source kern_db.cpp bz2.cpp) endif() -#if(MIOPEN_USE_COMPOSABLEKERNEL) -# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f16_instance.cpp) -# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm2d_fwd_f32_instance.cpp) -# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f16_instance.cpp) -# list(APPEND MIOpen_Source kernels/ck_header_only/layernorm/fwd/device_layernorm4d_fwd_f32_instance.cpp) -#endif() - if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_INCLUDE "kernels/static_composable_kernel/include/*/*.hpp") file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_SOURCE "kernels/static_composable_kernel/src/*/*.cpp") @@ -699,7 +690,7 @@ if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_ kernels/xform_bidirect_winograd_data.s kernels/xform_bidirect_winograd_filter.s kernels/xform_bidirect_winograd_out.s - kernels/UniversalTranspose.cl) + kernels/UniversalTranspose.cpp) # Kernels in development lists. # Should be ALWAYS empty in develop branch (at the time of PR merge) @@ -748,7 +739,6 @@ if(MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_ ocl/utilocl.cpp ocl/dropoutocl.cpp ocl/gcn_asm_utils.cpp -# ocl/rnn_util_ocl.cpp hip/hip_build_utils.cpp hip/batched_transpose_sol.cpp hip/general_tensor_reorder_sol.cpp @@ -961,6 +951,10 @@ if(BUILD_TESTING) target_compile_definitions(MIOpen PUBLIC $) endif() +if(MIOPEN_CK_BUILDER_EXPERIMENTAL) + target_compile_definitions(MIOpen PRIVATE CK_EXPERIMENTAL_BUILDER) +endif() + if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) target_link_libraries(MIOpen PRIVATE frugally-deep::fdeep Eigen3::Eigen) if(NOT TARGET nlohmann_json) diff --git a/projects/miopen/src/include/miopen/target_properties.hpp b/projects/miopen/src/include/miopen/target_properties.hpp index 9f2962c7cfb0..cab04861a8e3 100644 --- a/projects/miopen/src/include/miopen/target_properties.hpp +++ b/projects/miopen/src/include/miopen/target_properties.hpp @@ -29,12 +29,18 @@ #include #include #include +#include #define WORKAROUND_ISSUE_1204 1 // ROCm may incorrectly report "sramecc-" for gfx900. #define WORKAROUND_ISSUE_3001 1 namespace miopen { +static inline bool IsTF32Supported(const std::string& device_name) +{ + return device_name == "gfx942" || StartsWith(device_name, "gfx95"); +} + struct Handle; class TargetProperties From 2becfd6ae1932f558fe19609436e8a69f0a59888 Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 20 Jan 2026 16:46:11 +0100 Subject: [PATCH 19/33] Update CMakeLists.txt --- projects/miopen/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 4d3ae83985f7..7f3719aebd0a 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -493,7 +493,7 @@ if(MIOPEN_USE_HIPRTC) endif() option(Boost_USE_STATIC_LIBS "Use boost static libraries" ON) -#add_definitions(-DBOOST_ALL_NO_LIB=1) +add_definitions(-DBOOST_ALL_NO_LIB=1) find_package(Boost REQUIRED COMPONENTS filesystem CONFIG) find_path(HALF_INCLUDE_DIR half/half.hpp) From 28e96ffe6c1758d68ba202891a0fcd618ecdc924 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Thu, 22 Jan 2026 11:57:08 +0100 Subject: [PATCH 20/33] Removed non functional changes in op_kernel_args.hpp --- projects/miopen/CMakeLists.txt | 6 ++++-- projects/miopen/src/include/miopen/op_kernel_args.hpp | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 7f3719aebd0a..94f0b1272f64 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -77,6 +77,8 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +include(CTest) + find_package(Threads REQUIRED) find_package(ROCmCMakeBuildTools) @@ -339,8 +341,8 @@ if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_B if(MIOPEN_BUILD_CK) include(cmake/Dependencies.cmake) - execute_process( - COMMAND bash -c "grep -P -o '^[ \t]*(? @@ -22,7 +23,8 @@ struct OpKernelArg } template - OpKernelArg(T* arg) : buffer(sizeof(T*)) + OpKernelArg(T* arg) // NOLINT + : buffer(sizeof(T*)) { *(reinterpret_cast(buffer.data())) = arg; is_ptr = true; From b4b8ef5efebbb3644f0ab3424a3a0b247de26445 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Sat, 31 Jan 2026 19:34:09 +0100 Subject: [PATCH 21/33] Removed changes from MIOpenTensorKernels.cl file --- .../miopen/src/kernels/MIOpenTensorKernels.cl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernels.cl b/projects/miopen/src/kernels/MIOpenTensorKernels.cl index cebbe16d9081..b885e1e7b1b9 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernels.cl +++ b/projects/miopen/src/kernels/MIOpenTensorKernels.cl @@ -50,6 +50,7 @@ MIOPEN_TYPE miopenMax(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a > b) ? a : b); MIOPEN_TYPE miopenMin(MIOPEN_TYPE a, MIOPEN_TYPE b) { return ((a < b) ? a : b); } #ifdef USE_FWD_BIAS + __kernel void OpTensorFwdBias(global MIOPEN_TYPE* a, global MIOPEN_TYPE* b, const int b_c, @@ -124,9 +125,11 @@ __kernel void OpTensorFwdBias(global MIOPEN_TYPE* a, } } } + #endif #ifdef USE_FWD_BIAS_GENERIC + __kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -208,10 +211,12 @@ __kernel void OpTensorFwdBiasGeneric(global MIOPEN_TYPE* a, } } } + #endif // DLOWELL : cutting out this section #ifdef USE_LEADING_ONES + __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, global MIOPEN_TYPE* b, global MIOPEN_TYPE* c, @@ -230,9 +235,11 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, const int num_wg, const unsigned int bitmap) { + /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value */ + global MIOPEN_TYPE* a_off = a + Aoffset; global MIOPEN_TYPE* b_off = b + Boffset; global MIOPEN_TYPE* c_off = c + Coffset; @@ -296,9 +303,12 @@ __kernel void OpTensorLeadingOnes(global MIOPEN_TYPE* a, } } } + #endif +// DLOWELL : cutting out this section #ifdef USE_LEADING_ONES_GENERIC + __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -324,6 +334,7 @@ __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, const int num_wg, const unsigned int bitmap) { + /* Special case for leading ones where the total no. of threads is the * inner_product of the tensor dims. Each thread just updates one value */ @@ -415,9 +426,11 @@ __kernel void OpTensorLeadingOnesGeneric(global MIOPEN_TYPE* a, } } } + #endif #ifdef USE_4D_TENSOR_GENERIC + __kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a, const int a_nstride, const int a_cstride, @@ -525,6 +538,7 @@ __kernel void Op4dTensorGeneric(global MIOPEN_TYPE* a, } } } + #endif #ifdef USE_5D_TENSOR_GENERIC @@ -657,6 +671,7 @@ __kernel void Op5dTensorGeneric(global MIOPEN_TYPE* a, } } } + #endif #ifdef USE_2D_TENSOR_LITE @@ -766,4 +781,5 @@ __kernel void Op2dTensorLite(const global MIOPEN_TYPE* a, } } } + #endif From 0d95f99c0477a88e0a0c5687eddc6b2e58579bc3 Mon Sep 17 00:00:00 2001 From: stulebae Date: Mon, 2 Feb 2026 23:12:31 +0100 Subject: [PATCH 22/33] Update tensor_leading_ones_ocl_hip.cpp --- projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index ce30fb11e7e8..51db2e2efe0c 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -332,7 +332,7 @@ struct OpTensorLeadingOnesTest void verify() // verify if the output tensors are same { auto error = miopen::rms_range(tensC_ocl, tensC_hip); - EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; + EXPECT_NEAR(error, 0, 0.1) << "GPU outputs do not match each other. Error: " << error; } void TearDown() override From 1526e94d4869d14d0599976eb02e19fe66e5ef75 Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 3 Feb 2026 11:23:56 +0100 Subject: [PATCH 23/33] Update tensor_leading_ones_generic_ocl_hip.cpp --- .../miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 12c159e6fbab..39b2c85e1a42 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -337,7 +337,7 @@ struct OpTensorLeadingOnesGenericTest void verify() // verify if the output tensors are same { auto error = miopen::rms_range(tensC_ocl, tensC_hip); - EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; + EXPECT_NEAR(error, 0, 0.1) << "GPU outputs do not match each other. Error: " << error; } void TearDown() override From a54506a0c17a1577e3841498bca1e23521398f41 Mon Sep 17 00:00:00 2001 From: stulebae Date: Tue, 3 Feb 2026 23:05:29 +0100 Subject: [PATCH 24/33] Update CMakeLists.txt --- projects/miopen/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 94f0b1272f64..7e895a8c2369 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -341,8 +341,8 @@ if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_B if(MIOPEN_BUILD_CK) include(cmake/Dependencies.cmake) - execute_process ( - COMMAND bash -c "grep -P -o '^[ \t]*(? Date: Sun, 8 Feb 2026 15:20:07 +0100 Subject: [PATCH 25/33] Checking code for gfx1151 arch --- .../src/kernels/MIOpenTensorKernelsHip.cpp | 14 ++++---- .../tensor_leading_ones_generic_ocl_hip.cpp | 30 ++++++++-------- .../gtest/tensor_leading_ones_ocl_hip.cpp | 35 ++++++++++++------- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index b27f5f903538..9a8178adf3bf 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -919,9 +919,9 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, + const long Aoffset, + const long Boffset, + const long Coffset, const int num_wg, const unsigned int bitmap) { @@ -965,17 +965,14 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, #ifdef USE_LEADING_ONES_GENERIC extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, - const uint64_t Aoffset, const int a_nstride, const int a_cstride, const int a_hstride, MIOPEN_TYPE* b, - const uint64_t Boffset, const int b_nstride, const int b_cstride, const int b_hstride, MIOPEN_TYPE* c, - const uint64_t Coffset, const int c_c, const int c_h, const int c_w, @@ -985,8 +982,11 @@ extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const int num_wg, const int work_per_wg, + const long Aoffset, + const long Boffset, + const long Coffset, + const int num_wg, const unsigned int bitmap) { /* Special case for leading ones where the total no. of threads is the diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 39b2c85e1a42..ac7c8e5481a2 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -221,9 +221,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0LL, // Aoffset - 0LL, // Boffset - 0LL, // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, bitmap); @@ -252,9 +252,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0LL, - 0LL, - 0LL, + 0L, + 0L, + 0L, num_wg, bitmap); #endif @@ -277,17 +277,14 @@ struct OpTensorLeadingOnesGenericTest handle.AddKernel( kernel_name, network_config_hip, program_name, kernel_name, vld, vgd, params)( tensA_dev.get(), - uint64_t(0), // Aoffset tensorsConfig.acstrides[0], // a_nstride tensorsConfig.acstrides[1], // a_cstride tensorsConfig.acstrides[2], // a_hstride tensB_dev.get(), - uint64_t(0), // Boffset tensorsConfig.bstrides[0], // b_nstride tensorsConfig.bstrides[1], // b_cstride tensorsConfig.bstrides[2], // b_hstride tensC_dev.get(), - uint64_t(0), // Coffset tensorsConfig.aclens[1], // c_c tensorsConfig.aclens[2], // c_h tensorsConfig.aclens[3], // c_w @@ -297,8 +294,11 @@ struct OpTensorLeadingOnesGenericTest alpha0, alpha1, beta, - num_wg, work_per_wg, + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset + num_wg, bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); @@ -308,17 +308,14 @@ struct OpTensorLeadingOnesGenericTest kernel_name, network_config_hip, tensA_dev.get(), - uint64_t(0), tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], tensorsConfig.acstrides[2], tensB_dev.get(), - uint64_t(0), tensorsConfig.bstrides[0], tensorsConfig.bstrides[1], tensorsConfig.bstrides[2], tensC_dev.get(), - uint64_t(0), tensorsConfig.aclens[1], tensorsConfig.aclens[2], tensorsConfig.aclens[3], @@ -328,8 +325,11 @@ struct OpTensorLeadingOnesGenericTest alpha0, alpha1, beta, - num_wg, work_per_wg, + 0L, + 0L, + 0L, + num_wg, bitmap); #endif } @@ -337,7 +337,7 @@ struct OpTensorLeadingOnesGenericTest void verify() // verify if the output tensors are same { auto error = miopen::rms_range(tensC_ocl, tensC_hip); - EXPECT_NEAR(error, 0, 0.1) << "GPU outputs do not match each other. Error: " << error; + EXPECT_NEAR(error, 0, 0.001) << "GPU outputs do not match each other. Error: " << error; } void TearDown() override diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 51db2e2efe0c..712f0767d43a 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -92,12 +92,12 @@ std::vector TensorsConfigs() C = 4; H = 8; W = 8; - insertTestCase(N, C, H, W); + // insertTestCase(N, C, H, W); N = 2048; C = 4; H = 16; W = 16; - insertTestCase(N, C, H, W); + // insertTestCase(N, C, H, W); #endif return configs; } @@ -244,6 +244,10 @@ struct OpTensorLeadingOnesTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); + std::cout << "OpenCL data:\n"; + for(int i = 0; i < 16; ++i) + std::cout << tensC_ocl.data[i] << ' '; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -297,13 +301,17 @@ struct OpTensorLeadingOnesTest (T)alpha0, (T)alpha1, (T)beta, - uint64_t(0), // Aoffset - uint64_t(0), // Boffset - uint64_t(0), // Coffset + 0L, // Aoffset + 0L, // Boffset + 0L, // Coffset num_wg, bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); + std::cout << "HIP data:\n"; + for(int i = 0; i < 16; ++i) + std::cout << tensC_hip.data[i] << ' '; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -321,9 +329,9 @@ struct OpTensorLeadingOnesTest (T)alpha0, (T)alpha1, (T)beta, - uint64_t(0), - uint64_t(0), - uint64_t(0), + 0L, + 0L, + 0L, num_wg, bitmap); #endif @@ -332,7 +340,7 @@ struct OpTensorLeadingOnesTest void verify() // verify if the output tensors are same { auto error = miopen::rms_range(tensC_ocl, tensC_hip); - EXPECT_NEAR(error, 0, 0.1) << "GPU outputs do not match each other. Error: " << error; + EXPECT_NEAR(error, 0, 0.001) << "GPU outputs do not match each other. Error: " << error; } void TearDown() override @@ -390,7 +398,7 @@ struct OpTensorLeadingOnesTest PerfHelper ph; #endif }; - +/* using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -406,14 +414,14 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +*/ using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) { runOCL(); runHIP(); - verify(); + // verify(); } INSTANTIATE_TEST_SUITE_P(Smoke, @@ -422,7 +430,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +/* using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) @@ -438,3 +446,4 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); +*/ From 4d9ec3222d265107ea0f144e64beb9704521e893 Mon Sep 17 00:00:00 2001 From: stulebae Date: Sun, 8 Feb 2026 22:07:14 +0100 Subject: [PATCH 26/33] Update tensor_leading_ones_generic_ocl_hip.cpp --- .../tensor_leading_ones_generic_ocl_hip.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index ac7c8e5481a2..15c16f2740a9 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -98,7 +98,7 @@ std::vector TensorsConfigs() } } #else - int N = 1; + int N = 16; int C = 1; int H = 1; int W = 1; @@ -228,6 +228,10 @@ struct OpTensorLeadingOnesGenericTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); + std::cout << "OpenCL data:\n"; + for(int i = 0; i < 16; ++i) + std::cout << tensC_ocl.data[i] << ' '; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -302,6 +306,10 @@ struct OpTensorLeadingOnesGenericTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); + std::cout << "HIP data:\n"; + for(int i = 0; i < 16; ++i) + std::cout << tensC_hip.data[i] << ' '; + std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -396,7 +404,7 @@ struct OpTensorLeadingOnesGenericTest PerfHelper ph; #endif }; - +/* using GPU_OpTensorLeadingOnesGenericTest_FP16 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP16, PortTest) @@ -412,14 +420,14 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +*/ using GPU_OpTensorLeadingOnesGenericTest_FP32 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP32, PortTest) { runOCL(); runHIP(); - verify(); + // verify(); } INSTANTIATE_TEST_SUITE_P(Smoke, @@ -428,7 +436,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - +/* using GPU_OpTensorLeadingOnesGenericTest_FP64 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP64, PortTest) @@ -444,3 +452,4 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); +*/ From ce84dff4aece7fbab834995310824582bfc9fc6e Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Mon, 9 Feb 2026 02:06:07 +0100 Subject: [PATCH 27/33] Removed debug purposes code --- .../tensor_leading_ones_generic_ocl_hip.cpp | 18 +++++---------- .../gtest/tensor_leading_ones_ocl_hip.cpp | 22 ++++++------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 15c16f2740a9..092df4d77944 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -228,10 +228,6 @@ struct OpTensorLeadingOnesGenericTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); - std::cout << "OpenCL data:\n"; - for(int i = 0; i < 16; ++i) - std::cout << tensC_ocl.data[i] << ' '; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -306,10 +302,6 @@ struct OpTensorLeadingOnesGenericTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); - std::cout << "HIP data:\n"; - for(int i = 0; i < 16; ++i) - std::cout << tensC_hip.data[i] << ' '; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -404,7 +396,7 @@ struct OpTensorLeadingOnesGenericTest PerfHelper ph; #endif }; -/* + using GPU_OpTensorLeadingOnesGenericTest_FP16 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP16, PortTest) @@ -420,14 +412,14 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + using GPU_OpTensorLeadingOnesGenericTest_FP32 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP32, PortTest) { runOCL(); runHIP(); - // verify(); + verify(); } INSTANTIATE_TEST_SUITE_P(Smoke, @@ -436,7 +428,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -/* + using GPU_OpTensorLeadingOnesGenericTest_FP64 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP64, PortTest) @@ -452,4 +444,4 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 712f0767d43a..5c6ad5b3811e 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -92,12 +92,12 @@ std::vector TensorsConfigs() C = 4; H = 8; W = 8; - // insertTestCase(N, C, H, W); + insertTestCase(N, C, H, W); N = 2048; C = 4; H = 16; W = 16; - // insertTestCase(N, C, H, W); + insertTestCase(N, C, H, W); #endif return configs; } @@ -244,10 +244,6 @@ struct OpTensorLeadingOnesTest bitmap); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); - std::cout << "OpenCL data:\n"; - for(int i = 0; i < 16; ++i) - std::cout << tensC_ocl.data[i] << ' '; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -308,10 +304,6 @@ struct OpTensorLeadingOnesTest bitmap); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); - std::cout << "HIP data:\n"; - for(int i = 0; i < 16; ++i) - std::cout << tensC_hip.data[i] << ' '; - std::cout << '\n'; #if PERF_ENABLE ph.perfTest(handle, @@ -398,7 +390,7 @@ struct OpTensorLeadingOnesTest PerfHelper ph; #endif }; -/* + using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) @@ -414,14 +406,14 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) { runOCL(); runHIP(); - // verify(); + verify(); } INSTANTIATE_TEST_SUITE_P(Smoke, @@ -430,7 +422,7 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -/* + using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP64, PortTest) @@ -446,4 +438,4 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); -*/ + From 6477a096c79fbf54603bb3496a729ae3afc1032b Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Mon, 9 Feb 2026 02:39:25 +0100 Subject: [PATCH 28/33] Fixed clang-format violations --- .../miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp | 1 - projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 092df4d77944..4d517b2ffecd 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -444,4 +444,3 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 5c6ad5b3811e..0f39b00ce15f 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -438,4 +438,3 @@ INSTANTIATE_TEST_SUITE_P(Smoke, testing::Values(1.0), testing::Values(1.0), testing::Values(0.0, 1.0))); - From 50ca4ebbfbdaec1afebcca6d589f8c50562910d0 Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Wed, 11 Feb 2026 03:16:02 +0100 Subject: [PATCH 29/33] Removed unrelated changes in CMakeLists.txt --- projects/miopen/CMakeLists.txt | 22 +++--- projects/miopen/src/CMakeLists.txt | 29 ++++--- .../src/kernels/MIOpenTensorKernelsHip.cpp | 24 +++--- projects/miopen/test/gtest/CMakeLists.txt | 16 +++- .../tensor_leading_ones_generic_ocl_hip.cpp | 49 ++++++------ .../gtest/tensor_leading_ones_ocl_hip.cpp | 75 ++++++++++--------- 6 files changed, 119 insertions(+), 96 deletions(-) diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 7e895a8c2369..43eabf0db0de 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -122,6 +122,7 @@ option( BUILD_DEV "Build for development only" OFF) option(MIOPEN_ENABLE_FIN "Enable the fin driver for MIOpen" OFF) option(MIOPEN_ENABLE_FIN_INTERFACE "Enable internal interface for the fin" ${MIOPEN_ENABLE_FIN}) option(MIOPEN_STRIP_SYMBOLS "Strip symbols in release mode" ON) +option(MIOPEN_CK_BUILDER_EXPERIMENTAL "Enable experimental support for CK Builder" OFF) option(MIOPEN_INCBIN "Include kernels using incbin" OFF) if(MIOPEN_INCBIN) @@ -341,22 +342,21 @@ if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_B if(MIOPEN_BUILD_CK) include(cmake/Dependencies.cmake) - execute_process( + execute_process ( COMMAND bash -c "grep -P -o '^[ \t]*(? in requirements.txt") + message(FATAL_ERROR "Could not find composablekernel in requirements.txt") endif() - miopen_add_dependency(composable_kernel HASH ${COMPOSABLE_KERNEL_HASH} options NO_LOCAL) + miopen_add_dependency(composable_kernel options NO_LOCAL) else() - find_package(composable_kernel COMPONENTS device_conv_operations) + find_package(composable_kernel 1.0.0 COMPONENTS device_conv_operations) endif() endif() - if(MIOPEN_BACKEND STREQUAL "HIPNOGPU") + if( MIOPEN_BACKEND STREQUAL "HIPNOGPU") set(MIOPEN_MODE_NOGPU 1) endif() set(MIOPEN_BACKEND_HIP 1) @@ -501,7 +501,7 @@ find_package(Boost REQUIRED COMPONENTS filesystem CONFIG) find_path(HALF_INCLUDE_DIR half/half.hpp) message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") -option(MIOPEN_DEBUG_FIND_DB_CACHING "Use system find-db caching" ON) +option( MIOPEN_DEBUG_FIND_DB_CACHING "Use system find-db caching" ON) if(WIN32) set( DATA_INSTALL_DIR bin ) @@ -553,8 +553,8 @@ if(NOT WIN32 AND MIOPEN_USE_ROCTRACER) find_path(ROCTRACER_INCLUDE_DIR "roctracer/roctx.h") find_library(rocTracer roctx64) if(rocTracer AND ROCTRACER_INCLUDE_DIR) - message(STATUS "Build with rocTracer: " ${rocTracer}) - message(STATUS "rocTracer include directory: " ${ROCTRACER_INCLUDE_DIR}) + MESSAGE(STATUS "Build with rocTracer: " ${rocTracer}) + MESSAGE(STATUS "rocTracer include directory: " ${ROCTRACER_INCLUDE_DIR}) set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, roctracer") else() message(WARNING "rocTracer cannot be found! Build without rocTracer") @@ -685,7 +685,7 @@ endfunction() # install for all targets; this was the behaviour before MIOPEN_INSTALL_GPU_DATABASES # was added. set(ALL_GPU_DATABASES gfx803 gfx900 gfx906 gfx908 gfx90a gfx942 gfx1030) -set(MIOPEN_INSTALL_GPU_DATABASES "${ALL_GPU_DATABASES}" CACHE STRING "List of AMDGPU targets for which to install databases") +set(MIOPEN_INSTALL_GPU_DATABASES "${ALL_GPU_DATABASES}" CACHE LIST "List of AMDGPU targets for which to install databases") # This list should be updated when a new KDB file is added to the project set(KDB_BZ2_FILES gfx942.kdb.bz2 gfx90a.kdb.bz2 gfx1030.kdb.bz2 gfx908.kdb.bz2 gfx906.kdb.bz2 gfx900.kdb.bz2) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index e7da34485ed4..fc537d3f05ab 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -38,9 +38,11 @@ endif() # 2. How BF16 kernels (which are kind of mixed-precision now and expected to # remain in the future) perform final conversion (and rounding) of FP32 # to BF16 results. This affects the main functionality of the library. -option(MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) -option(MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) -option(MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) +option( MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) +option( MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) +option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) + +option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) configure_file("${PROJECT_SOURCE_DIR}/include/miopen/config.h.in" "${PROJECT_BINARY_DIR}/include/miopen/config.h") @@ -396,6 +398,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_INCLUDE "kernels/gpu_general_tensor_reorder_kernel/*.hpp") file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_SOURCE "kernels/gpu_general_tensor_reorder_kernel/*.cpp") + set(MIOPEN_KERNEL_INCLUDES ${STATIC_COMPOSABLE_KERNEL_INCLUDE} ${COMPOSABLE_KERNEL_INCLUDE} @@ -947,6 +950,12 @@ if(MIOPEN_USE_COMPOSABLEKERNEL) else() # Use the aliased targets when we pull CK from /opt/rocm or other place on disc. set(MIOPEN_CK_LINK_FLAGS composable_kernel::device_conv_operations hip::host) + + if(MIOPEN_CK_BUILDER_EXPERIMENTAL) + add_subdirectory(kernels/ck_builder) + list(APPEND MIOPEN_CK_LINK_FLAGS ck_builder) + add_definitions(-DCK_EXPERIMENTAL_BUILDER) + endif() endif() if(GPU_TARGETS MATCHES "gfx942" OR GPU_TARGETS MATCHES "gfx950") target_compile_definitions(MIOpen PRIVATE CK_ENABLE_TF32) @@ -1089,7 +1098,13 @@ local: # set_target_properties(MIOpen PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(MIOpen PROPERTIES VISIBILITY_INLINES_HIDDEN 1) endif() - +####################################### +if(MIOPEN_ENABLE_SQLITE) + # MIOpen depends on SQLite + target_link_libraries(MIOpen PRIVATE SQLite::SQLite3) + target_link_libraries(MIOpen INTERFACE $) +endif() +############################################################ # MIOpen depends on librt for Boost.Interprocess if(NOT WIN32 AND NOT APPLE) find_library(LIBRT rt) @@ -1099,12 +1114,6 @@ if(NOT WIN32 AND NOT APPLE) endif() endif() -if(MIOPEN_ENABLE_SQLITE) - # MIOpen depends on SQLite - target_link_libraries(MIOpen PRIVATE SQLite::SQLite3) - target_link_libraries(MIOpen INTERFACE $) -endif() - if(MIOPEN_USE_ROCTRACER) target_include_directories(MIOpen PRIVATE ${ROCTRACER_INCLUDE_DIR}) target_include_directories(MIOpen INTERFACE $) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 9a8178adf3bf..9a484ed9507c 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -640,9 +640,9 @@ extern "C" __global__ void OpTensorFwdBias(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const long Aoffset, - const long Boffset, - const long Coffset, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, const int num_wg, const int incr_wg) { @@ -724,9 +724,9 @@ extern "C" __global__ void OpTensorFwdBiasGeneric(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, const int work_per_wg, - const long Aoffset, - const long Boffset, - const long Coffset, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, const int num_wg, const int incr_wg) { @@ -919,9 +919,9 @@ extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha0, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, - const long Aoffset, - const long Boffset, - const long Coffset, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, const int num_wg, const unsigned int bitmap) { @@ -983,9 +983,9 @@ extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, const MIOPEN_TYPE alpha1, const MIOPEN_TYPE beta, const int work_per_wg, - const long Aoffset, - const long Boffset, - const long Coffset, + const uint64_t Aoffset, + const uint64_t Boffset, + const uint64_t Coffset, const int num_wg, const unsigned int bitmap) { diff --git a/projects/miopen/test/gtest/CMakeLists.txt b/projects/miopen/test/gtest/CMakeLists.txt index 07d329909f6c..2b804721fbe1 100644 --- a/projects/miopen/test/gtest/CMakeLists.txt +++ b/projects/miopen/test/gtest/CMakeLists.txt @@ -26,7 +26,13 @@ if(MIOPEN_BACKEND_HIP) endif() if(MIOPEN_BACKEND_OPENCL) - set(SKIP_TESTS dumpTensorTest.cpp) + list(APPEND SKIP_TESTS dumpTensorTest.cpp) +endif() + +if(MIOPEN_CK_BUILDER_EXPERIMENTAL) + add_definitions(-DCK_EXPERIMENTAL_BUILDER) +else() + list(APPEND SKIP_TESTS ck_builder_xdl.cpp ck_builder_shared.cpp) endif() function(add_gtest_negative_filter NEGATIVE_FILTER_TO_ADD) @@ -75,6 +81,12 @@ function(add_gtest TEST_NAME TEST_CPP) if(NOT MIOPEN_EMBED_DB STREQUAL "") target_link_libraries(${TEST_NAME} $) endif() + + if(MIOPEN_CK_BUILDER_EXPERIMENTAL) + find_package(composable_kernel 1.0.0 COMPONENTS device_conv_operations) + target_link_libraries(${TEST_NAME} ck_builder composable_kernel::device_conv_operations) + endif() + if(MIOPEN_TEST_DISCRETE) string(CONCAT TEST_ENVIRONMENT_VARIABLES "ENVIRONMENT;MIOPEN_USER_DB_PATH=${CMAKE_CURRENT_BINARY_DIR};" @@ -229,7 +241,7 @@ else() add_gtest(miopen_gtest "${TESTS_CPP}") if(MIOPEN_USE_COMPOSABLEKERNEL) - if(MIOPEN_BUILD_CK) + if(MIOPEN_BUILD_CK ) # The include directories also dont propagate correctly when using the non aliased targets. # Manually including them fixes this problem for inline builds. target_include_directories(miopen_gtest SYSTEM PRIVATE diff --git a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp index 4d517b2ffecd..3fa57e218df7 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_generic_ocl_hip.cpp @@ -116,15 +116,14 @@ std::vector TensorsConfigs() } template -struct OpTensorLeadingOnesGenericTest - : public ::testing::TestWithParam> +struct OpTensorLeadingOnesGenericTest : testing::TestWithParam> { protected: void SetUp() override { data_type = miopen_type{}; - std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + std::tie(tensorsConfig, alpha0, alpha1, beta) = this->GetParam(); // Generate elements in tensors tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( @@ -132,7 +131,7 @@ struct OpTensorLeadingOnesGenericTest tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( tensor_elem_gen_integer{}); tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( - [](auto...) { return 1; }); + [](auto...) { return 0; }); auto&& handle = get_handle(); // Write the device tensors @@ -221,9 +220,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, bitmap); @@ -252,9 +251,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0L, - 0L, - 0L, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, bitmap); #endif @@ -295,9 +294,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, bitmap); @@ -326,9 +325,9 @@ struct OpTensorLeadingOnesGenericTest alpha1, beta, work_per_wg, - 0L, - 0L, - 0L, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, bitmap); #endif @@ -397,7 +396,9 @@ struct OpTensorLeadingOnesGenericTest #endif }; -using GPU_OpTensorLeadingOnesGenericTest_FP16 = OpTensorLeadingOnesGenericTest; +using float16 = half_float::half; + +using GPU_OpTensorLeadingOnesGenericTest_FP16 = OpTensorLeadingOnesGenericTest; TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP16, PortTest) { @@ -408,10 +409,10 @@ TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP16, PortTest) INSTANTIATE_TEST_SUITE_P(Smoke, GPU_OpTensorLeadingOnesGenericTest_FP16, - testing::Combine(testing::ValuesIn(TensorsConfigs()), - testing::Values(1.0), - testing::Values(1.0), - testing::Values(0.0, 1.0))); + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(float16(1)), + testing::Values(float16(1)), + testing::Values(float16(0), float16(1)))); using GPU_OpTensorLeadingOnesGenericTest_FP32 = OpTensorLeadingOnesGenericTest; @@ -425,9 +426,9 @@ TEST_P(GPU_OpTensorLeadingOnesGenericTest_FP32, PortTest) INSTANTIATE_TEST_SUITE_P(Smoke, GPU_OpTensorLeadingOnesGenericTest_FP32, testing::Combine(testing::ValuesIn(TensorsConfigs()), - testing::Values(1.0), - testing::Values(1.0), - testing::Values(0.0, 1.0))); + testing::Values(1.f), + testing::Values(1.f), + testing::Values(0.f, 1.f))); using GPU_OpTensorLeadingOnesGenericTest_FP64 = OpTensorLeadingOnesGenericTest; diff --git a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp index 0f39b00ce15f..4663434bf0dd 100644 --- a/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_leading_ones_ocl_hip.cpp @@ -103,14 +103,13 @@ std::vector TensorsConfigs() } template -struct OpTensorLeadingOnesTest - : public ::testing::TestWithParam> +struct OpTensorLeadingOnesTest : testing::TestWithParam> { protected: void SetUp() override { prng::reset_seed(); - std::tie(tensorsConfig, alpha0, alpha1, beta) = GetParam(); + std::tie(tensorsConfig, alpha0, alpha1, beta) = this->GetParam(); // Generate elements in tensors tensA = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( @@ -118,7 +117,7 @@ struct OpTensorLeadingOnesTest tensB = tensor{tensorsConfig.blens, tensorsConfig.bstrides}.generate( tensor_elem_gen_integer{}); tensC = tensor{tensorsConfig.aclens, tensorsConfig.acstrides}.generate( - [](auto...) { return 1; }); + [](auto...) { return 0; }); auto&& handle = get_handle(); // Write the device tensors @@ -234,12 +233,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, - (T)alpha0, - (T)alpha1, - (T)beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + alpha0, + alpha1, + beta, + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, bitmap); @@ -258,12 +257,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], work_per_wg, - (T)alpha0, - (T)alpha1, - (T)beta, - 0L, - 0L, - 0L, + alpha0, + alpha1, + beta, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, bitmap); #endif @@ -294,12 +293,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], // c_nstride tensorsConfig.acstrides[1], // c_cstride work_per_wg, - (T)alpha0, - (T)alpha1, - (T)beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + alpha0, + alpha1, + beta, + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, bitmap); @@ -318,12 +317,12 @@ struct OpTensorLeadingOnesTest tensorsConfig.acstrides[0], tensorsConfig.acstrides[1], work_per_wg, - (T)alpha0, - (T)alpha1, - (T)beta, - 0L, - 0L, - 0L, + alpha0, + alpha1, + beta, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, bitmap); #endif @@ -384,14 +383,16 @@ struct OpTensorLeadingOnesTest miopen::Allocator::ManageDataPtr tensC_dev; TensorsConfig tensorsConfig; - double alpha0, alpha1, beta; + T alpha0, alpha1, beta; #if PERF_ENABLE PerfHelper ph; #endif }; -using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; +using float16 = half_float::half; + +using GPU_OpTensorLeadingOnesTest_FP16 = OpTensorLeadingOnesTest; TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) { @@ -402,10 +403,10 @@ TEST_P(GPU_OpTensorLeadingOnesTest_FP16, PortTest) INSTANTIATE_TEST_SUITE_P(Smoke, GPU_OpTensorLeadingOnesTest_FP16, - testing::Combine(testing::ValuesIn(TensorsConfigs()), - testing::Values(1.0), - testing::Values(1.0), - testing::Values(0.0, 1.0))); + testing::Combine(testing::ValuesIn(TensorsConfigs()), + testing::Values(float16(1)), + testing::Values(float16(1)), + testing::Values(float16(0), float16(1)))); using GPU_OpTensorLeadingOnesTest_FP32 = OpTensorLeadingOnesTest; @@ -419,9 +420,9 @@ TEST_P(GPU_OpTensorLeadingOnesTest_FP32, PortTest) INSTANTIATE_TEST_SUITE_P(Smoke, GPU_OpTensorLeadingOnesTest_FP32, testing::Combine(testing::ValuesIn(TensorsConfigs()), - testing::Values(1.0), - testing::Values(1.0), - testing::Values(0.0, 1.0))); + testing::Values(1.f), + testing::Values(1.f), + testing::Values(0.f, 1.f))); using GPU_OpTensorLeadingOnesTest_FP64 = OpTensorLeadingOnesTest; From b58f0001dc88610102ab02babffd5065885b7b86 Mon Sep 17 00:00:00 2001 From: stulebae Date: Thu, 12 Feb 2026 00:46:17 +0100 Subject: [PATCH 30/33] Update MIOpenTensorKernelsHip.cpp --- projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 9a484ed9507c..76d4fd1aa651 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -97,6 +97,7 @@ extern "C" __global__ void Op1dTensorGeneric(const MIOPEN_TYPE* a, c_ptr += c_step; } } + #endif #ifdef USE_2D_TENSOR_GENERIC @@ -153,6 +154,7 @@ extern "C" __global__ void Op2dTensorGeneric(const MIOPEN_TYPE* a, gid += step; } } + #endif #ifdef USE_2D_TENSOR_SQUASH @@ -418,6 +420,7 @@ extern "C" __global__ void Op3dTensorGeneric(const MIOPEN_TYPE* a, gid += step; } } + #endif #ifdef USE_4D_TENSOR_GENERIC From a03b9c4406f54e4ec16ad3317f2c7b42de3711c4 Mon Sep 17 00:00:00 2001 From: stulebae Date: Thu, 12 Feb 2026 00:48:29 +0100 Subject: [PATCH 31/33] Update CMakeLists.txt --- projects/miopen/src/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index fc537d3f05ab..663cc36be027 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -42,8 +42,6 @@ option( MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) option( MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) -option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) - configure_file("${PROJECT_SOURCE_DIR}/include/miopen/config.h.in" "${PROJECT_BINARY_DIR}/include/miopen/config.h") # configure a header file to pass the CMake version settings to the source, and package the header files in the output archive From c7b367194c59718e04a7c5e551ff462cce58a5ab Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Thu, 26 Mar 2026 23:23:22 +0100 Subject: [PATCH 32/33] Added GTest versions of LSTM tests --- projects/miopen/.clang-format | 2 +- projects/miopen/CMakeLists.txt | 7 +- projects/miopen/Dockerfile | 2 + projects/miopen/README.md | 15 +- projects/miopen/addkernels/addkernels.cpp | 50 +- projects/miopen/cmake/CppCheck.cmake | 10 +- .../miopen/cmake/EnableCompilerWarnings.cmake | 12 - projects/miopen/driver/InputFlags.cpp | 14 +- projects/miopen/driver/adam_driver.hpp | 3 +- projects/miopen/driver/bn_driver.hpp | 32 +- projects/miopen/driver/driver.hpp | 4 +- projects/miopen/driver/getitem_driver.hpp | 20 +- projects/miopen/driver/rope_driver.hpp | 19 +- projects/miopen/driver/timer.hpp | 14 +- .../driver/transformers_adam_w_driver.hpp | 5 +- projects/miopen/include/miopen/config.h.in | 9 - projects/miopen/requirements.txt | 2 +- projects/miopen/src/CMakeLists.txt | 75 +- projects/miopen/src/api/find2_0_commons.cpp | 35 +- projects/miopen/src/binary_cache.cpp | 5 +- projects/miopen/src/buffer_info.cpp | 8 + projects/miopen/src/bz2.cpp | 14 +- projects/miopen/src/comgr.cpp | 15 +- .../ai_conv_3d_kernel_tuning_utils.cpp | 4 - .../src/conv/heuristics/ai_heuristics.cpp | 38 +- .../src/conv/invokers/mlir_impl_gemm.cpp | 2 - .../miopen/src/conv/problem_description.cpp | 11 +- projects/miopen/src/conv/solver_finders.cpp | 10 +- projects/miopen/src/convolution.cpp | 17 +- projects/miopen/src/ctc.cpp | 50 +- projects/miopen/src/db_path.cpp.in | 13 +- projects/miopen/src/db_record.cpp | 48 +- projects/miopen/src/env_debug.cpp | 4 + projects/miopen/src/execution_context.cpp | 4 +- projects/miopen/src/expanduser.cpp | 6 +- projects/miopen/src/find_db.cpp | 107 +- projects/miopen/src/fusion.cpp | 3 +- projects/miopen/src/gemm_v2.cpp | 9 +- .../miopen/src/hip/batched_transpose_sol.cpp | 13 +- .../src/hip/general_tensor_reorder_sol.cpp | 15 +- projects/miopen/src/hip/handlehip.cpp | 30 +- projects/miopen/src/hipoc/hipoc_kernel.cpp | 14 +- projects/miopen/src/hipoc/hipoc_program.cpp | 12 +- projects/miopen/src/include/miopen/activ.hpp | 61 +- .../miopen/activ/problem_description.hpp | 2 +- .../miopen/adam/problem_description.hpp | 2 +- .../miopen/src/include/miopen/anyramdb.hpp | 2 +- .../miopen/batchnorm/problem_description.hpp | 6 +- .../src/include/miopen/batchnorm/solvers.hpp | 132 +- .../miopen/src/include/miopen/bfloat16.hpp | 56 +- .../src/include/miopen/binary_cache.hpp | 16 +- projects/miopen/src/include/miopen/bz2.hpp | 2 +- .../src/include/miopen/check_numerics.hpp | 7 +- projects/miopen/src/include/miopen/config.hpp | 5 - .../miopen/conv/data_invoke_params.hpp | 119 +- .../src/include/miopen/conv/db_getter.hpp | 3 +- .../ai_conv_3d_kernel_tuning_utils.hpp | 6 +- .../miopen/conv/problem_description.hpp | 76 +- .../include/miopen/conv/solver_finders.hpp | 4 +- .../src/include/miopen/conv/solvers.hpp | 2307 ++++---- .../miopen/src/include/miopen/convolution.hpp | 355 +- projects/miopen/src/include/miopen/ctc.hpp | 1 + projects/miopen/src/include/miopen/db.hpp | 33 +- .../miopen/src/include/miopen/db_path.hpp | 2 +- .../miopen/src/include/miopen/db_record.hpp | 11 +- .../miopen/src/include/miopen/dropout.hpp | 36 +- projects/miopen/src/include/miopen/env.hpp | 10 +- projects/miopen/src/include/miopen/errors.hpp | 1 - .../src/include/miopen/execution_context.hpp | 4 +- .../src/include/miopen/fin/fin_interface.hpp | 20 +- .../src/include/miopen/find_controls.hpp | 8 +- projects/miopen/src/include/miopen/fusion.hpp | 4 +- .../src/include/miopen/fusion/solvers.hpp | 221 +- .../miopen/src/include/miopen/fusion_plan.hpp | 9 +- .../miopen/general_tensor_reorder_sol.hpp | 1 + .../src/include/miopen/generic_search.hpp | 18 +- .../miopen/generic_search_controls.hpp | 5 +- .../include/miopen/graphapi/convolution.hpp | 11 +- .../src/include/miopen/graphapi/engine.hpp | 4 +- .../src/include/miopen/graphapi/enginecfg.hpp | 4 +- .../miopen/graphapi/execution_plan.hpp | 13 +- .../src/include/miopen/graphapi/matmul.hpp | 6 +- .../src/include/miopen/graphapi/opgraph.hpp | 12 +- .../src/include/miopen/graphapi/pointwise.hpp | 6 +- .../src/include/miopen/graphapi/reduction.hpp | 6 +- .../src/include/miopen/graphapi/reshape.hpp | 2 +- .../src/include/miopen/graphapi/rng.hpp | 2 +- .../src/include/miopen/graphapi/tensor.hpp | 6 +- .../src/include/miopen/groupnorm/solvers.hpp | 10 +- projects/miopen/src/include/miopen/handle.hpp | 3 +- .../src/include/miopen/hipoc_kernel.hpp | 14 +- .../src/include/miopen/hipoc_program.hpp | 10 +- .../src/include/miopen/invoke_params.hpp | 7 + .../src/include/miopen/invoker_cache.hpp | 14 +- .../miopen/src/include/miopen/kern_db.hpp | 2 - .../include/miopen/kernel_build_params.hpp | 15 +- .../miopen/src/include/miopen/kernel_info.hpp | 6 +- .../miopen/src/include/miopen/load_file.hpp | 3 +- .../miopen/src/include/miopen/lock_file.hpp | 4 +- projects/miopen/src/include/miopen/logger.hpp | 57 +- projects/miopen/src/include/miopen/lrn.hpp | 57 +- .../miopen/src/include/miopen/manage_ptr.hpp | 10 +- .../miopen/src/include/miopen/mha/solvers.hpp | 38 +- .../src/include/miopen/mlo_internal.hpp | 38 +- .../src/include/miopen/nogpu/handle_impl.hpp | 2 +- .../src/include/miopen/op_kernel_args.hpp | 7 +- .../miopen/pooling/problem_description.hpp | 6 - .../src/include/miopen/pooling/solvers.hpp | 98 +- .../miopen/src/include/miopen/problem.hpp | 13 +- projects/miopen/src/include/miopen/ramdb.hpp | 37 +- .../src/include/miopen/readonlyramdb.hpp | 5 +- .../src/include/miopen/reducetensor.hpp | 44 +- projects/miopen/src/include/miopen/rnn.hpp | 137 +- .../include/miopen/rnn/multi_stream_utils.hpp | 8 +- .../include/miopen/rnn/tmp_buffer_utils.hpp | 2 +- .../miopen/src/include/miopen/rnn_util.hpp | 10 +- projects/miopen/src/include/miopen/scalar.hpp | 4 +- .../miopen/src/include/miopen/seq_tensor.hpp | 52 +- .../miopen/src/include/miopen/sequences.hpp | 33 +- .../miopen/src/include/miopen/softmax.hpp | 5 +- .../miopen/softmax/problem_description.hpp | 85 +- .../src/include/miopen/softmax/solvers.hpp | 117 +- .../miopen/src/include/miopen/solution.hpp | 11 +- projects/miopen/src/include/miopen/solver.hpp | 13 + .../miopen/solver/conv_direct_naive_conv.hpp | 2 +- .../miopen/solver/implicitgemm_ck_util.hpp | 8 +- .../solver/implicitgemm_static_ck_util.hpp | 2 +- .../miopen/src/include/miopen/solver_id.hpp | 16 +- .../miopen/src/include/miopen/sqlite_db.hpp | 24 +- .../miopen/src/include/miopen/subbuffers.hpp | 5 +- .../src/include/miopen/target_properties.hpp | 3 +- .../miopen/src/include/miopen/temp_file.hpp | 4 +- projects/miopen/src/include/miopen/tensor.hpp | 131 +- .../miopen/src/include/miopen/tensor_ops.hpp | 76 +- .../miopen/src/include/miopen/tmp_dir.hpp | 9 +- projects/miopen/src/include/miopen/util.hpp | 96 +- .../miopen/utility/transposing_solver.hpp | 179 +- .../miopen/src/kernels/MIOpenCol2Im3d.cpp | 109 + projects/miopen/src/kernels/MIOpenLRNBwd.cl | 472 -- projects/miopen/src/kernels/MIOpenLRNFwd.cl | 713 --- projects/miopen/src/kernels/MIOpenPooling.cpp | 30 +- .../miopen/src/kernels/MIOpenPoolingBwd.cpp | 30 +- .../miopen/src/kernels/MIOpenPoolingBwdND.cpp | 30 +- .../src/kernels/MIOpenPoolingForwardNaive.cpp | 30 +- .../miopen/src/kernels/MIOpenPoolingND.cpp | 30 +- .../src/kernels/MIOpenTensorKernelsHip.cpp | 180 - .../miopen/src/kernels/gfx942.kdb.bz2.dvc | 4 +- projects/miopen/src/kernels/hip_math_ops.hpp | 5 + projects/miopen/src/kernels/miopen_math.hpp | 4 +- ...ad_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s | 34 - ...4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc | 4572 ---------------- .../Conv_Winograd_Rage_v4_6_0_metadata.inc | 212 - ...ad_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s | 34 - ...4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc | 4620 ----------------- .../Conv_Winograd_Rage_v4_7_0_metadata.inc | 212 - projects/miopen/src/logger.cpp | 52 +- projects/miopen/src/mlo_dir_conv.cpp | 43 +- projects/miopen/src/nogpu/handle.cpp | 2 + projects/miopen/src/ocl/convolutionocl.cpp | 8 +- projects/miopen/src/ocl/lrn_ocl.cpp | 6 +- projects/miopen/src/ocl/mloNorm.cpp | 244 +- projects/miopen/src/ocl/rnnocl.cpp | 32 +- projects/miopen/src/ocl/utilocl.cpp | 136 + projects/miopen/src/problem.cpp | 69 +- projects/miopen/src/reducetensor.cpp | 4 +- projects/miopen/src/reducetensor_api.cpp | 2 +- .../rnn/Solutions/Base/bw_data_modular.cpp | 2 +- .../rnn/Solutions/Base/fw_data_modular.cpp | 2 +- .../src/rnn/Solutions/bwd_multi_stream.cpp | 8 +- projects/miopen/src/softmax.cpp | 38 +- projects/miopen/src/solution.cpp | 66 +- projects/miopen/src/solver.cpp | 123 +- projects/miopen/src/solver/activ/fwd_0.cpp | 3 +- .../conv/conv_MP_bidirectional_winograd.cpp | 30 +- .../src/solver/conv/conv_bin_wino3x3U.cpp | 9 +- .../src/solver/conv/conv_bin_winoRxS.cpp | 16 +- ...ip_implicit_gemm_3d_grouped_bwd_xdlops.cpp | 64 +- ...ip_implicit_gemm_3d_grouped_fwd_xdlops.cpp | 78 +- ...ip_implicit_gemm_3d_grouped_wrw_xdlops.cpp | 88 +- ...conv_hip_implicit_gemm_bwd_data_xdlops.cpp | 76 +- .../conv_hip_implicit_gemm_fwd_xdlops.cpp | 78 +- ...v_hip_implicit_gemm_grouped_bwd_xdlops.cpp | 78 + ...v_hip_implicit_gemm_grouped_fwd_xdlops.cpp | 97 + ...v_hip_implicit_gemm_grouped_wrw_xdlops.cpp | 83 + .../solver/conv/conv_multipass_wino3x3WrW.cpp | 37 +- .../solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp | 10 +- .../solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp | 9 +- .../src/solver/conv/conv_ocl_dir2Dfwd.cpp | 12 +- .../miopen/src/solver/conv/conv_winoRxS.cpp | 26 +- .../src/solver/conv/conv_wino_fury_RxS.cpp | 1 + .../src/solver/conv/conv_wino_rage_RxS.cpp | 483 +- projects/miopen/src/solver/conv/gemm.cpp | 75 + projects/miopen/src/solver/conv/gemm_bwd.cpp | 124 +- .../src/solver/getitem/backward_getitem.cpp | 14 +- .../solver/layernorm/backward_layernorm.cpp | 16 +- .../solver/layernorm/backward_t5layernorm.cpp | 16 +- .../src/solver/mha/mha_solver_backward.cpp | 22 +- .../src/solver/mha/mha_solver_forward.cpp | 12 +- .../forward_multimarginloss.cpp | 14 +- .../prelu/backward_prelu_multi_weights.cpp | 14 +- .../prelu/backward_prelu_single_weight.cpp | 14 +- .../miopen/src/solver/reduce/forward_prod.cpp | 12 +- .../miopen/src/solver/reduce/forward_sum.cpp | 12 +- .../softmarginloss/forward_softmarginloss.cpp | 14 +- .../miopen/src/solver/softmax/softmax.cpp | 172 +- projects/miopen/src/target_properties.cpp | 6 +- projects/miopen/src/tensor.cpp | 65 +- projects/miopen/test/CMakeLists.txt | 8 +- projects/miopen/test/args.hpp | 2 +- projects/miopen/test/driver.hpp | 35 +- projects/miopen/test/gru_common.hpp | 2 +- projects/miopen/test/gtest/CMakeLists.txt | 49 +- projects/miopen/test/gtest/activ_common.hpp | 40 +- projects/miopen/test/gtest/bn_activ_infer.cpp | 16 +- projects/miopen/test/gtest/bn_fwd_infer.cpp | 4 +- .../miopen/test/gtest/cba_find2_infer.cpp | 5 - projects/miopen/test/gtest/cba_infer.cpp | 4 - projects/miopen/test/gtest/compare_helper.hpp | 30 +- projects/miopen/test/gtest/db_paths.cpp | 8 +- projects/miopen/test/gtest/db_sync.cpp | 22 +- projects/miopen/test/gtest/deepbench_lstm.cpp | 147 +- .../miopen/test/gtest/execute_primitive.cpp | 6 +- projects/miopen/test/gtest/find_2_conv.cpp | 13 + projects/miopen/test/gtest/fusion_test.cpp | 2 + projects/miopen/test/gtest/getitem.hpp | 7 +- .../test/gtest/gpu_reference_kernel.cpp | 12 +- .../graphapi_operationgraph_descriptor.cpp | 22 +- projects/miopen/test/gtest/gtest_common.hpp | 1 + .../miopen/test/gtest/handle_hip_device.cpp | 4 +- projects/miopen/test/gtest/hipblaslt_gemm.cpp | 1 + .../miopen/test/gtest/layout_transpose.cpp | 2 +- projects/miopen/test/gtest/log.cpp | 63 +- projects/miopen/test/gtest/log.hpp | 6 +- projects/miopen/test/gtest/log_test.cpp | 6 +- projects/miopen/test/gtest/lstm.cpp | 139 + .../test/{lstm_common.hpp => gtest/lstm.hpp} | 456 +- projects/miopen/test/gtest/lstm_dropout.cpp | 99 + projects/miopen/test/gtest/lstm_extra.cpp | 199 +- projects/miopen/test/gtest/main_hip.cpp | 22 + projects/miopen/test/gtest/mha_helper.hpp | 2 - .../test/gtest/pooling2d_asymmetric.cpp | 2 +- projects/miopen/test/gtest/pooling2d_wide.cpp | 14 +- .../miopen/test/gtest/reducecalculation.hpp | 2 +- projects/miopen/test/gtest/rnn_seq_api.hpp | 2 +- .../miopen/test/gtest/rnn_vanilla_common.hpp | 2 +- projects/miopen/test/gtest/t5layernorm.hpp | 7 +- .../test/gtest/tensor_4d_generic_ocl_hip.cpp | 94 +- .../gtest/tensor_fwd_bias_generic_ocl_hip.cpp | 28 +- .../test/gtest/tensor_fwd_bias_ocl_hip.cpp | 51 +- .../miopen/test/gtest/test_categories.yaml | 12 +- .../miopen/test/gtest/unit_FinInterface.cpp | 29 +- .../miopen/test/gtest/unit_conv_solver.cpp | 81 +- .../miopen/test/gtest/unit_conv_solver.hpp | 5 +- .../unit_conv_solver_ConvBinWinograd3x3U.cpp | 62 + .../unit_conv_solver_ConvBinWinogradRxS.cpp | 136 + ...nit_conv_solver_ConvBinWinogradRxSf2x3.cpp | 101 + ...t_conv_solver_ConvBinWinogradRxSf2x3g1.cpp | 169 + ...nit_conv_solver_ConvBinWinogradRxSf3x2.cpp | 125 + .../unit_conv_solver_ConvCkGroupedConvFwd.cpp | 2 +- ...nit_conv_solver_ConvMPBidirectWinograd.hpp | 115 + .../unit_conv_solver_ConvWinoFuryRxS.cpp | 64 + .../unit_conv_solver_ConvWinoRageRxS.cpp | 206 +- ...er_ConvWinograd3x3MultipassWrWF1x1x7x2.cpp | 67 + ...er_ConvWinograd3x3MultipassWrWF1x1x7x3.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF3x2.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF3x3.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF3x4.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF3x5.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF3x6.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF5x3.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF5x4.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF7x2.cpp | 67 + ...er_ConvWinograd3x3MultipassWrWF7x2x1x1.cpp | 67 + ...solver_ConvWinograd3x3MultipassWrWF7x3.cpp | 67 + ...er_ConvWinograd3x3MultipassWrWF7x3x1x1.cpp | 67 + .../gtest/unit_conv_solver_group_xdlops.hpp | 2 +- .../test/gtest/unit_implicitgemm_ck_util.cpp | 4 +- .../miopen/test/gtest/unit_lib_env_var.cpp | 2 +- projects/miopen/test/lstm.cpp | 43 - projects/miopen/test/lstm.hpp | 74 - projects/miopen/test/lstm_dropout.cpp | 69 - projects/miopen/test/rnn_util.hpp | 4 +- projects/miopen/test/sqlite_perfdb.cpp | 34 +- projects/miopen/test/tensor_holder.hpp | 19 +- shared/ctest/parse_test_categories.py | 38 +- 285 files changed, 8321 insertions(+), 15551 deletions(-) delete mode 100644 projects/miopen/src/kernels/MIOpenLRNBwd.cl delete mode 100644 projects/miopen/src/kernels/MIOpenLRNFwd.cl delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_metadata.inc delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc delete mode 100644 projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_metadata.inc create mode 100644 projects/miopen/test/gtest/lstm.cpp rename projects/miopen/test/{lstm_common.hpp => gtest/lstm.hpp} (87%) create mode 100644 projects/miopen/test/gtest/lstm_dropout.cpp delete mode 100644 projects/miopen/test/lstm.cpp delete mode 100644 projects/miopen/test/lstm.hpp delete mode 100644 projects/miopen/test/lstm_dropout.cpp diff --git a/projects/miopen/.clang-format b/projects/miopen/.clang-format index 660d6b044551..0d9e7127e9d0 100644 --- a/projects/miopen/.clang-format +++ b/projects/miopen/.clang-format @@ -44,7 +44,7 @@ Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +ForEachMacros: [ foreach, Q_FOREACH ] IncludeCategories: - Regex: '^"(llvm|llvm-c|clang|clang-c)/' Priority: 2 diff --git a/projects/miopen/CMakeLists.txt b/projects/miopen/CMakeLists.txt index 1c29c1b90460..989cb438e040 100644 --- a/projects/miopen/CMakeLists.txt +++ b/projects/miopen/CMakeLists.txt @@ -187,8 +187,7 @@ set( MIOPEN_BACKEND ${MIOPEN_DEFAULT_BACKEND} CACHE STRING set_property( CACHE MIOPEN_BACKEND PROPERTY STRINGS OpenCL HIP HIPOC HIPNOGPU) -# On Windows, internal symbols are only exported when BUILD_TESTING=true. As a result, building driver with BUILD_TESTING=false will result in failure. -set_var_to_condition(MIOPEN_BUILD_DRIVER_DEFAULT (NOT MIOPEN_EMBED_BUILD) AND (NOT (MIOPEN_BACKEND STREQUAL "HIPNOGPU")) AND ((NOT WIN32) OR BUILD_TESTING)) +set_var_to_condition(MIOPEN_BUILD_DRIVER_DEFAULT (NOT MIOPEN_EMBED_BUILD) AND (NOT (MIOPEN_BACKEND STREQUAL "HIPNOGPU"))) option(MIOPEN_BUILD_DRIVER "Build MIOpenDriver (and use it in tests)" ${MIOPEN_BUILD_DRIVER_DEFAULT}) message(STATUS "MIOPEN_BUILD_DRIVER: ${MIOPEN_BUILD_DRIVER}" ) @@ -472,10 +471,6 @@ if(MIOPEN_USE_HIPRTC) message(STATUS "Build with hiprtc ${hiprtc_VERSION} ${hiprtc_DIR}") endif() -option(Boost_USE_STATIC_LIBS "Use boost static libraries" ON) -add_definitions(-DBOOST_ALL_NO_LIB=1) -find_package(Boost REQUIRED COMPONENTS filesystem CONFIG) - find_path(HALF_INCLUDE_DIR half/half.hpp) message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") diff --git a/projects/miopen/Dockerfile b/projects/miopen/Dockerfile index e33d520b1e8a..56f9b96ff090 100755 --- a/projects/miopen/Dockerfile +++ b/projects/miopen/Dockerfile @@ -188,6 +188,7 @@ RUN if [ "$THEROCK_BUILD_MODE" = "Preset" ]; then \ -DTHEROCK_ENABLE_RAND=ON \ -DTHEROCK_ENABLE_ROCPROFSYS=ON \ -DTHEROCK_ENABLE_ROCPROF_TRACE_DECODER_BINARY=ON \ + -DTHEROCK_ENABLE_ROCGDB=ON \ -DTHEROCK_AMDGPU_DIST_BUNDLE_NAME=multiarch \ -DTHEROCK_AMDGPU_FAMILIES=$THEROCK_ASIC; \ else \ @@ -212,6 +213,7 @@ RUN if [ "$THEROCK_BUILD_MODE" = "Preset" ]; then \ -DTHEROCK_ENABLE_RAND=ON \ -DTHEROCK_ENABLE_ROCPROFSYS=ON \ -DTHEROCK_ENABLE_ROCPROF_TRACE_DECODER_BINARY=ON \ + -DTHEROCK_ENABLE_ROCGDB=ON \ -DTHEROCK_AMDGPU_DIST_BUNDLE_NAME=multiarch \ -DTHEROCK_AMDGPU_FAMILIES=$THEROCK_ASIC; \ fi diff --git a/projects/miopen/README.md b/projects/miopen/README.md index 92727dcb4d58..96d6bb46d0d2 100755 --- a/projects/miopen/README.md +++ b/projects/miopen/README.md @@ -38,10 +38,6 @@ To install MIOpen, you must first install these prerequisites: * [ROCm CMake](https://github.com/ROCm/rocm-cmake): provides CMake modules for common build tasks needed for the ROCm software stack * [Half](http://half.sourceforge.net/): IEEE 754-based, half-precision floating-point library -* [Boost](http://www.boost.org/): Version 1.79 is recommended, as older versions may need patches to - work on newer systems - * MIOpen uses `boost-system` and `boost-filesystem` packages to enable persistent - [kernel cache](https://rocm.docs.amd.com/projects/MIOpen/en/latest/cache.html) * [SQLite3](https://sqlite.org/index.html): A reading and writing performance database * lbzip2: A multi-threaded compress or decompress utility * [rocBLAS](https://github.com/ROCm/rocm-libraries/tree/develop/projects/rocblas): AMD's library for Basic Linear Algebra Subprograms @@ -330,19 +326,12 @@ switch branches or merge changes in Git to ensure any large binaries are kept in ## Installing the dependencies manually -If you're using Ubuntu v16, you can install the `Boost` packages using: +If you're using Ubuntu, you can install the `BZip2` packages using: ```shell -sudo apt-get install libboost-dev -sudo apt-get install libboost-system-dev -sudo apt-get install libboost-filesystem-dev +sudo apt-get install libbz2-dev ``` ->[!NOTE] ->By default, MIOpen attempts to build with Boost statically linked libraries. If required, you can build -with dynamically linked Boost libraries using the `-DBoost_USE_STATIC_LIBS=Off` flag during the -configuration stage. However, this is not recommended. - You must install the `half` header from the [half website](http://half.sourceforge.net/). ## Using Docker diff --git a/projects/miopen/addkernels/addkernels.cpp b/projects/miopen/addkernels/addkernels.cpp index 7477a2032c8b..074ae60f95ce 100644 --- a/projects/miopen/addkernels/addkernels.cpp +++ b/projects/miopen/addkernels/addkernels.cpp @@ -1,28 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + #include "include_inliner.hpp" #include "miopen/filesystem.hpp" #include @@ -316,7 +294,27 @@ int main(int argc, char* argv[]) std::string arg(argv[i]); std::transform(arg.begin(), arg.end(), arg.begin(), ::tolower); - if(arg == "-s" || arg == "-source") + // Handle response file (starts with @) + if(arg.starts_with('@')) + { + std::string response_file = std::string(argv[i]).substr(1); + std::ifstream file(response_file); + if(!file.is_open()) + { + std::cerr << "Error: Cannot open response file: " << response_file << std::endl; + return 1; + } + + std::string line; + while(std::getline(file, line)) + { + if(!line.empty() && !line.starts_with('#')) + { + sourceFiles.emplace_back(line); + } + } + } + else if(arg == "-s" || arg == "-source") { while(++i < argc && *argv[i] != '-') sourceFiles.emplace_back(argv[i]); diff --git a/projects/miopen/cmake/CppCheck.cmake b/projects/miopen/cmake/CppCheck.cmake index dae138104158..722ad98022dc 100644 --- a/projects/miopen/cmake/CppCheck.cmake +++ b/projects/miopen/cmake/CppCheck.cmake @@ -90,9 +90,8 @@ macro(enable_cppcheck) file(GLOB_RECURSE GSRCS ${GLOBS}) set(CPPCHECK_COMMAND ${CPPCHECK_EXE} - -q - # -v - # --report-progress + -v + --checkers-report=${CMAKE_BINARY_DIR}/cppcheck-checkers-report.txt ${CPPCHECK_FORCE} --cppcheck-build-dir=${CPPCHECK_BUILD_DIR} --platform=native @@ -115,6 +114,11 @@ macro(enable_cppcheck) RESULT_VARIABLE RESULT ) if(NOT RESULT EQUAL 0) + message(WARNING \"Cppcheck failed with exit code \${RESULT}\") + if(EXISTS \"${CMAKE_BINARY_DIR}/cppcheck-checkers-report.txt\") + file(READ \"${CMAKE_BINARY_DIR}/cppcheck-checkers-report.txt\" CHECKERS_REPORT) + message(WARNING \"\${CHECKERS_REPORT}\") + endif() message(FATAL_ERROR \"Cppcheck failed\") endif() ") diff --git a/projects/miopen/cmake/EnableCompilerWarnings.cmake b/projects/miopen/cmake/EnableCompilerWarnings.cmake index c133d0e4cbb1..3b487676b751 100644 --- a/projects/miopen/cmake/EnableCompilerWarnings.cmake +++ b/projects/miopen/cmake/EnableCompilerWarnings.cmake @@ -70,7 +70,6 @@ set(__clang_cxx_compile_options -Wno-unused-command-line-argument -Wno-weak-vtables -Wno-covered-switch-default - -Wno-unused-result -Wno-unsafe-buffer-usage -Wno-deprecated-declarations -Wno-shadow-uncaptured-local @@ -89,7 +88,6 @@ set(__clang_cxx_compile_options -Wno-float-equal -Wno-redundant-parens -Wno-format-nonliteral - -Wno-unused-template -Wno-comma -Wno-suggest-destructor-override -Wno-switch-enum @@ -102,8 +100,6 @@ set(__clang_cxx_compile_options -Wno-documentation -Wno-deprecated-builtins -Wno-enum-constexpr-conversion - -Wno-unused-value - -Wno-unused-parameter -Wno-missing-noreturn -Wno-tautological-constant-out-of-range-compare -Wno-nrvo @@ -120,17 +116,9 @@ if(WIN32) -fms-compatibility) endif() -set(__gnu_cxx_compile_options - -Wno-missing-field-initializers -) - add_compile_options( - "$<$,$>:${__msvc_cxx_compile_options}>" "$<$,$>:${__default_cxx_compile_options};${__clang_cxx_compile_options}>" - "$<$,$>:${__default_cxx_compile_options};${__gnu_cxx_compile_options}>" ) -unset(__msvc_cxx_compile_options) unset(__default_cxx_compile_options) -unset(__gnu_cxx_compile_options) unset(__clang_cxx_compile_options) diff --git a/projects/miopen/driver/InputFlags.cpp b/projects/miopen/driver/InputFlags.cpp index f1f9e7e993d1..412145ad487e 100644 --- a/projects/miopen/driver/InputFlags.cpp +++ b/projects/miopen/driver/InputFlags.cpp @@ -257,7 +257,7 @@ TensorParameters InputFlags::GetValueTensor(const std::string& long_name) const if(components.empty()) return {}; - auto parse = [](auto line) { + auto parse = [](const std::string& line) { auto ret = std::vector{}; const auto strs = miopen::SplitDelim(line, 'x'); for(auto&& str : strs) @@ -301,7 +301,7 @@ TensorParametersUint64 InputFlags::GetValueTensorUint64(const std::string& long_ if(components.size() < 1) return {}; - auto parse = [](auto line) { + auto parse = [](const std::string& line) { auto ret = std::vector{}; const auto strs = miopen::SplitDelim(line, 'x'); for(auto&& str : strs) @@ -386,12 +386,12 @@ InputFlags::GetValue2dVectorInt(const std::string& long_name) const { const auto& input = MapInputs.at(FindShortName(long_name)); const auto components = miopen::SplitDelim(input.value.c_str(), ','); - auto output = std::vector>{}; + std::vector> output; if(components.size() < 1) - return {}; + return output; - auto parse = [](auto line) { + auto parse = [](const std::string& line) { auto ret = std::vector{}; const auto strs = miopen::SplitDelim(line, 'x'); for(auto&& str : strs) @@ -421,12 +421,11 @@ InputFlags::GetValue2dVectorUint64(const std::string& long_name) const { const auto& input = MapInputs.at(FindShortName(long_name)); const auto components = miopen::SplitDelim(input.value.c_str(), ','); - auto output = std::vector>{}; if(components.size() < 1) return {}; - auto parse = [](auto line) { + auto parse = [](const std::string& line) { auto ret = std::vector{}; const auto strs = miopen::SplitDelim(line, 'x'); for(auto&& str : strs) @@ -443,6 +442,7 @@ InputFlags::GetValue2dVectorUint64(const std::string& long_name) const return ret; }; + auto output = std::vector>{}; for(auto&& component : components) { output.push_back(parse(component)); diff --git a/projects/miopen/driver/adam_driver.hpp b/projects/miopen/driver/adam_driver.hpp index 27d44011d615..2d1237b43ff3 100644 --- a/projects/miopen/driver/adam_driver.hpp +++ b/projects/miopen/driver/adam_driver.hpp @@ -333,11 +333,10 @@ int AdamDriver::AddCmdLineArgs() template std::vector AdamDriver::GetInputTensorLengthsFromCmdLine() { - std::vector ret; auto tensor = inflags.GetValueTensor("dims"); if(!tensor.lengths.empty()) return tensor.lengths; - return ret; + return {}; } template diff --git a/projects/miopen/driver/bn_driver.hpp b/projects/miopen/driver/bn_driver.hpp index 5a49543b959c..4feafdd98cf3 100644 --- a/projects/miopen/driver/bn_driver.hpp +++ b/projects/miopen/driver/bn_driver.hpp @@ -907,14 +907,14 @@ void BatchNormDriver::runGPUFwdTrain(Tref if(usePingPongBuffers) { // copy data from current running mean/var to previous running mean/var - hipMemcpy(prevRunMean.GetDevicePtr(), - runMean.GetDevicePtr(), - runMean.GetTensor().desc.GetElementSize() * sizeof(TAcc), - hipMemcpyDeviceToDevice); - hipMemcpy(prevRunVariance.GetDevicePtr(), - runVariance.GetDevicePtr(), - runVariance.GetTensor().desc.GetElementSize() * sizeof(TAcc), - hipMemcpyDeviceToDevice); + (void)hipMemcpy(prevRunMean.GetDevicePtr(), + runMean.GetDevicePtr(), + runMean.GetTensor().desc.GetElementSize() * sizeof(TAcc), + hipMemcpyDeviceToDevice); + (void)hipMemcpy(prevRunVariance.GetDevicePtr(), + runVariance.GetDevicePtr(), + runVariance.GetTensor().desc.GetElementSize() * sizeof(TAcc), + hipMemcpyDeviceToDevice); // not required, but just in case for consistency prevRunMean.CopyFromDeviceToHost(GetStream()); prevRunVariance.CopyFromDeviceToHost(GetStream()); @@ -993,14 +993,14 @@ void BatchNormDriver::runGPUFwdTrain(Tref if(usePingPongBuffers) { // copy data from current running mean/var to previous running mean/var - hipMemcpy(prevRunMean.GetDevicePtr(), - runMean.GetDevicePtr(), - runMean.GetTensor().desc.GetElementSize() * sizeof(TAcc), - hipMemcpyDeviceToDevice); - hipMemcpy(prevRunVariance.GetDevicePtr(), - runVariance.GetDevicePtr(), - runVariance.GetTensor().desc.GetElementSize() * sizeof(TAcc), - hipMemcpyDeviceToDevice); + (void)hipMemcpy(prevRunMean.GetDevicePtr(), + runMean.GetDevicePtr(), + runMean.GetTensor().desc.GetElementSize() * sizeof(TAcc), + hipMemcpyDeviceToDevice); + (void)hipMemcpy(prevRunVariance.GetDevicePtr(), + runVariance.GetDevicePtr(), + runVariance.GetTensor().desc.GetElementSize() * sizeof(TAcc), + hipMemcpyDeviceToDevice); // not required, but just in case for consistency prevRunMean.CopyFromDeviceToHost(GetStream()); prevRunVariance.CopyFromDeviceToHost(GetStream()); diff --git a/projects/miopen/driver/driver.hpp b/projects/miopen/driver/driver.hpp index 1df0c33801d4..841084ad90da 100644 --- a/projects/miopen/driver/driver.hpp +++ b/projects/miopen/driver/driver.hpp @@ -132,7 +132,7 @@ struct GPUMem } int FromGPU(hipStream_t q, void* p) { - hipDeviceSynchronize(); + (void)hipDeviceSynchronize(); _q = q; return static_cast(hipMemcpy(p, buf, GetSize(), hipMemcpyDeviceToHost)); } @@ -518,7 +518,7 @@ class Driver miopenCreate(&handle); #elif MIOPEN_BACKEND_HIP hipStream_t s; - hipStreamCreate(&s); + (void)hipStreamCreate(&s); miopenCreateWithStream(&handle, s); #endif diff --git a/projects/miopen/driver/getitem_driver.hpp b/projects/miopen/driver/getitem_driver.hpp index c48c9a0520b8..7dd32cb92d29 100644 --- a/projects/miopen/driver/getitem_driver.hpp +++ b/projects/miopen/driver/getitem_driver.hpp @@ -26,29 +26,21 @@ #ifndef GUARD_MIOPEN_GETITEM_DRIVER_HPP #define GUARD_MIOPEN_GETITEM_DRIVER_HPP -#include "InputFlags.hpp" #include "driver.hpp" #include "tensor_driver.hpp" #include "timer.hpp" -#include "random.hpp" -#include -#include -#include -#include -#include -#include +#include "../test/verify.hpp" + #include -#include #include -#include <../test/tensor_holder.hpp> -#include <../test/verify.hpp> +#include template int32_t mloGetitemBackwardRunHost(miopenTensorDescriptor_t dyDesc, uint32_t indexCount, miopenTensorDescriptor_t* indexDescs, miopenTensorDescriptor_t dxDesc, - miopenTensorDescriptor_t errorDesc, + miopenTensorDescriptor_t errorDesc Tgpu* dy, int32_t** indexs, Tcheck* dxhost, @@ -470,7 +462,6 @@ int GetitemDriver::RunBackwardCPU() indexDescs.size(), indexDescs.data(), dxDesc, - errorDesc, dy.data(), indexs_ptr.data(), dxhost.data(), @@ -478,8 +469,7 @@ int GetitemDriver::RunBackwardCPU() dims.size(), dims.data(), slices.size(), - slices_flat.data(), - offset); + slices_flat.data()); return miopenStatusSuccess; } diff --git a/projects/miopen/driver/rope_driver.hpp b/projects/miopen/driver/rope_driver.hpp index ecaf55ac647e..57534392748c 100644 --- a/projects/miopen/driver/rope_driver.hpp +++ b/projects/miopen/driver/rope_driver.hpp @@ -26,27 +26,18 @@ #ifndef GUARD_MIOPEN_ROPE_DRIVER_HPP #define GUARD_MIOPEN_ROPE_DRIVER_HPP -#include "InputFlags.hpp" #include "driver.hpp" #include "tensor_driver.hpp" #include "timer.hpp" -#include "random.hpp" -#include -#include -#include +#include "../test/verify.hpp" + #include -#include -#include #include #include -#include <../test/tensor_holder.hpp> -#include <../test/verify.hpp> template int32_t mloRoPEForwardRunHost(miopenTensorDescriptor_t xDesc, miopenTensorDescriptor_t cosDesc, - miopenTensorDescriptor_t sinDesc, - miopenTensorDescriptor_t yDesc, Tgpu* x, Tgpu* cos, Tgpu* sin, @@ -82,8 +73,6 @@ int32_t mloRoPEForwardRunHost(miopenTensorDescriptor_t xDesc, template int32_t mloRoPEBackwardRunHost(miopenTensorDescriptor_t dyDesc, miopenTensorDescriptor_t cosDesc, - miopenTensorDescriptor_t sinDesc, - miopenTensorDescriptor_t dxDesc, Tgpu* dy, Tgpu* cos, Tgpu* sin, @@ -329,7 +318,7 @@ template int RoPEDriver::RunForwardCPU() { mloRoPEForwardRunHost( - x_dyDesc, cosDesc, sinDesc, y_dxDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); + x_dyDesc, cosDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); return miopenStatusSuccess; } @@ -387,7 +376,7 @@ template int RoPEDriver::RunBackwardCPU() { mloRoPEBackwardRunHost( - x_dyDesc, cosDesc, sinDesc, y_dxDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); + x_dyDesc, cosDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); return miopenStatusSuccess; } diff --git a/projects/miopen/driver/timer.hpp b/projects/miopen/driver/timer.hpp index e879a6ab12cd..4af4c9a2ab9f 100644 --- a/projects/miopen/driver/timer.hpp +++ b/projects/miopen/driver/timer.hpp @@ -175,7 +175,7 @@ class RNNCombTimeLogger return; } - hipEventRecord(startEvent[launchCount].get(), stream); + (void)hipEventRecord(startEvent[launchCount].get(), stream); st = std::chrono::steady_clock::now(); } void StopAndPush() @@ -193,14 +193,14 @@ class RNNCombTimeLogger printf("Executed more iterations than planned\n"); return; } - hipEventRecord(endEvent[launchCount].get(), stream); + (void)hipEventRecord(endEvent[launchCount].get(), stream); if(clockMode == ClockMode::OldWallClock) { std::chrono::time_point st2 = std::chrono::steady_clock::now(); - hipEventSynchronize(endEvent[launchCount].get()); + (void)hipEventSynchronize(endEvent[launchCount].get()); std::chrono::time_point end2 = std::chrono::steady_clock::now(); @@ -214,7 +214,7 @@ class RNNCombTimeLogger else { if(clockMode == ClockMode::SeparateClocksSynced) - hipEventSynchronize(endEvent[launchCount].get()); + (void)hipEventSynchronize(endEvent[launchCount].get()); hostTimePerLaunch.push_back( std::chrono::duration_cast>(end - st) @@ -235,11 +235,11 @@ class RNNCombTimeLogger float gpu_time = 0.0f; if(clockMode == ClockMode::SeparateClocksNotSynced) - hipEventSynchronize(endEvent[n_iter - 1].get()); + (void)hipEventSynchronize(endEvent[n_iter - 1].get()); for(auto i = 0ull; i < n_iter; ++i) { - hipEventElapsedTime(&gpu_time, startEvent[i].get(), endEvent[i].get()); + (void)hipEventElapsedTime(&gpu_time, startEvent[i].get(), endEvent[i].get()); if(clockMode == ClockMode::SeparateClocksNotSynced || clockMode == ClockMode::SeparateClocksSynced) @@ -258,7 +258,7 @@ class RNNCombTimeLogger } if(n_iter == 1) - hipEventElapsedTime(&gpu_time, startEvent[0].get(), endEvent[0].get()); + (void)hipEventElapsedTime(&gpu_time, startEvent[0].get(), endEvent[0].get()); printf("GPU Kernel Time Elapsed: %f ms\n", n_iter > 1 ? gpu_avg / (n_iter - 1) : gpu_time); printf("Wall-clock Time Elapsed: %f ms\n", diff --git a/projects/miopen/driver/transformers_adam_w_driver.hpp b/projects/miopen/driver/transformers_adam_w_driver.hpp index 3ae7eda364a0..dfd82a3284c6 100644 --- a/projects/miopen/driver/transformers_adam_w_driver.hpp +++ b/projects/miopen/driver/transformers_adam_w_driver.hpp @@ -232,11 +232,8 @@ int TransformersAdamWDriver::AddCmdLineArgs() template std::vector TransformersAdamWDriver::GetInputTensorLengthsFromCmdLine() { - std::vector ret; auto tensor = inflags.GetValueTensor("dims"); - if(!tensor.lengths.empty()) - return tensor.lengths; - return ret; + return tensor.lengths; } template diff --git a/projects/miopen/include/miopen/config.h.in b/projects/miopen/include/miopen/config.h.in index 79fb689587a4..0390214dc93a 100644 --- a/projects/miopen/include/miopen/config.h.in +++ b/projects/miopen/include/miopen/config.h.in @@ -171,15 +171,6 @@ const char* getOffloadBundlerBinPath(); MIOPEN_HIPBLASLT_VERSION_PATCH) #endif // MIOPEN_USE_HIPBLASLT -/// WORKAROUND_BOOST_ISSUE_392 -/// Workaround for https://github.com/boostorg/config/issues/392#issuecomment-1109889533 -/// See also https://github.com/ROCm/MIOpen/pull/1490#issuecomment-1109928102, -/// https://github.com/ROCm/MIOpen/pull/1543 -/// TODO: Remove the W/A as soon we switch to the properly fixed boost. -#if MIOPEN_BACKEND_HIP -#include -#endif - #define MIOPEN_GOLDEN_DB_VERSION 21 #endif // GUARD_CONFIG_H_IN diff --git a/projects/miopen/requirements.txt b/projects/miopen/requirements.txt index 99fe1f479e8b..226e32915316 100755 --- a/projects/miopen/requirements.txt +++ b/projects/miopen/requirements.txt @@ -1,5 +1,5 @@ +bzip2@1.0.8 sqlite3@3.50.4 -DCMAKE_POSITION_INDEPENDENT_CODE=On -boost@1.83 -DCMAKE_POSITION_INDEPENDENT_CODE=On --build -DCMAKE_CXX_FLAGS=" -std=c++20 -Wno-enum-constexpr-conversion -Wno-deprecated-builtins -Wno-deprecated-declarations " facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake # ROCm/half@10abd99e7815f0ca5d892f58dd7d15a23b7cf92c --build ROCm/rocMLIR@rocm-5.5.0 -H sha256:a5f62769d28a73e60bc8d61022820f050e97c977c8f6f6275488db31512e1f42 -DBUILD_FAT_LIBROCKCOMPILER=1 -DCMAKE_IGNORE_PATH="/opt/conda/envs/py_3.8;/opt/conda/envs/py_3.9;/opt/conda/envs/py_3.10" -DCMAKE_IGNORE_PREFIX_PATH=/opt/conda -DCMAKE_CXX_FLAGS=" -Wno-deprecated-declarations " diff --git a/projects/miopen/src/CMakeLists.txt b/projects/miopen/src/CMakeLists.txt index 60162471d2e9..fa8383e82c05 100644 --- a/projects/miopen/src/CMakeLists.txt +++ b/projects/miopen/src/CMakeLists.txt @@ -19,6 +19,8 @@ option( MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) option( MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) +option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) + configure_file("${PROJECT_SOURCE_DIR}/include/miopen/config.h.in" "${PROJECT_BINARY_DIR}/include/miopen/config.h") # configure a header file to pass the CMake version settings to the source, and package the header files in the output archive @@ -528,10 +530,13 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/winograd/Conv_Winograd_Fury_v4_6_0_gfx12_1536vgprs_fp16_fp32acc_f2x3_c16_stride1.inc kernels/winograd/Conv_Winograd_Fury_v4_6_0_gfx12_1536vgprs_fp16_fp32acc_f2x3_c32_stride1.inc kernels/winograd/Conv_Winograd_Fury_v2_X_X_X_metadata.inc - kernels/winograd/Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc - kernels/winograd/Conv_Winograd_Rage_v4_6_0_metadata.inc - kernels/winograd/Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc - kernels/winograd/Conv_Winograd_Rage_v4_7_0_metadata.inc + kernels/winograd/Conv_Winograd_Rage_v4_6_1_gfx12x_fp16_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_6_1_gfx94x_fp16_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_9_0_gfx12x_fp16_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_9_0_gfx94x_bf16_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_9_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_9_0_gfx94x_fp32_fp32acc_f2x3_stride1.inc + kernels/winograd/Conv_Winograd_Rage_v4_X_X_metadata.inc kernels/xform_bidirect_winograd_code.inc kernels/xform_data_filter.inc kernels/xform_kd_cov2.inc @@ -569,8 +574,8 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/MIOpenGetitem.cpp kernels/MIOpenKthvalue.cpp kernels/MIOpenLayerNorm.cpp - kernels/MIOpenLRNBwd.cl - kernels/MIOpenLRNFwd.cl + kernels/MIOpenLRNBwd.cpp + kernels/MIOpenLRNFwd.cpp kernels/MIOpenMultiMarginLoss.cpp kernels/MIOpenNeuron.cpp kernels/MIOpenPReLU.cpp @@ -679,8 +684,10 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/winograd/Conv_Winograd_Fury_v2_4_1_fp16_fp16acc_f2x3_c32_stride1.s kernels/winograd/Conv_Winograd_Fury_v4_6_0_fp16_fp32acc_f2x3_c16_stride1.s kernels/winograd/Conv_Winograd_Fury_v4_6_0_fp16_fp32acc_f2x3_c32_stride1.s - kernels/winograd/Conv_Winograd_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s - kernels/winograd/Conv_Winograd_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s + kernels/winograd/Conv_Winograd_Rage_v4_6_1_fp16_fp32acc_f2x3_stride1.s + kernels/winograd/Conv_Winograd_Rage_v4_9_0_fp16_fp32acc_f2x3_stride1.s + kernels/winograd/Conv_Winograd_Rage_v4_9_0_fp32_fp32acc_f2x3_stride1.s + kernels/winograd/Conv_Winograd_Rage_v4_9_0_bf16_fp32acc_f2x3_stride1.s kernels/xform_data.s kernels/xform_filter.s kernels/xform_bidirect_winograd_data.s @@ -811,12 +818,19 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN set(KERNEL_SRC_ASM_PATH ${PROJECT_BINARY_DIR}/inlined_kernels/batch_${KERNELS_BATCH_ID}.S) set(KERNEL_SRC_BIN_PATH ${PROJECT_BINARY_DIR}/inlined_kernels/batch_${KERNELS_BATCH_ID}.bin) + # Generate response file for this batch to avoid command line length limits + set(KERNEL_RESPONSE_FILE ${PROJECT_BINARY_DIR}/inlined_kernels/batch_${KERNELS_BATCH_ID}_files.txt) + file(WRITE ${KERNEL_RESPONSE_FILE} "") + foreach(kernel_file ${KERNELS_BATCH}) + file(APPEND ${KERNEL_RESPONSE_FILE} "${kernel_file}\n") + endforeach() + if(MIOPEN_INCBIN) add_custom_command( OUTPUT ${KERNEL_SRC_HPP_PATH} ${KERNEL_SRC_ASM_PATH} ${KERNEL_SRC_BIN_PATH} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS addkernels ${KERNELS_BATCH} ${KERNEL_INCLUDES} - COMMAND $ -asm ${KERNEL_SRC_ASM_PATH} ${KERNEL_SRC_BIN_PATH} -target ${KERNEL_SRC_HPP_PATH} -extern ${EXTRA_OPTIONS} -source ${KERNELS_BATCH} + DEPENDS addkernels ${KERNELS_BATCH} ${KERNEL_INCLUDES} ${KERNEL_RESPONSE_FILE} + COMMAND $ -asm ${KERNEL_SRC_ASM_PATH} ${KERNEL_SRC_BIN_PATH} -target ${KERNEL_SRC_HPP_PATH} -extern ${EXTRA_OPTIONS} @${KERNEL_RESPONSE_FILE} COMMENT "Inlining kernels batch #${KERNELS_BATCH_ID}${MESSAGE_SUFFIX}" ) list(APPEND MIOpen_Source ${KERNEL_SRC_HPP_PATH} ${KERNEL_SRC_ASM_PATH}) @@ -824,8 +838,8 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN add_custom_command( OUTPUT ${KERNEL_SRC_HPP_PATH} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS addkernels ${KERNELS_BATCH} ${KERNEL_INCLUDES} - COMMAND $ -target ${KERNEL_SRC_HPP_PATH} -extern ${EXTRA_OPTIONS} -source ${KERNELS_BATCH} + DEPENDS addkernels ${KERNELS_BATCH} ${KERNEL_INCLUDES} ${KERNEL_RESPONSE_FILE} + COMMAND $ -target ${KERNEL_SRC_HPP_PATH} -extern ${EXTRA_OPTIONS} @${KERNEL_RESPONSE_FILE} COMMENT "Inlining kernels batch #${KERNELS_BATCH_ID}${MESSAGE_SUFFIX}" ) configure_file(kernels/kernels_batch.cpp.in ${KERNEL_SRC_CPP_PATH}) @@ -954,13 +968,9 @@ if(WIN32) target_link_libraries(MIOpen PRIVATE ntdll) endif() +generate_export_header(MIOpen BASE_NAME MIOPEN_INTERNALS EXPORT_FILE_NAME ${CMAKE_BINARY_DIR}/include/miopen/export_internals.h) + if(BUILD_TESTING) - # On Windows, export selected internal symbols only when tests are built. The officially released - # binaries must not have internals exposed because doing so violates the threats model requirements. - # We cannot use the CMake property CMAKE_EXPORT_ALL_SYMBOLS here because the number of automatically - # exported symbols exceeds the maximum allowed number in a DLL library (64K). - # See details here: https://learn.microsoft.com/en-us/cpp/build/exporting-from-a-dll?view=msvc-170 - generate_export_header(MIOpen BASE_NAME MIOPEN_INTERNALS EXPORT_FILE_NAME ${CMAKE_BINARY_DIR}/include/miopen/export_internals.h) target_compile_definitions(MIOpen PUBLIC $) endif() @@ -1048,28 +1058,9 @@ endif() target_link_libraries(MIOpen PRIVATE nlohmann_json::nlohmann_json) target_link_libraries(MIOpen INTERFACE $) -target_internal_library(MIOpen Boost::filesystem) -list(APPEND PACKAGE_STATIC_DEPENDS PACKAGE Boost COMPONENTS filesystem) - if(NOT WIN32 AND NOT APPLE) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/lib.def " - MIOPEN_${MIOPEN_BACKEND}_1 - { - global: - miopen*; - extern \"C++\" { - miopen::*; - }; - local: - *boost*; - extern \"C++\" { - std::*; - }; - }; - ") - target_link_libraries(MIOpen PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_BINARY_DIR}/lib.def") target_link_libraries(MIOpen PRIVATE "-Wl,--exclude-libs,ALL") - # set_target_properties(MIOpen PROPERTIES CXX_VISIBILITY_PRESET hidden) + set_target_properties(MIOpen PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(MIOpen PROPERTIES VISIBILITY_INLINES_HIDDEN 1) endif() ####################################### @@ -1079,14 +1070,6 @@ if(MIOPEN_ENABLE_SQLITE) target_link_libraries(MIOpen INTERFACE $) endif() ############################################################ -# MIOpen depends on librt for Boost.Interprocess -if(NOT WIN32 AND NOT APPLE) - find_library(LIBRT rt) - if(LIBRT) - message(STATUS "Librt: " ${LIBRT}) - target_internal_library(MIOpen ${LIBRT}) - endif() -endif() if(MIOPEN_USE_ROCTRACER) target_include_directories(MIOpen PRIVATE ${ROCTRACER_INCLUDE_DIR}) diff --git a/projects/miopen/src/api/find2_0_commons.cpp b/projects/miopen/src/api/find2_0_commons.cpp index 51b8cd4dfd99..737a4594c1c1 100644 --- a/projects/miopen/src/api/find2_0_commons.cpp +++ b/projects/miopen/src/api/find2_0_commons.cpp @@ -1,34 +1,12 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #include #include #include #include +#include #include #include #include @@ -36,7 +14,6 @@ #include #include -#include template static miopenStatus_t MakeProblem(miopenProblem_t* problem, @@ -134,7 +111,7 @@ miopenStatus_t miopenFuseProblems(miopenProblem_t problem1, miopenProblem_t prob auto& problem1_deref = miopen::deref(problem1); auto emplace_problem2 = [problem2](auto& problems) { - const auto impl2 = boost::hof::match( + const auto impl2 = miopen::hof_match( [&](miopen::Problem& problem2_inner) { problems.emplace_back(problem2_inner); }, [&](const miopen::FusedProblem& problem2_inner) { problems.reserve(problems.size() + problem2_inner.problems.size()); @@ -146,7 +123,7 @@ miopenStatus_t miopenFuseProblems(miopenProblem_t problem1, miopenProblem_t prob std::visit(impl2, miopen::deref(problem2).item); }; - std::visit(boost::hof::match( + std::visit(miopen::hof_match( [&](miopen::Problem& problem1_inner) { auto tmp = miopen::FusedProblem{}; tmp.problems.reserve(2); @@ -176,7 +153,7 @@ miopenStatus_t miopenSetProblemTensorDescriptor(miopenProblem_t problem, MIOPEN_LOG_FUNCTION(problem, id, descriptor); return miopen::try_([&] { - const auto impl = boost::hof::match( + const auto impl = miopen::hof_match( [&](miopen::Problem& problem) { if(!problem.RegisterTensorDescriptor(id, miopen::deref(descriptor))) MIOPEN_THROW(miopenStatusBadParm, diff --git a/projects/miopen/src/binary_cache.cpp b/projects/miopen/src/binary_cache.cpp index 7dda191d538d..f40399188f43 100644 --- a/projects/miopen/src/binary_cache.cpp +++ b/projects/miopen/src/binary_cache.cpp @@ -54,9 +54,8 @@ static fs::path ComputeSysCachePath() { auto p = miopen::ExpandUser(GetSystemDbPath()); if(!fs::exists(p)) - return {}; - else - return p; + p.clear(); + return p; } static fs::path ComputeUserCachePath() diff --git a/projects/miopen/src/buffer_info.cpp b/projects/miopen/src/buffer_info.cpp index ef567da1a2a1..162870b01553 100644 --- a/projects/miopen/src/buffer_info.cpp +++ b/projects/miopen/src/buffer_info.cpp @@ -69,6 +69,10 @@ MemLayout_t GetGroupConvLayout(MemLayout_t layout, bool IsDataBuffer) case MemLayout_t::CNHW: return MemLayout_t::CGNHW; case MemLayout_t::NCHW: return MemLayout_t::NGCHW; case MemLayout_t::NHWC: + // For transposing solvers, NHWC data will be transposed to NCHW before + // group operations, so we return the layout unchanged and let the + // transpose mechanism handle the conversion + return MemLayout_t::NHWC; case MemLayout_t::CHWN: case MemLayout_t::HWCN: case MemLayout_t::HWNC: @@ -86,6 +90,10 @@ MemLayout_t GetGroupConvLayout(MemLayout_t layout, bool IsDataBuffer) case MemLayout_t::CNHW: return MemLayout_t::GCNHW; case MemLayout_t::NCHW: return MemLayout_t::GNCHW; case MemLayout_t::NHWC: + // For transposing solvers, NHWC weights will be transposed to NCHW before + // group operations, so we return the layout unchanged and let the + // transpose mechanism handle the conversion + return MemLayout_t::NHWC; case MemLayout_t::CHWN: case MemLayout_t::HWCN: case MemLayout_t::HWNC: diff --git a/projects/miopen/src/bz2.cpp b/projects/miopen/src/bz2.cpp index 71c3161b93cd..29dce44514a6 100644 --- a/projects/miopen/src/bz2.cpp +++ b/projects/miopen/src/bz2.cpp @@ -70,12 +70,16 @@ std::vector compress(const std::vector& v, bool* compressed) if(compressed != nullptr and e == BZ_OUTBUFF_FULL) { *compressed = false; - return v; + result = v; } - check_bz2_error(e, "BZ2_bzBuffToBuffCompress"); - result.resize(len); - if(compressed != nullptr) - *compressed = true; + else + { + check_bz2_error(e, "BZ2_bzBuffToBuffCompress"); + result.resize(len); + if(compressed != nullptr) + *compressed = true; + } + return result; } diff --git a/projects/miopen/src/comgr.cpp b/projects/miopen/src/comgr.cpp index 6fd0fe8e6161..75f8e407264b 100644 --- a/projects/miopen/src/comgr.cpp +++ b/projects/miopen/src/comgr.cpp @@ -525,17 +525,26 @@ static std::string GetLog(const Dataset& dataset, const bool comgr_error_handlin /// \todo Clarify API and update implementation. const auto count = dataset.GetDataCount(AMD_COMGR_DATA_KIND_LOG); if(count < 1) - return {comgr_error_handling ? "comgr warning: error log not found" : ""}; + { + text = comgr_error_handling ? "comgr warning: error log not found" : ""; + return text; + } const auto data = dataset.GetData(AMD_COMGR_DATA_KIND_LOG, 0); text = data.GetString(); if(text.empty()) - return {comgr_error_handling ? "comgr info: error log empty" : ""}; + { + text = comgr_error_handling ? "comgr info: error log empty" : ""; + return text; + } } catch(ComgrError&) { if(comgr_error_handling) - return {"comgr error: failed to get error log"}; + { + text = "comgr error: failed to get error log"; + return text; + } // deepcode ignore EmptyThrowOutsideCatch: false positive throw; /// \anchor catch_and_rethrow_in_getlog diff --git a/projects/miopen/src/conv/heuristics/ai_conv_3d_kernel_tuning_utils.cpp b/projects/miopen/src/conv/heuristics/ai_conv_3d_kernel_tuning_utils.cpp index bdfb0ccc7abe..d97b23b2e802 100644 --- a/projects/miopen/src/conv/heuristics/ai_conv_3d_kernel_tuning_utils.cpp +++ b/projects/miopen/src/conv/heuristics/ai_conv_3d_kernel_tuning_utils.cpp @@ -54,7 +54,6 @@ int LayoutStringToCode(const std::string& layout) } // Helper: Extract 3D convolution features -MIOPEN_INTERNALS_EXPORT std::map GetFeatures3D(const ProblemDescription& problem, int /*max_cu*/, const std::string& /*arch*/) { @@ -189,7 +188,6 @@ std::vector ProcessExplicitXdlParams(const std::vector } // Helper: Tokenize kernel string -MIOPEN_INTERNALS_EXPORT std::vector GetKernelAsTokens(const std::string& kernel) { std::vector tokens; @@ -302,7 +300,6 @@ std::vector GetKernelAsTokens(const std::string& kernel) } // Helper: Fill indexes and kernels from valid_kernels -MIOPEN_INTERNALS_EXPORT void FillHeuristicKernels(const std::vector& valid_kernels, std::vector& indexes, std::vector>& kernels) @@ -330,7 +327,6 @@ void FillHeuristicKernels(const std::vector& valid_kernels, // TODO: new CK functionality will use -1 for autodeduction, so we could add -1 to the list. // Note that the current models have not been trained with -1 in mind, so it may not work as // expected. -MIOPEN_INTERNALS_EXPORT std::vector GenerateSplitK(int max_split_k) { std::vector split_ks; diff --git a/projects/miopen/src/conv/heuristics/ai_heuristics.cpp b/projects/miopen/src/conv/heuristics/ai_heuristics.cpp index b0b0ac9e6e05..16449972571e 100644 --- a/projects/miopen/src/conv/heuristics/ai_heuristics.cpp +++ b/projects/miopen/src/conv/heuristics/ai_heuristics.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include @@ -641,6 +642,26 @@ PredictionResult ProcessPredictions(const std::vector& predictions, return result; } +static void PromoteSolverToFront(PredictionResult& result, const char* solver_name) +{ + const auto solver_id = solver::Id{solver_name}; + if(!solver_id.IsValid()) + return; + + const auto target = solver_id.Value(); + const auto it = std::find(result.solver_ids.begin(), result.solver_ids.end(), target); + if(it == result.solver_ids.end() || it == result.solver_ids.begin()) + return; + + result.solver_ids.erase(it); + result.solver_ids.insert(result.solver_ids.begin(), target); + + result.any_solver_ids.clear(); + result.any_solver_ids.reserve(result.solver_ids.size()); + for(const auto id : result.solver_ids) + result.any_solver_ids.push_back(id); +} + /** * @brief Common logic for running TunaNet prediction and caching results * @param problem Convolution problem description @@ -661,6 +682,11 @@ ProcessAndCachePredictions(const conv::ProblemDescription& problem, // Process predictions (sort by probability, filter invalid solvers) auto result = ProcessPredictions(predictions, solver_map, is3d); + if(is3d && conv::IsBwdDataPointOutput3dStrideEqFilter(problem)) + { + PromoteSolverToFront(result, "GemmBwdRest"); + MIOPEN_LOG_I2("3D TunaNet override: promoting GemmBwdRest for point-output backward-data"); + } // Cache results for future use StorePredictionCache(problem, device, is3d, result.any_solver_ids); @@ -696,14 +722,14 @@ std::vector PredictSolver(const conv::ProblemDescription& problem, std::unique_ptr model = conv3d::Get3DModel(device); if(!model || !model->IsProblemSupported(problem, ctx)) { - return {}; // Fallback: empty vector + return cached_result; // Fallback: empty vector } MIOPEN_LOG_I2("Evaluating 3D TunaNet"); std::vector predictions = model->Forward(problem); - return ProcessAndCachePredictions( - problem, device, true, predictions, model->GetSolverMap()); + cached_result = + ProcessAndCachePredictions(problem, device, true, predictions, model->GetSolverMap()); } else { @@ -711,15 +737,17 @@ std::vector PredictSolver(const conv::ProblemDescription& problem, std::unique_ptr model = GetModel(device); if(!model || !model->IsProblemSupported(problem, ctx)) { - return {}; // Fallback: empty vector + return cached_result; // Fallback: empty vector } MIOPEN_LOG_I2("Evaluating TunaNet"); std::vector predictions = model->Forward(problem); - return ProcessAndCachePredictions( + cached_result = ProcessAndCachePredictions( problem, device, false, predictions, model->metadata.solver_map); } + + return cached_result; } } // namespace immed_mode diff --git a/projects/miopen/src/conv/invokers/mlir_impl_gemm.cpp b/projects/miopen/src/conv/invokers/mlir_impl_gemm.cpp index 98bcfc4dc4e3..9d6a6f92e5d6 100644 --- a/projects/miopen/src/conv/invokers/mlir_impl_gemm.cpp +++ b/projects/miopen/src/conv/invokers/mlir_impl_gemm.cpp @@ -34,8 +34,6 @@ #include -#include - #if MIIR_VERSION_FLAT >= 6 #define MIIR_BARE_POINTER_ABI 1 #else diff --git a/projects/miopen/src/conv/problem_description.cpp b/projects/miopen/src/conv/problem_description.cpp index c94f2a7ab94a..5d005c418a3a 100644 --- a/projects/miopen/src/conv/problem_description.cpp +++ b/projects/miopen/src/conv/problem_description.cpp @@ -135,6 +135,9 @@ void ProblemDescription::HeuristicUpdateLayouts() // If we have preset layouts that are valid, and they are consistent with each other, then we do // not need to change them. + // Note: For transposed solvers that modify strides (e.g., NHWC→NCHW), the cached layout string + // may not be updated here. This is acceptable for degenerate dimensions (N=1, C=1, etc.) where + // strides satisfy multiple layouts, as solvers use actual strides rather than layout strings. if(!in_layout.empty() && in_layout == out_layout && in_layout == weights_layout && std::find(supported_layouts.begin(), supported_layouts.end(), in_layout) != supported_layouts.end() && @@ -150,9 +153,13 @@ void ProblemDescription::HeuristicUpdateLayouts() { for(const std::string& layout : supported_layouts) { - if(in.IsPossibleLayout4D5D(layout, mode) && out.IsPossibleLayout4D5D(layout, mode) && - weights.IsPossibleLayout4D5D(layout, mode)) + bool in_ok = in.IsPossibleLayout4D5D(layout, mode); + bool out_ok = out.IsPossibleLayout4D5D(layout, mode); + bool wei_ok = weights.IsPossibleLayout4D5D(layout, mode); + + if(in_ok && out_ok && wei_ok) { + // Update the cached layout strings to match the detected layout in_layout = layout; weights_layout = layout; out_layout = layout; diff --git a/projects/miopen/src/conv/solver_finders.cpp b/projects/miopen/src/conv/solver_finders.cpp index 1789656388c8..240cbcd41914 100644 --- a/projects/miopen/src/conv/solver_finders.cpp +++ b/projects/miopen/src/conv/solver_finders.cpp @@ -227,15 +227,16 @@ std::vector EvaluateInvokers(const Handle& handle, FindCoreResult& core_result, bool force_attach_binary) { + std::vector ret; + const auto arch = env::value(MIOPEN_DEVICE_ARCH); if(!arch.empty()) - return {}; + return ret; bool using_search_cutoff = env::value(MIOPEN_SEARCH_CUTOFF); auto selected = miopen::solver::ConvSolution{miopenStatusUnknownError}; auto best = std::numeric_limits::max(); auto best_invoker = Invoker{}; - auto ret = std::vector{}; std::vector samples; for(const auto& sol : solutions) @@ -355,7 +356,10 @@ std::vector EvaluateInvokers(const Handle& handle, } if(!selected.Succeeded()) - return {}; + { + ret.clear(); + return ret; + } handle.RegisterInvoker(best_invoker, network_config, selected.solver_id, algorithm_name); MIOPEN_LOG_I("Selected: " << selected << ": " << best diff --git a/projects/miopen/src/convolution.cpp b/projects/miopen/src/convolution.cpp index 15714b4c0de7..53b8b5fad639 100644 --- a/projects/miopen/src/convolution.cpp +++ b/projects/miopen/src/convolution.cpp @@ -26,8 +26,7 @@ #include #include #include - -#include +#include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_DIRECT) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM) @@ -257,9 +256,14 @@ ConvolutionDescriptor::GetForwardOutputTensorWithLayout(const TensorDescriptor& MIOPEN_THROW(miopenStatusBadParm, "Channels do not match for the filter"); } - if(miopen::any_of(boost::combine(GetTransposeConvPads(), GetConvStrides()), [](auto v) { - auto trans_conv_pad = boost::get<0>(v); - auto stride = boost::get<1>(v); + const auto& pads_ = GetTransposeConvPads(); + const auto& strides_ = GetConvStrides(); + auto zip_ = std::views::iota(std::size_t(0), pads_.size()) | + std::views::transform( + [&](std::size_t i) { return std::make_tuple(pads_[i], strides_[i]); }); + + if(std::ranges::any_of(zip_, [](auto v) { + auto [trans_conv_pad, stride] = v; return trans_conv_pad >= stride; })) { @@ -376,7 +380,8 @@ bool ConvolutionDescriptor::IsWinograd3x3SupportedAndFast( if(!(problem.GetOutChannels() >= 16 && problem.GetOutChannels() % 2 == 0)) return false; - return solver::conv::ConvBinWinograd3x3U{}.IsApplicable(ctx, problem); + return solver::conv::ConvBinWinograd3x3U{}.IsApplicable(ctx, problem) || + solver::conv::TransposedConvBinWinograd3x3U{}.IsApplicable(ctx, problem); } std::size_t ConvolutionDescriptor::GetWorkSpaceSize(ExecutionContext ctx, diff --git a/projects/miopen/src/ctc.cpp b/projects/miopen/src/ctc.cpp index cf268db4a0f1..1b997d2775b2 100644 --- a/projects/miopen/src/ctc.cpp +++ b/projects/miopen/src/ctc.cpp @@ -90,31 +90,31 @@ void CTCLossDescriptor::CTCLoss(const Handle& handle, int beta_offset = alpha_offset + max_time_step * batch_size * max_S_len; int batch_bytes = 4 * batch_size; // batch size multiples sizeof(int) - hipMemcpyWithStream(static_cast(workSpace), - inputLengths, - batch_bytes, - hipMemcpyHostToDevice, - handle.GetStream()); - hipMemcpyWithStream(static_cast(workSpace) + batch_size, - labelLengths, - batch_bytes, - hipMemcpyHostToDevice, - handle.GetStream()); - hipMemcpyWithStream(static_cast(workSpace) + 2 * static_cast(batch_size), - labels_offset.data(), - batch_bytes, - hipMemcpyHostToDevice, - handle.GetStream()); - hipMemcpyWithStream(static_cast(workSpace) + 3 * static_cast(batch_size), - repeat.data(), - batch_bytes, - hipMemcpyHostToDevice, - handle.GetStream()); - hipMemcpyWithStream(static_cast(workSpace) + 4 * static_cast(batch_size), - labels, - total_label_len * sizeof(int), - hipMemcpyHostToDevice, - handle.GetStream()); + (void)hipMemcpyWithStream(static_cast(workSpace), + inputLengths, + batch_bytes, + hipMemcpyHostToDevice, + handle.GetStream()); + (void)hipMemcpyWithStream(static_cast(workSpace) + batch_size, + labelLengths, + batch_bytes, + hipMemcpyHostToDevice, + handle.GetStream()); + (void)hipMemcpyWithStream(static_cast(workSpace) + 2 * static_cast(batch_size), + labels_offset.data(), + batch_bytes, + hipMemcpyHostToDevice, + handle.GetStream()); + (void)hipMemcpyWithStream(static_cast(workSpace) + 3 * static_cast(batch_size), + repeat.data(), + batch_bytes, + hipMemcpyHostToDevice, + handle.GetStream()); + (void)hipMemcpyWithStream(static_cast(workSpace) + 4 * static_cast(batch_size), + labels, + total_label_len * sizeof(int), + hipMemcpyHostToDevice, + handle.GetStream()); std::string network_config = "t" + std::to_string(max_time_step) + "n" + std::to_string(batch_size) + "a" + diff --git a/projects/miopen/src/db_path.cpp.in b/projects/miopen/src/db_path.cpp.in index 35772e3d6c27..e4cd30c967c1 100644 --- a/projects/miopen/src/db_path.cpp.in +++ b/projects/miopen/src/db_path.cpp.in @@ -91,7 +91,10 @@ fs::path PrepareUserDbPath() /// the library to use exactly that path. const auto p = env::value(MIOPEN_USER_DB_PATH); if(!p.empty()) - return ExpandUser(p); + { + auto result = ExpandUser(p); + return result; + } /// \anchor nfs-detection /// Otherwise, check if the user-db-path denotes a network filesystem. If this is the case, then /// let's assume that the library is running on a cluster system, where home directory is shared @@ -100,14 +103,14 @@ fs::path PrepareUserDbPath() /// WARNING: This makes all the library's user databases non-persistent. However, for now, we /// assume that the performance hit associated with the usage of shared home dir is worse than /// the deletion of user databases. - const auto udb_path = ExpandUser("${MIOPEN_USER_DB_PATH}"); + auto result = ExpandUser("${MIOPEN_USER_DB_PATH}"); #if !MIOPEN_BUILD_DEV - if(GetFilesystemChecker().IsNetworkedFilesystem(udb_path)) + if(GetFilesystemChecker().IsNetworkedFilesystem(result)) { - return fs::temp_directory_path() / ".config" / "miopen"; + result = fs::temp_directory_path() / ".config" / "miopen"; } #endif - return udb_path; + return result; } } // namespace diff --git a/projects/miopen/src/db_record.cpp b/projects/miopen/src/db_record.cpp index 6ba1efd5827d..0b70b367ec32 100644 --- a/projects/miopen/src/db_record.cpp +++ b/projects/miopen/src/db_record.cpp @@ -1,32 +1,12 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include +#include // std::isspace() #include #include #include +#include #include #include #include @@ -139,7 +119,21 @@ bool DbRecord::ParseContents(std::istream& contents) auto id = id_and_values.substr(0, id_size); auto values = id_and_values.substr(id_size + 1); - boost::trim(values); + + const auto ltrim = [](std::string& str) { + auto e = std::ranges::find_if_not(str, [](char c) { return std::isspace(c); }); + str.erase(str.begin(), e); + return str; + }; + const auto rtrim = [](std::string& str) { + auto e = std::ranges::find_if_not(str | std::views::reverse, + [](char c) { return std::isspace(c); }); + str.erase(e.base(), str.end()); + return str; + }; + + ltrim(values); + rtrim(values); #if WORKAROUND_ISSUE_1987 // Detect legacy find-db item (v.1.0 ID:VALUES) and transform it to the current format. diff --git a/projects/miopen/src/env_debug.cpp b/projects/miopen/src/env_debug.cpp index b919a8ee7762..1807a32862fe 100644 --- a/projects/miopen/src/env_debug.cpp +++ b/projects/miopen/src/env_debug.cpp @@ -47,6 +47,7 @@ MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X4) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X5) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X6) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3) MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DEBUG_CHECK_SUB_BUFFER_OOB_MEMORY_ACCESS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_CK_IGEMM_FWD_V6R1_DLOPS_NCHW) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_DIRECT) @@ -69,6 +70,7 @@ MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_FIND_MODE_FUSION) MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_USER_DB_PATH) MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_CUSTOM_CACHE_DIR) MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_LOG_LEVEL) +MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_LOG_BUFFER_SIZE, 0); MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DEBUG_CONV_DIRECT_ASM_WRW1X1_PERF_VALS) namespace miopen { @@ -184,6 +186,7 @@ const LibEnvVar& FindEnvVariable(std::string_view name) {MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X5.GetName(), MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X5}, {MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X6.GetName(), MIOPEN_DEBUG_AMD_WINOGRAD_MPASS_F3X6}, {MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1.GetName(), MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1}, + {MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3.GetName(), MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3}, {MIOPEN_DEBUG_CHECK_SUB_BUFFER_OOB_MEMORY_ACCESS.GetName(), MIOPEN_DEBUG_CHECK_SUB_BUFFER_OOB_MEMORY_ACCESS}, {MIOPEN_DEBUG_CONV_CK_IGEMM_FWD_V6R1_DLOPS_NCHW.GetName(), MIOPEN_DEBUG_CONV_CK_IGEMM_FWD_V6R1_DLOPS_NCHW}, {MIOPEN_DEBUG_CONV_DIRECT.GetName(), MIOPEN_DEBUG_CONV_DIRECT}, @@ -206,6 +209,7 @@ const LibEnvVar& FindEnvVariable(std::string_view name) {MIOPEN_FIND_MODE.GetName(), MIOPEN_FIND_MODE}, {MIOPEN_FIND_MODE_FUSION.GetName(), MIOPEN_FIND_MODE_FUSION}, {MIOPEN_LOG_LEVEL.GetName(), MIOPEN_LOG_LEVEL}, + {MIOPEN_LOG_BUFFER_SIZE.GetName(), MIOPEN_LOG_BUFFER_SIZE}, {MIOPEN_USER_DB_PATH.GetName(), MIOPEN_USER_DB_PATH}, {MIOPEN_CUSTOM_CACHE_DIR.GetName(), MIOPEN_CUSTOM_CACHE_DIR}, {MIOPEN_DEBUG_CONV_DIRECT_ASM_WRW1X1_PERF_VALS.GetName(), MIOPEN_DEBUG_CONV_DIRECT_ASM_WRW1X1_PERF_VALS} diff --git a/projects/miopen/src/execution_context.cpp b/projects/miopen/src/execution_context.cpp index 5a48c043a8ae..b82f81395c29 100644 --- a/projects/miopen/src/execution_context.cpp +++ b/projects/miopen/src/execution_context.cpp @@ -136,12 +136,12 @@ static bool CalculateIsAmdRocmOpencl(const miopen::ExecutionContext& context) static rocm_meta_version AmdRocmMetadataVersionGetEnv() { - const rocm_meta_version val{env::value(MIOPEN_DEBUG_AMD_ROCM_METADATA_ENFORCE)}; + rocm_meta_version val{env::value(MIOPEN_DEBUG_AMD_ROCM_METADATA_ENFORCE)}; if(!val.IsValid()) { MIOPEN_LOG_W("Incorrect MIOPEN_DEBUG_AMD_ROCM_ENFORCE_MDVERSION = " << val.getValue() << ", using default."); - return rocm_meta_version::Unknown; + val = rocm_meta_version::Unknown; } return val; } diff --git a/projects/miopen/src/expanduser.cpp b/projects/miopen/src/expanduser.cpp index dabae941e098..7cae646cd924 100644 --- a/projects/miopen/src/expanduser.cpp +++ b/projects/miopen/src/expanduser.cpp @@ -31,7 +31,6 @@ #include #ifdef _WIN32 #include -#include #endif #ifdef __linux__ @@ -53,7 +52,7 @@ namespace miopen { namespace { std::string GetHomeDir() { - const auto p = env::value(HOME); + auto p = env::value(HOME); if(!(p.empty() || p == std::string("/"))) { return p; @@ -62,7 +61,8 @@ std::string GetHomeDir() // need to figure out what is the correct thing to do here // in tensoflow unit tests run via bazel, $HOME is not set, so this can happen // setting home_dir to the /tmp for now - return fs::temp_directory_path().string(); + p = fs::temp_directory_path().string(); + return p; } } // namespace diff --git a/projects/miopen/src/find_db.cpp b/projects/miopen/src/find_db.cpp index 997a6e149735..ed4a98f8ba6c 100644 --- a/projects/miopen/src/find_db.cpp +++ b/projects/miopen/src/find_db.cpp @@ -127,76 +127,77 @@ template fs::path FindDbRecord_t::GetInstalledPathFile(const Handle& handle, const std::string& path_suffix) { - static const auto installed_path = [&] { + static const fs::path installed_path = [&] { const std::string ext = ".fdb.txt"; const auto root_path = GetSystemDbPath(); const auto base_name = handle.GetDbBasename(); const auto suffix = GetSystemFindDbSuffix() + (path_suffix.empty() ? "" : ('.' + path_suffix)); - const auto file_path = root_path / (base_name + "." + suffix + ext); + auto file_path = root_path / (base_name + "." + suffix + ext); if(fs::exists(file_path)) { MIOPEN_LOG_I2("Found exact find database file: " << file_path); return file_path; } - else + + MIOPEN_LOG_I2("inexact find database search"); + if(!fs::is_directory(root_path)) { - MIOPEN_LOG_I2("inexact find database search"); - if(fs::is_directory(root_path)) - { - MIOPEN_LOG_I2("Iterating over find db directory " << root_path); - std::vector all_files; - std::vector contents; - std::copy(fs::directory_iterator(root_path), - fs::directory_iterator(), - std::back_inserter(contents)); - for(auto const& filepath : contents) - { - if(fs::is_regular_file(filepath) && - filepath.extension() == path_suffix + ".fdb.txt") - all_files.push_back(filepath); - } + MIOPEN_LOG_I("Database directory does not exist"); + file_path = fs::path{}; + return file_path; + } - const auto db_id = handle.GetTargetProperties().DbId(); - const int real_cu_count = handle.GetMaxComputeUnits(); - int closest_cu = std::numeric_limits::max(); - fs::path best_path; - for(const auto& entry : all_files) - { - const auto fname = entry.stem().string(); - MIOPEN_LOG_I("Checking find db file: " << fname); - // Check for alternate back end same ASIC - if(fname.starts_with(base_name)) - { - return entry; - } - if(db_id.empty() || !miopen::StartsWith(db_id, "gfx") || real_cu_count == 0) - return fs::path{}; - // Check for alternate ASIC any back end - if(fname.starts_with(db_id)) - { - const auto pos = fname.find('_'); - int cur_count = -1; - if(pos != std::string::npos) - cur_count = std::stoi(fname.substr(pos + 1)); - else - cur_count = std::stoi(fname.substr(db_id.length()), nullptr, 16); - if(abs(cur_count - real_cu_count) < (closest_cu)) - { - best_path = entry; - closest_cu = abs(cur_count - real_cu_count); - } - } - } - return best_path; + MIOPEN_LOG_I2("Iterating over find db directory " << root_path); + std::vector all_files; + std::vector contents; + std::copy(fs::directory_iterator(root_path), + fs::directory_iterator(), + std::back_inserter(contents)); + for(auto const& filepath : contents) + { + if(fs::is_regular_file(filepath) && filepath.extension() == path_suffix + ".fdb.txt") + all_files.push_back(filepath); + } + + const auto db_id = handle.GetTargetProperties().DbId(); + const int real_cu_count = handle.GetMaxComputeUnits(); + int closest_cu = std::numeric_limits::max(); + + for(const auto& entry : all_files) + { + const auto fname = entry.stem().string(); + MIOPEN_LOG_I("Checking find db file: " << fname); + // Check for alternate back end same ASIC + if(fname.starts_with(base_name)) + { + file_path = entry; + return file_path; + } + if(db_id.empty() || !miopen::StartsWith(db_id, "gfx") || real_cu_count == 0) + { + file_path = fs::path{}; + return file_path; // empty } - else + // Check for alternate ASIC any back end + if(fname.starts_with(db_id)) { - MIOPEN_LOG_I("Database directory does not exist"); - return fs::path{}; + const auto pos = fname.find('_'); + int cur_count = -1; + if(pos != std::string::npos) + cur_count = std::stoi(fname.substr(pos + 1)); + else + cur_count = std::stoi(fname.substr(db_id.length()), nullptr, 16); + if(abs(cur_count - real_cu_count) < (closest_cu)) + { + file_path = entry; + closest_cu = abs(cur_count - real_cu_count); + } } } + return file_path; }(); + return installed_path; } #endif diff --git a/projects/miopen/src/fusion.cpp b/projects/miopen/src/fusion.cpp index 2270210ae470..b2e7e6863232 100644 --- a/projects/miopen/src/fusion.cpp +++ b/projects/miopen/src/fusion.cpp @@ -765,8 +765,7 @@ static auto GetFusedWinogradSolvers() { return solver::SolverContainer, - solver::fusion::ConvWinoRageRxSFused<2, 3>>{}; + solver::fusion::ConvWinoFuryRxSFused<2, 3>>{}; } static auto GetAllFusionSolvers() diff --git a/projects/miopen/src/gemm_v2.cpp b/projects/miopen/src/gemm_v2.cpp index 0d4dfec912b5..006f26714271 100644 --- a/projects/miopen/src/gemm_v2.cpp +++ b/projects/miopen/src/gemm_v2.cpp @@ -138,7 +138,6 @@ rocblas_status miopen_rocblas_gemm_ex3(const miopen::Handle& handle, #if USE_ROCBLAS_GEMM_EX3 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdisabled-macro-expansion" -#pragma clang diagnostic ignored "-Wdeprecated-declarations" float alpha = gemm_desc.alpha; float beta = gemm_desc.beta; auto flags = FlagsForRocblasFp32Fp16Call(gemm_desc); @@ -301,15 +300,15 @@ inline void ProfilingRecordStart(const Handle& handle, HipEventPtr& start, HipEv { start = make_hip_event(); stop = make_hip_event(); - hipEventRecord(start.get(), handle.GetStream()); + (void)hipEventRecord(start.get(), handle.GetStream()); } inline void ProfilingRecordStop(const Handle& handle, HipEventPtr& start, HipEventPtr& stop) { - hipEventRecord(stop.get(), handle.GetStream()); - hipEventSynchronize(stop.get()); + (void)hipEventRecord(stop.get(), handle.GetStream()); + (void)hipEventSynchronize(stop.get()); float mS = 0; - hipEventElapsedTime(&mS, start.get(), stop.get()); + (void)hipEventElapsedTime(&mS, start.get(), stop.get()); handle.ResetKernelTime(); handle.AccumKernelTime(mS); } diff --git a/projects/miopen/src/hip/batched_transpose_sol.cpp b/projects/miopen/src/hip/batched_transpose_sol.cpp index 4c0ee0b29910..3419a61b9ea9 100644 --- a/projects/miopen/src/hip/batched_transpose_sol.cpp +++ b/projects/miopen/src/hip/batched_transpose_sol.cpp @@ -243,19 +243,20 @@ static inline BatchedTransposeParam HeuristicGet(const ExecutionContext& ctx, // Early heuristic for cases that has very large width, very small height (or vice versa) if(hw_radio <= 48) { - return (width <= 8) ? BatchedTransposeParam{4, 64, 1, 1, 1, 1} - : BatchedTransposeParam{64, 4, 1, 1, 1, 1}; + best_kernel = (width <= 8) ? BatchedTransposeParam{4, 64, 1, 1, 1, 1} + : BatchedTransposeParam{64, 4, 1, 1, 1, 1}; } else if(hw_radio <= 128) { - return (width <= 8) ? BatchedTransposeParam{4, 128, 1, 1, 1, 1} - : BatchedTransposeParam{128, 4, 1, 1, 1, 1}; + best_kernel = (width <= 8) ? BatchedTransposeParam{4, 128, 1, 1, 1, 1} + : BatchedTransposeParam{128, 4, 1, 1, 1, 1}; } else { - return (width <= 8) ? BatchedTransposeParam{4, 256, 1, 1, 1, 1} - : BatchedTransposeParam{256, 4, 1, 1, 1, 1}; + best_kernel = (width <= 8) ? BatchedTransposeParam{4, 256, 1, 1, 1, 1} + : BatchedTransposeParam{256, 4, 1, 1, 1, 1}; } + return best_kernel; } for(const auto& it : std::ranges::reverse_view(kernel_list)) diff --git a/projects/miopen/src/hip/general_tensor_reorder_sol.cpp b/projects/miopen/src/hip/general_tensor_reorder_sol.cpp index 291326431d08..f208debba538 100644 --- a/projects/miopen/src/hip/general_tensor_reorder_sol.cpp +++ b/projects/miopen/src/hip/general_tensor_reorder_sol.cpp @@ -90,29 +90,26 @@ HeuristicGet(std::size_t data_size, uint32_t dim_0, uint32_t dim_1, uint32_t dim { if(dim_3 >= 16) { - return GeneralReorderParam{16, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; + default_kernel = GeneralReorderParam{16, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; } else if(dim_3 >= 8) { - return GeneralReorderParam{8, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; + default_kernel = GeneralReorderParam{8, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; } else if(dim_3 >= 4) { - return GeneralReorderParam{4, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; + default_kernel = GeneralReorderParam{4, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; } else if(dim_3 >= 2) { - return GeneralReorderParam{2, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; + default_kernel = GeneralReorderParam{2, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; } else { - return GeneralReorderParam{1, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; + default_kernel = GeneralReorderParam{1, TENSOR_REORDER_BLOCK_SIZE, 1, 1, 1, 1}; } } - else - { - return default_kernel; - } + return default_kernel; } } // namespace tensor_reorder diff --git a/projects/miopen/src/hip/handlehip.cpp b/projects/miopen/src/hip/handlehip.cpp index de02513e8ed5..dfb668935da0 100644 --- a/projects/miopen/src/hip/handlehip.cpp +++ b/projects/miopen/src/hip/handlehip.cpp @@ -122,6 +122,8 @@ void default_deallocator(void*, void* mem) } // namespace +// MIOPEN_INTERNALS_EXPORT set here because the function isn't present in headers +// but called from test/gtest/handle_hip_device.cpp MIOPEN_INTERNALS_EXPORT int get_device_id() // Get random device { int device; @@ -131,6 +133,8 @@ MIOPEN_INTERNALS_EXPORT int get_device_id() // Get random device return device; } +// MIOPEN_INTERNALS_EXPORT set here because the function isn't present in headers +// but called from test/gtest/handle_hip_device.cpp MIOPEN_INTERNALS_EXPORT void set_device(int id) { auto status = hipSetDevice(id); @@ -160,7 +164,7 @@ struct HandleImpl // typedef MIOPEN_MANAGE_PTR(hipStream_t, hipStreamDestroy) StreamPtr; using StreamPtr = std::shared_ptr::type>; - HandleImpl() { hipInit(0); } + HandleImpl() { (void)hipInit(0); } StreamPtr create_stream() { @@ -185,7 +189,7 @@ struct HandleImpl void elapsed_time(hipEvent_t start, hipEvent_t stop) { if(enable_profiling) - hipEventElapsedTime(&this->profiling_result, start, stop); + (void)hipEventElapsedTime(&this->profiling_result, start, stop); } std::function elapsed_time_handler() @@ -199,7 +203,7 @@ struct HandleImpl std::string get_device_name() const { hipDeviceProp_t props{}; - hipGetDeviceProperties(&props, device); + (void)hipGetDeviceProperties(&props, device); const std::string name(props.gcnArchName); MIOPEN_LOG_NQI("Raw device name: " << name); return name; // NOLINT (performance-no-automatic-move) @@ -404,14 +408,18 @@ Allocator::ManageDataPtr Handle::Create(std::size_t sz) const Allocator::ManageDataPtr& Handle::WriteTo(const void* data, Allocator::ManageDataPtr& ddata, std::size_t sz) const +{ + WriteTo(data, ddata.get(), sz); + return ddata; +} + +void Handle::WriteTo(const void* data, Data_t ddata, std::size_t sz) const { MIOPEN_HANDLE_LOCK this->Finish(); - auto status = - hipMemcpyWithStream(ddata.get(), data, sz, hipMemcpyHostToDevice, this->GetStream()); + auto status = hipMemcpyWithStream(ddata, data, sz, hipMemcpyHostToDevice, this->GetStream()); if(status != hipSuccess) MIOPEN_THROW_HIP_STATUS(status, "Hip error writing to buffer: "); - return ddata; } void Handle::ReadTo(void* data, const Allocator::ManageDataPtr& ddata, std::size_t sz) const @@ -663,9 +671,11 @@ void Handle::Finish() const } #else // hipStreamSynchronize is broken, so we use hipEventSynchronize instead - auto ev = make_hip_event(); - hipEventRecord(ev.get(), this->GetStream()); - auto status = hipEventSynchronize(ev.get()); + auto ev = make_hip_event(); + auto status = hipEventRecord(ev.get(), this->GetStream()); + if(status != hipSuccess) + MIOPEN_THROW_HIP_STATUS(status, "Failed hip event recording"); + status = hipEventSynchronize(ev.get()); if(status != hipSuccess) MIOPEN_THROW_HIP_STATUS(status, "Failed hip sychronization"); #endif @@ -727,7 +737,7 @@ std::size_t Handle::GetImage3dMaxWidth() const std::size_t Handle::GetWavefrontWidth() const { hipDeviceProp_t props{}; - hipGetDeviceProperties(&props, this->impl->device); + (void)hipGetDeviceProperties(&props, this->impl->device); auto result = static_cast(props.warpSize); return result; } diff --git a/projects/miopen/src/hipoc/hipoc_kernel.cpp b/projects/miopen/src/hipoc/hipoc_kernel.cpp index 47c1754db87e..7af1b50686be 100644 --- a/projects/miopen/src/hipoc/hipoc_kernel.cpp +++ b/projects/miopen/src/hipoc/hipoc_kernel.cpp @@ -47,7 +47,7 @@ HipEventProfiler::HipEventProfiler(const Handle& handle_) { start = make_hip_event(); stop = make_hip_event(); - hipEventRecord(start.get(), handle.GetStream()); + (void)hipEventRecord(start.get(), handle.GetStream()); } } @@ -55,10 +55,10 @@ HipEventProfiler::~HipEventProfiler() { if(start) { - hipEventRecord(stop.get(), handle.GetStream()); - hipEventSynchronize(stop.get()); + (void)hipEventRecord(stop.get(), handle.GetStream()); + (void)hipEventSynchronize(stop.get()); float event_time = 0.0f; - hipEventElapsedTime(&event_time, start.get(), stop.get()); + (void)hipEventElapsedTime(&event_time, start.get(), stop.get()); handle.ResetKernelTime(); handle.AccumKernelTime(event_time); } @@ -141,7 +141,7 @@ void HIPOCKernelInvoke::run(void* args, std::size_t size) const } } #else - hipEventSynchronize(stop.get()); + (void)hipEventSynchronize(stop.get()); #endif callback(start.get(), stop.get()); } @@ -215,7 +215,9 @@ void HIPOCKernelInvoke::run_cooperative(void** kern_args) const if(callback) { - hipEventSynchronize(stop.get()); + status = hipEventSynchronize(stop.get()); + if(status != hipSuccess) + MIOPEN_THROW_HIP_STATUS(status, "hipEventSynchronize() failed"); callback(start.get(), stop.get()); } } diff --git a/projects/miopen/src/hipoc/hipoc_program.cpp b/projects/miopen/src/hipoc/hipoc_program.cpp index 067c2e8882cb..84c5042b4c93 100644 --- a/projects/miopen/src/hipoc/hipoc_program.cpp +++ b/projects/miopen/src/hipoc/hipoc_program.cpp @@ -175,22 +175,22 @@ HIPOCProgramImpl::HIPOCProgramImpl(const fs::path& program_name, const fs::path& } HIPOCProgramImpl::HIPOCProgramImpl(const fs::path& program_name, const std::vector& blob) - : program(program_name) ///, module(CreateModuleInMem(blob)) + : program(program_name), binary(blob) // Store the binary data to prevent use-after-free { const auto& arch = env::value(MIOPEN_DEVICE_ARCH); if(!arch.empty()) return; - module = CreateModuleInMem(blob); + module = CreateModuleInMem(binary); // Use stored binary instead of parameter } -HIPOCProgramImpl::HIPOCProgramImpl(const fs::path& program_name, - const std::vector& blob) - : program(program_name) ///, module(CreateModuleInMem(blob)) +HIPOCProgramImpl::HIPOCProgramImpl(const fs::path& program_name, const std::vector& blob) + : program(program_name), + binary(blob.begin(), blob.end()) // Store the binary data to prevent use-after-free { const auto& arch = env::value(MIOPEN_DEVICE_ARCH); if(!arch.empty()) return; - module = CreateModuleInMem(blob); + module = CreateModuleInMem(binary); // Use stored binary instead of parameter } HIPOCProgramImpl::HIPOCProgramImpl(const fs::path& program_name, diff --git a/projects/miopen/src/include/miopen/activ.hpp b/projects/miopen/src/include/miopen/activ.hpp index b7db13b47827..dca41ff18807 100644 --- a/projects/miopen/src/include/miopen/activ.hpp +++ b/projects/miopen/src/include/miopen/activ.hpp @@ -39,42 +39,43 @@ namespace miopen { struct Handle; struct TensorDescriptor; -struct MIOPEN_INTERNALS_EXPORT ActivationDescriptor : miopenActivationDescriptor +struct ActivationDescriptor : miopenActivationDescriptor { - ActivationDescriptor(); + MIOPEN_INTERNALS_EXPORT ActivationDescriptor(); ActivationDescriptor(miopenActivationMode_t m, const double* pparms); + MIOPEN_INTERNALS_EXPORT ActivationDescriptor(miopenActivationMode_t m, double alpha, double beta, double gamma); - miopenActivationMode_t GetMode() const; - double GetAlpha() const; - double GetBeta() const; - double GetGamma() const; + MIOPEN_INTERNALS_EXPORT miopenActivationMode_t GetMode() const; + MIOPEN_INTERNALS_EXPORT double GetAlpha() const; + MIOPEN_INTERNALS_EXPORT double GetBeta() const; + MIOPEN_INTERNALS_EXPORT double GetGamma() const; - miopenStatus_t Forward(const Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - size_t xOffset = 0, - size_t yOffset = 0) const; + MIOPEN_INTERNALS_EXPORT miopenStatus_t Forward(const Handle& handle, + const void* alpha, + const TensorDescriptor& xDesc, + ConstData_t x, + const void* beta, + const TensorDescriptor& yDesc, + Data_t y, + size_t xOffset = 0, + size_t yOffset = 0) const; - miopenStatus_t Backward(const Handle& handle, - const void* alpha, - const TensorDescriptor& yDesc, - ConstData_t y, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - size_t yOffset = 0, - size_t dyOffset = 0, - size_t xOffset = 0, - size_t dxOffset = 0) const; + MIOPEN_INTERNALS_EXPORT miopenStatus_t Backward(const Handle& handle, + const void* alpha, + const TensorDescriptor& yDesc, + ConstData_t y, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& xDesc, + ConstData_t x, + const void* beta, + const TensorDescriptor& dxDesc, + Data_t dx, + size_t yOffset = 0, + size_t dyOffset = 0, + size_t xOffset = 0, + size_t dxOffset = 0) const; friend std::ostream& operator<<(std::ostream& stream, const ActivationDescriptor& x); diff --git a/projects/miopen/src/include/miopen/activ/problem_description.hpp b/projects/miopen/src/include/miopen/activ/problem_description.hpp index 1b3f5e46e335..823f69263011 100644 --- a/projects/miopen/src/include/miopen/activ/problem_description.hpp +++ b/projects/miopen/src/include/miopen/activ/problem_description.hpp @@ -44,7 +44,7 @@ enum class Direction Backward, }; -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase +struct ProblemDescription : ProblemDescriptionBase { // Forward constructor ProblemDescription(const ActivationDescriptor& activ, diff --git a/projects/miopen/src/include/miopen/adam/problem_description.hpp b/projects/miopen/src/include/miopen/adam/problem_description.hpp index 12599f2735ff..27a9ea86f740 100644 --- a/projects/miopen/src/include/miopen/adam/problem_description.hpp +++ b/projects/miopen/src/include/miopen/adam/problem_description.hpp @@ -37,7 +37,7 @@ struct NetworkConfig; namespace adam { -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase +struct ProblemDescription : ProblemDescriptionBase { ProblemDescription(const TensorDescriptor& paramInDesc_, const TensorDescriptor& paramOutDesc_, diff --git a/projects/miopen/src/include/miopen/anyramdb.hpp b/projects/miopen/src/include/miopen/anyramdb.hpp index 4158a731c34d..a0632750c389 100644 --- a/projects/miopen/src/include/miopen/anyramdb.hpp +++ b/projects/miopen/src/include/miopen/anyramdb.hpp @@ -40,7 +40,7 @@ namespace miopen { class LockFile; -struct MIOPEN_INTERNALS_EXPORT AnyRamDb +struct AnyRamDb { using TRecord = std::vector; diff --git a/projects/miopen/src/include/miopen/batchnorm/problem_description.hpp b/projects/miopen/src/include/miopen/batchnorm/problem_description.hpp index 865209a63936..7230d93709e6 100644 --- a/projects/miopen/src/include/miopen/batchnorm/problem_description.hpp +++ b/projects/miopen/src/include/miopen/batchnorm/problem_description.hpp @@ -445,17 +445,13 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, std::string GetDirectionStr() const { - std::string s; - switch(direction) { - case Direction::ForwardInference: return "Inf"; ; + case Direction::ForwardInference: return "Inf"; case Direction::ForwardTraining: return "Trn"; case Direction::Backward: return "Bwd"; default: MIOPEN_THROW(miopenStatusInvalidValue, "Wrong Batchnorm Direction provided"); } - - return s; } std::string GetModeStr() const diff --git a/projects/miopen/src/include/miopen/batchnorm/solvers.hpp b/projects/miopen/src/include/miopen/batchnorm/solvers.hpp index 07c358efba32..6ca31b0bd1ba 100644 --- a/projects/miopen/src/include/miopen/batchnorm/solvers.hpp +++ b/projects/miopen/src/include/miopen/batchnorm/solvers.hpp @@ -49,19 +49,17 @@ using BatchNormTunableSolver = TunableSolverMixin; ; -struct BnFwdTrainingPerActivation final : BatchnormSolver +struct MIOPEN_INTERNALS_EXPORT BnFwdTrainingPerActivation final : BatchnormSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; }; struct PerformanceConfigBnBwdBackward : PerfConfigBase @@ -69,25 +67,23 @@ struct PerformanceConfigBnBwdBackward : PerfConfigBase valid_kernels; - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnBwdBackward(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) + PerformanceConfigBnBwdBackward(int idx, std::string kernl_id) : index(idx), kernel_id(kernl_id) { } PerformanceConfigBnBwdBackward() : PerformanceConfigBnBwdBackward(0, "") {} PerformanceConfigBnBwdBackward(bool) : PerformanceConfigBnBwdBackward(0, "") {} - MIOPEN_INTERNALS_EXPORT void - HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool - IsValid(const ExecutionContext&, const miopen::batchnorm::ProblemDescription& problem) const; + void HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); + bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, + const miopen::batchnorm::ProblemDescription& problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigBnBwdBackward& other) const; + bool operator==(const PerformanceConfigBnBwdBackward& other) const; }; struct PerformanceConfigBnFwdTraining : PerfConfigBase @@ -95,107 +91,95 @@ struct PerformanceConfigBnFwdTraining : PerfConfigBase valid_kernels; - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnFwdTraining(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) + PerformanceConfigBnFwdTraining(int idx, std::string kernl_id) : index(idx), kernel_id(kernl_id) { } PerformanceConfigBnFwdTraining() : PerformanceConfigBnFwdTraining(0, "") {} PerformanceConfigBnFwdTraining(bool) : PerformanceConfigBnFwdTraining(0, "") {} - MIOPEN_INTERNALS_EXPORT void - HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool - IsValid(const ExecutionContext&, const miopen::batchnorm::ProblemDescription& problem) const; + void HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); + bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, + const miopen::batchnorm::ProblemDescription& problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigBnFwdTraining& other) const; + bool operator==(const PerformanceConfigBnFwdTraining& other) const; }; -struct BnBwdTrainingSpatial final : BatchNormTunableSolver +struct MIOPEN_INTERNALS_EXPORT BnBwdTrainingSpatial final + : BatchNormTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnBwdBackward GetDefaultPerformanceConfig( + PerformanceConfigBnBwdBackward GetDefaultPerformanceConfig( const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext& ctx, + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnBwdBackward& config) const override; + + PerformanceConfigBnBwdBackward Search(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const AnyInvokeParams& invoke_ctx) const override; + + bool IsApplicable(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem, const PerformanceConfigBnBwdBackward& config) const override; - - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnBwdBackward - Search(const ExecutionContext& ctx, - const miopen::batchnorm::ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx) const override; - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem, - const PerformanceConfigBnBwdBackward& config) const override; }; -struct BnFwdTrainingSpatial final : BatchNormTunableSolver +struct MIOPEN_INTERNALS_EXPORT BnFwdTrainingSpatial final + : BatchNormTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnFwdTraining GetDefaultPerformanceConfig( + PerformanceConfigBnFwdTraining GetDefaultPerformanceConfig( const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext& ctx, + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnFwdTraining& config) const override; + + PerformanceConfigBnFwdTraining Search(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const AnyInvokeParams& invoke_ctx) const override; + + bool IsApplicable(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem, const PerformanceConfigBnFwdTraining& config) const override; - - MIOPEN_INTERNALS_EXPORT PerformanceConfigBnFwdTraining - Search(const ExecutionContext& ctx, - const miopen::batchnorm::ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx) const override; - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem, - const PerformanceConfigBnFwdTraining& config) const override; }; -struct BnBwdTrainingPerActivation final : BatchnormSolver +struct MIOPEN_INTERNALS_EXPORT BnBwdTrainingPerActivation final : BatchnormSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; }; -struct BnFwdInference final : BatchnormSolver +struct MIOPEN_INTERNALS_EXPORT BnFwdInference final : BatchnormSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsDynamic() const override { return true; } + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::batchnorm::ProblemDescription& problem) const override; }; } // namespace batchnorm diff --git a/projects/miopen/src/include/miopen/bfloat16.hpp b/projects/miopen/src/include/miopen/bfloat16.hpp index 86c9d734a5ae..3e3a184a72d1 100644 --- a/projects/miopen/src/include/miopen/bfloat16.hpp +++ b/projects/miopen/src/include/miopen/bfloat16.hpp @@ -1,35 +1,13 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + #ifndef BFLOAT16_H_ #define BFLOAT16_H_ -#include + #include #include -class bfloat16 : boost::totally_ordered> +class bfloat16 { public: bfloat16() : data_{0} {} @@ -152,6 +130,30 @@ class bfloat16 : boost::totally_ordered> std::uint16_t data_; }; +inline bfloat16 operator+(bfloat16 a, const bfloat16& b) +{ + a += b; + return a; +} + +inline bfloat16 operator-(bfloat16 a, const bfloat16& b) +{ + a -= b; + return a; +} + +inline bfloat16 operator*(bfloat16 a, const bfloat16& b) +{ + a *= b; + return a; +} + +inline bfloat16 operator/(bfloat16 a, const bfloat16& b) +{ + a /= b; + return a; +} + namespace std { template <> class numeric_limits diff --git a/projects/miopen/src/include/miopen/binary_cache.hpp b/projects/miopen/src/include/miopen/binary_cache.hpp index 856938368033..53ce80a406dc 100644 --- a/projects/miopen/src/include/miopen/binary_cache.hpp +++ b/projects/miopen/src/include/miopen/binary_cache.hpp @@ -44,20 +44,20 @@ GetCacheFile(const std::string& device, const fs::path& name, const std::string& MIOPEN_INTERNALS_EXPORT fs::path GetCachePath(bool is_system); #if !MIOPEN_ENABLE_SQLITE_KERN_CACHE -fs::path LoadBinary(const TargetProperties& target, - std::size_t num_cu, - const fs::path& name, - const std::string& args); +MIOPEN_INTERNALS_EXPORT fs::path LoadBinary(const TargetProperties& target, + std::size_t num_cu, + const fs::path& name, + const std::string& args); fs::path SaveBinary(const fs::path& binary_path, const TargetProperties& target, const fs::path& name, const std::string& args); #else -std::vector LoadBinary(const TargetProperties& target, - std::size_t num_cu, - const fs::path& name, - const std::string& args); +MIOPEN_INTERNALS_EXPORT std::vector LoadBinary(const TargetProperties& target, + std::size_t num_cu, + const fs::path& name, + const std::string& args); void SaveBinary(const std::vector& hsaco, const TargetProperties& target, diff --git a/projects/miopen/src/include/miopen/bz2.hpp b/projects/miopen/src/include/miopen/bz2.hpp index 546b28efa9c2..cdeb3f2d16bc 100644 --- a/projects/miopen/src/include/miopen/bz2.hpp +++ b/projects/miopen/src/include/miopen/bz2.hpp @@ -31,7 +31,7 @@ #include namespace miopen { -MIOPEN_INTERNALS_EXPORT void check_bz2_error(int e, const std::string& name); +void check_bz2_error(int e, const std::string& name); MIOPEN_INTERNALS_EXPORT std::vector compress(const std::vector& v, bool* compressed = nullptr); MIOPEN_INTERNALS_EXPORT std::vector decompress(const std::vector& v, unsigned int size); diff --git a/projects/miopen/src/include/miopen/check_numerics.hpp b/projects/miopen/src/include/miopen/check_numerics.hpp index c305e2793120..347b7ebd389e 100644 --- a/projects/miopen/src/include/miopen/check_numerics.hpp +++ b/projects/miopen/src/include/miopen/check_numerics.hpp @@ -20,13 +20,12 @@ struct CheckNumerics static const int ComputeStats = 0x10; // Print mean/absmean/min/max (slow) }; -MIOPEN_INTERNALS_EXPORT bool CheckNumericsEnabled(int bitMask = -1); +bool CheckNumericsEnabled(int bitMask = -1); MIOPEN_INTERNALS_EXPORT bool checkNumericsInput(const Handle& handle, const TensorDescriptor& dDesc, ConstData_t data); -MIOPEN_INTERNALS_EXPORT bool -checkNumericsOutput(const Handle& handle, const TensorDescriptor& dDesc, ConstData_t data); -MIOPEN_INTERNALS_EXPORT bool checkNumericsImpl( +bool checkNumericsOutput(const Handle& handle, const TensorDescriptor& dDesc, ConstData_t data); +bool checkNumericsImpl( const Handle& handle, int mode, const TensorDescriptor& dDesc, ConstData_t data, bool isInput); } // namespace miopen diff --git a/projects/miopen/src/include/miopen/config.hpp b/projects/miopen/src/include/miopen/config.hpp index dcf005fbc390..6de46175c3d2 100644 --- a/projects/miopen/src/include/miopen/config.hpp +++ b/projects/miopen/src/include/miopen/config.hpp @@ -28,11 +28,6 @@ #include #include - -#ifdef MIOPEN_BUILD_TESTING #include -#else -#define MIOPEN_INTERNALS_EXPORT -#endif #endif // GUARD_MIOPEN_CONFIG_HPP diff --git a/projects/miopen/src/include/miopen/conv/data_invoke_params.hpp b/projects/miopen/src/include/miopen/conv/data_invoke_params.hpp index 892c84ff38e2..cf7b8146cf72 100644 --- a/projects/miopen/src/include/miopen/conv/data_invoke_params.hpp +++ b/projects/miopen/src/include/miopen/conv/data_invoke_params.hpp @@ -29,7 +29,8 @@ #include #include #include - +#include +#include namespace miopen { namespace conv { @@ -78,5 +79,121 @@ struct DataInvokeParams : InvokeParams Data_t GetWorkspace() const { return workSpace; } }; +struct TransposeConvInvokeParams : InvokeParams +{ + // Flat tensor descriptors (required by TransposingSolver member pointers) + TensorDescriptor inDesc; + TensorDescriptor wDesc; + TensorDescriptor outDesc; + + // Flat data pointers (required by TransposingSolver member pointers) + ConstData_t in = nullptr; + ConstData_t w = nullptr; + Data_t out = nullptr; + + // Workspace + Data_t workspace = nullptr; + std::size_t workspaceSize = 0; + + // Additional conv params + bool gfx90aFp16alt = false; + Scalar alpha{1.0}; + Scalar beta{0.0}; + + // WrW-specific data pointers (used when is_wrw == true) + // For WrW: w (weights slot) holds dw which is the output (Data_t), + // out (output slot) holds x which is an input (ConstData_t) + Data_t w_as_output = nullptr; // dw buffer pointer for WrW output transpose + ConstData_t out_as_input = nullptr; // x buffer pointer for WrW input transpose + bool is_wrw = false; + + TransposeConvInvokeParams() = default; + + // Constructor from DataInvokeParams only (uses tensor descs from params.tensors) + explicit TransposeConvInvokeParams(const DataInvokeParams& params) + : InvokeParams{params.type}, + inDesc(params.tensors.inDesc), + wDesc(params.tensors.wDesc), + outDesc(params.tensors.outDesc), + in(params.tensors.in), + w(params.tensors.w), + out(params.tensors.out), + workspace(params.workSpace), + workspaceSize(params.workSpaceSize), + gfx90aFp16alt(params.gfx90aFp16alt), + alpha(params.alpha), + beta(params.beta) + { + } + + // Constructor from WrWInvokeParams + // Maps: in/inDesc = dy, w/wDesc = dw, out/outDesc = x + // Also sets w_as_output and out_as_input for correct transpose I/O semantics + explicit TransposeConvInvokeParams(const WrWInvokeParams& params) + : InvokeParams{params.type}, + inDesc(params.tensors.dyDesc), + wDesc(params.tensors.dwDesc), + outDesc(params.tensors.xDesc), + in(params.tensors.dy), + w(params.tensors.dw), + out(nullptr), // not used for WrW — x is accessed via out_as_input + workspace(params.workSpace), + workspaceSize(params.workSpaceSize), + gfx90aFp16alt(params.gfx90aFp16alt), + alpha(params.alpha), + beta(params.beta), + w_as_output(params.tensors.dw), + out_as_input(params.tensors.x), + is_wrw(true) + { + } + + // Constructor from existing DataInvokeParams + ProblemDescription + TransposeConvInvokeParams(const DataInvokeParams& params, const ProblemDescription& problem) + : InvokeParams{params.type}, + inDesc(problem.GetIn()), + wDesc(problem.GetWeights()), + outDesc(problem.GetOut()), + in(params.tensors.in), + w(params.tensors.w), + out(params.tensors.out), + workspace(params.workSpace), + workspaceSize(params.workSpaceSize), + gfx90aFp16alt(params.gfx90aFp16alt), + alpha(params.alpha), + beta(params.beta) + { + } + + std::size_t GetWorkspaceSize() const { return workspaceSize; } + Data_t GetWorkspace() const { return workspace; } + + /// Convert to DataInvokeParams for inner solver invocation (Fwd/Bwd). + DataInvokeParams ToDataInvokeParams() const + { + return DataInvokeParams{type, + ConvDataTensors{inDesc, in, wDesc, w, outDesc, out}, + workspace, + workspaceSize, + gfx90aFp16alt, + alpha, + beta}; + } + + /// Convert to WrWInvokeParams for inner solver invocation (WrW). + /// Maps back: inDesc/in → dy, wDesc/w_as_output → dw, outDesc/out_as_input → x + WrWInvokeParams ToWrWInvokeParams() const + { + return WrWInvokeParams{ + type, + ConvWrwTensors{inDesc, in, outDesc, out_as_input, wDesc, w_as_output}, + workspace, + workspaceSize, + gfx90aFp16alt, + alpha, + beta}; + } +}; + } // namespace conv } // namespace miopen diff --git a/projects/miopen/src/include/miopen/conv/db_getter.hpp b/projects/miopen/src/include/miopen/conv/db_getter.hpp index 2590be10ab1b..7d20e6a03406 100644 --- a/projects/miopen/src/include/miopen/conv/db_getter.hpp +++ b/projects/miopen/src/include/miopen/conv/db_getter.hpp @@ -60,6 +60,5 @@ class [[nodiscard]] DbGetter final std::optional db; }; -[[nodiscard]] MIOPEN_INTERNALS_EXPORT auto -MakeConvDbGetter(const ExecutionContext& ctx) -> DbGetter; +[[nodiscard]] auto MakeConvDbGetter(const ExecutionContext& ctx) -> DbGetter; } // namespace miopen diff --git a/projects/miopen/src/include/miopen/conv/heuristics/ai_conv_3d_kernel_tuning_utils.hpp b/projects/miopen/src/include/miopen/conv/heuristics/ai_conv_3d_kernel_tuning_utils.hpp index 2fe478cfca71..1834fb6b1178 100644 --- a/projects/miopen/src/include/miopen/conv/heuristics/ai_conv_3d_kernel_tuning_utils.hpp +++ b/projects/miopen/src/include/miopen/conv/heuristics/ai_conv_3d_kernel_tuning_utils.hpp @@ -43,7 +43,8 @@ namespace solver { namespace conv { const miopen::ExecutionContext& GetDummyCtx(); -MIOPEN_INTERNALS_EXPORT std::map +MIOPEN_INTERNALS_EXPORT +std::map GetFeatures3D(const miopen::conv::ProblemDescription&, int max_cu, const std::string& arch); MIOPEN_INTERNALS_EXPORT std::vector GetKernelAsTokens(const std::string& kernel); @@ -53,7 +54,8 @@ MIOPEN_INTERNALS_EXPORT void FillHeuristicKernels(const std::vector std::vector& indexes, std::vector>& kernels); -MIOPEN_INTERNALS_EXPORT std::vector GenerateSplitK(int max_split_k); +MIOPEN_INTERNALS_EXPORT +std::vector GenerateSplitK(int max_split_k); // Main template implementation with validation function template diff --git a/projects/miopen/src/include/miopen/conv/problem_description.hpp b/projects/miopen/src/include/miopen/conv/problem_description.hpp index 120c1117a194..d6cbcee3b35f 100644 --- a/projects/miopen/src/include/miopen/conv/problem_description.hpp +++ b/projects/miopen/src/include/miopen/conv/problem_description.hpp @@ -104,10 +104,10 @@ namespace conv { MIOPEN_INTERNALS_EXPORT miopenAlphaBetaCase_t ClassifyAlphaBeta(const Scalar& alpha, const Scalar& beta); -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase +struct ProblemDescription : ProblemDescriptionBase #if MIOPEN_ENABLE_SQLITE , - SQLiteSerializable + SQLiteSerializable #endif { ProblemDescription() = default; @@ -222,7 +222,7 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase bool IsDirectionForward() const { return direction == conv::Direction::Forward; } bool IsDirectionBackwardData() const { return direction == conv::Direction::BackwardData; } bool IsDirectionBackwardWrW() const { return direction == conv::Direction::BackwardWeights; } - std::string GetDirectionStr() const; + MIOPEN_INTERNALS_EXPORT std::string GetDirectionStr() const; const Scalar& GetAlpha() const { return alpha; } const Scalar& GetBeta() const { return beta; } @@ -340,9 +340,9 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase out.AllLengthsFitIntoInt(); } - void HeuristicUpdateLayouts(); + MIOPEN_INTERNALS_EXPORT void HeuristicUpdateLayouts(); - void MakeNetworkConfig(std::string& conf_key) const; + MIOPEN_INTERNALS_EXPORT void MakeNetworkConfig(std::string& conf_key) const; NetworkConfig MakeNetworkConfig() const override { @@ -354,7 +354,7 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase // Todo: remove after fixing fin [[deprecated]] NetworkConfig BuildConfKey() const { return MakeNetworkConfig(); } - void Serialize(std::ostream& stream) const; + MIOPEN_INTERNALS_EXPORT void Serialize(std::ostream& stream) const; friend std::ostream& operator<<(std::ostream& os, const ProblemDescription& obj) { @@ -409,14 +409,14 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase [&](std::string value, std::string name) { f(value, name); }); } - void SetupFloats(ExecutionContext& ctx) const; - void SetupComputeType(const ExecutionContext& ctx) const; + MIOPEN_INTERNALS_EXPORT void SetupFloats(ExecutionContext& ctx) const; + MIOPEN_INTERNALS_EXPORT void SetupComputeType(const ExecutionContext& ctx) const; private: std::string ComputeLayout(const TensorDescriptor& td) const; - std::string ComputeInLayout() const; - std::string ComputeOutLayout() const; - std::string ComputeWeightsLayout() const; + MIOPEN_INTERNALS_EXPORT std::string ComputeInLayout() const; + MIOPEN_INTERNALS_EXPORT std::string ComputeOutLayout() const; + MIOPEN_INTERNALS_EXPORT std::string ComputeWeightsLayout() const; TensorDescriptor in; TensorDescriptor weights; @@ -433,5 +433,59 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase mutable bool use_tf32 = false; }; +inline bool IsPointOutput3dStrideEqFilter(const ProblemDescription& problem, + Direction direction, + bool require_input_spatial_eq_filter) +{ + if(problem.GetDirection() != direction) + return false; + + const auto& conv = problem.GetConv(); + if(conv.GetSpatialDimension() != 3 || conv.group_count != 1) + return false; + + const auto& in_desc = problem.GetIn(); + const auto& out_desc = problem.GetOut(); + const auto& point_desc = direction == Direction::Forward ? out_desc : in_desc; + const auto& w_desc = problem.GetWeights(); + + const auto& point_lens = point_desc.GetLengths(); + if(point_lens.size() != 5 || point_lens[2] != 1 || point_lens[3] != 1 || point_lens[4] != 1) + return false; + + const auto& in_lens = in_desc.GetLengths(); + + const auto& pads = conv.GetConvPads(); + const auto& strides = conv.GetConvStrides(); + const auto& dilations = conv.GetConvDilations(); + const auto& w_lens = w_desc.GetLengths(); + if(w_lens.size() != 5 || pads.size() != 3 || strides.size() != 3 || dilations.size() != 3) + return false; + + for(int i = 0; i < 3; ++i) + { + if(pads[i] != 0 || dilations[i] != 1) + return false; + if(static_cast(w_lens[2 + i]) != strides[i]) + return false; + if(require_input_spatial_eq_filter && + static_cast(in_lens[2 + i]) != static_cast(w_lens[2 + i])) + return false; + } + + return true; +} + +inline bool IsBwdDataPointOutput3dStrideEqFilter(const ProblemDescription& problem) +{ + return IsPointOutput3dStrideEqFilter(problem, Direction::BackwardData, false); +} + +inline bool IsFwdDataPointOutput3dStrideEqFilter(const ProblemDescription& problem) +{ + // For direct GEMM without Im2Col, input spatial must equal filter spatial. + return IsPointOutput3dStrideEqFilter(problem, Direction::Forward, true); +} + } // namespace conv } // namespace miopen diff --git a/projects/miopen/src/include/miopen/conv/solver_finders.hpp b/projects/miopen/src/include/miopen/conv/solver_finders.hpp index 4bebb4d09ce9..6d9704ef5215 100644 --- a/projects/miopen/src/include/miopen/conv/solver_finders.hpp +++ b/projects/miopen/src/include/miopen/conv/solver_finders.hpp @@ -181,8 +181,8 @@ FindCoreResult FindCore(const AnyInvokeParams& invoke_ctx, bool force_attach_binary = false); namespace conv { -bool MIOPEN_INTERNALS_EXPORT IsAlgorithmDisabled(miopenConvAlgorithm_t algo); -bool MIOPEN_INTERNALS_EXPORT IsEnoughWorkspace(std::string_view where, +MIOPEN_INTERNALS_EXPORT bool IsAlgorithmDisabled(miopenConvAlgorithm_t algo); +MIOPEN_INTERNALS_EXPORT bool IsEnoughWorkspace(std::string_view where, const miopen::solver::Id& solver_id, std::size_t required_size, const miopen::AnyInvokeParams* invokeParams, diff --git a/projects/miopen/src/include/miopen/conv/solvers.hpp b/projects/miopen/src/include/miopen/conv/solvers.hpp index b5a634bea3ed..82af3b1dfb13 100644 --- a/projects/miopen/src/include/miopen/conv/solvers.hpp +++ b/projects/miopen/src/include/miopen/conv/solvers.hpp @@ -35,6 +35,8 @@ #include #include #include +#include +#include #include #include @@ -87,7 +89,7 @@ struct PerformanceConfigConvAsm3x3U : PerfConfigBase +struct MIOPEN_INTERNALS_EXPORT ConvAsm3x3U final : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConfigConvAsm3x3U + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm3x3U&) const override; + PerformanceConfigConvAsm3x3U Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigConvAsm3x3U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm3x3U&) const override; }; struct PerformanceConfigConvAsm1x1U : PerfConfigBase @@ -152,7 +152,7 @@ struct PerformanceConfigConvAsm1x1U : PerfConfigBase static void Visit(Self&& self, F f) @@ -181,14 +181,14 @@ struct PerformanceConfigConvAsm1x1U : PerfConfigBase +struct MIOPEN_INTERNALS_EXPORT ConvAsm1x1U final : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceConfigConvAsm1x1U + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1U&) const override; + PerformanceConfigConvAsm1x1U Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigConvAsm1x1U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1U&) const override; }; struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase @@ -265,7 +263,7 @@ struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase static void Visit(Self&& self, F f) @@ -298,93 +296,92 @@ struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase +struct MIOPEN_INTERNALS_EXPORT ConvAsm1x1UV2 final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceConfigConvAsm1x1UV2 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1UV2&) const override; + PerformanceConfigConvAsm1x1UV2 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigConvAsm1x1UV2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1UV2&) const override; }; -struct ConvAsm5x10u2v2f1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsm5x10u2v2f1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsm5x10u2v2b1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsm5x10u2v2b1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsm7x7c3h224w224k64u2v2p3q3f1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsm7x7c3h224w224k64u2v2p3q3f1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvOclDirectFwd11x11 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwd11x11 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvOclDirectFwdGen final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwdGen final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; struct PerformanceImplicitGemm : PerfConfigBase @@ -422,7 +419,7 @@ struct PerformanceImplicitGemm : PerfConfigBase { } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemm(bool spare); + PerformanceImplicitGemm(bool spare); template static void Visit(Self&& self, F f) @@ -445,13 +442,11 @@ struct PerformanceImplicitGemm : PerfConfigBase f(self.WeiBlockCopyClusterLengths_K, "WeiBlockCopyClusterLengths_K"); } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemm& other) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceImplicitGemm& other) const; }; struct PerformanceImplicitGemmV4R1 : public PerformanceImplicitGemm @@ -485,8 +480,7 @@ struct PerformanceImplicitGemmV4R1 : public PerformanceImplicitGemm PerformanceImplicitGemmV4R1(bool spare) : PerformanceImplicitGemm(spare) {} - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; }; struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase @@ -502,7 +496,7 @@ struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase static void Visit(Self&& self, F f) @@ -528,27 +522,23 @@ struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateBlockGemmPerformanceParameters() const; + std::tuple CalculateGemmABlockCopyPerformanceParameters() const; + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); }; struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase @@ -564,7 +554,7 @@ struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase static void Visit(Self&& self, F f) @@ -590,27 +580,23 @@ struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateBlockGemmPerformanceParameters() const; + std::tuple CalculateGemmABlockCopyPerformanceParameters() const; + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); }; struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase @@ -626,7 +612,7 @@ struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase static void Visit(Self&& self, F f) @@ -653,27 +639,24 @@ struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGridSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + std::tuple CalculateBlockGemmPerformanceParameters() const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); }; struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase @@ -689,7 +672,7 @@ struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase static void Visit(Self&& self, F f) @@ -716,27 +699,24 @@ struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateBlockGemmPerformanceParameters() const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple MIOPEN_INTERNALS_EXPORT + std::tuple MIOPEN_INTERNALS_EXPORT CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); }; struct PerformanceImplicitGemmBwdDataV4R1Xdlops @@ -759,16 +739,15 @@ struct PerformanceImplicitGemmBwdDataV4R1Xdlops MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(int, int, int, int, int, int, bool, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(bool spare); + PerformanceImplicitGemmBwdDataV4R1Xdlops(); + PerformanceImplicitGemmBwdDataV4R1Xdlops(bool spare); PerformanceImplicitGemmBwdDataV4R1Xdlops( int a, int b, int c, int d, int e, int f, bool g, bool h) : PerformanceImplicitGemmBwdDataV4R1Xdlops(a, b, c, d, e, f, g, h, false) { } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmBwdDataV4R1Xdlops& other) const; + bool operator==(const PerformanceImplicitGemmBwdDataV4R1Xdlops& other) const; template static void Visit(Self&& self, F f) @@ -783,73 +762,68 @@ struct PerformanceImplicitGemmBwdDataV4R1Xdlops f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); } - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + bool IsFastToBeUsedForTuning(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); }; -struct ConvHipImplicitGemmV4R1Fwd final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmV4R1Fwd final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmV4R1 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; + PerformanceImplicitGemmV4R1 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; }; -struct ConvHipImplicitGemmV4R4Fwd final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmV4R4Fwd final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceImplicitGemmV4R4Fwd + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4Fwd&) const override; + PerformanceImplicitGemmV4R4Fwd Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmV4R4Fwd&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4Fwd&) const override; private: static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); @@ -875,7 +849,7 @@ struct PerformanceConvMlirIgemm : PerfConfigBase return heur; } - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(int, int, int, int, int, int, bool); + PerformanceConvMlirIgemm(int, int, int, int, int, int, bool); PerformanceConvMlirIgemm(int a, int b, int c, int d, int e, int f) : PerformanceConvMlirIgemm(a, b, c, d, e, f, false) @@ -884,9 +858,9 @@ struct PerformanceConvMlirIgemm : PerfConfigBase PerformanceConvMlirIgemm() : PerformanceConvMlirIgemm(-1, -1, -1, -1, -1, -1, false) {} - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(bool spare); + PerformanceConvMlirIgemm(bool spare); - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemm& other) const; + bool operator==(const PerformanceConvMlirIgemm& other) const; template static void Visit(Self&& self, F f) @@ -899,9 +873,8 @@ struct PerformanceConvMlirIgemm : PerfConfigBase f(self.GemmNPerThread, "GemmNPerThread"); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool SetNextValue(const miopen::conv::ProblemDescription&); private: void SetMlirHeuristicInitRequest(); @@ -911,22 +884,20 @@ struct ConvMlirIgemmFwd final : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemm + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + PerformanceConvMlirIgemm Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; }; struct PerformanceConvMlirIgemmXdlops : PerfConfigBase @@ -955,14 +926,14 @@ struct PerformanceConvMlirIgemmXdlops : PerfConfigBase static void Visit(Self&& self, F f) @@ -977,9 +948,8 @@ struct PerformanceConvMlirIgemmXdlops : PerfConfigBase(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemmXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + PerformanceConvMlirIgemmXdlops Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; }; -struct ConvHipImplicitGemmV4R4WrW final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmV4R4WrW final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceImplicitGemmV4R4WrW + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4WrW&) const override; + PerformanceImplicitGemmV4R4WrW Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmV4R4WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4WrW&) const override; private: static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); @@ -1044,22 +1011,20 @@ struct ConvMlirIgemmWrW final : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemm + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + PerformanceConvMlirIgemm Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; }; struct ConvMlirIgemmWrWXdlops final : ConvTunableSolver @@ -1069,25 +1034,23 @@ struct ConvMlirIgemmWrWXdlops final : ConvTunableSolver(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemmXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + PerformanceConvMlirIgemmXdlops Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; }; struct PerformanceImplicitGemmForwardV4R4Xdlops @@ -1105,7 +1068,7 @@ struct PerformanceImplicitGemmForwardV4R4Xdlops MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops(int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops(); + PerformanceImplicitGemmForwardV4R4Xdlops(); PerformanceImplicitGemmForwardV4R4Xdlops(bool) : PerformanceImplicitGemmForwardV4R4Xdlops() {} template @@ -1122,27 +1085,23 @@ struct PerformanceImplicitGemmForwardV4R4Xdlops f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops& other) const; + bool operator==(const PerformanceImplicitGemmForwardV4R4Xdlops& other) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + bool IsFastToBeUsedForTuning(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; @@ -1163,8 +1122,8 @@ struct PerformanceImplicitGemmForwardV4R5Xdlops MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(int, int, int, int, int, int, bool, bool, int, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(bool spare); + PerformanceImplicitGemmForwardV4R5Xdlops(); + PerformanceImplicitGemmForwardV4R5Xdlops(bool spare); PerformanceImplicitGemmForwardV4R5Xdlops( int a, int b, int c, int d, int e, int f, bool g, bool h, int i) @@ -1186,27 +1145,23 @@ struct PerformanceImplicitGemmForwardV4R5Xdlops f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R5Xdlops& other) const; + bool operator==(const PerformanceImplicitGemmForwardV4R5Xdlops& other) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + bool IsFastToBeUsedForTuning(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; @@ -1226,9 +1181,9 @@ struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm bool GemmBThreadCopyMoreGemmKPack; int GemmBThreadDataPerRead_GemmN; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm( + PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm( int, int, int, int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(); + PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(); PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(bool) : PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm() { @@ -1251,27 +1206,23 @@ struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm& other) const; + bool operator==(const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm& other) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + bool IsFastToBeUsedForTuning(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; @@ -1288,7 +1239,7 @@ struct PerformanceImplicitGemmBwdV1R1Xdlops : PerfConfigBase @@ -1304,31 +1255,27 @@ struct PerformanceImplicitGemmBwdV1R1Xdlops : PerfConfigBase CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const miopen::conv::ProblemDescription&) const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; -struct ConvHipImplicitGemmForwardV4R4Xdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmForwardV4R4Xdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -1336,19 +1283,18 @@ struct ConvHipImplicitGemmForwardV4R4Xdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmForwardV4R4Xdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops + PerformanceImplicitGemmForwardV4R4Xdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; @@ -1360,7 +1306,7 @@ struct ConvHipImplicitGemmForwardV4R4Xdlops final friend struct PerformanceImplicitGemmForwardV4R4Xdlops; }; -struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -1368,20 +1314,20 @@ struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm + PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm + PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; @@ -1393,7 +1339,7 @@ struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final friend struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm; }; -struct ConvHipImplicitGemmForwardV4R5Xdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmForwardV4R5Xdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -1401,75 +1347,72 @@ struct ConvHipImplicitGemmForwardV4R5Xdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmForwardV4R5Xdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops + PerformanceImplicitGemmForwardV4R5Xdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; }; -struct ConvHipImplicitGemmV4R1WrW final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmV4R1WrW final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmV4R1 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; + PerformanceImplicitGemmV4R1 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; }; -struct ConvHipImplicitGemmBwdDataV1R1 final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmBwdDataV1R1 final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceImplicitGemmBwdDataV1R1 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV1R1&) const override; + PerformanceImplicitGemmBwdDataV1R1 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } private: static std::tuple CalculateGemmSize(const ExecutionContext&, @@ -1482,22 +1425,20 @@ struct ConvMlirIgemmBwd final : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemm + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + PerformanceConvMlirIgemm Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; }; struct ConvMlirIgemmBwdXdlops final : ConvTunableSolver @@ -1507,47 +1448,44 @@ struct ConvMlirIgemmBwdXdlops final : ConvTunableSolver(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceConvMlirIgemmXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + PerformanceConvMlirIgemmXdlops Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; }; -struct ConvHipImplicitGemmBwdDataV4R1 final : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmBwdDataV4R1 final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + PerformanceImplicitGemmBwdDataV4R1 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1&) const override; + PerformanceImplicitGemmBwdDataV4R1 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmBwdDataV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1&) const override; private: static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); @@ -1557,7 +1495,7 @@ struct ConvHipImplicitGemmBwdDataV4R1 final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -1565,19 +1503,18 @@ struct ConvHipImplicitGemmBwdDataV4R1Xdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmBwdDataV4R1Xdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops + PerformanceImplicitGemmBwdDataV4R1Xdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; @@ -1590,7 +1527,7 @@ struct ConvHipImplicitGemmBwdDataV4R1Xdlops final friend struct PerformanceImplicitGemmBwdDataV4R1Xdlops; }; -struct ConvHipImplicitGemmBwdDataV1R1Xdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmBwdDataV1R1Xdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -1598,25 +1535,23 @@ struct ConvHipImplicitGemmBwdDataV1R1Xdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceImplicitGemmBwdV1R1Xdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + PerformanceImplicitGemmBwdV1R1Xdlops Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; private: static std::tuple @@ -1625,131 +1560,132 @@ struct ConvHipImplicitGemmBwdDataV1R1Xdlops final friend struct PerformanceImplicitGemmBwdV1R1Xdlops; }; -struct ConvAsmImplicitGemmV4R1DynamicFwd final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmV4R1DynamicFwd final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmV4R1DynamicFwd_1x1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmV4R1DynamicFwd_1x1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmV4R1DynamicWrw final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmV4R1DynamicWrw final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmGTCDynamicWrwXdlops final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmGTCDynamicWrwXdlops final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmV4R1DynamicBwd final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmV4R1DynamicBwd final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmGTCDynamicFwdXdlops final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmGTCDynamicFwdXdlops final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvAsmImplicitGemmGTCDynamicBwdXdlops final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmGTCDynamicBwdXdlops final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; /// Holds common member functions for the Solvers which share the same /// "legacy exhaustive search" machinery. -struct ConvOclDirectFwdLegacyExhaustiveSearch : ConvTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwdLegacyExhaustiveSearch + : ConvTunableSolver { LegacyPerformanceConfig GetDefaultPerformanceConfig(const ExecutionContext&, @@ -1801,36 +1737,36 @@ struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwd1x1 final : ConvOclDirectFwdLegac } }; -struct ConvBinWinograd3x3U final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvBinWinograd3x3U final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvBinWinogradRxS final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvBinWinogradRxS final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; struct PerformanceConfigConvBinWinogradRxS : PerfConfigBase { int n_groups; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS(int n_groups_); + PerformanceConfigConvBinWinogradRxS(int n_groups_); PerformanceConfigConvBinWinogradRxS() : PerformanceConfigConvBinWinogradRxS(-1) {} PerformanceConfigConvBinWinogradRxS(bool) : PerformanceConfigConvBinWinogradRxS(1) {} @@ -1843,23 +1779,20 @@ struct PerformanceConfigConvBinWinogradRxS : PerfConfigBase void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); bool IsValid(const ExecutionContext& ctx, const miopen::conv::ProblemDescription&) const { return IsValid(ctx); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvBinWinogradRxS& other) const; + bool IsValid(const ExecutionContext&) const; + bool operator==(const PerformanceConfigConvBinWinogradRxS& other) const; }; template struct ConvBinWinoRxS final : ConvTunableSolver { - MIOPEN_INTERNALS_EXPORT const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } + const std::string& SolverDbId() const override { return GetSolverDbId(); } static const std::string& GetSolverDbId() { @@ -1870,23 +1803,21 @@ struct ConvBinWinoRxS final : ConvTunableSolver; extern template struct ConvBinWinoRxS<3, 2>; -#if defined(__clang__) -#pragma clang diagnostic pop #endif -struct ConvBinWinogradRxSf2x3g1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvBinWinogradRxSf2x3g1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; template @@ -1985,8 +1910,8 @@ struct ConvMPBidirectWinograd_xdlops final ConvMPBidirectWinograd_xdlops>(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { @@ -2032,14 +1957,13 @@ struct ConvMPBidirectWinograd_xdlops final bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops + PerformanceImplicitGemmForwardV4R4Xdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; private: ExecutionContext @@ -2203,35 +2127,32 @@ struct PerformanceConfigAsmDirect3x3WrW : PerfConfigBase +struct MIOPEN_INTERNALS_EXPORT ConvAsmBwdWrW3x3 final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceConfigAsmDirect3x3WrW + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmDirect3x3WrW&) const override; + PerformanceConfigAsmDirect3x3WrW Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW& config) const override; + const PerformanceConfigAsmDirect3x3WrW& config) const override; }; template @@ -2311,17 +2232,17 @@ struct PerformanceConfigConvAsmBwdWrW1x1 : PerfConfigBase { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 MIOPEN_INTERNALS_EXPORT - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + PerformanceConfigConvAsmBwdWrW1x1 MIOPEN_INTERNALS_EXPORT GetDefaultPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsmBwdWrW1x1&) const override; + PerformanceConfigConvAsmBwdWrW1x1 Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsmBwdWrW1x1&) const override; }; /// N_BATCH_LOOPS - {1,2,4,8,16} Num batches processed in single workitem. @@ -2462,31 +2377,30 @@ struct PerformanceConfigConvOclBwdWrw2 template struct ConvOclBwdWrW2 final : ConvTunableSolver> { - MIOPEN_INTERNALS_EXPORT const std::string& SolverDbId() const override + const std::string& SolverDbId() const override { return this->template GetSolverDbId>(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 + PerformanceConfigConvOclBwdWrw2 GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool + bool IsValidPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigConvOclBwdWrw2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 + PerformanceConfigConvOclBwdWrw2 Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvOclBwdWrw2&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvOclBwdWrw2&) const override; protected: bool IsApplicableBase(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; @@ -2495,10 +2409,7 @@ struct ConvOclBwdWrW2 final : ConvTunableSolver; extern template struct PerformanceConfigConvOclBwdWrw2<2>; @@ -2512,8 +2423,6 @@ extern template struct ConvOclBwdWrW2<4>; extern template struct ConvOclBwdWrW2<8>; extern template struct ConvOclBwdWrW2<16>; -#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) -#pragma clang diagnostic pop #endif /// A separate solver from ConvOclBwdWrW2 to disable auto-tuning for certain configs. @@ -2536,46 +2445,46 @@ struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2NonTunable final : ConvSolver const miopen::conv::ProblemDescription&) const override; }; -struct ConvOclBwdWrW53 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW53 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvOclBwdWrW1x1 final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW1x1 final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct fft final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT fft final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; struct PerformanceImplicitGemmWrwV4R4Xdlops : PerfConfigBase @@ -2592,8 +2501,8 @@ struct PerformanceImplicitGemmWrwV4R4Xdlops : PerfConfigBase + std::tuple CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; -struct ConvHipImplicitGemmWrwV4R4Xdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmWrwV4R4Xdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -2648,25 +2553,23 @@ struct ConvHipImplicitGemmWrwV4R4Xdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + PerformanceImplicitGemmWrwV4R4Xdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; + PerformanceImplicitGemmWrwV4R4Xdlops Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; }; struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm @@ -2684,9 +2587,9 @@ struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm bool GemmAThreadCopyMoreGemmK; bool GemmBThreadCopyMoreGemmK; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm( + PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm( int, int, int, int, int, int, int, int, int, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(); + PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(); PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(bool) : PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm() { @@ -2708,35 +2611,31 @@ struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm& other) const; + bool operator==(const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm& other) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsReallyValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool IsFastToBeUsedForTuning(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + std::tuple CalculateBlockSize() const; std::tuple CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple + std::tuple CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; }; -struct ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -2744,23 +2643,23 @@ struct ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm + PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm + PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; @@ -2783,12 +2682,12 @@ struct PerformanceConvCkIgemmFwdV6r1DlopsNchw f(self.ck_tunable_list_id, "ck_tunable_list_id"); } - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; bool operator==(const PerformanceConvCkIgemmFwdV6r1DlopsNchw& config) const { return ck_tunable_list_id == config.ck_tunable_list_id; @@ -2802,36 +2701,34 @@ struct ConvCkIgemmFwdV6r1DlopsNchw final : ConvTunableSolver(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, + PerformanceConvCkIgemmFwdV6r1DlopsNchw + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; + PerformanceConvCkIgemmFwdV6r1DlopsNchw Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; }; -struct ConvDirectNaiveConvFwd final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvDirectNaiveConvFwd final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } /// Use very small fixed value enough to backup GEMM for cases when /// GEMM is disabled. @@ -2839,19 +2736,19 @@ struct ConvDirectNaiveConvFwd final : ConvSolver { return 0.01f; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvDirectNaiveConvBwd final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvDirectNaiveConvBwd final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } /// Use very small fixed value enough to backup GEMM for cases when /// GEMM is disabled. @@ -2859,19 +2756,19 @@ struct ConvDirectNaiveConvBwd final : ConvSolver { return 0.01f; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct ConvDirectNaiveConvWrw final : ConvSolver +struct MIOPEN_INTERNALS_EXPORT ConvDirectNaiveConvWrw final : ConvSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } /// Use very small fixed value enough to backup GEMM for cases when /// GEMM is disabled. @@ -2879,11 +2776,11 @@ struct ConvDirectNaiveConvWrw final : ConvSolver { return 0.01f; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; }; -struct GemmFwdBase : ConvSolver +struct MIOPEN_INTERNALS_EXPORT GemmFwdBase : ConvSolver { bool IsDynamic() const override { return true; } float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; @@ -2968,7 +2865,7 @@ struct MIOPEN_INTERNALS_EXPORT GemmFwdRest final : GemmFwdBase const miopen::conv::ProblemDescription&) const override; }; -struct GemmBwdBase : ConvSolver +struct MIOPEN_INTERNALS_EXPORT GemmBwdBase : ConvSolver { bool IsDynamic() const override { return true; } float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; @@ -3034,7 +2931,7 @@ struct MIOPEN_INTERNALS_EXPORT GemmBwdRest final : GemmBwdBase const miopen::conv::ProblemDescription&) const override; }; -struct GemmWrwBase : ConvSolver +struct MIOPEN_INTERNALS_EXPORT GemmWrwBase : ConvSolver { bool IsDynamic() const override { return true; } float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; @@ -3076,7 +2973,8 @@ struct MIOPEN_INTERNALS_EXPORT GemmWrwUniversal final : GemmWrwBase const miopen::conv::ProblemDescription&) const override; }; -struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase +struct MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC + : PerfConfigBase { std::string direction; std::string tensor_layout; @@ -3110,56 +3008,56 @@ struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); + PerformanceConfigAsmImplicitGemmGTC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); + PerformanceConfigAsmImplicitGemmGTC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); PerformanceConfigAsmImplicitGemmGTC() : PerformanceConfigAsmImplicitGemmGTC("fwd", "nchw", @@ -3271,12 +3169,12 @@ struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase { const std::string& SolverDbId() const override @@ -3461,24 +3358,24 @@ struct ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; @@ -3646,18 +3543,17 @@ struct PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC : PerformanceConfigAsmIm spare) { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; }; -struct ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -3665,24 +3561,24 @@ struct ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; @@ -3851,23 +3747,22 @@ struct PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC : PerformanceConfigAsmIm { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT size_t ComputeKernelOccupancy() const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + size_t ComputeKernelOccupancy() const; private: void SetParamsForKSplit(const miopen::conv::ProblemDescription& problem, const size_t& occupancy); }; -struct ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final +struct MIOPEN_INTERNALS_EXPORT ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -3875,24 +3770,24 @@ struct ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; @@ -4068,14 +3963,12 @@ struct PerformanceConfigAsmImplicitGemmGTCvector bool IsValidValue() const = delete; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; - MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigAsmImplicitGemmGTCvector& other) const; - MIOPEN_INTERNALS_EXPORT void - CopyParameters(const PerformanceConfigAsmImplicitGemmGTCvector& other); - MIOPEN_INTERNALS_EXPORT std::string ToString() const override; - MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT int BlockSize() const; + bool IsDefaultConstructed() const; + bool operator==(const PerformanceConfigAsmImplicitGemmGTCvector& other) const; + void CopyParameters(const PerformanceConfigAsmImplicitGemmGTCvector& other); + std::string ToString() const override; + std::string ToKernelName(const ExecutionContext&) const; + int BlockSize() const; }; struct PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC : PerformanceConfigAsmImplicitGemmGTCvector { @@ -4213,14 +4106,14 @@ struct PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC : PerformanceConfigAsmIm { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void HeuristicInit(const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; }; struct ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC final @@ -4230,21 +4123,21 @@ struct ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC final { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; @@ -4268,16 +4161,16 @@ struct PerformanceConfigHipImplicitGemmFwdXdlops : PerformanceConfigHipImplicitGemmFwdXdlops(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void DefaultKernelFromList(const ExecutionContext& ctx); + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmFwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemmFwdXdlops& other) const; private: template @@ -4286,7 +4179,7 @@ struct PerformanceConfigHipImplicitGemmFwdXdlops bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; }; -struct ConvHipImplicitGemmFwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmFwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4294,23 +4187,22 @@ struct ConvHipImplicitGemmFwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops + PerformanceConfigHipImplicitGemmFwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; + PerformanceConfigHipImplicitGemmFwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; /// \anchor igemm_get_wti_magic_number // Magic Number Alert: // Naive convolutions have GetWti() that return very small value (0.01f). @@ -4350,16 +4242,16 @@ struct PerformanceConfigHipImplicitGemmBwdXdlops : PerformanceConfigHipImplicitGemmBwdXdlops(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void DefaultKernelFromList(const ExecutionContext& ctx); + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmBwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemmBwdXdlops& other) const; private: template @@ -4368,7 +4260,7 @@ struct PerformanceConfigHipImplicitGemmBwdXdlops bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; }; -struct ConvHipImplicitGemmBwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmBwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4376,23 +4268,22 @@ struct ConvHipImplicitGemmBwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops + PerformanceConfigHipImplicitGemmBwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; + PerformanceConfigHipImplicitGemmBwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; /// \ref igemm_get_wti_magic_number float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override { @@ -4422,17 +4313,18 @@ struct PerformanceConfigHipImplicitGemmGroupFwdXdlops : PerformanceConfigHipImplicitGemmGroupFwdXdlops(0, "") { } + + void DefaultKernelFromList(const ExecutionContext& ctx); MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupFwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemmGroupFwdXdlops& other) const; MIOPEN_INTERNALS_EXPORT bool IsModelApplicable(const ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem) const; @@ -4455,7 +4347,7 @@ struct PerformanceConfigHipImplicitGemmGroupFwdXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemmGroupFwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmGroupFwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4463,32 +4355,31 @@ struct ConvHipImplicitGemmGroupFwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops + PerformanceConfigHipImplicitGemmGroupFwdXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool + bool IsValidPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops + PerformanceConfigHipImplicitGemmGroupFwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; /// \ref igemm_get_wti_magic_number float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override { return 0.02f; }; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4515,17 +4406,17 @@ struct PerformanceConfigHipImplicitGemm3DGroupFwdXdlops : PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(0, "") { } + void DefaultKernelFromList(const ExecutionContext& ctx); MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops& other) const; bool UseTF32() const { return use_tf32; } private: @@ -4537,7 +4428,7 @@ struct PerformanceConfigHipImplicitGemm3DGroupFwdXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemm3DGroupFwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemm3DGroupFwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4545,29 +4436,28 @@ struct ConvHipImplicitGemm3DGroupFwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops + PerformanceConfigHipImplicitGemm3DGroupFwdXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops + PerformanceConfigHipImplicitGemm3DGroupFwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4594,17 +4484,16 @@ struct PerformanceConfigHipImplicitGemm3DGroupWrwXdlops : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void DefaultKernelFromList(const ExecutionContext& ctx); + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops& other) const; bool UseTF32() const { return use_tf32; } private: @@ -4616,7 +4505,7 @@ struct PerformanceConfigHipImplicitGemm3DGroupWrwXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemm3DGroupWrwXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemm3DGroupWrwXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4624,21 +4513,21 @@ struct ConvHipImplicitGemm3DGroupWrwXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops + PerformanceConfigHipImplicitGemm3DGroupWrwXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops + PerformanceConfigHipImplicitGemm3DGroupWrwXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; @@ -4648,8 +4537,8 @@ struct ConvHipImplicitGemm3DGroupWrwXdlops final return 0.02f; }; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4680,17 +4569,16 @@ struct PerformanceConfigHipImplicitGemm3DGroupBwdXdlops : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + void DefaultKernelFromList(const ExecutionContext& ctx); + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops& other) const; bool UseTF32() const { return use_tf32; } private: @@ -4702,7 +4590,7 @@ struct PerformanceConfigHipImplicitGemm3DGroupBwdXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemm3DGroupBwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemm3DGroupBwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4710,21 +4598,21 @@ struct ConvHipImplicitGemm3DGroupBwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops + PerformanceConfigHipImplicitGemm3DGroupBwdXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops + PerformanceConfigHipImplicitGemm3DGroupBwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution + ConvSolution GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; @@ -4734,8 +4622,8 @@ struct ConvHipImplicitGemm3DGroupBwdXdlops final return 0.02f; }; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4763,17 +4651,17 @@ struct PerformanceConfigHipImplicitGemmGroupBwdXdlops { } + void DefaultKernelFromList(const ExecutionContext& ctx); MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupBwdXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemmGroupBwdXdlops& other) const; MIOPEN_INTERNALS_EXPORT bool IsModelApplicable(const ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem) const; @@ -4799,7 +4687,7 @@ struct PerformanceConfigHipImplicitGemmGroupBwdXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemmGroupBwdXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmGroupBwdXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4807,32 +4695,31 @@ struct ConvHipImplicitGemmGroupBwdXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops + PerformanceConfigHipImplicitGemmGroupBwdXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool + bool IsValidPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops + PerformanceConfigHipImplicitGemmGroupBwdXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; /// \ref igemm_get_wti_magic_number float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override { return 0.02f; }; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4861,17 +4748,18 @@ struct PerformanceConfigHipImplicitGemmGroupWrwXdlops : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") { } + + void DefaultKernelFromList(const ExecutionContext& ctx); MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupWrwXdlops& other) const; + bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigHipImplicitGemmGroupWrwXdlops& other) const; MIOPEN_INTERNALS_EXPORT bool IsModelApplicable(const ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem) const; @@ -4897,7 +4785,7 @@ struct PerformanceConfigHipImplicitGemmGroupWrwXdlops mutable bool use_tf32 = false; }; -struct ConvHipImplicitGemmGroupWrwXdlops final +struct MIOPEN_INTERNALS_EXPORT ConvHipImplicitGemmGroupWrwXdlops final : ConvTunableSolver { const std::string& SolverDbId() const override @@ -4905,32 +4793,31 @@ struct ConvHipImplicitGemmGroupWrwXdlops final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops + PerformanceConfigHipImplicitGemmGroupWrwXdlops GetDefaultPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool + bool IsValidPerformanceConfig(const ExecutionContext&, const miopen::conv::ProblemDescription&, const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops + PerformanceConfigHipImplicitGemmGroupWrwXdlops Search(const ExecutionContext&, const miopen::conv::ProblemDescription&, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; /// \ref igemm_get_wti_magic_number float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override { return 0.02f; }; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool MayNeedWorkspace() const override { return true; } private: @@ -4971,23 +4858,23 @@ struct PerformanceConfigConvDepthwiseFwd2D : PerfConfigBaseCK +struct MIOPEN_INTERNALS_EXPORT ConvDepthwiseFwd2D final + : ConvTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvDepthwiseFwd2D GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvDepthwiseFwd2D&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvDepthwiseFwd2D - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; + PerformanceConfigConvDepthwiseFwd2D + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvDepthwiseFwd2D&) const override; + PerformanceConfigConvDepthwiseFwd2D Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; bool IsDynamic() const override { return true; } /// Use very small fixed value enough to backup GEMM for cases when /// GEMM is disabled. @@ -5002,14 +4889,270 @@ struct ConvDepthwiseFwd2D final : ConvTunableSolver +template +struct ConvWinogradNHWCTransposingBase : TransposingSolver +{ + using Problem = miopen::conv::ProblemDescription; + using InvokeParams = miopen::conv::TransposeConvInvokeParams; + using Base = TransposingSolver; + + /// Convert from API params to TransposeConvInvokeParams. + /// The API passes DataInvokeParams for Fwd/Bwd and WrWInvokeParams for WrW. + static InvokeParams ConvertFromApiParams(const AnyInvokeParams& any_params) + { + if(any_params.IsOfType()) + { + const auto& wrw_params = any_params.CastTo(); + return InvokeParams{wrw_params}; + } + const auto& data_params = any_params.CastTo(); + return InvokeParams{data_params}; + } + + /// Convert TransposeConvInvokeParams back to the correct type for inner solver. + /// Inner Winograd solvers expect DataInvokeParams for Fwd/Bwd and WrWInvokeParams for WrW. + static AnyInvokeParams ConvertForInnerSolver(const InvokeParams& params) + { + if(params.is_wrw) + return params.ToWrWInvokeParams(); + return params.ToDataInvokeParams(); + } + + /// Override Transpose to recompute layout strings after transposing tensors. + /// This is needed because conv::ProblemDescription caches layout strings at construction, + /// and they must be updated to reflect the new NCHW-like strides after transposition. + inline static Problem Transpose(const Problem& problem) + { + auto transposed_problem = Base::Transpose(problem); + // CRITICAL: Attempt to update cached layout strings to match transposed strides. + // Some inner solvers (e.g., RxSf2x3g1) check IsLayoutDefault() which validates + // cached layout strings. Note: For degenerate dimensions (N=1, C=1, etc.), the cached + // string may not be updated since strides satisfy multiple layouts. Solvers using + // IsPossibleLayout4D5D() will work correctly regardless as they check actual strides. + transposed_problem.HeuristicUpdateLayouts(); + return transposed_problem; + } + + inline static auto GetTransposes(const Problem& problem) + { + const bool is_wrw = problem.IsDirectionBackwardWrW(); + + // Layout string "NCDHW" supports both 4D (NCHW) and 5D (NCDHW) tensors: + // - For 4D tensors: Automatically interpreted as NCHW (D dimension is implicit/1) + // - For 5D tensors: Full NCDHW layout for 3D convolutions + // This makes the transposing solver future-proof for both 2D and 3D convolutions. + auto ret = std::array, 3>{{ + { + &Problem::GetIn, + &InvokeParams::inDesc, + &InvokeParams::in, // in (dy for WrW): always an input + nullptr, + "NCDHW", // transpose NHWC/NDHWC->NCHW/NCDHW + true, + }, + { + &Problem::GetWeights, + &InvokeParams::wDesc, + is_wrw ? nullptr : &InvokeParams::w, // Fwd/Bwd: w is input + is_wrw ? &InvokeParams::w_as_output : nullptr, // WrW: dw is output + "NCDHW", // weights: layout adapts to tensor dimensionality + !is_wrw, // Fwd/Bwd: input; WrW: output + }, + { + &Problem::GetOut, + &InvokeParams::outDesc, + is_wrw ? &InvokeParams::out_as_input : nullptr, // WrW: x is input + is_wrw ? nullptr : &InvokeParams::out, // Fwd/Bwd: out is output + "NCDHW", // out: layout adapts to tensor dimensionality + is_wrw, // Fwd/Bwd: output; WrW: input + }, + }}; + + return ret; + } +}; + +/// Non-tunable transposing wrapper for NHWC winograd solvers. +template +struct ConvWinogradNHWCTransposingSolver + : ConvWinogradNHWCTransposingBase, Inner, ConvSolver> +{ +}; + +/// Tunable transposing wrapper for NHWC winograd solvers. +/// Uses ConvTunableSolver as base so that the transposed +/// solver properly exposes tuning methods (GetDefaultPerformanceConfig, IsValidPerformanceConfig, +/// Search, GetSolution with config). The TransposingSolverGetSolution<..., true> specialization +/// handles delegation of all tunable methods to the inner solver with transposed problems. +template +struct ConvWinogradNHWCTransposingTunableSolver + : ConvWinogradNHWCTransposingBase, + Inner, + ConvTunableSolver> +{ +}; + +struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinograd3x3U final + : ConvWinogradNHWCTransposingSolver +{ + const std::string& SolverDbId() const { return GetSolverDbId(); } +}; + +struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinogradRxS final + : ConvWinogradNHWCTransposingSolver +{ + const std::string& SolverDbId() const { return GetSolverDbId(); } +}; + +struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinogradRxSf2x3g1 final + : ConvWinogradNHWCTransposingSolver +{ + const std::string& SolverDbId() const + { + return GetSolverDbId(); + } +}; +template +struct TransposedConvMPBidirectWinograd final + : ConvWinogradNHWCTransposingSolver< + ConvMPBidirectWinograd> +{ + const std::string& SolverDbId() const + { + return this->template GetSolverDbId< + TransposedConvMPBidirectWinograd>(); + } +}; + +#ifndef CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP +extern template struct TransposedConvMPBidirectWinograd<2, 3>; +extern template struct TransposedConvMPBidirectWinograd<3, 3>; +extern template struct TransposedConvMPBidirectWinograd<4, 3>; +extern template struct TransposedConvMPBidirectWinograd<5, 3>; +extern template struct TransposedConvMPBidirectWinograd<6, 3>; +#endif + +template +struct TransposedConvWinograd3x3MultipassWrW final + : ConvWinogradNHWCTransposingSolver< + ConvWinograd3x3MultipassWrW> +{ + const std::string& SolverDbId() const + { + return this->template GetSolverDbId>(); + } +}; + +#ifndef CONV_MULTIPASS_WINO3X3WRW_CPP +extern template struct TransposedConvWinograd3x3MultipassWrW<3, 2>; +extern template struct TransposedConvWinograd3x3MultipassWrW<3, 3>; +extern template struct TransposedConvWinograd3x3MultipassWrW<3, 4>; +extern template struct TransposedConvWinograd3x3MultipassWrW<3, 5>; +extern template struct TransposedConvWinograd3x3MultipassWrW<3, 6>; +extern template struct TransposedConvWinograd3x3MultipassWrW<7, 2>; +extern template struct TransposedConvWinograd3x3MultipassWrW<7, 3>; +extern template struct TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>; +extern template struct TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>; +extern template struct TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>; +extern template struct TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>; +extern template struct TransposedConvWinograd3x3MultipassWrW<5, 3>; +extern template struct TransposedConvWinograd3x3MultipassWrW<5, 4>; +#endif + +template +struct TransposedConvWinoFuryRxS final + : ConvWinogradNHWCTransposingSolver> +{ + const std::string& SolverDbId() const + { + return this->template GetSolverDbId>(); + } +}; + +#ifndef CONV_WINO_FURY_RXS_CPP +extern template struct TransposedConvWinoFuryRxS<2, 3>; +#endif + +template +struct TransposedConvWinoRageRxS final + : ConvWinogradNHWCTransposingSolver> +{ + const std::string& SolverDbId() const + { + return this->template GetSolverDbId>(); + } +}; + +#ifndef CONV_WINO_RAGE_RXS_CPP +extern template struct TransposedConvWinoRageRxS<2, 3>; +#endif + +// Tunable transposed Winograd solvers for NHWC layout support +template +struct TransposedConvBinWinoRxS final + : ConvWinogradNHWCTransposingTunableSolver> +{ + const std::string& SolverDbId() const override + { + return this->template GetSolverDbId>(); + } +}; + +// Suppress misleading clang warnings +#ifndef CONV_BIN_WINO_RXS_CPP + +extern template struct TransposedConvBinWinoRxS<2, 3>; +extern template struct TransposedConvBinWinoRxS<3, 2>; + +#endif + +template +struct TransposedConvMPBidirectWinograd_xdlops final + : ConvWinogradNHWCTransposingTunableSolver< + ConvMPBidirectWinograd_xdlops> +{ + const std::string& SolverDbId() const + { + return this->template GetSolverDbId>(); + } +}; + +// To suppress misleading clang warnings +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif +extern template struct TransposedConvMPBidirectWinograd_xdlops<2, 3>; +extern template struct TransposedConvMPBidirectWinograd_xdlops<3, 3>; +extern template struct TransposedConvMPBidirectWinograd_xdlops<4, 3>; +extern template struct TransposedConvMPBidirectWinograd_xdlops<5, 3>; +extern template struct TransposedConvMPBidirectWinograd_xdlops<6, 3>; +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic pop +#endif // Test helper functions for metadata validation // These functions return all CK kernel TypeStrings without problem-based filtering diff --git a/projects/miopen/src/include/miopen/convolution.hpp b/projects/miopen/src/include/miopen/convolution.hpp index eb16142ef1cb..374e123c16b2 100644 --- a/projects/miopen/src/include/miopen/convolution.hpp +++ b/projects/miopen/src/include/miopen/convolution.hpp @@ -71,7 +71,7 @@ struct ExecutionContext; struct Handle; struct TensorDescriptor; -struct MIOPEN_INTERNALS_EXPORT ConvolutionAttribute +struct ConvolutionAttribute { class Gfx90aFp16alt { @@ -151,7 +151,7 @@ struct MIOPEN_INTERNALS_EXPORT ConvolutionAttribute /// * -1: Default (attribute-specific). /// * 0: Disabled/Yes. /// * 1: Enabled/No. - void Set(miopenConvolutionAttrib_t attr, int value); + MIOPEN_INTERNALS_EXPORT void Set(miopenConvolutionAttrib_t attr, int value); int Get(miopenConvolutionAttrib_t attr) const; friend void to_json(nlohmann::json& json, const ConvolutionAttribute& conv); @@ -174,198 +174,205 @@ enum class FallbackPath Default_ = None }; -struct MIOPEN_INTERNALS_EXPORT ConvolutionDescriptor : miopenConvolutionDescriptor +struct ConvolutionDescriptor : miopenConvolutionDescriptor { - ConvolutionDescriptor(std::size_t spatial_dim, - miopenConvolutionMode_t c_mode, - miopenPaddingMode_t p_mode, - const std::vector& p_pads = {0, 0}, - const std::vector& p_strides = {1, 1}, - const std::vector& p_dilations = {1, 1}, - const std::vector& p_trans_output_pads = {0, 0}, - int p_group_count = 1, - float p_lowp_quant = float(1)); + MIOPEN_INTERNALS_EXPORT ConvolutionDescriptor(std::size_t spatial_dim, + miopenConvolutionMode_t c_mode, + miopenPaddingMode_t p_mode, + const std::vector& p_pads = {0, 0}, + const std::vector& p_strides = {1, 1}, + const std::vector& p_dilations = {1, 1}, + const std::vector& p_trans_output_pads = {0, + 0}, + int p_group_count = 1, + float p_lowp_quant = float(1)); - ConvolutionDescriptor(const std::vector& p_pads = {0, 0}, - const std::vector& p_strides = {1, 1}, - const std::vector& p_dilations = {1, 1}, - const std::vector& p_trans_output_pads = {0, 0}, - int p_group_count = 1, - float p_lowp_quant = float(1)); + MIOPEN_INTERNALS_EXPORT ConvolutionDescriptor(const std::vector& p_pads = {0, 0}, + const std::vector& p_strides = {1, 1}, + const std::vector& p_dilations = {1, 1}, + const std::vector& p_trans_output_pads = {0, + 0}, + int p_group_count = 1, + float p_lowp_quant = float(1)); - std::size_t GetSpatialDimension() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetSpatialDimension() const; - const std::vector& GetConvPads() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetConvPads() const; - const std::vector& GetConvStrides() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetConvStrides() const; - const std::vector& GetConvDilations() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetConvDilations() const; - const std::vector& GetTransposeConvPads() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetTransposeConvPads() const; - int GetGroupCount() const; + MIOPEN_INTERNALS_EXPORT int GetGroupCount() const; - TensorDescriptor GetForwardOutputTensorWithLayout(const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - const std::string& yLayout, - miopenDataType_t yType = miopenFloat) const; - - TensorDescriptor GetForwardOutputTensor(const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - miopenDataType_t yType = miopenFloat) const; - - bool IsWinograd3x3SupportedAndFast(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) const; - - std::size_t GetWorkSpaceSize(ExecutionContext ctx, - const conv::ProblemDescription& problem) const; - - void FindConvFwdAlgorithm(const Handle& handle, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& yDesc, - Data_t y, - int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - std::size_t workSpaceSize, - bool exhaustiveSearch) const; - - void ConvolutionForward(const Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& wDesc, - ConstData_t w, - miopenConvFwdAlgorithm_t algo, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - std::size_t workSpaceSize) const; - - std::size_t GetSolutionCount(const ExecutionContext& ctx, - const conv::ProblemDescription& problem) const; + MIOPEN_INTERNALS_EXPORT TensorDescriptor + GetForwardOutputTensorWithLayout(const TensorDescriptor& xDesc, + const TensorDescriptor& wDesc, + const std::string& yLayout, + miopenDataType_t yType = miopenFloat) const; - std::vector + MIOPEN_INTERNALS_EXPORT TensorDescriptor + GetForwardOutputTensor(const TensorDescriptor& xDesc, + const TensorDescriptor& wDesc, + miopenDataType_t yType = miopenFloat) const; + + MIOPEN_INTERNALS_EXPORT bool + IsWinograd3x3SupportedAndFast(const miopen::ExecutionContext& ctx, + const conv::ProblemDescription& problem) const; + + MIOPEN_INTERNALS_EXPORT std::size_t + GetWorkSpaceSize(ExecutionContext ctx, const conv::ProblemDescription& problem) const; + + MIOPEN_INTERNALS_EXPORT void FindConvFwdAlgorithm(const Handle& handle, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& wDesc, + ConstData_t w, + const TensorDescriptor& yDesc, + Data_t y, + int requestAlgoCount, + int* returnedAlgoCount, + miopenConvAlgoPerf_t* perfResults, + Data_t workSpace, + std::size_t workSpaceSize, + bool exhaustiveSearch) const; + + MIOPEN_INTERNALS_EXPORT void ConvolutionForward(const Handle& handle, + const void* alpha, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& wDesc, + ConstData_t w, + miopenConvFwdAlgorithm_t algo, + const void* beta, + const TensorDescriptor& yDesc, + Data_t y, + Data_t workSpace, + std::size_t workSpaceSize) const; + + MIOPEN_INTERNALS_EXPORT std::size_t + GetSolutionCount(const ExecutionContext& ctx, const conv::ProblemDescription& problem) const; + + MIOPEN_INTERNALS_EXPORT std::vector GetSolutions(const ExecutionContext& ctx, const conv::ProblemDescription& problem, size_t maxSolutionCount, FallbackPath* fallbackPathTaken, const AnyInvokeParams* invokeParams = nullptr) const; - void CompileSolution(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - solver::Id solver_id) const; - - std::size_t GetForwardSolutionWorkspaceSize(const Handle& handle, - const TensorDescriptor& wDesc, - const TensorDescriptor& xDesc, - const TensorDescriptor& yDesc, - solver::Id solver_id) const; + MIOPEN_INTERNALS_EXPORT void CompileSolution(const ExecutionContext& ctx, + const conv::ProblemDescription& problem, + solver::Id solver_id) const; - void ConvolutionForwardImmediate(const Handle& handle, + MIOPEN_INTERNALS_EXPORT std::size_t + GetForwardSolutionWorkspaceSize(const Handle& handle, + const TensorDescriptor& wDesc, + const TensorDescriptor& xDesc, + const TensorDescriptor& yDesc, + solver::Id solver_id) const; + + MIOPEN_INTERNALS_EXPORT void ConvolutionForwardImmediate(const Handle& handle, + const TensorDescriptor& wDesc, + ConstData_t w, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + Data_t workSpace, + std::size_t workSpaceSize, + solver::Id solver_id) const; + + MIOPEN_INTERNALS_EXPORT void FindConvBwdDataAlgorithm(const Handle& handle, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& wDesc, + ConstData_t w, + const TensorDescriptor& dxDesc, + Data_t dx, + int requestAlgoCount, + int* returnedAlgoCount, + miopenConvAlgoPerf_t* perfResults, + Data_t workSpace, + std::size_t workSpaceSize, + bool exhaustiveSearch) const; + + MIOPEN_INTERNALS_EXPORT void ConvolutionBackwardData(const Handle& handle, + const void* alpha, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& wDesc, + ConstData_t w, + miopenConvBwdDataAlgorithm_t algo, + const void* beta, + const TensorDescriptor& dxDesc, + Data_t dx, + Data_t workSpace, + std::size_t workSpaceSize) const; + + MIOPEN_INTERNALS_EXPORT std::size_t + GetBackwardSolutionWorkspaceSize(const Handle& handle, + const TensorDescriptor& dyDesc, const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - std::size_t workSpaceSize, + const TensorDescriptor& dxDesc, solver::Id solver_id) const; - void FindConvBwdDataAlgorithm(const Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& dxDesc, - Data_t dx, - int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - std::size_t workSpaceSize, - bool exhaustiveSearch) const; - - void ConvolutionBackwardData(const Handle& handle, - const void* alpha, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - miopenConvBwdDataAlgorithm_t algo, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t workSpace, - std::size_t workSpaceSize) const; - - std::size_t GetBackwardSolutionWorkspaceSize(const Handle& handle, - const TensorDescriptor& dyDesc, - const TensorDescriptor& wDesc, - const TensorDescriptor& dxDesc, - solver::Id solver_id) const; - - void ConvolutionBackwardImmediate(const Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t workSpace, - std::size_t workSpaceSize, - solver::Id solver_id) const; - - std::size_t GetWrwSolutionWorkspaceSize(const Handle& handle, - const TensorDescriptor& dyDesc, - const TensorDescriptor& xDesc, - const TensorDescriptor& dwDesc, - solver::Id solver_id) const; - - void ConvolutionWrwImmediate(const Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - std::size_t workSpaceSize, - solver::Id solver_id) const; - - void FindConvBwdWeightsAlgorithm(const Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& dwDesc, - Data_t dw, - int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - std::size_t workSpaceSize, - bool exhaustiveSearch) const; - - void ConvolutionBackwardWeights(const Handle& handle, - const void* alpha, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - miopenConvBwdWeightsAlgorithm_t algo, - const void* beta, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - std::size_t workSpaceSize) const; - miopenMathType_t GetMathType() const; + MIOPEN_INTERNALS_EXPORT void ConvolutionBackwardImmediate(const Handle& handle, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& wDesc, + ConstData_t w, + const TensorDescriptor& dxDesc, + Data_t dx, + Data_t workSpace, + std::size_t workSpaceSize, + solver::Id solver_id) const; + + MIOPEN_INTERNALS_EXPORT std::size_t GetWrwSolutionWorkspaceSize(const Handle& handle, + const TensorDescriptor& dyDesc, + const TensorDescriptor& xDesc, + const TensorDescriptor& dwDesc, + solver::Id solver_id) const; + + MIOPEN_INTERNALS_EXPORT void ConvolutionWrwImmediate(const Handle& handle, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& dwDesc, + Data_t dw, + Data_t workSpace, + std::size_t workSpaceSize, + solver::Id solver_id) const; + + MIOPEN_INTERNALS_EXPORT void FindConvBwdWeightsAlgorithm(const Handle& handle, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& dwDesc, + Data_t dw, + int requestAlgoCount, + int* returnedAlgoCount, + miopenConvAlgoPerf_t* perfResults, + Data_t workSpace, + std::size_t workSpaceSize, + bool exhaustiveSearch) const; + + MIOPEN_INTERNALS_EXPORT void ConvolutionBackwardWeights(const Handle& handle, + const void* alpha, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& xDesc, + ConstData_t x, + miopenConvBwdWeightsAlgorithm_t algo, + const void* beta, + const TensorDescriptor& dwDesc, + Data_t dw, + Data_t workSpace, + std::size_t workSpaceSize) const; + MIOPEN_INTERNALS_EXPORT miopenMathType_t GetMathType() const; std::size_t spatialDim; miopenConvolutionMode_t mode; diff --git a/projects/miopen/src/include/miopen/ctc.hpp b/projects/miopen/src/include/miopen/ctc.hpp index 755cc542306d..ba48906334cf 100644 --- a/projects/miopen/src/include/miopen/ctc.hpp +++ b/projects/miopen/src/include/miopen/ctc.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include diff --git a/projects/miopen/src/include/miopen/db.hpp b/projects/miopen/src/include/miopen/db.hpp index 66bbb4924f50..78352602506e 100644 --- a/projects/miopen/src/include/miopen/db.hpp +++ b/projects/miopen/src/include/miopen/db.hpp @@ -26,6 +26,7 @@ #ifndef GUARD_MIOPEN_DB_HPP_ #define GUARD_MIOPEN_DB_HPP_ +#include #include #include #include @@ -47,13 +48,14 @@ class LockFile; constexpr bool DisableUserDbFileIO = MIOPEN_DISABLE_USERDB; /// No instance of this class should be used from several threads at the same time. -class MIOPEN_INTERNALS_EXPORT PlainTextDb +class PlainTextDb { public: + MIOPEN_INTERNALS_EXPORT PlainTextDb(DbKinds db_kind_, const fs::path& filename_, bool is_system = false); /// Searches db for provided key and returns found record or none if key not found in database - std::optional FindRecord(const std::string& key); + MIOPEN_INTERNALS_EXPORT std::optional FindRecord(const std::string& key); template inline std::optional FindRecord(const T& problem_config) @@ -66,19 +68,19 @@ class MIOPEN_INTERNALS_EXPORT PlainTextDb /// replaced by provided record. /// /// Returns true if store was successful, false otherwise. - bool StoreRecord(const DbRecord& record); + MIOPEN_INTERNALS_EXPORT bool StoreRecord(const DbRecord& record); /// Stores provided record in database. If record with same key is already in database it is /// updated with values from provided record. Provided records data is also updated via /// DbRecord::Merge(). /// /// Returns true if update was successful, false otherwise. - bool UpdateRecord(DbRecord& record); + MIOPEN_INTERNALS_EXPORT bool UpdateRecord(DbRecord& record); /// Removes record with provided key from db /// /// Returns true if remove was successful, false otherwise. - bool RemoveRecord(const std::string& key); + MIOPEN_INTERNALS_EXPORT bool RemoveRecord(const std::string& key); /// Removes ID with associated VALUES from record with key PROBLEM_CONFIG from db. /// If payload of a record becomes empty after that, also removes the entire record @@ -92,7 +94,7 @@ class MIOPEN_INTERNALS_EXPORT PlainTextDb return Remove(key, id); } - bool Remove(const std::string& key, const std::string& id); + MIOPEN_INTERNALS_EXPORT bool Remove(const std::string& key, const std::string& id); template inline bool RemoveRecord(const T& problem_config) @@ -340,14 +342,17 @@ class DbTimer template static auto Measure(const std::string& funcName, TFunc&& func) { - if(!miopen::IsLogging(LoggingLevel::Info2)) - return func(); - - const auto start = std::chrono::high_resolution_clock::now(); - auto ret = func(); - const auto end = std::chrono::high_resolution_clock::now(); - MIOPEN_LOG_I2("Db::" << funcName << " time: " << (end - start).count() * .000001f << " ms"); - return ret; + const bool logging = miopen::IsLogging(LoggingLevel::Info2); + const auto start = logging ? std::chrono::high_resolution_clock::now() + : std::chrono::high_resolution_clock::time_point{}; + auto ret = func(); + if(logging) + { + const auto end = std::chrono::high_resolution_clock::now(); + MIOPEN_LOG_I2("Db::" << funcName << " time: " << (end - start).count() * .000001f + << " ms"); + } + return std::move(ret); // NOLINT(clang-analyzer-cplusplus.Move) } }; } // namespace miopen diff --git a/projects/miopen/src/include/miopen/db_path.hpp b/projects/miopen/src/include/miopen/db_path.hpp index 14dfdceaa263..0e1b45a100c6 100644 --- a/projects/miopen/src/include/miopen/db_path.hpp +++ b/projects/miopen/src/include/miopen/db_path.hpp @@ -35,7 +35,7 @@ namespace miopen { MIOPEN_INTERNALS_EXPORT fs::path GetSystemDbPath(); MIOPEN_INTERNALS_EXPORT const fs::path& GetUserDbPath(); MIOPEN_INTERNALS_EXPORT std::string GetUserDbSuffix(); -MIOPEN_INTERNALS_EXPORT std::string GetSystemFindDbSuffix(); +std::string GetSystemFindDbSuffix(); #ifdef MIOPEN_BUILD_TESTING namespace testing { diff --git a/projects/miopen/src/include/miopen/db_record.hpp b/projects/miopen/src/include/miopen/db_record.hpp index edeb202cd046..9ccaef49ca59 100644 --- a/projects/miopen/src/include/miopen/db_record.hpp +++ b/projects/miopen/src/include/miopen/db_record.hpp @@ -26,6 +26,7 @@ #ifndef GUARD_MIOPEN_DB_RECORD_HPP_ #define GUARD_MIOPEN_DB_RECORD_HPP_ +#include #include #include @@ -71,7 +72,7 @@ enum class DbKinds : std::uint8_t /// Upon construction, allows getting and modifying contents of a record (IDs and VALUES). /// /// All operations are MP- and MT-safe. -class MIOPEN_INTERNALS_EXPORT DbRecord +class DbRecord { public: template @@ -196,11 +197,11 @@ class MIOPEN_INTERNALS_EXPORT DbRecord return ss.str(); } - bool ParseContents(std::istream& contents); + MIOPEN_INTERNALS_EXPORT bool ParseContents(std::istream& contents); void WriteContents(std::ostream& stream) const; void WriteIdsAndValues(std::ostream& stream) const; - bool SetValues(const std::string& id, const std::string& values); - bool GetValues(const std::string& id, std::string& values) const; + MIOPEN_INTERNALS_EXPORT bool SetValues(const std::string& id, const std::string& values); + MIOPEN_INTERNALS_EXPORT bool GetValues(const std::string& id, std::string& values) const; DbRecord(const std::string& key_) : key(key_) {} @@ -232,7 +233,7 @@ class MIOPEN_INTERNALS_EXPORT DbRecord /// E.g. this = {ID1:VALUE1} /// that = {ID1:VALUE3, ID2:VALUE2} /// this.Merge(that) = {ID1:VALUE1, ID2:VALUE2} - void Merge(const DbRecord& that); + MIOPEN_INTERNALS_EXPORT void Merge(const DbRecord& that); /// Obtains VALUES from an object of class T and sets it in record (in association with ID, /// under the current KEY). diff --git a/projects/miopen/src/include/miopen/dropout.hpp b/projects/miopen/src/include/miopen/dropout.hpp index 4effed166581..588b5ad334b5 100644 --- a/projects/miopen/src/include/miopen/dropout.hpp +++ b/projects/miopen/src/include/miopen/dropout.hpp @@ -54,9 +54,9 @@ using prngStates = xorwowStates; namespace miopen { -struct MIOPEN_INTERNALS_EXPORT DropoutDescriptor : miopenDropoutDescriptor +struct DropoutDescriptor : miopenDropoutDescriptor { - DropoutDescriptor(); + MIOPEN_INTERNALS_EXPORT DropoutDescriptor(); float dropout; Data_t pstates; @@ -68,23 +68,23 @@ struct MIOPEN_INTERNALS_EXPORT DropoutDescriptor : miopenDropoutDescriptor miopenDataType_t dataType_; - void InitPRNGState(const Handle& handle, - Data_t prng_states, - size_t prng_stateSizeInBytes, - unsigned long long prng_seed) const; + MIOPEN_INTERNALS_EXPORT void InitPRNGState(const Handle& handle, + Data_t prng_states, + size_t prng_stateSizeInBytes, + unsigned long long prng_seed) const; - void Dropout(const Handle& handle, - const TensorDescriptor& noise_shape, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - Data_t reserveSpace, - size_t reserveSpaceSizeInBytes, - size_t in_offset = 0, - size_t out_offset = 0, - size_t rsvsp_offset = 0, - bool is_backward = false) const; + MIOPEN_INTERNALS_EXPORT void Dropout(const Handle& handle, + const TensorDescriptor& noise_shape, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + Data_t reserveSpace, + size_t reserveSpaceSizeInBytes, + size_t in_offset = 0, + size_t out_offset = 0, + size_t rsvsp_offset = 0, + bool is_backward = false) const; }; std::ostream& operator<<(std::ostream& stream, const DropoutDescriptor& x); diff --git a/projects/miopen/src/include/miopen/env.hpp b/projects/miopen/src/include/miopen/env.hpp index 99c9eeb80051..81a08d7bd7aa 100644 --- a/projects/miopen/src/include/miopen/env.hpp +++ b/projects/miopen/src/include/miopen/env.hpp @@ -178,31 +178,31 @@ inline bool enabled(const detail::EnvVar& t) { return t.template value inline U value(T t) { - return std::forward(t).ref().template value(); + return t.ref().template value(); } template inline U value_or(T t, U k) { - return std::forward(t).ref().template value(std::forward(k)); + return t.ref().template value(k); } template inline std::string name(T t) { - return std::string{std::forward(t).ref().name()}; + return std::string{t.ref().name()}; } template void clear(T t) { - std::forward(t).ref().clear(); + t.ref().clear(); } template inline void update(T t, U k) { - std::forward(t).ref().template update(std::forward(k)); + t.ref().template update(k); } } // namespace miopen::env diff --git a/projects/miopen/src/include/miopen/errors.hpp b/projects/miopen/src/include/miopen/errors.hpp index 9538f10c640a..c435a4c344c4 100644 --- a/projects/miopen/src/include/miopen/errors.hpp +++ b/projects/miopen/src/include/miopen/errors.hpp @@ -65,7 +65,6 @@ template [[noreturn]] void MIOpenThrow(const std::string& file, int line, Params&&... args) { auto exe = miopen::Exception(std::forward(args)...); - MIOPEN_LOG_E_FROM(file + ":" + std::to_string(line), exe.message); throw exe.SetContext(file, line); } diff --git a/projects/miopen/src/include/miopen/execution_context.hpp b/projects/miopen/src/include/miopen/execution_context.hpp index 3e4676c0506b..79e4efccf698 100644 --- a/projects/miopen/src/include/miopen/execution_context.hpp +++ b/projects/miopen/src/include/miopen/execution_context.hpp @@ -77,7 +77,7 @@ MIOPEN_EXPORT extern bool } // namespace debug -struct MIOPEN_INTERNALS_EXPORT ExecutionContext +struct ExecutionContext { // Solution-specific std::string general_compile_options; @@ -297,7 +297,7 @@ struct MIOPEN_INTERNALS_EXPORT ExecutionContext private: const Handle* stream = nullptr; - void DetectRocm(); + MIOPEN_INTERNALS_EXPORT void DetectRocm(); }; struct [[deprecated]] ConvolutionContext : ExecutionContext diff --git a/projects/miopen/src/include/miopen/fin/fin_interface.hpp b/projects/miopen/src/include/miopen/fin/fin_interface.hpp index b816a3038e8f..cd9d450824a8 100644 --- a/projects/miopen/src/include/miopen/fin/fin_interface.hpp +++ b/projects/miopen/src/include/miopen/fin/fin_interface.hpp @@ -58,7 +58,7 @@ namespace fin_interface { // ================== Interface for Fin ================== // Base classes for solvers. -class MIOPEN_INTERNALS_EXPORT Solver +class Solver { public: // GetId(), IsDynamic() and IsTunable() throw miopenStatusNotInitialized if the solver is not @@ -124,7 +124,7 @@ extern template class SolverMixin; // Convolution solver -class MIOPEN_INTERNALS_EXPORT ConvSolver final +class ConvSolver final : public SolverMixin { public: @@ -146,7 +146,7 @@ class MIOPEN_INTERNALS_EXPORT ConvSolver final }; // Batch normalization solver -class MIOPEN_INTERNALS_EXPORT BatchNormSolver final +class BatchNormSolver final : public SolverMixin { protected: @@ -162,16 +162,14 @@ class MIOPEN_INTERNALS_EXPORT BatchNormSolver final // dummy if a solver with specified name does not exist. // Convolution -MIOPEN_INTERNALS_EXPORT const std::vector& GetAllConvSolvers(); -MIOPEN_INTERNALS_EXPORT std::vector -GetConvSolvers(const std::vector& names); -MIOPEN_INTERNALS_EXPORT ConvSolver GetConvSolver(const std::string& name); +const std::vector& GetAllConvSolvers(); +std::vector GetConvSolvers(const std::vector& names); +ConvSolver GetConvSolver(const std::string& name); // Batch normalization -MIOPEN_INTERNALS_EXPORT const std::vector& GetAllBatchNormSolvers(); -MIOPEN_INTERNALS_EXPORT std::vector -GetBatchNormSolvers(const std::vector& names); -MIOPEN_INTERNALS_EXPORT BatchNormSolver GetBatchNormSolver(const std::string& name); +const std::vector& GetAllBatchNormSolvers(); +std::vector GetBatchNormSolvers(const std::vector& names); +BatchNormSolver GetBatchNormSolver(const std::string& name); // Examples: // diff --git a/projects/miopen/src/include/miopen/find_controls.hpp b/projects/miopen/src/include/miopen/find_controls.hpp index 4766d0aeaf8c..75dcd4c559ba 100644 --- a/projects/miopen/src/include/miopen/find_controls.hpp +++ b/projects/miopen/src/include/miopen/find_controls.hpp @@ -60,7 +60,7 @@ enum class FindEnforceAction EnforcedLast_ = DbClean, }; -class MIOPEN_INTERNALS_EXPORT FindEnforce +class FindEnforce { FindEnforceAction action; @@ -72,7 +72,7 @@ class MIOPEN_INTERNALS_EXPORT FindEnforce } public: - FindEnforce(); + MIOPEN_INTERNALS_EXPORT FindEnforce(); explicit FindEnforce(FindEnforceAction action_) : action(action_) {} template @@ -107,7 +107,7 @@ class MIOPEN_INTERNALS_EXPORT FindEnforce MIOPEN_INTERNALS_EXPORT std::optional> GetEnvFindOnlySolver(); -class MIOPEN_INTERNALS_EXPORT FindMode +class FindMode { public: enum class Values @@ -140,7 +140,7 @@ class MIOPEN_INTERNALS_EXPORT FindMode public: // Todo: remove default value of primitive - FindMode(solver::Primitive primitive = solver::Primitive::Convolution); + MIOPEN_INTERNALS_EXPORT FindMode(solver::Primitive primitive = solver::Primitive::Convolution); Values Get() const { return value; } void Set(Values const v) { value = v; } diff --git a/projects/miopen/src/include/miopen/fusion.hpp b/projects/miopen/src/include/miopen/fusion.hpp index bb02cae0e225..b1d2bc7b9f5f 100644 --- a/projects/miopen/src/include/miopen/fusion.hpp +++ b/projects/miopen/src/include/miopen/fusion.hpp @@ -78,7 +78,7 @@ struct MIOPEN_INTERNALS_EXPORT BiasFusionOpDescriptor : FusionOpDescriptor TensorDescriptor base_desc; }; -struct MIOPEN_INTERNALS_EXPORT TensorScaleAddOpDescriptor : public FusionOpDescriptor +struct TensorScaleAddOpDescriptor : public FusionOpDescriptor { TensorScaleAddOpDescriptor(const TensorDescriptor& desc) : tensor_desc(desc) {} miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; @@ -103,7 +103,7 @@ struct MIOPEN_INTERNALS_EXPORT ActivFwdFusionOpDescriptor : FusionOpDescriptor miopenActivationMode_t activMode; }; -struct MIOPEN_INTERNALS_EXPORT ActivBwdFusionOpDescriptor : FusionOpDescriptor +struct ActivBwdFusionOpDescriptor : FusionOpDescriptor { ActivBwdFusionOpDescriptor(miopenActivationMode_t mode) : activMode(mode) {} miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; diff --git a/projects/miopen/src/include/miopen/fusion/solvers.hpp b/projects/miopen/src/include/miopen/fusion/solvers.hpp index ec472b455a9f..fc3c8bc7c2c2 100644 --- a/projects/miopen/src/include/miopen/fusion/solvers.hpp +++ b/projects/miopen/src/include/miopen/fusion/solvers.hpp @@ -53,66 +53,62 @@ struct PerformanceConfigConvBiasActivAsm1x1U : conv::PerformanceConfigConvAsm1x1 : PerformanceConfigConvAsm1x1U(-1, -1, -1, -1, -1, -1, -1, -1, false) { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionContext& ctx, - const FusionDescription& problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& problem); + void HeuristicInit(const FusionContext& ctx, const FusionDescription& problem); + bool SetNextValue(const FusionDescription& problem); bool IsValid(const FusionContext&, const FusionDescription& problem) const { return IsValid(problem); } - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionDescription& problem) const; + bool IsValid(const FusionDescription& problem) const; }; -struct ConvBiasActivAsm1x1U : FusionTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvBiasActivAsm1x1U + : FusionTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, const FusionDescription& problem, const PerformanceConfigConvBiasActivAsm1x1U& /*config*/) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBiasActivAsm1x1U + PerformanceConfigConvBiasActivAsm1x1U GetDefaultPerformanceConfig(const FusionContext&, const FusionDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBiasActivAsm1x1U + PerformanceConfigConvBiasActivAsm1x1U Search(const FusionContext& context, const FusionDescription& problem, const AnyInvokeParams& invoke_params) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const FusionContext&, - const FusionDescription&, - const PerformanceConfigConvBiasActivAsm1x1U&) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; + bool IsValidPerformanceConfig(const FusionContext&, + const FusionDescription&, + const PerformanceConfigConvBiasActivAsm1x1U&) const override; + float GetWti(const FusionContext&, const FusionDescription&) const override; }; using PerformanceConfigConvOclDirectFwdFused = LegacyPerformanceConfig; -struct ConvOclDirectFwdFused final : FusionTunableSolver +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwdFused final + : FusionTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, - const FusionDescription& problem, - const PerformanceConfigConvOclDirectFwdFused&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclDirectFwdFused + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& problem, + const PerformanceConfigConvOclDirectFwdFused&) const override; + PerformanceConfigConvOclDirectFwdFused GetDefaultPerformanceConfig(const FusionContext&, const FusionDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclDirectFwdFused + PerformanceConfigConvOclDirectFwdFused Search(const FusionContext&, const FusionDescription&, const AnyInvokeParams& invoke_params) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const FusionContext&, - const FusionDescription&, - const PerformanceConfigConvOclDirectFwdFused&) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription& problem) const override; + bool IsValidPerformanceConfig(const FusionContext&, + const FusionDescription&, + const PerformanceConfigConvOclDirectFwdFused&) const override; + float GetWti(const FusionContext&, const FusionDescription& problem) const override; }; struct PerformanceConfigConvCKIgemmFwdBiasActivFused @@ -133,19 +129,17 @@ struct PerformanceConfigConvCKIgemmFwdBiasActivFused : PerformanceConfigConvCKIgemmFwdBiasActivFused(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; + void HeuristicInit(const FusionDescription& fdesc_problem); + bool SetNextValue(const FusionDescription& fdesc_problem); + bool IsValidValue() const; + bool IsValid(const FusionContext&, const FusionDescription& fdesc_problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvCKIgemmFwdBiasActivFused& other) const; + bool operator==(const PerformanceConfigConvCKIgemmFwdBiasActivFused& other) const; private: template @@ -154,7 +148,7 @@ struct PerformanceConfigConvCKIgemmFwdBiasActivFused bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; }; -struct ConvCKIgemmFwdBiasActivFused final +struct MIOPEN_INTERNALS_EXPORT ConvCKIgemmFwdBiasActivFused final : FusionTunableSolver { const std::string& SolverDbId() const override @@ -162,20 +156,20 @@ struct ConvCKIgemmFwdBiasActivFused final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmFwdBiasActivFused + PerformanceConfigConvCKIgemmFwdBiasActivFused GetDefaultPerformanceConfig(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmFwdBiasActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmFwdBiasActivFused + PerformanceConfigConvCKIgemmFwdBiasActivFused Search(const FusionContext& ctx, const FusionDescription& fdesc_problem, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const FusionContext& ctx, + const FusionDescription& fdesc_problem) const override; + ConvSolution GetSolution(const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmFwdBiasActivFused& config) const override; @@ -203,19 +197,17 @@ struct PerformanceConfigConvCKIgemmGrpFwdActivFused : PerformanceConfigConvCKIgemmGrpFwdActivFused(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; + void HeuristicInit(const FusionDescription& fdesc_problem); + bool SetNextValue(const FusionDescription& fdesc_problem); + bool IsValidValue() const; + bool IsValid(const FusionContext&, const FusionDescription& fdesc_problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvCKIgemmGrpFwdActivFused& other) const; + bool operator==(const PerformanceConfigConvCKIgemmGrpFwdActivFused& other) const; private: template @@ -224,7 +216,7 @@ struct PerformanceConfigConvCKIgemmGrpFwdActivFused bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; }; -struct ConvCKIgemmGrpFwdActivFused final +struct MIOPEN_INTERNALS_EXPORT ConvCKIgemmGrpFwdActivFused final : FusionTunableSolver { const std::string& SolverDbId() const override @@ -232,26 +224,25 @@ struct ConvCKIgemmGrpFwdActivFused final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmGrpFwdActivFused + PerformanceConfigConvCKIgemmGrpFwdActivFused GetDefaultPerformanceConfig(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmGrpFwdActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmGrpFwdActivFused + PerformanceConfigConvCKIgemmGrpFwdActivFused Search(const FusionContext& ctx, const FusionDescription& fdesc_problem, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const FusionContext& ctx, + const FusionDescription& fdesc_problem) const override; + ConvSolution GetSolution(const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmGrpFwdActivFused& config) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize(const FusionContext&, - const FusionDescription&) const override; + size_t GetWorkspaceSize(const FusionContext&, const FusionDescription&) const override; private: template @@ -276,19 +267,17 @@ struct PerformanceConfigConvCKIgemmGrpFwdBiasActivFused : PerformanceConfigConvCKIgemmGrpFwdBiasActivFused(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; + void HeuristicInit(const FusionDescription& fdesc_problem); + bool SetNextValue(const FusionDescription& fdesc_problem); + bool IsValidValue() const; + bool IsValid(const FusionContext&, const FusionDescription& fdesc_problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvCKIgemmGrpFwdBiasActivFused& other) const; + bool operator==(const PerformanceConfigConvCKIgemmGrpFwdBiasActivFused& other) const; private: template @@ -297,7 +286,7 @@ struct PerformanceConfigConvCKIgemmGrpFwdBiasActivFused bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; }; -struct ConvCKIgemmGrpFwdBiasActivFused final +struct MIOPEN_INTERNALS_EXPORT ConvCKIgemmGrpFwdBiasActivFused final : FusionTunableSolver { const std::string& SolverDbId() const override @@ -305,26 +294,25 @@ struct ConvCKIgemmGrpFwdBiasActivFused final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmGrpFwdBiasActivFused + PerformanceConfigConvCKIgemmGrpFwdBiasActivFused GetDefaultPerformanceConfig(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmGrpFwdBiasActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmGrpFwdBiasActivFused + PerformanceConfigConvCKIgemmGrpFwdBiasActivFused Search(const FusionContext& ctx, const FusionDescription& fdesc_problem, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const FusionContext& ctx, + const FusionDescription& fdesc_problem) const override; + ConvSolution GetSolution(const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerformanceConfigConvCKIgemmGrpFwdBiasActivFused& config) const override; bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize(const FusionContext&, - const FusionDescription&) const override; + size_t GetWorkspaceSize(const FusionContext&, const FusionDescription&) const override; private: template @@ -349,19 +337,17 @@ struct PerfConfigConvCKIgemmFwdBiasResAddActivFused : PerfConfigConvCKIgemmFwdBiasResAddActivFused(0, "") { } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; + void HeuristicInit(const FusionDescription& fdesc_problem); + bool SetNextValue(const FusionDescription& fdesc_problem); + bool IsValidValue() const; + bool IsValid(const FusionContext&, const FusionDescription& fdesc_problem) const; template static void Visit(Self&& s, F f) { f(s.kernel_id, "kernel_id"); } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerfConfigConvCKIgemmFwdBiasResAddActivFused& other) const; + bool operator==(const PerfConfigConvCKIgemmFwdBiasResAddActivFused& other) const; private: template @@ -378,20 +364,20 @@ struct ConvCKIgemmFwdBiasResAddActivFused final return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT PerfConfigConvCKIgemmFwdBiasResAddActivFused + PerfConfigConvCKIgemmFwdBiasResAddActivFused GetDefaultPerformanceConfig(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + bool IsValidPerformanceConfig( const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerfConfigConvCKIgemmFwdBiasResAddActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerfConfigConvCKIgemmFwdBiasResAddActivFused + PerfConfigConvCKIgemmFwdBiasResAddActivFused Search(const FusionContext& ctx, const FusionDescription& fdesc_problem, const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution + bool IsApplicable(const FusionContext& ctx, + const FusionDescription& fdesc_problem) const override; + ConvSolution GetSolution(const FusionContext& ctx, const FusionDescription& fdesc_problem, const PerfConfigConvCKIgemmFwdBiasResAddActivFused& config) const override; @@ -401,35 +387,32 @@ struct ConvCKIgemmFwdBiasResAddActivFused final bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; }; -struct ConvBinWinogradRxSFused final : FusionSolverBase +struct MIOPEN_INTERNALS_EXPORT ConvBinWinogradRxSFused final : FusionSolverBase { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& context, - const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution( - const FusionContext& context, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; + bool IsApplicable(const FusionContext& context, + const FusionDescription& fdesc_problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& fdesc_problem) const override; + float GetWti(const FusionContext&, const FusionDescription&) const override; }; -struct ConvBinWinogradRxSf2x3g1Fused final : FusionSolverBase +struct MIOPEN_INTERNALS_EXPORT ConvBinWinogradRxSf2x3g1Fused final : FusionSolverBase { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& problem) const override; + float GetWti(const FusionContext&, const FusionDescription&) const override; }; template @@ -468,21 +451,21 @@ struct ConvWinoRageRxSFused final : FusionSolverBase ConvSolution GetSolution(const FusionContext&, const FusionDescription&) const override; }; -#ifndef CONV_WINO_FURY_RXS_CPP +#ifndef CONV_WINO_RAGE_RXS_CPP extern template struct ConvWinoRageRxSFused<2, 3>; #endif -struct BnFwdInferActivationFused final : FusionSolverBase +struct MIOPEN_INTERNALS_EXPORT BnFwdInferActivationFused final : FusionSolverBase { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& problem) const override; }; struct BnFwdTrgActivationFused final : FusionSolverBase @@ -492,10 +475,10 @@ struct BnFwdTrgActivationFused final : FusionSolverBase return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& problem) const override; }; struct BnBwdTrgActivationFused final : FusionSolverBase @@ -505,10 +488,10 @@ struct BnBwdTrgActivationFused final : FusionSolverBase return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; + bool IsApplicable(const FusionContext& context, + const FusionDescription& problem) const override; + ConvSolution GetSolution(const FusionContext& context, + const FusionDescription& problem) const override; }; } // namespace fusion diff --git a/projects/miopen/src/include/miopen/fusion_plan.hpp b/projects/miopen/src/include/miopen/fusion_plan.hpp index 07e2ab71407a..6eae25d321d6 100644 --- a/projects/miopen/src/include/miopen/fusion_plan.hpp +++ b/projects/miopen/src/include/miopen/fusion_plan.hpp @@ -44,12 +44,13 @@ struct Exec_arg_t }; struct FusionContext; -struct MIOPEN_INTERNALS_EXPORT FusionPlanDescriptor : miopenFusionPlanDescriptor +struct FusionPlanDescriptor : miopenFusionPlanDescriptor { FusionPlanDescriptor() {} - FusionPlanDescriptor(miopenFusionDirection_t dir, const TensorDescriptor& inDesc); + MIOPEN_INTERNALS_EXPORT FusionPlanDescriptor(miopenFusionDirection_t dir, + const TensorDescriptor& inDesc); bool isValid() const { return is_valid; }; - miopenStatus_t AddOp(std::shared_ptr desc); + MIOPEN_INTERNALS_EXPORT miopenStatus_t AddOp(std::shared_ptr desc); TensorDescriptor DeriveOutputDescriptor(); miopenStatus_t GetWorkspaceSizeImmed(const Handle& handle, size_t& workSpaceSize, @@ -62,7 +63,7 @@ struct MIOPEN_INTERNALS_EXPORT FusionPlanDescriptor : miopenFusionPlanDescriptor const OperatorArgs& op_args, Data_t workspace, size_t workspace_size); - miopenStatus_t Compile(const Handle& handle); + MIOPEN_INTERNALS_EXPORT miopenStatus_t Compile(const Handle& handle); std::vector Find(const Handle& handle, const std::function& invoke_params, diff --git a/projects/miopen/src/include/miopen/general_tensor_reorder_sol.hpp b/projects/miopen/src/include/miopen/general_tensor_reorder_sol.hpp index 723b14115f49..1fefb10e286b 100644 --- a/projects/miopen/src/include/miopen/general_tensor_reorder_sol.hpp +++ b/projects/miopen/src/include/miopen/general_tensor_reorder_sol.hpp @@ -26,6 +26,7 @@ #ifndef GUARD_GENERAL_MIOPEN_TENSOR_REORDER_SOL_HPP #define GUARD_GENERAL_MIOPEN_TENSOR_REORDER_SOL_HPP +#include #include #include #include diff --git a/projects/miopen/src/include/miopen/generic_search.hpp b/projects/miopen/src/include/miopen/generic_search.hpp index 77e0eb7dda07..9c7b571ad2a8 100644 --- a/projects/miopen/src/include/miopen/generic_search.hpp +++ b/projects/miopen/src/include/miopen/generic_search.hpp @@ -441,7 +441,7 @@ auto GenericSearch(const Solver s, const auto invoke_ctx = [invoke_ctx_]() { auto copy = invoke_ctx_; copy.SetInvokeType(InvokeType::AutoTune); - return copy; + return std::move(copy); }(); // list of sampled solutions @@ -524,6 +524,8 @@ auto GenericSearch(const Solver s, size_t last_imprv = 0; auto threads_remaining = total_threads; std::vector samples; + float tuning_tolerance = + 1.0f + static_cast(env::value(MIOPEN_TUNING_FOLLOWUP_TOLERANCE_PCT)) / 100.0f; while(true) { if(n_current >= n_runs_total) @@ -616,19 +618,18 @@ auto GenericSearch(const Solver s, } // Smooth the jitter of measurements: - // If the 1st probe is NOT too bad (measured time <= 1.10 * worst sample of the best - // config), then gather 9 more samples, and remove positive z-score outliers. Use - // the mean value with outliers removed for calculating best config. - constexpr int N_RUNS = 10; + // If the 1st probe is NOT too bad (measured time <= tuning_tolerance * worst sample + // of the best config), then gather more samples, and remove positive z-score + // outliers. Use the mean value with outliers removed for calculating best config. last_imprv++; - if(elapsed_time < worst_time * 1.10f && elapsed_time < skip_time) + if(elapsed_time < worst_time * tuning_tolerance && elapsed_time < skip_time) { MIOPEN_LOG_I2("Finding average for: " << elapsed_time << " / " << best_time << " = " << (elapsed_time / best_time)); try { - for(int i = 1; i < N_RUNS; ++i) + for(int i = 1; i < env::value(MIOPEN_TUNING_ITERATIONS); ++i) { invoker(profile_h, invoke_ctx); samples.push_back(profile_h.GetKernelTime()); @@ -711,7 +712,8 @@ auto GenericSearch(const Solver s, { MIOPEN_LOG_I( "Search cutoff or skipped for all kernels. Last config returned: " << last_config); - return last_config; + best_config = std::move(last_config); + return best_config; } if(!is_passed) diff --git a/projects/miopen/src/include/miopen/generic_search_controls.hpp b/projects/miopen/src/include/miopen/generic_search_controls.hpp index a515f27fd3ee..9f70852ca1ac 100644 --- a/projects/miopen/src/include/miopen/generic_search_controls.hpp +++ b/projects/miopen/src/include/miopen/generic_search_controls.hpp @@ -27,7 +27,6 @@ #pragma once #include #include -#include #include #include @@ -42,6 +41,10 @@ MIOPEN_DECLARE_ENV_VAR_UINT64( MIOPEN_TUNING_PATIENCE, MIOPEN_DEFAULT_TUNING_PATIENCE) // End tuning if no improvement in X iterations +MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_TUNING_FOLLOWUP_TOLERANCE_PCT, 10) + +MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_TUNING_ITERATIONS, 10) + #if MIOPEN_USE_COMGR MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_COMPILE_PARALLEL_LEVEL, 1) // COMGR is not parallelizable #else diff --git a/projects/miopen/src/include/miopen/graphapi/convolution.hpp b/projects/miopen/src/include/miopen/graphapi/convolution.hpp index e1831ef4a9a5..cfb0690437c7 100644 --- a/projects/miopen/src/include/miopen/graphapi/convolution.hpp +++ b/projects/miopen/src/include/miopen/graphapi/convolution.hpp @@ -186,7 +186,7 @@ class MIOPEN_INTERNALS_EXPORT ConvolutionBuilder Convolution build() &&; private: - bool validate() const; + MIOPEN_INTERNALS_NO_EXPORT bool validate() const; }; class MIOPEN_INTERNALS_EXPORT BackendConvolutionDescriptor : public BackendDescriptor @@ -410,7 +410,7 @@ class OperationConvolutionForwardBuilder : public OperationConvolutionBuilder } }; -class MIOPEN_INTERNALS_EXPORT BackendOperationConvolutionDescriptor : public BackendDescriptor +class BackendOperationConvolutionDescriptor : public BackendDescriptor { protected: miopenBackendDescriptor_t mConvolutionDescriptor = nullptr; @@ -462,8 +462,7 @@ class MIOPEN_INTERNALS_EXPORT BackendOperationConvolutionDescriptor : public Bac void* arrayOfElements); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationConvolutionForwardDescriptor - : public BackendOperationConvolutionDescriptor +class BackendOperationConvolutionForwardDescriptor : public BackendOperationConvolutionDescriptor { private: OperationConvolutionForwardBuilder mBuilder; @@ -554,7 +553,7 @@ class OperationConvolutionBackwardDataBuilder : public OperationConvolutionBuild } }; -class MIOPEN_INTERNALS_EXPORT BackendOperationConvolutionBackwardDataDescriptor +class BackendOperationConvolutionBackwardDataDescriptor : public BackendOperationConvolutionDescriptor { private: @@ -646,7 +645,7 @@ class OperationConvolutionBackwardFilterBuilder : public OperationConvolutionBui } }; -class MIOPEN_INTERNALS_EXPORT BackendOperationConvolutionBackwardFilterDescriptor +class BackendOperationConvolutionBackwardFilterDescriptor : public BackendOperationConvolutionDescriptor { private: diff --git a/projects/miopen/src/include/miopen/graphapi/engine.hpp b/projects/miopen/src/include/miopen/graphapi/engine.hpp index 0c2f25bfaf43..7cc7058452bc 100644 --- a/projects/miopen/src/include/miopen/graphapi/engine.hpp +++ b/projects/miopen/src/include/miopen/graphapi/engine.hpp @@ -43,7 +43,7 @@ class Engine; class OpGraph; // Pattern is a family of solvers for the same graph shape -class MIOPEN_INTERNALS_EXPORT GraphPatternMatcher +class GraphPatternMatcher { public: @@ -182,7 +182,7 @@ class MIOPEN_INTERNALS_EXPORT EngineBuilder EngineBuilder& setGlobalIndex(int64_t globalIndex); - EngineBuilder& setSmCount(int32_t smCount); + MIOPEN_INTERNALS_NO_EXPORT EngineBuilder& setSmCount(int32_t smCount); EngineBuilder& setExecutor(const std::shared_ptr& e); diff --git a/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp b/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp index e9a426f80524..ff71f0ba3ee3 100644 --- a/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp +++ b/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp @@ -73,7 +73,7 @@ class EngineCfg /* For now we don't support tuning and a builder is not needed, * but in future it will be needed. */ -class MIOPEN_INTERNALS_EXPORT EngineCfgBuilder +class EngineCfgBuilder { EngineCfg mEngineCfg; bool mEngineSet = false; @@ -97,7 +97,7 @@ class MIOPEN_INTERNALS_EXPORT EngineCfgBuilder return std::move(setEngine(std::move(engine))); } EngineCfg build() &; - EngineCfg build() &&; + MIOPEN_INTERNALS_EXPORT EngineCfg build() &&; }; class MIOPEN_INTERNALS_EXPORT BackendEngineCfgDescriptor : public BackendDescriptor diff --git a/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp b/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp index f8935612fedc..629ac1f061e8 100644 --- a/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp +++ b/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp @@ -38,7 +38,7 @@ namespace miopen { namespace graphapi { -class MIOPEN_INTERNALS_EXPORT ExecutionPlan +class ExecutionPlan { private: /* we don't use a pointer for mEngineCfg @@ -62,7 +62,7 @@ class MIOPEN_INTERNALS_EXPORT ExecutionPlan const EngineCfg& getEngineCfg() const noexcept { return mEngineCfg; } EngineCfg& getEngineCfg() noexcept { return mEngineCfg; } const std::vector& getIntermediateIds() const noexcept { return mIntermediateIds; } - std::string getJsonRepresentation() const; + MIOPEN_INTERNALS_EXPORT std::string getJsonRepresentation() const; void execute(miopenHandle_t handle, const VariantPack& variantPack) { @@ -160,8 +160,9 @@ class MIOPEN_INTERNALS_EXPORT ExecutionPlanBuilder public: ExecutionPlanBuilder& setHandle(miopenHandle_t handle) &; ExecutionPlanBuilder& setEngineCfg(const EngineCfg& engineCfg) &; - ExecutionPlanBuilder& setEngineCfg(EngineCfg&& engineCfg) &; - ExecutionPlanBuilder& setIntermediateIds(const std::vector& ids) &; + MIOPEN_INTERNALS_NO_EXPORT ExecutionPlanBuilder& setEngineCfg(EngineCfg&& engineCfg) &; + MIOPEN_INTERNALS_NO_EXPORT ExecutionPlanBuilder& + setIntermediateIds(const std::vector& ids) &; ExecutionPlanBuilder& setIntermediateIds(std::vector&& ids) &; ExecutionPlanBuilder& setJsonRepresentation(const std::string_view& s) &; @@ -190,11 +191,11 @@ class MIOPEN_INTERNALS_EXPORT ExecutionPlanBuilder return std::move(setJsonRepresentation(s)); } - ExecutionPlan build() &; + MIOPEN_INTERNALS_NO_EXPORT ExecutionPlan build() &; ExecutionPlan build() &&; }; -class MIOPEN_INTERNALS_EXPORT BackendExecutionPlanDescriptor : public BackendDescriptor +class BackendExecutionPlanDescriptor : public BackendDescriptor { private: ExecutionPlanBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/matmul.hpp b/projects/miopen/src/include/miopen/graphapi/matmul.hpp index 7215e9d346eb..ee07d2d8ae54 100644 --- a/projects/miopen/src/include/miopen/graphapi/matmul.hpp +++ b/projects/miopen/src/include/miopen/graphapi/matmul.hpp @@ -45,7 +45,7 @@ class Matmul friend class MatmulBuilder; }; -class MIOPEN_INTERNALS_EXPORT MatmulBuilder +class MatmulBuilder { private: @@ -60,7 +60,7 @@ class MIOPEN_INTERNALS_EXPORT MatmulBuilder return *this; } - Matmul build() const; + MIOPEN_INTERNALS_EXPORT Matmul build() const; }; class MIOPEN_INTERNALS_EXPORT BackendMatmulDescriptor : public BackendDescriptor @@ -167,7 +167,7 @@ class MIOPEN_INTERNALS_EXPORT OperationMatmulBuilder OperationMatmul build(); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationMatmulDescriptor : public BackendDescriptor +class BackendOperationMatmulDescriptor : public BackendDescriptor { private: OperationMatmulBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/opgraph.hpp b/projects/miopen/src/include/miopen/graphapi/opgraph.hpp index 771795899474..c75c0551c064 100644 --- a/projects/miopen/src/include/miopen/graphapi/opgraph.hpp +++ b/projects/miopen/src/include/miopen/graphapi/opgraph.hpp @@ -158,7 +158,7 @@ class MIOPEN_INTERNALS_EXPORT OpNode using Edge = OpNode::Edge; -class MIOPEN_INTERNALS_EXPORT SourceOpNode : public OpNode +class SourceOpNode : public OpNode { protected: std::vector mOutTensors; @@ -188,7 +188,7 @@ class MIOPEN_INTERNALS_EXPORT SourceOpNode : public OpNode } }; -class MIOPEN_INTERNALS_EXPORT SinkOpNode : public OpNode +class SinkOpNode : public OpNode { protected: std::vector mInTensors; @@ -220,7 +220,7 @@ using VecOfPaths = std::vector; class Engine; -class MIOPEN_INTERNALS_EXPORT OpGraph +class OpGraph { // NOTE: mSrcNode and mSinkNode need to reside on the heap because the graph may move // to a new memory location after building, while the nodes maintain address @@ -391,7 +391,7 @@ class MIOPEN_INTERNALS_EXPORT OpGraph } }; -class MIOPEN_INTERNALS_EXPORT OpGraphBuilder +class OpGraphBuilder { private: std::vector mNodes; @@ -428,12 +428,12 @@ class MIOPEN_INTERNALS_EXPORT OpGraphBuilder }; // r-value method that consumes *this - OpGraph build() &&; + MIOPEN_INTERNALS_EXPORT OpGraph build() &&; }; MIOPEN_INTERNALS_EXPORT bool isIsomorphic(const OpGraph& left, const OpGraph& right); -MIOPEN_INTERNALS_EXPORT std::string pathToStr(const Path& path); +std::string pathToStr(const Path& path); class MIOPEN_INTERNALS_EXPORT BackendOperationGraphDescriptor : public BackendDescriptor { diff --git a/projects/miopen/src/include/miopen/graphapi/pointwise.hpp b/projects/miopen/src/include/miopen/graphapi/pointwise.hpp index c92db8d03747..343a096a3e30 100644 --- a/projects/miopen/src/include/miopen/graphapi/pointwise.hpp +++ b/projects/miopen/src/include/miopen/graphapi/pointwise.hpp @@ -96,7 +96,7 @@ class Pointwise friend class PointwiseBuilder; }; -class MIOPEN_INTERNALS_EXPORT PointwiseBuilder +class PointwiseBuilder { private: Pointwise mPointwise; @@ -157,7 +157,7 @@ class MIOPEN_INTERNALS_EXPORT PointwiseBuilder return *this; } - Pointwise build(); + MIOPEN_INTERNALS_EXPORT Pointwise build(); }; class MIOPEN_INTERNALS_EXPORT BackendPointwiseDescriptor : public BackendDescriptor @@ -274,7 +274,7 @@ class MIOPEN_INTERNALS_EXPORT OperationPointwiseBuilder OperationPointwise build(); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationPointwiseDescriptor : public BackendDescriptor +class BackendOperationPointwiseDescriptor : public BackendDescriptor { private: OperationPointwiseBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/reduction.hpp b/projects/miopen/src/include/miopen/graphapi/reduction.hpp index 5323e16d5d3d..5720bc0c0de2 100644 --- a/projects/miopen/src/include/miopen/graphapi/reduction.hpp +++ b/projects/miopen/src/include/miopen/graphapi/reduction.hpp @@ -52,7 +52,7 @@ class Reduction miopenDataType_t getCompType() const { return mCompType; } }; -class MIOPEN_INTERNALS_EXPORT ReductionBuilder +class ReductionBuilder { private: Reduction mReduction; @@ -74,7 +74,7 @@ class MIOPEN_INTERNALS_EXPORT ReductionBuilder return *this; } - Reduction build(); + MIOPEN_INTERNALS_EXPORT Reduction build(); }; class MIOPEN_INTERNALS_EXPORT BackendReductionDescriptor : public BackendDescriptor @@ -136,7 +136,7 @@ class MIOPEN_INTERNALS_EXPORT OperationReductionBuilder OperationReduction build(); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationReductionDescriptor : public BackendDescriptor +class BackendOperationReductionDescriptor : public BackendDescriptor { private: OperationReductionBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/reshape.hpp b/projects/miopen/src/include/miopen/graphapi/reshape.hpp index bf4bcf0ab93d..3f097fe708b1 100644 --- a/projects/miopen/src/include/miopen/graphapi/reshape.hpp +++ b/projects/miopen/src/include/miopen/graphapi/reshape.hpp @@ -73,7 +73,7 @@ class MIOPEN_INTERNALS_EXPORT OperationReshapeBuilder OperationReshape build(); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationReshapeDescriptor : public BackendDescriptor +class BackendOperationReshapeDescriptor : public BackendDescriptor { private: OperationReshapeBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/rng.hpp b/projects/miopen/src/include/miopen/graphapi/rng.hpp index e56b43e5895c..1420240ec99b 100644 --- a/projects/miopen/src/include/miopen/graphapi/rng.hpp +++ b/projects/miopen/src/include/miopen/graphapi/rng.hpp @@ -178,7 +178,7 @@ class MIOPEN_INTERNALS_EXPORT OperationRngBuilder OperationRng build(); }; -class MIOPEN_INTERNALS_EXPORT BackendOperationRngDescriptor : public BackendDescriptor +class BackendOperationRngDescriptor : public BackendDescriptor { private: OperationRngBuilder mBuilder; diff --git a/projects/miopen/src/include/miopen/graphapi/tensor.hpp b/projects/miopen/src/include/miopen/graphapi/tensor.hpp index 1c83ef1708d8..f079de84b73a 100644 --- a/projects/miopen/src/include/miopen/graphapi/tensor.hpp +++ b/projects/miopen/src/include/miopen/graphapi/tensor.hpp @@ -107,9 +107,9 @@ class MIOPEN_INTERNALS_EXPORT TensorBuilder public: TensorBuilder& setDataType(miopenDataType_t dataType) &; TensorBuilder& setDim(const std::vector& dimensions) &; - TensorBuilder& setDim(std::vector&& dimensions) &; + MIOPEN_INTERNALS_NO_EXPORT TensorBuilder& setDim(std::vector&& dimensions) &; TensorBuilder& setStride(const std::vector& strides) &; - TensorBuilder& setStride(std::vector&& strides) &; + MIOPEN_INTERNALS_NO_EXPORT TensorBuilder& setStride(std::vector&& strides) &; TensorBuilder& setId(int64_t id) &; TensorBuilder& setVirtual(bool isVirtual) &; @@ -136,7 +136,7 @@ class MIOPEN_INTERNALS_EXPORT TensorBuilder TensorBuilder&& setId(int64_t id) && { return std::move(setId(id)); } TensorBuilder&& setVirtual(bool isVirtual) && { return std::move(setVirtual(isVirtual)); } - Tensor build() const&; + MIOPEN_INTERNALS_NO_EXPORT Tensor build() const&; Tensor build() &&; }; diff --git a/projects/miopen/src/include/miopen/groupnorm/solvers.hpp b/projects/miopen/src/include/miopen/groupnorm/solvers.hpp index 0bb6111fa468..70ede100d087 100644 --- a/projects/miopen/src/include/miopen/groupnorm/solvers.hpp +++ b/projects/miopen/src/include/miopen/groupnorm/solvers.hpp @@ -44,12 +44,10 @@ struct GroupNormForward final : NormalizationSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::groupnorm::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::groupnorm::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::groupnorm::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::groupnorm::ProblemDescription& problem) const override; }; } // namespace groupnorm diff --git a/projects/miopen/src/include/miopen/handle.hpp b/projects/miopen/src/include/miopen/handle.hpp index 2156b4bf15c7..6fa5d1782373 100644 --- a/projects/miopen/src/include/miopen/handle.hpp +++ b/projects/miopen/src/include/miopen/handle.hpp @@ -40,8 +40,6 @@ #include #include -#include - #include #include #include @@ -186,6 +184,7 @@ struct MIOPEN_EXPORT Handle : miopenHandle Allocator::ManageDataPtr Create(std::size_t sz) const; Allocator::ManageDataPtr& WriteTo(const void* data, Allocator::ManageDataPtr& ddata, std::size_t sz) const; + void WriteTo(const void* data, Data_t ddata, std::size_t sz) const; void ReadTo(void* data, const Allocator::ManageDataPtr& ddata, std::size_t sz) const; void ReadTo(void* data, ConstData_t ddata, std::size_t sz) const; shared CreateSubBuffer(Data_t data, std::size_t offset, std::size_t size) const; diff --git a/projects/miopen/src/include/miopen/hipoc_kernel.hpp b/projects/miopen/src/include/miopen/hipoc_kernel.hpp index c1c3fba2765e..ae21d5f39ba2 100644 --- a/projects/miopen/src/include/miopen/hipoc_kernel.hpp +++ b/projects/miopen/src/include/miopen/hipoc_kernel.hpp @@ -43,7 +43,11 @@ using HipEventPtr = MIOPEN_MANAGE_PTR(hipEvent_t, hipEventDestroy); inline HipEventPtr make_hip_event() { hipEvent_t result = nullptr; - hipEventCreate(&result); + + const auto status = hipEventCreate(&result); + if(status != hipSuccess) + MIOPEN_THROW_HIP_STATUS(status, "hipEventCreate failed"); + return HipEventPtr{result}; } @@ -119,7 +123,7 @@ struct KernelArgs uint64_t hidden[6] = {}; }; -struct MIOPEN_INTERNALS_EXPORT HIPOCKernelInvoke +struct HIPOCKernelInvoke { HIPOCKernelInvoke() {} HIPOCKernelInvoke(hipStream_t pstream, @@ -185,8 +189,8 @@ struct MIOPEN_INTERNALS_EXPORT HIPOCKernelInvoke const std::string& GetName() const { return name; } private: - void run(void* args, std::size_t size) const; - void run_cooperative(void** kern_args) const; + MIOPEN_INTERNALS_EXPORT void run(void* args, std::size_t size) const; + MIOPEN_INTERNALS_EXPORT void run_cooperative(void** kern_args) const; hipStream_t stream = nullptr; hipFunction_t fun = nullptr; @@ -197,7 +201,7 @@ struct MIOPEN_INTERNALS_EXPORT HIPOCKernelInvoke bool coop_launch; }; -struct MIOPEN_INTERNALS_EXPORT HIPOCKernel +struct HIPOCKernel { HIPOCProgram program; std::string name; diff --git a/projects/miopen/src/include/miopen/hipoc_program.hpp b/projects/miopen/src/include/miopen/hipoc_program.hpp index 31d10b5851ad..b8fe1c76c855 100644 --- a/projects/miopen/src/include/miopen/hipoc_program.hpp +++ b/projects/miopen/src/include/miopen/hipoc_program.hpp @@ -26,6 +26,7 @@ #ifndef GUARD_MIOPEN_HIPOC_PROGRAM_HPP #define GUARD_MIOPEN_HIPOC_PROGRAM_HPP +#include #include #include #include @@ -50,18 +51,19 @@ struct HIPOCProgram const TargetProperties& target, const std::string& kernel_src); HIPOCProgram(const fs::path& program_name, const fs::path& hsaco); - HIPOCProgram(const fs::path& program_name, const std::vector& hsaco); + MIOPEN_INTERNALS_EXPORT HIPOCProgram(const fs::path& program_name, + const std::vector& hsaco); HIPOCProgram(const fs::path& program_name, const std::vector& hsaco); std::shared_ptr impl; hipModule_t GetModule() const; /// \return Pathname of CO file, if it resides on the filesystem. /// This function should not be called after FreeCodeObjectFileStorage(). - fs::path GetCodeObjectPathname() const; + MIOPEN_INTERNALS_EXPORT fs::path GetCodeObjectPathname() const; /// \return In-memory CO blob. - const std::vector& GetCodeObjectBlob() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetCodeObjectBlob() const; /// \return True if CO blob resides in-memory. /// False if CO resides on filesystem. - bool IsCodeObjectInMemory() const; + MIOPEN_INTERNALS_EXPORT bool IsCodeObjectInMemory() const; bool IsCodeObjectInFile() const; bool IsCodeObjectInTempFile() const; void FreeCodeObjectFileStorage(); diff --git a/projects/miopen/src/include/miopen/invoke_params.hpp b/projects/miopen/src/include/miopen/invoke_params.hpp index 4c30e2c63f50..ce16c1e8e95a 100644 --- a/projects/miopen/src/include/miopen/invoke_params.hpp +++ b/projects/miopen/src/include/miopen/invoke_params.hpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -123,6 +124,12 @@ struct AnyInvokeParams return *reinterpret_cast(impl->GetRawPtr()); } + template + bool IsOfType() const + { + return impl && impl->CanCastTo(typeid(Actual)); + } + operator bool() const { return impl != nullptr; } private: diff --git a/projects/miopen/src/include/miopen/invoker_cache.hpp b/projects/miopen/src/include/miopen/invoker_cache.hpp index 4fd791efd0f3..a748696d146f 100644 --- a/projects/miopen/src/include/miopen/invoker_cache.hpp +++ b/projects/miopen/src/include/miopen/invoker_cache.hpp @@ -43,18 +43,18 @@ class InvokerCache // network_config, solver_id using Key = std::pair; - std::optional operator[](const Key& key) const; + MIOPEN_INTERNALS_EXPORT std::optional operator[](const Key& key) const; // For find 1.0 - std::optional GetFound1_0(const std::string& network_config, - const std::string& algorithm) const; + MIOPEN_INTERNALS_EXPORT std::optional GetFound1_0(const std::string& network_config, + const std::string& algorithm) const; std::optional GetFound1_0SolverId(const std::string& network_config, const std::string& algorithm) const; - void Register(const Key& key, const Invoker& invoker); + MIOPEN_INTERNALS_EXPORT void Register(const Key& key, const Invoker& invoker); // For find 1.0 - void SetAsFound1_0(const std::string& network_config, - const std::string& algorithm, - const std::string& solver_id); + MIOPEN_INTERNALS_EXPORT void SetAsFound1_0(const std::string& network_config, + const std::string& algorithm, + const std::string& solver_id); private: struct Item diff --git a/projects/miopen/src/include/miopen/kern_db.hpp b/projects/miopen/src/include/miopen/kern_db.hpp index 3ede092f55d8..37d8781bebda 100644 --- a/projects/miopen/src/include/miopen/kern_db.hpp +++ b/projects/miopen/src/include/miopen/kern_db.hpp @@ -34,8 +34,6 @@ #include #include -#include - #include #include #include diff --git a/projects/miopen/src/include/miopen/kernel_build_params.hpp b/projects/miopen/src/include/miopen/kernel_build_params.hpp index 822ac91e1a6a..6596cf20dfde 100644 --- a/projects/miopen/src/include/miopen/kernel_build_params.hpp +++ b/projects/miopen/src/include/miopen/kernel_build_params.hpp @@ -135,19 +135,22 @@ class KernelBuildParameters }; namespace kbp { -struct MIOPEN_INTERNALS_EXPORT OpenCL +struct OpenCL { - static std::string Generate(const std::vector& options); + MIOPEN_INTERNALS_EXPORT static std::string + Generate(const std::vector& options); }; -struct MIOPEN_INTERNALS_EXPORT GcnAsm +struct GcnAsm { - static std::string Generate(const std::vector& options); + MIOPEN_INTERNALS_EXPORT static std::string + Generate(const std::vector& options); }; -struct MIOPEN_INTERNALS_EXPORT HIP +struct HIP { - static std::string Generate(const std::vector& options); + MIOPEN_INTERNALS_EXPORT static std::string + Generate(const std::vector& options); }; } // namespace kbp diff --git a/projects/miopen/src/include/miopen/kernel_info.hpp b/projects/miopen/src/include/miopen/kernel_info.hpp index d2571afa3239..d0fad6bb329a 100644 --- a/projects/miopen/src/include/miopen/kernel_info.hpp +++ b/projects/miopen/src/include/miopen/kernel_info.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include namespace miopen { @@ -50,9 +51,8 @@ struct KernelInfo friend std::ostream& operator<<(std::ostream& os, const KernelInfo& k); }; -std::vector PrecompileKernels(const Handle& h, - const std::vector& kernels, - bool force_attach_binary = false); +MIOPEN_INTERNALS_EXPORT std::vector PrecompileKernels( + const Handle& h, const std::vector& kernels, bool force_attach_binary = false); } // namespace solver } // namespace miopen diff --git a/projects/miopen/src/include/miopen/load_file.hpp b/projects/miopen/src/include/miopen/load_file.hpp index 04c0a62f2f3c..bf5c1f93862f 100644 --- a/projects/miopen/src/include/miopen/load_file.hpp +++ b/projects/miopen/src/include/miopen/load_file.hpp @@ -27,13 +27,14 @@ #ifndef MIOPEN_GUARD_MLOPEN_LOAD_FILE_HPP #define MIOPEN_GUARD_MLOPEN_LOAD_FILE_HPP +#include #include #include #include namespace miopen { -std::vector LoadFile(const fs::path& path); +MIOPEN_INTERNALS_EXPORT std::vector LoadFile(const fs::path& path); } // namespace miopen diff --git a/projects/miopen/src/include/miopen/lock_file.hpp b/projects/miopen/src/include/miopen/lock_file.hpp index 89d69feb60ff..58559c823973 100644 --- a/projects/miopen/src/include/miopen/lock_file.hpp +++ b/projects/miopen/src/include/miopen/lock_file.hpp @@ -45,7 +45,7 @@ MIOPEN_INTERNALS_EXPORT fs::path LockFilePath(const fs::path& filename_); // One process should never have more than one instance of this class with same path at the same // time. It may lead to undefined behaviour on Windows. // Also on windows mutex can be removed because file locks are MT-safe there. -class MIOPEN_INTERNALS_EXPORT LockFile +class LockFile { private: class PassKey @@ -116,7 +116,7 @@ class MIOPEN_INTERNALS_EXPORT LockFile access_mutex.unlock_shared(); } - static LockFile& Get(const fs::path& file); + MIOPEN_INTERNALS_EXPORT static LockFile& Get(const fs::path& file); template bool try_lock_for(TDuration duration) diff --git a/projects/miopen/src/include/miopen/logger.hpp b/projects/miopen/src/include/miopen/logger.hpp index 9bd3ba715cc7..036c91f8ca4b 100644 --- a/projects/miopen/src/include/miopen/logger.hpp +++ b/projects/miopen/src/include/miopen/logger.hpp @@ -34,8 +34,8 @@ #include #include -#include #include +#include #if MIOPEN_USE_ROCTRACER #include @@ -174,7 +174,7 @@ std::array make_array(T x, Ts... xs) #define MIOPEN_LOG_ENUM_EACH(x) std::pair(#x, x) #if WORKAROUND_ISSUE_PP_TRANSFORM_ARGS -#define MIOPEN_LOG_ENUM(os, ...) os +#define MIOPEN_LOG_ENUM(os, x, ...) os << x #else #define MIOPEN_LOG_ENUM(os, x, ...) \ miopen::LogEnum(os, x, make_array(MIOPEN_PP_TRANSFORM_ARGS(MIOPEN_LOG_ENUM_EACH, __VA_ARGS__))) @@ -367,33 +367,42 @@ constexpr std::string_view LoggingParseFunction(const std::string_view func, #define MIOPEN_GET_FN_NAME miopen::LoggingParseFunction(__func__, __PRETTY_FUNCTION__) #endif -MIOPEN_INTERNALS_EXPORT void ClearBufferLog(); +MIOPEN_INTERNALS_EXPORT bool IsLogBufferOn(); + +MIOPEN_INTERNALS_EXPORT void ClearLogBuffer(); MIOPEN_INTERNALS_EXPORT void BufferLog(std::string line); MIOPEN_INTERNALS_EXPORT void OutputBufferedLogs(); -#define MIOPEN_LOG_XQ_CUSTOM(level, disableQuieting, category, fn_name, ...) \ - do \ - { \ - std::ostringstream miopen_log_ss; \ - miopen_log_ss << miopen::LoggingPrefix() << category << " [" << fn_name << "] " \ - << __VA_ARGS__ << std::endl; \ - if(miopen::IsLogging(level, disableQuieting)) \ - { \ - std::cerr << miopen_log_ss.str(); \ - } \ - if(!miopen::IsLogging(miopen::LoggingLevel::Info2, disableQuieting)) \ - { \ - if(level < miopen::LoggingLevel::Trace) \ - { \ - miopen::BufferLog(miopen_log_ss.str()); \ - } \ - if(level == miopen::LoggingLevel::Error) \ - { \ - miopen::OutputBufferedLogs(); \ - } \ - } \ +#define MIOPEN_LOG_XQ_CUSTOM(level, disableQuieting, category, fn_name, ...) \ + do \ + { \ + std::ostringstream miopen_log_ss; \ + const bool is_logging = miopen::IsLogging(level, disableQuieting); \ + const bool is_logging_i2 = \ + miopen::IsLogging(miopen::LoggingLevel::Info2, disableQuieting); \ + const bool buffer_on = miopen::IsLogBufferOn(); \ + if(is_logging || (buffer_on && !is_logging_i2)) \ + { \ + miopen_log_ss << miopen::LoggingPrefix() << category << " [" << fn_name << "] " \ + << __VA_ARGS__ << std::endl; \ + if(is_logging) \ + { \ + std::cerr << miopen_log_ss.str(); \ + } \ + if(buffer_on && !is_logging_i2) \ + { \ + if(level < miopen::LoggingLevel::Trace) \ + { \ + miopen::BufferLog(miopen_log_ss.str()); \ + } \ + if(level == miopen::LoggingLevel::Error) \ + { \ + miopen::OutputBufferedLogs(); \ + } \ + } \ + } \ } while(false) #define MIOPEN_LOG_XQ_(level, disableQuieting, fn_name, ...) \ diff --git a/projects/miopen/src/include/miopen/lrn.hpp b/projects/miopen/src/include/miopen/lrn.hpp index 93dac6992434..c6164ca06849 100644 --- a/projects/miopen/src/include/miopen/lrn.hpp +++ b/projects/miopen/src/include/miopen/lrn.hpp @@ -35,40 +35,41 @@ namespace miopen { -struct MIOPEN_INTERNALS_EXPORT LRNDescriptor : miopenLRNDescriptor +struct LRNDescriptor : miopenLRNDescriptor { - LRNDescriptor(); + MIOPEN_INTERNALS_EXPORT LRNDescriptor(); LRNDescriptor(miopenLRNMode_t m, unsigned int pn, const double* pparms); + MIOPEN_INTERNALS_EXPORT LRNDescriptor(miopenLRNMode_t m, unsigned int pn, std::vector pparms); - miopenLRNMode_t GetMode() const; - unsigned int GetN() const; - double GetAlpha() const; - double GetBeta() const; - double GetK() const; + MIOPEN_INTERNALS_EXPORT miopenLRNMode_t GetMode() const; + MIOPEN_INTERNALS_EXPORT unsigned int GetN() const; + MIOPEN_INTERNALS_EXPORT double GetAlpha() const; + MIOPEN_INTERNALS_EXPORT double GetBeta() const; + MIOPEN_INTERNALS_EXPORT double GetK() const; - miopenStatus_t Forward(const Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - bool do_backward, - Data_t workSpace) const; + MIOPEN_INTERNALS_EXPORT miopenStatus_t Forward(const Handle& handle, + const void* alpha, + const TensorDescriptor& xDesc, + ConstData_t x, + const void* beta, + const TensorDescriptor& yDesc, + Data_t y, + bool do_backward, + Data_t workSpace) const; - miopenStatus_t Backward(const Handle& handle, - const void* alpha, - const TensorDescriptor& yDesc, - ConstData_t y, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - ConstData_t workSpace) const; + MIOPEN_INTERNALS_EXPORT miopenStatus_t Backward(const Handle& handle, + const void* alpha, + const TensorDescriptor& yDesc, + ConstData_t y, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& xDesc, + ConstData_t x, + const void* beta, + const TensorDescriptor& dxDesc, + Data_t dx, + ConstData_t workSpace) const; friend std::ostream& operator<<(std::ostream& stream, const LRNDescriptor& x); diff --git a/projects/miopen/src/include/miopen/manage_ptr.hpp b/projects/miopen/src/include/miopen/manage_ptr.hpp index 3c6cfd6fe083..e4ecd65f7bc9 100644 --- a/projects/miopen/src/include/miopen/manage_ptr.hpp +++ b/projects/miopen/src/include/miopen/manage_ptr.hpp @@ -39,7 +39,15 @@ struct manage_deleter { if(x != nullptr) { - f(x); // NOLINT (cppcoreguidelines-owning-memory) + if constexpr(std::is_void_v) + { + f(x); // NOLINT (cppcoreguidelines-owning-memory) + } + else + { + [[maybe_unused]] const auto status = + f(x); // NOLINT (cppcoreguidelines-owning-memory) + } } } }; diff --git a/projects/miopen/src/include/miopen/mha/solvers.hpp b/projects/miopen/src/include/miopen/mha/solvers.hpp index 6bac473a717f..2e49504c76cf 100644 --- a/projects/miopen/src/include/miopen/mha/solvers.hpp +++ b/projects/miopen/src/include/miopen/mha/solvers.hpp @@ -39,42 +39,36 @@ namespace mha { using MhaSolver = NonTunableSolverBase; -struct MhaForward final : MhaSolver +struct MIOPEN_INTERNALS_EXPORT MhaForward final : MhaSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT std::size_t - GetWorkspaceSize(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override; + bool MayNeedWorkspace() const override; }; -struct MhaBackward final : MhaSolver +struct MIOPEN_INTERNALS_EXPORT MhaBackward final : MhaSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + bool IsApplicable(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT std::size_t - GetWorkspaceSize(const ExecutionContext& context, - const miopen::mha::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override; + bool MayNeedWorkspace() const override; }; } // namespace mha diff --git a/projects/miopen/src/include/miopen/mlo_internal.hpp b/projects/miopen/src/include/miopen/mlo_internal.hpp index fcf64a007641..62ccab567922 100644 --- a/projects/miopen/src/include/miopen/mlo_internal.hpp +++ b/projects/miopen/src/include/miopen/mlo_internal.hpp @@ -173,74 +173,76 @@ auto mloConstruct(T& x) -> decltype(x.mloConstruct(), void()) x.mloConstruct(); } -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindAllGemmSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> AllGemmWorkspaceSize(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> AllDirectForwardBackwardDataWorkspaceSize(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> FindAllImplicitGemmWorkspaceSizes(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> FindAllWinogradWorkspaceSizes(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> FindWinogradWrWWorkspaceSizes(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> FindImplicitGemmWrWWorkspaceSizes(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> AllDirectBwdWrW2DWorkspaceSize(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector> +std::vector> AllFFTForwardBackwardDataWorkspaceSize(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem); -MIOPEN_INTERNALS_EXPORT std::vector +MIOPEN_INTERNALS_EXPORT +std::vector FindAllDirectSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindAllImplicitGemmSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindAllWinogradSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindWinogradWrWAllSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindImplicitGemmWrWAllSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +MIOPEN_INTERNALS_EXPORT +std::vector FindAllBwdWrW2DSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); -MIOPEN_INTERNALS_EXPORT std::vector +std::vector FindAllFFTSolutions(const miopen::ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem, const miopen::AnyInvokeParams& invoke_ctx); @@ -555,8 +557,8 @@ struct mlo_construct_norm : mlo_construct_activ_lrn_pooling_common void mloConstruct(); protected: - int mloConstructFwd(); - int mloConstructBwd(); + void mloConstructFwd(); + void mloConstructBwd(); int _norm_region = 0; int _norm_area = 0; double _normAlpha = 0.0; diff --git a/projects/miopen/src/include/miopen/nogpu/handle_impl.hpp b/projects/miopen/src/include/miopen/nogpu/handle_impl.hpp index 3a1ebdf29531..1a0aeba2e3d3 100644 --- a/projects/miopen/src/include/miopen/nogpu/handle_impl.hpp +++ b/projects/miopen/src/include/miopen/nogpu/handle_impl.hpp @@ -36,7 +36,7 @@ struct HandleImpl void elapsed_time(hipEvent_t start, hipEvent_t stop) { if(enable_profiling) - hipEventElapsedTime(&this->profiling_result, start, stop); + (void)hipEventElapsedTime(&this->profiling_result, start, stop); } std::function elapsed_time_handler() diff --git a/projects/miopen/src/include/miopen/op_kernel_args.hpp b/projects/miopen/src/include/miopen/op_kernel_args.hpp index 692f52def35d..6eb7caf4ac81 100644 --- a/projects/miopen/src/include/miopen/op_kernel_args.hpp +++ b/projects/miopen/src/include/miopen/op_kernel_args.hpp @@ -4,11 +4,12 @@ #ifndef MIOPEN_GUARD_MLOPEN_OP_KERNEL_ARGS_HPP #define MIOPEN_GUARD_MLOPEN_OP_KERNEL_ARGS_HPP -#include #include +#include +#include + #include #include -#include struct OpKernelArg { @@ -33,7 +34,7 @@ struct OpKernelArg } std::size_t size() const { return buffer.size(); }; - boost::container::small_vector buffer; + std::vector buffer{8}; bool is_ptr = false; }; diff --git a/projects/miopen/src/include/miopen/pooling/problem_description.hpp b/projects/miopen/src/include/miopen/pooling/problem_description.hpp index eb3b91f1deca..a2a97ca61f61 100644 --- a/projects/miopen/src/include/miopen/pooling/problem_description.hpp +++ b/projects/miopen/src/include/miopen/pooling/problem_description.hpp @@ -66,8 +66,6 @@ struct ProblemDescription : ProblemDescriptionBase, const PoolingDescriptor& GetPooling() const { return pooling; } const TensorDescriptor& GetXDesc() const { return xDesc; } const TensorDescriptor& GetYDesc() const { return yDesc; } - TensorDescriptor& GetXDesc() { return xDesc; } - TensorDescriptor& GetYDesc() { return yDesc; } const TensorDescriptor& GetDXDesc() const { @@ -183,16 +181,12 @@ struct ProblemDescription : ProblemDescriptionBase, } std::string GetDirectionStr() const { - std::string s; - switch(direction) { case Direction::Forward: return "Fwd"; case Direction::Backward: return "Bwd"; default: MIOPEN_THROW(miopenStatusInvalidValue, "Wrong pooling direction provided"); } - - return s; } std::string GetModeStr() const { diff --git a/projects/miopen/src/include/miopen/pooling/solvers.hpp b/projects/miopen/src/include/miopen/pooling/solvers.hpp index ec8a945de923..38f9019eaa8b 100644 --- a/projects/miopen/src/include/miopen/pooling/solvers.hpp +++ b/projects/miopen/src/include/miopen/pooling/solvers.hpp @@ -305,57 +305,48 @@ struct PoolingFwdNCHWTransposingSolver using Problem = miopen::pooling::ProblemDescription; using InvokeParams = miopen::pooling::FwdInvokeParams; - inline static auto GetTransposes() + inline static auto GetTransposes(const Problem& /*problem*/) { - auto ret = std::array, 2>{{ + return std::array, 2>{{ { - &Problem::GetXDesc, &Problem::GetXDesc, &InvokeParams::xDesc, - {&InvokeParams::x}, + &InvokeParams::x, // x is input + nullptr, "NCDHW", true, }, { - &Problem::GetYDesc, &Problem::GetYDesc, &InvokeParams::yDesc, - {}, + nullptr, + &InvokeParams::y, // y is output "NCDHW", false, }, }}; - - // Before C++20 you can't aggregate initialize non-first union element - ret[1].as_output = &InvokeParams::y; - - return ret; } }; struct TransposedPoolingFwd2d final : PoolingFwdNCHWTransposingSolver { - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } + const std::string& SolverDbId() const { return GetSolverDbId(); } PoolingForward2d::PerformanceConfigType GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const override + const miopen::pooling::ProblemDescription& problem) const { return PoolingForward2d{}.GetDefaultPerformanceConfig(ctx, problem); } - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingForward2d::PerformanceConfigType& config) const override + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::pooling::ProblemDescription& problem, + const PoolingForward2d::PerformanceConfigType& config) const { return PoolingForward2d{}.IsValidPerformanceConfig(ctx, problem, config); } PoolingForward2d::PerformanceConfigType Search(const ExecutionContext& context, const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override + const AnyInvokeParams& invoke_context) const { return PoolingForward2d{}.Search(context, problem, invoke_context); } @@ -363,27 +354,23 @@ struct TransposedPoolingFwd2d final : PoolingFwdNCHWTransposingSolver { - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } + const std::string& SolverDbId() const { return GetSolverDbId(); } PoolingForwardNd::PerformanceConfigType GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const override + const miopen::pooling::ProblemDescription& problem) const { return PoolingForwardNd{}.GetDefaultPerformanceConfig(ctx, problem); } - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingForwardNd::PerformanceConfigType& config) const override + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::pooling::ProblemDescription& problem, + const PoolingForwardNd::PerformanceConfigType& config) const { return PoolingForwardNd{}.IsValidPerformanceConfig(ctx, problem, config); } PoolingForwardNd::PerformanceConfigType Search(const ExecutionContext& context, const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override + const AnyInvokeParams& invoke_context) const { return PoolingForwardNd{}.Search(context, problem, invoke_context); } @@ -462,57 +449,48 @@ struct PoolingBwdNCHWTransposingSolver using Problem = miopen::pooling::ProblemDescription; using InvokeParams = miopen::pooling::BwdInvokeParams; - inline static auto GetTransposes() + inline static auto GetTransposes(const Problem& /*problem*/) { - auto ret = std::array, 2>{{ + return std::array, 2>{{ { - &Problem::GetXDesc, &Problem::GetXDesc, &InvokeParams::dxDesc, - {}, + nullptr, + &InvokeParams::dx, // dx is output "NCDHW", false, }, { - &Problem::GetYDesc, &Problem::GetYDesc, &InvokeParams::dyDesc, - {&InvokeParams::dy}, + &InvokeParams::dy, // dy is input + nullptr, "NCDHW", true, }, }}; - - // Before C++20 you can't aggregate initialize non-first union element - ret[0].as_output = &InvokeParams::dx; - - return ret; } }; struct TransposedPoolingBwd2d final : PoolingBwdNCHWTransposingSolver { - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } + const std::string& SolverDbId() const { return GetSolverDbId(); } PoolingBackward2d::PerformanceConfigType GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const override + const miopen::pooling::ProblemDescription& problem) const { return PoolingBackward2d{}.GetDefaultPerformanceConfig(ctx, problem); } - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingBackward2d::PerformanceConfigType& config) const override + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::pooling::ProblemDescription& problem, + const PoolingBackward2d::PerformanceConfigType& config) const { return PoolingBackward2d{}.IsValidPerformanceConfig(ctx, problem, config); } PoolingBackward2d::PerformanceConfigType Search(const ExecutionContext& context, const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override + const AnyInvokeParams& invoke_context) const { return PoolingBackward2d{}.Search(context, problem, invoke_context); } @@ -520,27 +498,23 @@ struct TransposedPoolingBwd2d final : PoolingBwdNCHWTransposingSolver { - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } + const std::string& SolverDbId() const { return GetSolverDbId(); } PoolingBackwardNd::PerformanceConfigType GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const override + const miopen::pooling::ProblemDescription& problem) const { return PoolingBackwardNd{}.GetDefaultPerformanceConfig(ctx, problem); } - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingBackwardNd::PerformanceConfigType& config) const override + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::pooling::ProblemDescription& problem, + const PoolingBackwardNd::PerformanceConfigType& config) const { return PoolingBackwardNd{}.IsValidPerformanceConfig(ctx, problem, config); } PoolingBackwardNd::PerformanceConfigType Search(const ExecutionContext& context, const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override + const AnyInvokeParams& invoke_context) const { return PoolingBackwardNd{}.Search(context, problem, invoke_context); } diff --git a/projects/miopen/src/include/miopen/problem.hpp b/projects/miopen/src/include/miopen/problem.hpp index df923c44e715..df214f6039d1 100644 --- a/projects/miopen/src/include/miopen/problem.hpp +++ b/projects/miopen/src/include/miopen/problem.hpp @@ -210,7 +210,7 @@ struct Problem void LogDriverCommand(const BatchnormDescriptor& descriptor) const; }; -struct MIOPEN_INTERNALS_EXPORT FusedProblem +struct FusedProblem { std::vector problems; @@ -219,11 +219,10 @@ struct MIOPEN_INTERNALS_EXPORT FusedProblem // Not implemented, but silently } - [[nodiscard]] std::vector FindSolutions(const Handle& handle, - const FindOptions& options, - std::size_t max_solutions) const; + [[nodiscard]] MIOPEN_INTERNALS_EXPORT std::vector FindSolutions( + const Handle& handle, const FindOptions& options, std::size_t max_solutions) const; - void PropagateDescriptors(); + MIOPEN_INTERNALS_EXPORT void PropagateDescriptors(); [[nodiscard]] miopenTensorArgumentId_t GetInputId() const { @@ -238,12 +237,12 @@ struct MIOPEN_INTERNALS_EXPORT FusedProblem [[nodiscard]] const TensorDescriptor& GetInput() const { return problems.front().GetInput(); } [[nodiscard]] const TensorDescriptor& GetOutput() const { return problems.back().GetOutput(); } - [[nodiscard]] FusionPlanDescriptor AsFusionPlan() const; + [[nodiscard]] MIOPEN_INTERNALS_EXPORT FusionPlanDescriptor AsFusionPlan() const; friend void to_json(nlohmann::json& j, const FusedProblem& problem); friend void from_json(const nlohmann::json& j, FusedProblem& problem); - [[nodiscard]] fusion::FusionInvokeParams + [[nodiscard]] MIOPEN_INTERNALS_EXPORT fusion::FusionInvokeParams MakeInvokeParams(const std::function& buffer_getter, const std::function& workspace_getter, diff --git a/projects/miopen/src/include/miopen/ramdb.hpp b/projects/miopen/src/include/miopen/ramdb.hpp index 7d05c2a85f03..9d3a222b36c1 100644 --- a/projects/miopen/src/include/miopen/ramdb.hpp +++ b/projects/miopen/src/include/miopen/ramdb.hpp @@ -25,6 +25,7 @@ *******************************************************************************/ #pragma once +#include #include #include @@ -44,7 +45,7 @@ using ramdb_clock = std::chrono::steady_clock; class LockFile; -class MIOPEN_INTERNALS_EXPORT RamDb : protected PlainTextDb +class RamDb : protected PlainTextDb { public: RamDb(DbKinds db_kind_, @@ -56,7 +57,7 @@ class MIOPEN_INTERNALS_EXPORT RamDb : protected PlainTextDb { } - RamDb(DbKinds db_kind_, const fs::path& path, bool is_system = false); + MIOPEN_INTERNALS_EXPORT RamDb(DbKinds db_kind_, const fs::path& path, bool is_system = false); RamDb(const RamDb&) = delete; RamDb(RamDb&&) = delete; @@ -64,7 +65,8 @@ class MIOPEN_INTERNALS_EXPORT RamDb : protected PlainTextDb RamDb& operator=(RamDb&&) = delete; static fs::path GetTimeFilePath(const fs::path& path); - static RamDb& GetCached(DbKinds db_kind_, const fs::path& path, bool is_system); + MIOPEN_INTERNALS_EXPORT static RamDb& + GetCached(DbKinds db_kind_, const fs::path& path, bool is_system); static RamDb& GetCached(DbKinds db_kind_, const fs::path& path, @@ -75,7 +77,7 @@ class MIOPEN_INTERNALS_EXPORT RamDb : protected PlainTextDb return GetCached(db_kind_, path, is_system); } - std::optional FindRecord(const std::string& problem); + MIOPEN_INTERNALS_EXPORT std::optional FindRecord(const std::string& problem); template std::optional FindRecord(const TProblem& problem) @@ -93,10 +95,10 @@ class MIOPEN_INTERNALS_EXPORT RamDb : protected PlainTextDb return record->GetValues(id, value); } - bool StoreRecord(const DbRecord& record); - bool UpdateRecord(DbRecord& record); - bool RemoveRecord(const std::string& key); - bool Remove(const std::string& key, const std::string& id); + MIOPEN_INTERNALS_EXPORT bool StoreRecord(const DbRecord& record); + MIOPEN_INTERNALS_EXPORT bool UpdateRecord(DbRecord& record); + MIOPEN_INTERNALS_EXPORT bool RemoveRecord(const std::string& key); + MIOPEN_INTERNALS_EXPORT bool Remove(const std::string& key, const std::string& id); template inline bool Remove(const T& problem_config, const std::string& id) @@ -155,14 +157,17 @@ class DbTimer template static auto Measure(const std::string& funcName, TFunc&& func) { - if(!miopen::IsLogging(LoggingLevel::Info2)) - return func(); - - const auto start = std::chrono::high_resolution_clock::now(); - auto ret = func(); - const auto end = std::chrono::high_resolution_clock::now(); - MIOPEN_LOG_I2("Db::" << funcName << " time: " << (end - start).count() * .000001f << " ms"); - return ret; + const bool logging = miopen::IsLogging(LoggingLevel::Info2); + const auto start = logging ? std::chrono::high_resolution_clock::now() + : std::chrono::high_resolution_clock::time_point{}; + auto ret = func(); + if(logging) + { + const auto end = std::chrono::high_resolution_clock::now(); + MIOPEN_LOG_I2("Db::" << funcName << " time: " << (end - start).count() * .000001f + << " ms"); + } + return std::move(ret); // NOLINT(clang-analyzer-cplusplus.Move) } public: diff --git a/projects/miopen/src/include/miopen/readonlyramdb.hpp b/projects/miopen/src/include/miopen/readonlyramdb.hpp index 670e3b5369ce..e1dc3f2bc69a 100644 --- a/projects/miopen/src/include/miopen/readonlyramdb.hpp +++ b/projects/miopen/src/include/miopen/readonlyramdb.hpp @@ -26,6 +26,7 @@ #ifndef MIOPEN_GUARD_MLOPEN_READONLYRAMDB_HPP #define MIOPEN_GUARD_MLOPEN_READONLYRAMDB_HPP +#include #include #include @@ -40,12 +41,12 @@ namespace debug { MIOPEN_INTERNALS_EXPORT bool& rordb_embed_fs_override(); } // namespace debug -class MIOPEN_INTERNALS_EXPORT ReadonlyRamDb +class ReadonlyRamDb { public: ReadonlyRamDb(DbKinds db_kind_, const fs::path& path) : db_kind(db_kind_), db_path(path) {} - static ReadonlyRamDb& + MIOPEN_INTERNALS_EXPORT static ReadonlyRamDb& GetCached(DbKinds db_kind_, const fs::path& path, bool warn_if_unreadable); std::optional FindRecord(const std::string& problem) const diff --git a/projects/miopen/src/include/miopen/reducetensor.hpp b/projects/miopen/src/include/miopen/reducetensor.hpp index 8b0d0337e53c..77bc8f049288 100644 --- a/projects/miopen/src/include/miopen/reducetensor.hpp +++ b/projects/miopen/src/include/miopen/reducetensor.hpp @@ -41,14 +41,14 @@ namespace miopen { -struct MIOPEN_INTERNALS_EXPORT ReduceTensorDescriptor : miopenReduceTensorDescriptor +struct ReduceTensorDescriptor : miopenReduceTensorDescriptor { ReduceTensorDescriptor() = default; - ReduceTensorDescriptor(miopenReduceTensorOp_t reduceTensorOp, - miopenDataType_t reduceTensorCompType, - miopenNanPropagation_t reduceTensorNanOpt, - miopenReduceTensorIndices_t reduceTensorIndices, - miopenIndicesType_t reduceTensorIndicesType); + MIOPEN_INTERNALS_EXPORT ReduceTensorDescriptor(miopenReduceTensorOp_t reduceTensorOp, + miopenDataType_t reduceTensorCompType, + miopenNanPropagation_t reduceTensorNanOpt, + miopenReduceTensorIndices_t reduceTensorIndices, + miopenIndicesType_t reduceTensorIndicesType); miopenReduceTensorOp_t reduceTensorOp_; miopenDataType_t reduceTensorCompType_; @@ -56,22 +56,22 @@ struct MIOPEN_INTERNALS_EXPORT ReduceTensorDescriptor : miopenReduceTensorDescri miopenReduceTensorIndices_t reduceTensorIndices_; miopenIndicesType_t reduceTensorIndicesType_; - std::size_t GetWorkspaceSize(const Handle& handle, - const TensorDescriptor& inDesc, - const TensorDescriptor& outDesc) const; - std::size_t GetIndicesSize(const TensorDescriptor& inDesc, - const TensorDescriptor& outDesc) const; - void ReduceTensor(const Handle& handle, - Data_t indices, - size_t indicesSizeInBytes, - Data_t workspace, - size_t workspaceSizeInBytes, - const void* alpha, - const TensorDescriptor& aDesc, - ConstData_t A, - const void* beta, - const TensorDescriptor& cDesc, - Data_t C) const; + MIOPEN_INTERNALS_EXPORT std::size_t GetWorkspaceSize(const Handle& handle, + const TensorDescriptor& inDesc, + const TensorDescriptor& outDesc) const; + MIOPEN_INTERNALS_EXPORT std::size_t GetIndicesSize(const TensorDescriptor& inDesc, + const TensorDescriptor& outDesc) const; + MIOPEN_INTERNALS_EXPORT void ReduceTensor(const Handle& handle, + Data_t indices, + size_t indicesSizeInBytes, + Data_t workspace, + size_t workspaceSizeInBytes, + const void* alpha, + const TensorDescriptor& aDesc, + ConstData_t A, + const void* beta, + const TensorDescriptor& cDesc, + Data_t C) const; }; std::ostream& operator<<(std::ostream& stream, const ReduceTensorDescriptor& c); diff --git a/projects/miopen/src/include/miopen/rnn.hpp b/projects/miopen/src/include/miopen/rnn.hpp index 49e31910eea4..940e657b78cf 100644 --- a/projects/miopen/src/include/miopen/rnn.hpp +++ b/projects/miopen/src/include/miopen/rnn.hpp @@ -65,10 +65,10 @@ struct c_array_view void profileRNNkernels(const Handle& handle, unsigned char select, float& ctime); -struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor +struct RNNDescriptor : miopenRNNDescriptor { - RNNDescriptor(); + MIOPEN_INTERNALS_EXPORT RNNDescriptor(); RNNDescriptor(int hsz, int layers, miopenRNNMode_t rmode, @@ -113,13 +113,14 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor std::vector pTensorLengthsCalculation(const TensorDescriptor& xDesc, int layer, int paramID) const; - static SeqTensorDescriptor makeSeqTensorDescriptor(miopenDataType_t t, - miopenRNNBaseLayout_t layout, - int maxSeqLength, - int batchSize, - int vectorSize, - const int* lensPerSeq, - const void* padding_marker_ptr); + MIOPEN_INTERNALS_EXPORT static SeqTensorDescriptor + makeSeqTensorDescriptor(miopenDataType_t t, + miopenRNNBaseLayout_t layout, + int maxSeqLength, + int batchSize, + int vectorSize, + const int* lensPerSeq, + const void* padding_marker_ptr); static SeqTensorDescriptor makeSeqTensorDescriptor( c_array_view descs, @@ -130,8 +131,9 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor std::vector& td, std::vector& ptd); - static miopenRNNBaseLayout_t getBaseLayoutFromDataTensor(const SeqTensorDescriptor& desc); - static std::tuple, bool> + MIOPEN_INTERNALS_EXPORT static miopenRNNBaseLayout_t + getBaseLayoutFromDataTensor(const SeqTensorDescriptor& desc); + MIOPEN_INTERNALS_EXPORT static std::tuple, bool> convertRNNBaseLayout(miopenRNNBaseLayout_t layout); size_t GetMainSolWorkspaceSize(size_t batchLenSum, @@ -150,10 +152,11 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor int seqLength, c_array_view xDesc) const; - size_t GetMaxWorkspaceSize(const Handle& handle, - const SeqTensorDescriptor& xDesc, - miopenRNNFWDMode_t fwdMode) const; - size_t GetMaxReserveSize(const Handle& handle, const SeqTensorDescriptor& xDesc) const; + MIOPEN_INTERNALS_EXPORT size_t GetMaxWorkspaceSize(const Handle& handle, + const SeqTensorDescriptor& xDesc, + miopenRNNFWDMode_t fwdMode) const; + MIOPEN_INTERNALS_EXPORT size_t GetMaxReserveSize(const Handle& handle, + const SeqTensorDescriptor& xDesc) const; std::tuple GetTmpSpaceSizeDynamicAlgo(const Handle& handle, const SeqTensorDescriptor& xDesc, @@ -236,24 +239,24 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor size_t GetRNNHiddenSuperTensorSize(const Handle& handle, c_array_view xDesc) const; - void RNNForward(const Handle& handle, - miopenRNNFWDMode_t fwdMode, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - Data_t hy, - const TensorDescriptor& cDesc, - ConstData_t cx, - Data_t cy, - const SeqTensorDescriptor& yDesc, - Data_t y, - ConstData_t w, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; + MIOPEN_INTERNALS_EXPORT void RNNForward(const Handle& handle, + miopenRNNFWDMode_t fwdMode, + const SeqTensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& hDesc, + ConstData_t hx, + Data_t hy, + const TensorDescriptor& cDesc, + ConstData_t cx, + Data_t cy, + const SeqTensorDescriptor& yDesc, + Data_t y, + ConstData_t w, + size_t weightSpaceSize, + Data_t workSpace, + size_t workSpaceSize, + Data_t reserveSpace, + size_t reserveSpaceSize) const; void RNNForwardTraining(const Handle& handle, int seqLen, @@ -295,26 +298,26 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor Data_t workSpace, size_t workSpaceSize) const; - void RNNBackwardData(const Handle& handle, - const SeqTensorDescriptor& yDesc, - ConstData_t y, - ConstData_t dy, - const TensorDescriptor& hDesc, - ConstData_t hx, - ConstData_t dhy, - Data_t dhx, - const TensorDescriptor& cDesc, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - const SeqTensorDescriptor& xDesc, - Data_t dx, - ConstData_t w, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; + MIOPEN_INTERNALS_EXPORT void RNNBackwardData(const Handle& handle, + const SeqTensorDescriptor& yDesc, + ConstData_t y, + ConstData_t dy, + const TensorDescriptor& hDesc, + ConstData_t hx, + ConstData_t dhy, + Data_t dhx, + const TensorDescriptor& cDesc, + ConstData_t cx, + ConstData_t dcy, + Data_t dcx, + const SeqTensorDescriptor& xDesc, + Data_t dx, + ConstData_t w, + size_t weightSpaceSize, + Data_t workSpace, + size_t workSpaceSize, + Data_t reserveSpace, + size_t reserveSpaceSize) const; void RNNBackwardData(const Handle& handle, int seqLen, @@ -343,19 +346,19 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor Data_t reserveSpace, size_t reserveSpaceSize) const; - void RNNBackwardWeights(const Handle& handle, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - const SeqTensorDescriptor& yDesc, - ConstData_t y, - Data_t dw, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; + MIOPEN_INTERNALS_EXPORT void RNNBackwardWeights(const Handle& handle, + const SeqTensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& hDesc, + ConstData_t hx, + const SeqTensorDescriptor& yDesc, + ConstData_t y, + Data_t dw, + size_t weightSpaceSize, + Data_t workSpace, + size_t workSpaceSize, + ConstData_t reserveSpace, + size_t reserveSpaceSize) const; void RNNBackwardWeights(const Handle& handle, int seqLen, @@ -620,7 +623,7 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor size_t reserveSpaceSize) const; }; -MIOPEN_INTERNALS_EXPORT std::ostream& operator<<(std::ostream& stream, const RNNDescriptor& r); +std::ostream& operator<<(std::ostream& stream, const RNNDescriptor& r); } // namespace miopen MIOPEN_DEFINE_OBJECT(miopenRNNDescriptor, miopen::RNNDescriptor); diff --git a/projects/miopen/src/include/miopen/rnn/multi_stream_utils.hpp b/projects/miopen/src/include/miopen/rnn/multi_stream_utils.hpp index 18f55e99d4e8..d0bb2f18daed 100644 --- a/projects/miopen/src/include/miopen/rnn/multi_stream_utils.hpp +++ b/projects/miopen/src/include/miopen/rnn/multi_stream_utils.hpp @@ -70,8 +70,8 @@ class MultiStreamController if(i != rootStreamId) { const miopen::HipEventPtr sync_event = make_hip_fast_event(); - RecordEvent(sync_event.get(), i); - SetWaitEvent(sync_event.get(), rootStreamId); + (void)RecordEvent(sync_event.get(), i); + (void)SetWaitEvent(sync_event.get(), rootStreamId); } } } @@ -80,12 +80,12 @@ class MultiStreamController { const miopen::HipEventPtr main_event = make_hip_fast_event(); - RecordEvent(main_event.get(), rootStreamId); + (void)RecordEvent(main_event.get(), rootStreamId); for(size_t i = 0, stream_cnt = size(); i < stream_cnt; i++) { if(i != rootStreamId) - SetWaitEvent(main_event.get(), i); + (void)SetWaitEvent(main_event.get(), i); } }; diff --git a/projects/miopen/src/include/miopen/rnn/tmp_buffer_utils.hpp b/projects/miopen/src/include/miopen/rnn/tmp_buffer_utils.hpp index d6cdb6963a6e..e58972707805 100644 --- a/projects/miopen/src/include/miopen/rnn/tmp_buffer_utils.hpp +++ b/projects/miopen/src/include/miopen/rnn/tmp_buffer_utils.hpp @@ -741,7 +741,7 @@ struct WeightsBufferDescriptor std::next(strides.rbegin()), std::multiplies{}); strides[0] *= bias_cnt; - return strides; + return std::move(strides); }(bias_lens, bias_mode * 2); return {input_vector_filter_sz, diff --git a/projects/miopen/src/include/miopen/rnn_util.hpp b/projects/miopen/src/include/miopen/rnn_util.hpp index 9e2a49e59141..fadc1c837924 100644 --- a/projects/miopen/src/include/miopen/rnn_util.hpp +++ b/projects/miopen/src/include/miopen/rnn_util.hpp @@ -74,17 +74,17 @@ struct RnnHipAutoProfiler attached_handle->EnableProfiling(false); start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), attached_handle->GetStream()); + (void)hipEventRecord(start.get(), attached_handle->GetStream()); #endif } void RNNProfilingEnd() { #if MIOPEN_BACKEND_HIP - hipEventRecord(stop.get(), attached_handle->GetStream()); - hipEventSynchronize(stop.get()); + (void)hipEventRecord(stop.get(), attached_handle->GetStream()); + (void)hipEventSynchronize(stop.get()); float eventTime_mS = 0; - hipEventElapsedTime(&eventTime_mS, start.get(), stop.get()); + (void)hipEventElapsedTime(&eventTime_mS, start.get(), stop.get()); attached_handle->EnableProfiling(true); attached_handle->ResetKernelTime(); @@ -104,7 +104,7 @@ struct RnnHipAutoProfiler inline miopen::HipEventPtr make_hip_fast_event() { hipEvent_t result = nullptr; - hipEventCreateWithFlags(&result, hipEventDisableTiming); + (void)hipEventCreateWithFlags(&result, hipEventDisableTiming); return miopen::HipEventPtr{result}; } #endif // #if MIOPEN_BACKEND_HIP diff --git a/projects/miopen/src/include/miopen/scalar.hpp b/projects/miopen/src/include/miopen/scalar.hpp index e0308102a042..1cb55d0147cd 100644 --- a/projects/miopen/src/include/miopen/scalar.hpp +++ b/projects/miopen/src/include/miopen/scalar.hpp @@ -33,11 +33,11 @@ namespace miopen { // Class store value in double for higher precision. -struct MIOPEN_INTERNALS_EXPORT Scalar +struct Scalar { explicit Scalar(double val) : mVal(val), mType(miopenDouble) {} - Scalar(ConstData_t ptr, miopenDataType_t type); + MIOPEN_INTERNALS_EXPORT Scalar(ConstData_t ptr, miopenDataType_t type); float GetAsFloat() const { return static_cast(mVal); } double GetAsDouble() const { return mVal; } diff --git a/projects/miopen/src/include/miopen/seq_tensor.hpp b/projects/miopen/src/include/miopen/seq_tensor.hpp index 57fbd4bcb48d..60d9f51ff1e1 100644 --- a/projects/miopen/src/include/miopen/seq_tensor.hpp +++ b/projects/miopen/src/include/miopen/seq_tensor.hpp @@ -44,7 +44,7 @@ namespace miopen { -struct MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor : miopenSeqTensorDescriptor +struct SeqTensorDescriptor : miopenSeqTensorDescriptor { SeqTensorDescriptor(); @@ -67,10 +67,10 @@ struct MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor : miopenSeqTensorDescriptor const std::vector& layout_in, const std::initializer_list& lens_in, bool with_padded_seq_layout); - SeqTensorDescriptor(miopenDataType_t t, - const std::vector& layout_in, - const std::vector& lens_in, - bool with_padded_seq_layout); + MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor(miopenDataType_t t, + const std::vector& layout_in, + const std::vector& lens_in, + bool with_padded_seq_layout); SeqTensorDescriptor(miopenDataType_t t, const std::vector& layout_in, @@ -79,13 +79,13 @@ struct MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor : miopenSeqTensorDescriptor const std::vector& padding_marker_in, bool use_seq_len, bool with_padded_seq_layout); - SeqTensorDescriptor(miopenDataType_t t, - const std::vector& layout_in, - const std::vector& lens_in, - const std::vector& seq_len, - const std::vector& padding_marker_in, - bool use_seq_len, - bool with_padded_seq_layout); + MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor(miopenDataType_t t, + const std::vector& layout_in, + const std::vector& lens_in, + const std::vector& seq_len, + const std::vector& padding_marker_in, + bool use_seq_len, + bool with_padded_seq_layout); SeqTensorDescriptor(miopenDataType_t t, const std::vector& layout_in, @@ -102,39 +102,39 @@ struct MIOPEN_INTERNALS_EXPORT SeqTensorDescriptor : miopenSeqTensorDescriptor bool use_seq_len, bool with_padded_seq_layout); - const std::vector& GetLayoutVector() const; - const std::vector& GetLengths() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetLayoutVector() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetLengths() const; const std::vector& GetPadding() const; - const std::vector& GetSequenceLengthsVector() const; - const std::vector& GetPaddingMarkerHolder() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetSequenceLengthsVector() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetPaddingMarkerHolder() const; // Get vector of strides only for padded tensor, // if IsPaddedSeqLayout()==false function returns an empty vector std::vector GetPaddedStrides() const; - bool IsPacked() const; - bool IsPaddedSeqLayout() const; - bool IsSequenceLengthsSorted() const; + MIOPEN_INTERNALS_EXPORT bool IsPacked() const; + MIOPEN_INTERNALS_EXPORT bool IsPaddedSeqLayout() const; + MIOPEN_INTERNALS_EXPORT bool IsSequenceLengthsSorted() const; bool IsZeroBytePadding() const; - bool IsPaddingMarkerSpecified() const; + MIOPEN_INTERNALS_EXPORT bool IsPaddingMarkerSpecified() const; - miopenDataType_t GetType() const; + MIOPEN_INTERNALS_EXPORT miopenDataType_t GetType() const; - std::size_t GetMaxCountOfSequences() const; - std::size_t GetMaxSequenceLength() const; - std::size_t GetTotalSequenceLen() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetMaxCountOfSequences() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetMaxSequenceLength() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetTotalSequenceLen() const; // Calculate the number of tensor significant elements in pieces std::size_t GetElementCount() const; // Calculating the size of the occupied disk space in bytes - std::size_t GetTensorRealByteSpace() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetTensorRealByteSpace() const; // Calculating the maximal tensor size in bytes if all sequences are of the maximum size std::size_t GetTensorMaxByteSpace() const; bool operator==(const SeqTensorDescriptor& rhs) const; bool operator!=(const SeqTensorDescriptor& rhs) const; - std::vector GetBatchesPerSequence() const; + MIOPEN_INTERNALS_EXPORT std::vector GetBatchesPerSequence() const; std::string ToString() const; friend std::ostream& operator<<(std::ostream& stream, const SeqTensorDescriptor& t); diff --git a/projects/miopen/src/include/miopen/sequences.hpp b/projects/miopen/src/include/miopen/sequences.hpp index 20157d5a06cf..39a2b1ee02be 100644 --- a/projects/miopen/src/include/miopen/sequences.hpp +++ b/projects/miopen/src/include/miopen/sequences.hpp @@ -1,36 +1,11 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #ifndef GUARD_MIOPEN_SEQUENCES_HPP_ #define GUARD_MIOPEN_SEQUENCES_HPP_ #include -#include - #include #include #include @@ -57,7 +32,7 @@ class Sequence constexpr end() const; // returns iterator pointing to element of sequence equal to value or end(). - // this method may not be implemented. boost::find will be used instead than. + // this method may not be implemented. std::ranges::find will be used instead than. constexpr find(TValue value) const; } @@ -172,7 +147,7 @@ auto GenericFindImpl(rank<1>, template auto GenericFindImpl(rank<0>, const TRange& range, const TValue& value) { - return boost::find(range, value); + return std::ranges::find(range, value); } template diff --git a/projects/miopen/src/include/miopen/softmax.hpp b/projects/miopen/src/include/miopen/softmax.hpp index 518d475d1315..19a5cc0b094d 100644 --- a/projects/miopen/src/include/miopen/softmax.hpp +++ b/projects/miopen/src/include/miopen/softmax.hpp @@ -37,7 +37,7 @@ namespace miopen { struct Handle; struct TensorDescriptor; -struct MIOPEN_INTERNALS_EXPORT SoftmaxDescriptor : miopenSoftmaxDescriptor +struct SoftmaxDescriptor : miopenSoftmaxDescriptor { SoftmaxDescriptor() {} @@ -57,8 +57,7 @@ struct MIOPEN_INTERNALS_EXPORT SoftmaxDescriptor : miopenSoftmaxDescriptor mode = mode_; } - MIOPEN_INTERNALS_EXPORT friend std::ostream& operator<<(std::ostream& stream, - const SoftmaxDescriptor& x); + friend std::ostream& operator<<(std::ostream& stream, const SoftmaxDescriptor& x); friend void to_json(nlohmann::json& json, const SoftmaxDescriptor& descriptor); friend void from_json(const nlohmann::json& json, SoftmaxDescriptor& descriptor); diff --git a/projects/miopen/src/include/miopen/softmax/problem_description.hpp b/projects/miopen/src/include/miopen/softmax/problem_description.hpp index 9a5e7150b4a2..d3f42e7ccdc1 100644 --- a/projects/miopen/src/include/miopen/softmax/problem_description.hpp +++ b/projects/miopen/src/include/miopen/softmax/problem_description.hpp @@ -1,33 +1,15 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #pragma once +#include +#if MIOPEN_ENABLE_SQLITE +#include +#endif #include #include +#include namespace miopen { @@ -35,7 +17,16 @@ struct NetworkConfig; namespace softmax { -struct ProblemDescription : ProblemDescriptionBase +struct ProblemDescriptionTag +{ +}; + +struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, + ProblemDescriptionTag +#if MIOPEN_ENABLE_SQLITE + , + SQLiteSerializable +#endif { // softmax forward constructor ProblemDescription(const void* alpha_, @@ -110,8 +101,44 @@ struct ProblemDescription : ProblemDescriptionBase const TensorDescriptor& GetdYDesc() const { return dyDesc; } const TensorDescriptor& GetdXDesc() const { return xdxDesc; } + void Serialize(std::ostream& stream) const { stream << MakeNetworkConfig().ToString(); } + NetworkConfig MakeNetworkConfig() const override; + template + static void Visit(Self&& self, std::function f) + { + // The column names match the driver command line argument names + f(static_cast(self.isForward), "forw"); + f(self.GetBatchSize(), "batchsize"); + f(self.GetChannels(), "in_channels"); + f(self.GetHeight(), "in_h"); + f(self.GetWidth(), "in_w"); + f(static_cast(self.algorithm), "algorithm"); + f(static_cast(self.mode), "mode"); + } + + template + static void Visit(Self&& self, std::function f) + { + f(GetDataTypeName(self.yDesc.GetType()), "data_type"); + f(self.GetLayout(), "layout"); + } + + template + static void VisitAll(Self&& self, const Visitor& f) + { + Visit(std::forward(self), [&](int64_t value, std::string name) { f(value, name); }); + Visit(std::forward(self), + [&](std::string value, std::string name) { f(value, name); }); + } + + // This declaration marks softmax as a primitive with tuning enabled. + // Any tunable solver would be able pick it and fetch a db instance in ExecutePrimitive. + // It has to be discoverable via ADL from problem description. + friend auto GetDb(const ExecutionContext& context, + const ProblemDescriptionTag&) -> PerformanceDb; + private: void CheckAndAssignAlphaBeta(const void* alpha_, const void* beta_) { @@ -141,6 +168,12 @@ struct ProblemDescription : ProblemDescriptionBase const miopenSoftmaxAlgorithm_t algorithm; const miopenSoftmaxMode_t mode; + + std::size_t GetBatchSize() const { return yDesc.GetLengths()[0]; } + std::size_t GetChannels() const { return yDesc.GetLengths()[1]; } + std::size_t GetHeight() const { return yDesc.GetLengths()[2]; } + std::size_t GetWidth() const { return yDesc.GetLengths()[3]; } + std::string GetLayout() const { return yDesc.GetLayout_str(); } }; } // namespace softmax diff --git a/projects/miopen/src/include/miopen/softmax/solvers.hpp b/projects/miopen/src/include/miopen/softmax/solvers.hpp index 425604c9e5ee..908ba50afb37 100644 --- a/projects/miopen/src/include/miopen/softmax/solvers.hpp +++ b/projects/miopen/src/include/miopen/softmax/solvers.hpp @@ -1,31 +1,13 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #pragma once +#include +#include +#include +#include +#include #include #include @@ -37,16 +19,91 @@ namespace softmax { using SoftmaxSolver = NonTunableSolverBase; -struct Softmax final : SoftmaxSolver +template +using SoftmaxTunableSolver = + TunableSolverMixin; + +struct PerformanceConfigSoftmax : PerfConfigBase { - const std::string& SolverDbId() const override { return GetSolverDbId(); } + int local_size; + bool initialized = false; + PerformanceConfigSoftmax(int _local_size) : local_size(_local_size) {} + PerformanceConfigSoftmax() : local_size(start_local_size) {} + PerformanceConfigSoftmax(bool) : local_size(start_local_size) {} + void HeuristicInit(const miopen::softmax::ProblemDescription& problem); + bool SetNextValue(const miopen::softmax::ProblemDescription& problem); + bool IsValidValue() const; + bool IsValid(const ExecutionContext& context, + const miopen::softmax::ProblemDescription& problem) const; + + template + static void Visit(Self&& s, F f) + { + f(s.local_size, "local_size"); + } + bool operator==(const PerformanceConfigSoftmax& other) const; +public: + static constexpr auto default_local_size = 1024; + static constexpr auto max_local_size = 1024; + static constexpr auto start_local_size = 1; +}; + +struct Softmax final : SoftmaxTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } bool IsApplicable(const ExecutionContext& context, const miopen::softmax::ProblemDescription& problem) const override; - + bool IsDynamic() const override { return true; } + PerformanceConfigSoftmax + GetDefaultPerformanceConfig(const ExecutionContext& context, + const miopen::softmax::ProblemDescription& problem) const override; + bool IsValidPerformanceConfig(const ExecutionContext& context, + const miopen::softmax::ProblemDescription& problem, + const PerformanceConfigSoftmax& config) const override; + PerformanceConfigSoftmax Search(const ExecutionContext& context, + const miopen::softmax::ProblemDescription& problem, + const AnyInvokeParams& invoke_context) const override + { + // Forward y and backward dx are both an input and an output, so we need to restore them + // after tuning to make sure tuning doesn't affect the output + auto invoke_context_softmax = invoke_context.CastTo(); + std::vector data_backup; + if(problem.IsForward() && invoke_context_softmax.forward_y != nullptr) + { + data_backup = std::vector(invoke_context_softmax.yDesc.GetNumBytes()); + context.GetStream().ReadTo(data_backup.data(), + invoke_context_softmax.forward_y, + invoke_context_softmax.yDesc.GetNumBytes()); + } + else if(!problem.IsForward() && invoke_context_softmax.dx != nullptr) + { + data_backup = std::vector(invoke_context_softmax.xdxDesc.GetNumBytes()); + context.GetStream().ReadTo(data_backup.data(), + invoke_context_softmax.dx, + invoke_context_softmax.xdxDesc.GetNumBytes()); + } + + auto result = GenericSearch(*this, context, problem, invoke_context); + + if(problem.IsForward() && invoke_context_softmax.forward_y != nullptr) + { + context.GetStream().WriteTo(data_backup.data(), + invoke_context_softmax.forward_y, + invoke_context_softmax.yDesc.GetNumBytes()); + } + else if(!problem.IsForward() && invoke_context_softmax.dx != nullptr) + { + context.GetStream().WriteTo(data_backup.data(), + invoke_context_softmax.dx, + invoke_context_softmax.xdxDesc.GetNumBytes()); + } + + return result; + } ConvSolution GetSolution(const ExecutionContext& context, - const miopen::softmax::ProblemDescription& problem) const override; - + const miopen::softmax::ProblemDescription& problem, + const PerformanceConfigSoftmax& config) const override; std::size_t GetWorkspaceSize(const ExecutionContext&, const miopen::softmax::ProblemDescription&) const override { diff --git a/projects/miopen/src/include/miopen/solution.hpp b/projects/miopen/src/include/miopen/solution.hpp index 1ad07dd01fed..3ebf8d4d29a5 100644 --- a/projects/miopen/src/include/miopen/solution.hpp +++ b/projects/miopen/src/include/miopen/solution.hpp @@ -45,7 +45,7 @@ namespace miopen { struct Handle; -struct MIOPEN_INTERNALS_EXPORT Solution : miopenSolution +struct Solution : miopenSolution { std::vector serialization_cache; @@ -107,10 +107,11 @@ struct MIOPEN_INTERNALS_EXPORT Solution : miopenSolution const ProblemContainer& GetProblem() const { return problem; } void SetProblem(ProblemContainer value) { problem = std::move(value); } - void Run(const Handle& handle, - const std::unordered_map& inputs, - Data_t workspace, - size_t workspace_size); + MIOPEN_INTERNALS_EXPORT void + Run(const Handle& handle, + const std::unordered_map& inputs, + Data_t workspace, + size_t workspace_size); void LogDriverCommand() const; diff --git a/projects/miopen/src/include/miopen/solver.hpp b/projects/miopen/src/include/miopen/solver.hpp index 545082cb10d8..d78e4034d93c 100644 --- a/projects/miopen/src/include/miopen/solver.hpp +++ b/projects/miopen/src/include/miopen/solver.hpp @@ -122,6 +122,12 @@ struct SolverBase } }; +#ifdef _WIN32 +// Suppress -Wundefined-func-template warning for Windows +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wundefined-func-template" +#endif + /// Common interface for tunable and non-tunable solvers template struct SolverInterface : SolverBase @@ -189,6 +195,8 @@ struct TunableSolverTrait template struct SolverBaseTunable : SolverInterfaceTunable, TunableSolverTrait { + using PerformanceConfigType = PerformanceConfig; + bool IsTunable() const final { return true; }; /// Initializes performance config to the default values. @@ -231,6 +239,11 @@ struct SolverBaseTunable : SolverInterfaceTunable, TunableSolv } }; +#ifdef _WIN32 +// Suppress -Wundefined-func-template warning for Windows +#pragma GCC diagnostic pop +#endif + // \todo Should be removed template using NonTunableSolverBase = SolverBaseNonTunable; diff --git a/projects/miopen/src/include/miopen/solver/conv_direct_naive_conv.hpp b/projects/miopen/src/include/miopen/solver/conv_direct_naive_conv.hpp index 4afc1b599927..1d6583a21996 100644 --- a/projects/miopen/src/include/miopen/solver/conv_direct_naive_conv.hpp +++ b/projects/miopen/src/include/miopen/solver/conv_direct_naive_conv.hpp @@ -147,7 +147,7 @@ auto MakeStrideArray(V vec) { ret[i] = static_cast(vec[i]); } - return ret; + return std::move(ret); } ::miopen::solver::ConvSolution diff --git a/projects/miopen/src/include/miopen/solver/implicitgemm_ck_util.hpp b/projects/miopen/src/include/miopen/solver/implicitgemm_ck_util.hpp index bf36a0242c02..5dee1ab50055 100644 --- a/projects/miopen/src/include/miopen/solver/implicitgemm_ck_util.hpp +++ b/projects/miopen/src/include/miopen/solver/implicitgemm_ck_util.hpp @@ -1160,7 +1160,8 @@ ConvSolution InitInvokerFactoryNCHW(const ExecutionContext& ctx, if(ptr_iter == conv_ptrs.end()) { MIOPEN_LOG_E("PerformanceConfig kernel '" + kernel_id + "' does not exist."); - return {miopenStatusInvalidValue}; + result = ConvSolution{miopenStatusInvalidValue}; + return result; } #ifdef CK_EXPERIMENTAL_BUILDER @@ -1302,6 +1303,7 @@ ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, const ProblemDescriptionType& problem, const std::string& kernel_id) { + ConvSolution result; auto conv_ptrs = DeviceOpType::GetInstances(); std::optional split_k = std::nullopt; std::string id_string = kernel_id; @@ -1317,10 +1319,10 @@ ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, if(ptr_iter == conv_ptrs.end()) { MIOPEN_LOG_E("PerformanceConfig kernel '" + kernel_id + "' does not exist."); - return {miopenStatusInvalidValue}; + result = ConvSolution{miopenStatusInvalidValue}; + return result; } - ConvSolution result; #ifdef CK_EXPERIMENTAL_BUILDER std::string description = (*ptr_iter)->describe()->detailed(); diff --git a/projects/miopen/src/include/miopen/solver/implicitgemm_static_ck_util.hpp b/projects/miopen/src/include/miopen/solver/implicitgemm_static_ck_util.hpp index 6d84b82258ed..509b7de237b1 100644 --- a/projects/miopen/src/include/miopen/solver/implicitgemm_static_ck_util.hpp +++ b/projects/miopen/src/include/miopen/solver/implicitgemm_static_ck_util.hpp @@ -186,7 +186,7 @@ inline static auto GetPerformanceConfigBase(const ExecutionContext& ctx, PerformanceImplicitGemm_t pp; pp.HeuristicInit(ctx, problem); MIOPEN_LOG_I(pp.ToString()); - return pp; + return std::move(pp); } static inline bool use_amd_inline_asm(const ExecutionContext& ctx, diff --git a/projects/miopen/src/include/miopen/solver_id.hpp b/projects/miopen/src/include/miopen/solver_id.hpp index 76a13b051c25..5b58150b9826 100644 --- a/projects/miopen/src/include/miopen/solver_id.hpp +++ b/projects/miopen/src/include/miopen/solver_id.hpp @@ -67,21 +67,21 @@ enum class Primitive MultiMarginLoss }; -struct MIOPEN_INTERNALS_EXPORT Id +struct Id { static constexpr uint64_t invalid_value = 0; Id() = default; - Id(uint64_t value_); + MIOPEN_INTERNALS_EXPORT Id(uint64_t value_); Id(ForceInit, uint64_t value_); - Id(const std::string& str); - Id(const char* str); + MIOPEN_INTERNALS_EXPORT Id(const std::string& str); + MIOPEN_INTERNALS_EXPORT Id(const char* str); - std::string ToString() const; - AnySolver GetSolver() const; + MIOPEN_INTERNALS_EXPORT std::string ToString() const; + MIOPEN_INTERNALS_EXPORT AnySolver GetSolver() const; const SolverBase* GetSolverBase() const; - std::string GetAlgo(conv::Direction dir) const; - miopenConvAlgorithm_t GetAlgo() const; + MIOPEN_INTERNALS_EXPORT std::string GetAlgo(conv::Direction dir) const; + MIOPEN_INTERNALS_EXPORT miopenConvAlgorithm_t GetAlgo() const; Primitive GetPrimitive() const; bool IsValid() const { return is_valid; } diff --git a/projects/miopen/src/include/miopen/sqlite_db.hpp b/projects/miopen/src/include/miopen/sqlite_db.hpp index 85e7d87f2f26..e3f1b949759c 100644 --- a/projects/miopen/src/include/miopen/sqlite_db.hpp +++ b/projects/miopen/src/include/miopen/sqlite_db.hpp @@ -1,5 +1,5 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT +// SPDX-License-Identifier: MIT #pragma once @@ -130,7 +130,7 @@ struct SQLiteSerializable } }; -class MIOPEN_INTERNALS_EXPORT SQLite +class SQLite { class impl; // do we need propagate const @@ -163,18 +163,18 @@ class MIOPEN_INTERNALS_EXPORT SQLite }; using result_type = std::vector>; - SQLite(); - SQLite(const fs::path& filename_, bool is_system); - ~SQLite(); - SQLite(SQLite&&) noexcept; - SQLite& operator=(SQLite&&) noexcept; + MIOPEN_INTERNALS_EXPORT SQLite(); + MIOPEN_INTERNALS_EXPORT SQLite(const fs::path& filename_, bool is_system); + MIOPEN_INTERNALS_EXPORT ~SQLite(); + MIOPEN_INTERNALS_EXPORT SQLite(SQLite&&) noexcept; + MIOPEN_INTERNALS_EXPORT SQLite& operator=(SQLite&&) noexcept; SQLite& operator=(const SQLite&) = delete; - bool Valid() const; - result_type Exec(const std::string& query) const; - int Changes() const; + MIOPEN_INTERNALS_EXPORT bool Valid() const; + MIOPEN_INTERNALS_EXPORT result_type Exec(const std::string& query) const; + MIOPEN_INTERNALS_EXPORT int Changes() const; int Retry(std::function) const; static int Retry(std::function f, fs::path filename); - std::string ErrorMessage() const; + MIOPEN_INTERNALS_EXPORT std::string ErrorMessage() const; }; template @@ -510,7 +510,7 @@ class SQLitePerfDb : public SQLiteBase } /// Updates record under key PROBLEM_CONFIG with data ID:VALUES in database. - /// Returns updated record or boost::none if insertion failed + /// Returns updated record or std::optional equals to nullopt_t if insertion failed template inline std::optional UpdateUnsafe(const T& problem_config, const std::string& id, const V& values) diff --git a/projects/miopen/src/include/miopen/subbuffers.hpp b/projects/miopen/src/include/miopen/subbuffers.hpp index 77dbdb143208..de2d69666806 100644 --- a/projects/miopen/src/include/miopen/subbuffers.hpp +++ b/projects/miopen/src/include/miopen/subbuffers.hpp @@ -26,12 +26,15 @@ #pragma once +#include #include namespace miopen { struct Handle; -std::size_t GetSubbufferAlignment(const Handle* handle = nullptr); +// Export required for test code to link. Unlike templates (weak linkage), +// regular functions need explicit export for visibility in shared library. +MIOPEN_INTERNALS_EXPORT std::size_t GetSubbufferAlignment(const Handle* handle = nullptr); } // namespace miopen diff --git a/projects/miopen/src/include/miopen/target_properties.hpp b/projects/miopen/src/include/miopen/target_properties.hpp index cab04861a8e3..32a3475faeb8 100644 --- a/projects/miopen/src/include/miopen/target_properties.hpp +++ b/projects/miopen/src/include/miopen/target_properties.hpp @@ -26,6 +26,7 @@ #ifndef GUARD_TARGET_PROPERTIES_HPP #define GUARD_TARGET_PROPERTIES_HPP +#include #include #include #include @@ -181,7 +182,7 @@ class TargetProperties static std::size_t GetMaxWaveScratchSize() { return MaxWaveScratchSize; } static std::size_t GetMaxLocalMemorySize() { return MaxLocalMemorySize; } - void Init(const Handle*); + MIOPEN_INTERNALS_EXPORT void Init(const Handle*); }; } // namespace miopen diff --git a/projects/miopen/src/include/miopen/temp_file.hpp b/projects/miopen/src/include/miopen/temp_file.hpp index 98d30f20ed43..5eeb8f1287c1 100644 --- a/projects/miopen/src/include/miopen/temp_file.hpp +++ b/projects/miopen/src/include/miopen/temp_file.hpp @@ -33,10 +33,10 @@ namespace miopen { -class MIOPEN_INTERNALS_EXPORT TempFile +class TempFile { public: - TempFile(const std::string& path_infix); + MIOPEN_INTERNALS_EXPORT TempFile(const std::string& path_infix); TempFile(TempFile&& other) noexcept = default; TempFile& operator=(TempFile&& other) noexcept = default; diff --git a/projects/miopen/src/include/miopen/tensor.hpp b/projects/miopen/src/include/miopen/tensor.hpp index f66b15275a01..518147960e15 100644 --- a/projects/miopen/src/include/miopen/tensor.hpp +++ b/projects/miopen/src/include/miopen/tensor.hpp @@ -135,14 +135,14 @@ std::ptrdiff_t integer_division_ceil(X x, Y y) return (tx + ty - 1) / ty; } -struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor +struct TensorDescriptor : miopenTensorDescriptor { - TensorDescriptor(); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(); // This constructor is only used in test/tensor_holder.hpp // clang-format off [[deprecated("Use constructor with lengths instead")]] - TensorDescriptor(miopenDataType_t t); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t); // clang-format on // It is preferable to use constructors with lengths and strides with the std::size_t @@ -151,42 +151,46 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor // The delegation constructor should be placed above the target constructor in the // code for better dependency tracking - TensorDescriptor(miopenDataType_t t, const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, std::vector&& lens_in); - - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - std::vector&& lens_in); - - TensorDescriptor(miopenDataType_t t, - const std::vector& lens_in, - const std::vector& strides_in); - TensorDescriptor(miopenDataType_t t, - const std::initializer_list& lens_in, - const std::initializer_list& strides_in); - TensorDescriptor(miopenDataType_t t, - const std::vector& lens_in, - const std::vector& strides_in); - TensorDescriptor(miopenDataType_t t, - std::vector&& lens_in, - std::vector&& strides_in); - - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in, - const std::vector& strides_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::initializer_list& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, const std::vector& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::initializer_list& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::vector& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + std::vector&& lens_in); + + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + const std::vector& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + const std::initializer_list& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + const std::vector& lens_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + std::vector&& lens_in); + + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::vector& lens_in, + const std::vector& strides_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::initializer_list& lens_in, + const std::initializer_list& strides_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + const std::vector& lens_in, + const std::vector& strides_in); + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + std::vector&& lens_in, + std::vector&& strides_in); + + MIOPEN_INTERNALS_EXPORT TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + const std::vector& lens_in, + const std::vector& strides_in); TensorDescriptor(miopenDataType_t t, miopenTensorLayout_t layout_in, std::vector&& lens_in, @@ -210,30 +214,30 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor bool IsVectorized() const; - const std::vector& GetLengths() const; - const std::vector& GetStrides() const; - unsigned GetNumDims() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetLengths() const; + MIOPEN_INTERNALS_EXPORT const std::vector& GetStrides() const; + MIOPEN_INTERNALS_EXPORT unsigned GetNumDims() const; - miopenDataType_t GetType() const; + MIOPEN_INTERNALS_EXPORT miopenDataType_t GetType() const; // clang-format off [[deprecated("Use GetLayoutEnum() instead")]] - miopenTensorLayout_t GetLayout_t() const; + MIOPEN_INTERNALS_EXPORT miopenTensorLayout_t GetLayout_t() const; // clang-format on - const std::optional& GetLayoutEnum() const; - static std::string LayoutEnumToStr(miopenTensorLayout_t layout); - const std::string& GetLayout_str() const; + MIOPEN_INTERNALS_EXPORT const std::optional& GetLayoutEnum() const; + MIOPEN_INTERNALS_EXPORT static std::string LayoutEnumToStr(miopenTensorLayout_t layout); + MIOPEN_INTERNALS_EXPORT const std::string& GetLayout_str() const; - std::size_t GetVectorLength() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetVectorLength() const; std::optional GetCastType() const; void SetCastType(miopenDataType_t cast_type_); - std::size_t GetElementSize() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetElementSize() const; - std::size_t GetElementSpace() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetElementSpace() const; - std::size_t GetNumBytes() const; + MIOPEN_INTERNALS_EXPORT std::size_t GetNumBytes() const; - std::size_t GetIndex(std::initializer_list l) const; + MIOPEN_INTERNALS_EXPORT std::size_t GetIndex(std::initializer_list l) const; template std::size_t GetIndex(Ts... is) const @@ -241,19 +245,19 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor return this->GetIndex({static_cast(is)...}); } - bool IsPacked() const; + MIOPEN_INTERNALS_EXPORT bool IsPacked() const; bool IsContiguous() const; /// Checks all lengths and strides. - bool AllDimsFitIntoInt() const; + MIOPEN_INTERNALS_EXPORT bool AllDimsFitIntoInt() const; /// Checks only lengths. - bool AllLengthsFitIntoInt() const; + MIOPEN_INTERNALS_EXPORT bool AllLengthsFitIntoInt() const; - bool operator==(const TensorDescriptor& rhs) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const TensorDescriptor& rhs) const; bool operator!=(const TensorDescriptor& rhs) const; bool operator<(const TensorDescriptor& rhs) const; bool operator>(const TensorDescriptor& rhs) const; - std::string ToString() const; + MIOPEN_INTERNALS_EXPORT std::string ToString() const; enum class LayoutValidationMode { @@ -263,23 +267,24 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor // For vectorized layouts storage_layout must be without the ending 'c' // \todo make private - bool IsPossibleLayout(const std::string& storage_layout, - const std::string& layout, - LayoutValidationMode validationMode) const; + MIOPEN_INTERNALS_EXPORT bool IsPossibleLayout(const std::string& storage_layout, + const std::string& layout, + LayoutValidationMode validationMode) const; // Layout could be NCHW, NHWC, NCDHW, NDHWC, NCHWc, ... - bool IsPossibleLayout4D5D(const std::string& layout, LayoutValidationMode validationMode) const; + MIOPEN_INTERNALS_EXPORT bool IsPossibleLayout4D5D(const std::string& layout, + LayoutValidationMode validationMode) const; static std::vector find_permutation(const std::vector& lens, const std::vector& strides); // storage_layout must be NCHW or NCHWc for NCHWc, CHWN or CHWNc for CHWNc, NCHW for other 4D // layouts, NCDHW for 5D layouts - std::string GetLayout(std::string storage_layout) const; + MIOPEN_INTERNALS_EXPORT std::string GetLayout(std::string storage_layout) const; static miopenTensorLayout_t StringToLayoutType(std::string layout_str, bool vectorized, int vector_length); - friend MIOPEN_INTERNALS_EXPORT std::ostream& operator<<(std::ostream& stream, + MIOPEN_INTERNALS_EXPORT friend std::ostream& operator<<(std::ostream& stream, const TensorDescriptor& t); friend void to_json(nlohmann::json& j, const TensorDescriptor& descriptor); diff --git a/projects/miopen/src/include/miopen/tensor_ops.hpp b/projects/miopen/src/include/miopen/tensor_ops.hpp index 25d838598b9c..932b6c9a21c7 100644 --- a/projects/miopen/src/include/miopen/tensor_ops.hpp +++ b/projects/miopen/src/include/miopen/tensor_ops.hpp @@ -1,54 +1,23 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + #ifndef GUARD_MIOPEN_TENSOR_OPPS_HPP_ #define GUARD_MIOPEN_TENSOR_OPPS_HPP_ #include #include -#include #include -#include + +#include +#include +#include #include -#include -#include namespace miopen { struct Handle; -struct f_length_is_not_1_t -{ - template - bool operator()(T&& v) - { - return boost::get<0>(v) > 1; - } -}; - -MIOPEN_INTERNALS_EXPORT TensorDescriptor GetFlattenedTensorDescriptor(const TensorDescriptor& desc); +TensorDescriptor GetFlattenedTensorDescriptor(const TensorDescriptor& desc); template std::tuple @@ -86,9 +55,20 @@ GetConsistentFlattenedTensorDescriptors(const TDescriptors&... real_descriptor_p } } - auto non1_length_strides = - boost::combine(real_descriptors[0]->GetLengths(), real_descriptor_pack.GetStrides()...) | - boost::adaptors::filtered(f_length_is_not_1_t()); + const auto& length_ = real_descriptors[0]->GetLengths(); + const size_t num_dims = length_.size(); + + const auto combine_length_and_strides = [&](std::size_t dim) { + return std::apply( + [&](const auto*... descs) { + return std::make_tuple(length_[dim], descs->GetStrides()[dim]...); + }, + real_descriptors); + }; + + auto indices = std::views::iota(std::size_t(0), num_dims) | + std::views::filter([&](const size_t i) { return length_[i] > 1; }); + auto non1_length_strides = indices | std::views::transform(combine_length_and_strides); if((is_all_packed && is_all_same_strided) || non1_length_strides.empty()) { @@ -104,19 +84,19 @@ GetConsistentFlattenedTensorDescriptors(const TDescriptors&... real_descriptor_p std::array, NTensor> array_of_flat_strides; auto i = non1_length_strides.begin(); - std::size_t flat_len = boost::get<0>(*i); + std::size_t flat_len = std::get<0>(*i); auto i_previous = i++; // the 0-th dimension full-length doesn't matter for(; i != non1_length_strides.end(); ++i) { - std::size_t len = boost::get<0>(*i); + std::size_t len = std::get<0>(*i); bool is_all_full_length = true; repeat_n( [&](auto itensor) { - const std::size_t stride = boost::get(*i); - const std::size_t previous_stride = boost::get(*i_previous); + const std::size_t stride = std::get(*i); + const std::size_t previous_stride = std::get(*i_previous); const std::size_t full_len = previous_stride / stride; is_all_full_length &= (len == full_len); }, @@ -132,7 +112,7 @@ GetConsistentFlattenedTensorDescriptors(const TDescriptors&... real_descriptor_p repeat_n( [&](auto itensor) { - std::size_t previous_stride = boost::get(*i_previous); + std::size_t previous_stride = std::get(*i_previous); array_of_flat_strides[itensor].push_back(previous_stride); }, NTensorConstant); @@ -146,7 +126,7 @@ GetConsistentFlattenedTensorDescriptors(const TDescriptors&... real_descriptor_p // strides of all flattend tensors are different repeat_n( [&](auto itensor) { - std::size_t previous_stride = boost::get(*i_previous); + std::size_t previous_stride = std::get(*i_previous); array_of_flat_strides[itensor].push_back(previous_stride); }, NTensorConstant); diff --git a/projects/miopen/src/include/miopen/tmp_dir.hpp b/projects/miopen/src/include/miopen/tmp_dir.hpp index a32998bc2634..f290d83d7b77 100644 --- a/projects/miopen/src/include/miopen/tmp_dir.hpp +++ b/projects/miopen/src/include/miopen/tmp_dir.hpp @@ -10,10 +10,10 @@ namespace miopen { -struct MIOPEN_INTERNALS_EXPORT TmpDir +struct TmpDir { fs::path path; - explicit TmpDir(std::string_view prefix = ""); + MIOPEN_INTERNALS_EXPORT explicit TmpDir(std::string_view prefix = ""); TmpDir(TmpDir&&) noexcept = default; TmpDir& operator=(TmpDir&&) noexcept = default; @@ -23,9 +23,10 @@ struct MIOPEN_INTERNALS_EXPORT TmpDir operator const fs::path&() const { return path; } operator std::string() const { return path.string(); } - int Execute(std::string_view cmd, std::string_view args, bool allowChangingCwd = true) const; + MIOPEN_INTERNALS_EXPORT int + Execute(std::string_view cmd, std::string_view args, bool allowChangingCwd = true) const; - ~TmpDir(); + MIOPEN_INTERNALS_EXPORT ~TmpDir(); }; } // namespace miopen diff --git a/projects/miopen/src/include/miopen/util.hpp b/projects/miopen/src/include/miopen/util.hpp index b9a7403261d0..6b1439266e06 100644 --- a/projects/miopen/src/include/miopen/util.hpp +++ b/projects/miopen/src/include/miopen/util.hpp @@ -13,33 +13,61 @@ namespace miopen { struct Handle; -MIOPEN_INTERNALS_EXPORT float Im2ColGPU(const Handle& handle, - std::size_t spatial_dim, - ConstData_t im, - std::size_t im_offset, - std::size_t in_c, - const std::vector& in_spatial, - const std::vector& wei_spatial, - const std::vector& out_spatial, - const std::vector& pad_spatial, - const std::vector& stride_spatial, - const std::vector& dilation_spatial, - Data_t col, - miopenDataType_t type); +float Im2ColGPU(const Handle& handle, + std::size_t spatial_dim, + ConstData_t im, + std::size_t im_offset, + std::size_t in_c, + const std::vector& in_spatial, + const std::vector& wei_spatial, + const std::vector& out_spatial, + const std::vector& pad_spatial, + const std::vector& stride_spatial, + const std::vector& dilation_spatial, + Data_t col, + miopenDataType_t type); -MIOPEN_INTERNALS_EXPORT float Col2ImGPU(const Handle& handle, - std::size_t spatial_dim, - ConstData_t col, - const std::vector& out_spatial, - const std::vector& wei_spatial, - const std::vector& pad_spatial, - const std::vector& stride_spatial, - const std::vector& dilation_spatial, - std::size_t in_c, - const std::vector& in_spatial, - Data_t im, - std::size_t im_offset, - miopenDataType_t type); +float Col2ImGPU(const Handle& handle, + std::size_t spatial_dim, + ConstData_t col, + const std::vector& out_spatial, + const std::vector& wei_spatial, + const std::vector& pad_spatial, + const std::vector& stride_spatial, + const std::vector& dilation_spatial, + std::size_t in_c, + const std::vector& in_spatial, + Data_t im, + std::size_t im_offset, + miopenDataType_t type); + +float Col2Im3dGPUBatched(const Handle& handle, + ConstData_t col, + uint32_t out_d, + uint32_t out_h, + uint32_t out_w, + uint32_t wei_d, + uint32_t wei_h, + uint32_t wei_w, + uint32_t pad_d, + uint32_t pad_h, + uint32_t pad_w, + uint32_t stride_d, + uint32_t stride_h, + uint32_t stride_w, + uint32_t dilation_d, + uint32_t dilation_h, + uint32_t dilation_w, + uint32_t in_c, + uint32_t in_d, + uint32_t in_h, + uint32_t in_w, + uint32_t batch_count, + uint64_t col_batch_stride, + Data_t im, + uint64_t im_batch_stride, + uint64_t im_offset, + miopenDataType_t type); MIOPEN_INTERNALS_EXPORT float transpose_NCHW2CNHW(const Handle& handle, int n, @@ -81,14 +109,14 @@ MIOPEN_INTERNALS_EXPORT float transpose_NCHW2Vec(const Handle& handle, const void* alpha, const void* beta); -MIOPEN_INTERNALS_EXPORT float transpose_packed_MN2NM(const Handle& handle, - int m, - int n, - std::size_t in_offset, - std::size_t out_offset, - ConstData_t in, - Data_t out, - miopenDataType_t type); +float transpose_packed_MN2NM(const Handle& handle, + int m, + int n, + std::size_t in_offset, + std::size_t out_offset, + ConstData_t in, + Data_t out, + miopenDataType_t type); } // namespace miopen #endif // _MIOPEN_UTIL_HPP_ diff --git a/projects/miopen/src/include/miopen/utility/transposing_solver.hpp b/projects/miopen/src/include/miopen/utility/transposing_solver.hpp index 6a9f32f53726..67e48662805c 100644 --- a/projects/miopen/src/include/miopen/utility/transposing_solver.hpp +++ b/projects/miopen/src/include/miopen/utility/transposing_solver.hpp @@ -162,15 +162,17 @@ struct UniversalTransposeSolver : TransposePseudoSolver auto sln = ConvSolution{}; { - const auto tensor_space = problem.input.GetElementSize(); + const auto tensor_space = problem.input.GetElementSpace(); const auto cus = ctx.GetStream().GetMaxComputeUnits(); const auto build_params = GetDataTypeKBP(problem.input.GetType()); constexpr std::size_t max_block_size = 256; const auto local_size = max_block_size; + // Note: std::max/min without explicit template params to avoid cppcheck false + // positives when testing with -Dmax/-Dmin preprocessor defines const auto num_blocks = - std::max(1, (tensor_space + local_size - 1) / local_size); - const auto capped_blocks = std::min(num_blocks, cus * 4); + std::max(std::size_t{1}, (tensor_space + local_size - 1) / local_size); + const auto capped_blocks = std::min(num_blocks, cus * 4); const auto global_size = capped_blocks * local_size; auto transposeKernel = KernelInfo{}; @@ -295,18 +297,19 @@ struct BatchedTransposeSolverImpl : TransposePseudoSolver static TransposeSolution CreateTransposeSolution(const ExecutionContext& ctx, const TensorDescriptor& desc) { - const auto& lens = desc.GetLengths(); + const auto& lens = desc.GetLengths(); + constexpr int n_dims = BatchedTransposeTraits::ndims; + const uint32_t n = static_cast(lens[0]); const uint32_t c = static_cast(lens[1]); - constexpr int expected_dims = BatchedTransposeTraits::ndims; - if constexpr(expected_dims == 4) + if constexpr(n_dims == 4) { const uint32_t h = static_cast(lens[2]); const uint32_t w = static_cast(lens[3]); return TransposeSolution(ctx, desc.GetType(), n, c, h, w); } - else // expected_dims == 5 + else // n_dims == 5 { const uint32_t d = static_cast(lens[2]); const uint32_t h = static_cast(lens[3]); @@ -371,18 +374,13 @@ inline std::string SyncLayoutDims(const char* from, const char* to) template struct ProblemTensorTransposeDescriptor { - using DescriptorGetter = TensorDescriptor& (Problem::*)(); using ConstDescriptorGetter = const TensorDescriptor& (Problem::*)() const; - DescriptorGetter descriptor; ConstDescriptorGetter cdescriptor; TensorDescriptor InvokeParams::*rt_descriptor; - union - { - ConstData_t InvokeParams::*as_input; - Data_t InvokeParams::*as_output; - }; + ConstData_t InvokeParams::*as_input = nullptr; + Data_t InvokeParams::*as_output = nullptr; const char* to; bool is_input; @@ -391,8 +389,10 @@ struct ProblemTensorTransposeDescriptor inline void Transpose(const Problem& src, Problem_& dest) const { const auto& desc_from = (src.*cdescriptor)(); - auto& desc_to = (dest.*descriptor)(); - desc_to = Transpose(desc_from); + // Use const_cast on the copy (dest) only - this is safe because we're mutating a copy, + // not the original problem. The copy is owned by the transposing solver and not shared. + auto& desc_to = const_cast((dest.*cdescriptor)()); + desc_to = Transpose(desc_from); } inline void Transpose(const InvokeParams& src, InvokeParams& dest) const @@ -526,17 +526,93 @@ class ProblemTensorTransposeGroup std::vector outputs; }; +/// Helper base that provides the correct GetSolution override based on whether +/// the solver Base is tunable or non-tunable. Tunable solvers have +/// GetSolution(ctx, problem, config) while non-tunable solvers have +/// GetSolution(ctx, problem). This class inherits from Base so that +/// TransposingSolver can inherit from it and get both Base and GetSolution. +template ::value> +struct TransposingSolverGetSolution; + +// Forward declaration +template +struct TransposingSolver; + +/// Non-tunable specialization: provides GetSolution(ctx, problem) +template +struct TransposingSolverGetSolution : Base +{ + ConvSolution GetSolution(const ExecutionContext& ctx, const Problem& problem) const override + { + auto transposed_problem = Derived::Transpose(problem); + ConvSolution sln = Inner{}.GetSolution(ctx, transposed_problem); + return static_cast(this)->WrapSolutionWithTranspose( + ctx, problem, transposed_problem, std::move(sln)); + } +}; + +/// Tunable specialization: provides GetSolution(ctx, problem, config) and delegates +/// GetDefaultPerformanceConfig, IsValidPerformanceConfig, and Search to the inner solver. +template +struct TransposingSolverGetSolution : Base +{ + using PerformanceConfigType = typename Inner::PerformanceConfigType; + + PerformanceConfigType GetDefaultPerformanceConfig(const ExecutionContext& ctx, + const Problem& problem) const override + { + auto transposed_problem = Derived::Transpose(problem); + return Inner{}.GetDefaultPerformanceConfig(ctx, transposed_problem); + } + + bool IsValidPerformanceConfig(const ExecutionContext& ctx, + const Problem& problem, + const PerformanceConfigType& config) const override + { + auto transposed_problem = Derived::Transpose(problem); + return Inner{}.IsValidPerformanceConfig(ctx, transposed_problem, config); + } + + PerformanceConfigType Search(const ExecutionContext& ctx, + const Problem& problem, + const AnyInvokeParams& invoke_ctx) const override + { + auto transposed_problem = Derived::Transpose(problem); + return Inner{}.Search(ctx, transposed_problem, invoke_ctx); + } + + ConvSolution GetSolution(const ExecutionContext& ctx, + const Problem& problem, + const PerformanceConfigType& config) const override + { + auto transposed_problem = Derived::Transpose(problem); + ConvSolution sln = Inner{}.GetSolution(ctx, transposed_problem, config); + return static_cast(this)->WrapSolutionWithTranspose( + ctx, problem, transposed_problem, std::move(sln)); + } +}; + template -struct TransposingSolver : Base +struct TransposingSolver : TransposingSolverGetSolution { + // Allow the GetSolution helper to access Transpose and WrapSolutionWithTranspose + friend struct TransposingSolverGetSolution::value>; + using TransposeDescriptor = ProblemTensorTransposeDescriptor; /// TransposingSolver always needs workspace for transpose buffers. bool MayNeedWorkspace() const override { return true; } - /// Convert invoke params for inner solver invocation. - /// Override in derived class if inner solver expects a different params type. - /// Default: return params as AnyInvokeParams (pass-through). /// Convert from API invoke params to TransposingSolver invoke params. /// Override in derived class if API passes a different type than InvokeParams. /// Default: cast AnyInvokeParams directly to InvokeParams. @@ -545,6 +621,9 @@ struct TransposingSolver : Base return any_params.CastTo(); } + /// Convert invoke params for inner solver invocation. + /// Override in derived class if inner solver expects a different params type. + /// Default: return params as AnyInvokeParams (pass-through). static AnyInvokeParams ConvertForInnerSolver(const InvokeParams& params) { return params; } static std::vector GetTransposeSolvers() @@ -572,7 +651,7 @@ struct TransposingSolver : Base const auto transpose_solvers = Derived::GetTransposeSolversMap(); auto any_difference = false; - for(auto transpose : Derived::GetTransposes()) + for(auto transpose : Derived::GetTransposes(problem)) { decltype(auto) descriptor = (problem.*(transpose.cdescriptor))(); const auto layout = descriptor.GetLayout_str(); @@ -631,25 +710,41 @@ struct TransposingSolver : Base { // Use Derived::Transpose to allow derived classes to override (CRTP pattern) const auto transposed_problem = Derived::Transpose(problem); - auto ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); - for(const auto& transpose : Derived::GetTransposes()) + // Calculate workspace needed by inner solver plus transpose buffers + auto ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); + for(const auto& transpose : Derived::GetTransposes(problem)) { const auto& descriptor = (transposed_problem.*(transpose.cdescriptor))(); const auto e_size = get_data_size(descriptor.GetType()); - ws_size += descriptor.GetElementSpace() * e_size; + const auto tensor_size = descriptor.GetElementSpace() * e_size; + ws_size += tensor_size; } - return ws_size; } - ConvSolution GetSolution(const ExecutionContext& ctx, - const Problem& problem, - const typename Inner::PerformanceConfigType& config) const override + bool IsDynamic() const override + { + // Transposed solvers ALWAYS require workspace for transpose buffers, + // so they can ONLY work in Find mode (not immediate mode with workspace=0). + return false; + } + + float GetWti(const ExecutionContext& ctx, const Problem& problem) const override + { + // Delegate to inner solver's GetWti() to ensure transposed solvers + // are included in WTI fallback path when TunaNet doesn't predict them + const auto transposed_problem = Derived::Transpose(problem); + return Inner{}.GetWti(ctx, transposed_problem); + } + + /// Wraps an inner solver's ConvSolution with transpose kernels. + /// Called by the GetSolution helper specializations. + ConvSolution WrapSolutionWithTranspose(const ExecutionContext& ctx, + const Problem& problem, + Problem& transposed_problem, + ConvSolution sln) const { - // Use Derived::Transpose to allow derived classes to override (CRTP pattern) - auto transposed_problem = Derived::Transpose(problem); - ConvSolution sln = Inner{}.GetSolution(ctx, transposed_problem, config); // NOLINTNEXTLINE (bugprone-unchecked-optional-access) auto old_factory = sln.invoker_factory.value(); const auto old_kernels_end = sln.construction_params.size(); @@ -658,7 +753,7 @@ struct TransposingSolver : Base std::vector> in_transpose_ifs, out_transpose_ifs; - for(auto transpose : Derived::GetTransposes()) + for(auto transpose : Derived::GetTransposes(problem)) { // For input transposes: use original problem's layout as source // For output transposes: use transposed problem's layout as source @@ -729,10 +824,14 @@ struct TransposingSolver : Base if(in_transpose_ifs.size() + out_transpose_ifs.size() == 0) return sln; - const auto ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); - + // Inner solver workspace is used as the starting offset for SegmentedGpuBuffer. + // The allocator carves out transpose buffers starting after the inner solver's region. + // Total workspace (inner + transpose) is reported via sln.workspace_sz so the caller + // allocates enough memory for both. + const auto inner_ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); + const auto total_ws_size = this->GetWorkspaceSize(ctx, problem); sln.invoker_factory = - [old_factory, old_kernels_end, ws_size, in_transpose_ifs, out_transpose_ifs]( + [old_factory, old_kernels_end, in_transpose_ifs, out_transpose_ifs, inner_ws_size]( const std::vector& kernels) { const auto inner_kernels = std::vector{kernels.begin(), kernels.begin() + old_kernels_end}; @@ -757,14 +856,16 @@ struct TransposingSolver : Base auto invoker = old_factory(inner_kernels); - return [invoker, in_transpose_invokers, out_transpose_invokers, ws_size]( + return [invoker, in_transpose_invokers, out_transpose_invokers, inner_ws_size]( const Handle& handle, const AnyInvokeParams& any_params) { const auto invoke_params = Derived::ConvertFromApiParams(any_params); auto transposed_params = invoke_params; handle.ResetKernelTime(); - SegmentedGpuBuffer allocator{handle, invoke_params.workspace, ws_size}; + // Start allocating transpose buffers after the inner solver's workspace. + // SegmentedGpuBuffer third parameter is the starting OFFSET, not size. + SegmentedGpuBuffer allocator{handle, invoke_params.workspace, inner_ws_size}; ProblemTensorTransposeGroup transposeGroup{handle, allocator, @@ -782,6 +883,8 @@ struct TransposingSolver : Base }; }; + sln.workspace_sz = total_ws_size; + return sln; } @@ -789,7 +892,7 @@ struct TransposingSolver : Base inline static Problem Transpose(const Problem& problem) { auto transposed_problem = problem; - for(const auto& transpose : Derived::GetTransposes()) + for(const auto& transpose : Derived::GetTransposes(problem)) transpose.Transpose(problem, transposed_problem); return transposed_problem; } diff --git a/projects/miopen/src/kernels/MIOpenCol2Im3d.cpp b/projects/miopen/src/kernels/MIOpenCol2Im3d.cpp index 5da567e95974..8313f17c40f4 100644 --- a/projects/miopen/src/kernels/MIOpenCol2Im3d.cpp +++ b/projects/miopen/src/kernels/MIOpenCol2Im3d.cpp @@ -138,3 +138,112 @@ extern "C" __global__ void Col2Im3dU(FLOAT* col, im_off[gid] = tmp; #endif } + +extern "C" __global__ void Col2Im3dUBatched(FLOAT* col, + const unsigned int col_d, + const unsigned int col_h, + const unsigned int col_w, + const unsigned int wei_d, + const unsigned int wei_h, + const unsigned int wei_w, + const unsigned int pad_d, + const unsigned int pad_h, + const unsigned int pad_w, + const unsigned int stride_d, + const unsigned int stride_h, + const unsigned int stride_w, + const unsigned int dilation_d, + const unsigned int dilation_h, + const unsigned int dilation_w, + const unsigned int channels, + const unsigned int depth, + const unsigned int height, + const unsigned int width, + const unsigned int batch_count, + const uint64_t col_batch_stride, + FLOAT* im, + const uint64_t im_batch_stride, + const uint64_t im_offset) +{ + const uint64_t per_image_size = static_cast(channels) * depth * height * width; + const uint64_t gid = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const uint64_t global_size = static_cast(batch_count) * per_image_size; + + if(gid >= global_size) + return; + + const uint64_t n = gid / per_image_size; + const uint64_t localid = gid - n * per_image_size; + + const unsigned int im_ch = localid / (width * height * depth); + unsigned int itmp = localid % (width * height * depth); + unsigned int im_d = itmp / (width * height); + itmp = itmp % (width * height); + unsigned int im_h = itmp / width; + unsigned int im_w = itmp % width; + + im_d += pad_d; + im_h += pad_h; + im_w += pad_w; + + const unsigned int start_d = (im_d < dilation_d * (wei_d - 1) + 1) + ? 0 + : (im_d - (dilation_d * (wei_d - 1) + 1)) / stride_d + 1; + const unsigned int end_d = min(col_d, im_d / stride_d + 1); + + const unsigned int start_h = (im_h < dilation_h * (wei_h - 1) + 1) + ? 0 + : (im_h - (dilation_h * (wei_h - 1) + 1)) / stride_h + 1; + const unsigned int end_h = min(col_h, im_h / stride_h + 1); + + const unsigned int start_w = (im_w < dilation_w * (wei_w - 1) + 1) + ? 0 + : (im_w - (dilation_w * (wei_w - 1) + 1)) / stride_w + 1; + const unsigned int end_w = min(col_w, im_w / stride_w + 1); + +#if MIOPEN_USE_64BIT_INDEX + const uint64_t ch_offset = + static_cast(im_ch) * col_d * col_w * col_h * wei_d * wei_w * wei_h; +#else + const uint32_t ch_offset = im_ch * col_d * col_w * col_h * wei_d * wei_w * wei_h; +#endif + + FLOAT_ACCUM tmp = (FLOAT_ACCUM)0; + FLOAT* col_off = col + n * col_batch_stride + ch_offset; + + for(unsigned int cz = start_d; cz < end_d; cz++) + { + for(unsigned int cy = start_h; cy < end_h; cy++) + { + for(unsigned int cx = start_w; cx < end_w; cx++) + { + if((im_d - cz * stride_d) % dilation_d == 0 && + (im_h - cy * stride_h) % dilation_h == 0 && + (im_w - cx * stride_w) % dilation_w == 0) + { + const unsigned int z = (im_d - cz * stride_d) / dilation_d; + const unsigned int y = (im_h - cy * stride_h) / dilation_h; + const unsigned int x = (im_w - cx * stride_w) / dilation_w; + +#if MIOPEN_USE_64BIT_INDEX + const uint64_t col_index = + ((((((uint64_t)z * wei_h) + y) * wei_w + x) * col_d + cz) * col_h + cy) * + col_w + + cx; +#else + const uint32_t col_index = + (((((z * wei_h) + y) * wei_w + x) * col_d + cz) * col_h + cy) * col_w + cx; +#endif + tmp += CVT_FLOAT2ACCUM(col_off[col_index]); + } + } + } + } + + FLOAT* im_off = im + im_offset + n * im_batch_stride; +#if ACCUMULATOR_NEEDS_CONVERSION + im_off[localid] = tmp > CVT_FLOAT2ACCUM(MAX_VAL) ? MAX_VAL : CVT_ACCUM2FLOAT(tmp); +#else + im_off[localid] = tmp; +#endif +} diff --git a/projects/miopen/src/kernels/MIOpenLRNBwd.cl b/projects/miopen/src/kernels/MIOpenLRNBwd.cl deleted file mode 100644 index 4e60bf133a66..000000000000 --- a/projects/miopen/src/kernels/MIOpenLRNBwd.cl +++ /dev/null @@ -1,472 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#define PPCAT_NX(A, B) A##B -#define PPCAT(A, B) PPCAT_NX(A, B) -#define TWO 2 -#define THREE 3 -#define FOUR 4 -#define EIGHT 8 - -#define DBG_RANGE 0 -#if MIOPEN_USE_FP16 == 1 -#pragma OPENCL EXTENSION cl_khr_fp16 : enable -#define _FLOAT half -#endif -#if MIOPEN_USE_FP32 == 1 -#define _FLOAT float -#endif - -#define _FLOAT2 PPCAT(_FLOAT, TWO) -#define _FLOAT3 PPCAT(_FLOAT, THREE) -#define _FLOAT4 PPCAT(_FLOAT, FOUR) -#define _FLOAT8 PPCAT(_FLOAT, EIGHT) - -#define UNUSED __attribute__((__unused__)) - -#define MLO_LRN_GROUP_SZ2 1 -#define MLO_LRN_STRIDE 1 - -#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_KERNEL_SZ - 1) -#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1) -#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0) -// #define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2 - -struct LRNForwardParam -{ - _FLOAT alphaoverarea; - _FLOAT alpha; - _FLOAT beta; - _FLOAT K; -}; - -struct LRNBackwardParam -{ - _FLOAT ratio; - _FLOAT alpha; - _FLOAT beta; -}; - -/* - -This is a naive implementation. -The "sliding window" -based implementation is in MIOpenLRNFwd.cl file - -*/ - -__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2))) -__kernel void -MIOpenLRNWithinChannelBwd(const __global _FLOAT* top, - const __global _FLOAT* bot, - const __global _FLOAT* top_df, - const __global _FLOAT* scale, - __global _FLOAT* bot_df, - UNUSED _FLOAT ratio, - _FLOAT alpha, - _FLOAT beta) -{ - __local _FLOAT top_df_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT]; - __local _FLOAT ratio_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT]; - int x = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX; - int y = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX; - int lcl_id0 = get_local_id(0); - int lcl_id1 = get_local_id(1); - int ob = get_global_id(2); // output * batch_sz - int o = ob / MLO_LRN_BATCH_SZ; - int b = ob - o * MLO_LRN_BATCH_SZ; - int top_x = x; - int top_y = y; - int top_df_off = b * MLO_LRN_TOPDF_BATCH_STRIDE + o * MLO_LRN_TOPDF_CHANNEL_STRIDE; - int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE; - int bot_x = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX; - int bot_y = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX; - - _FLOAT prv_exp_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; - // _FLOAT prv_top_df[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; - - // load top_diff and scale tiles - for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1) - { - int top_y_act = top_y + b_j - MLO_LRN_PAD; - - bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT); - - top_y_act = (invisibleY) ? 0 : top_y_act; - - int top_df_y_off = top_y_act * MLO_LRN_TOPDF_STRIDE; - int scale_y_off = top_y_act * MLO_LRN_SCALE_STRIDE; - - int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH; - - for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0) - { - - int top_x_act = top_x + b_i - MLO_LRN_PAD; - - bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH); - - top_x_act = (invisibleX) ? 0 : top_x_act; -#if DBG_RANGE - if(top_df_off + top_df_y_off + top_x_act >= - MLO_LRN_BATCH_SZ * MLO_LRN_TOPDF_BATCH_STRIDE) - { - printf("K:err:topdf-off_range\n"); - } -#endif - _FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act]; - _FLOAT scale_val = scale[scale_off + scale_y_off + top_x_act]; - - top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val; - scale_val = (invisibleX || invisibleY) ? (_FLOAT)1.f : scale_val; - - top_df_data[lcl_off_v + b_i] = top_df_val; - ratio_data[lcl_off_v + b_i] = scale_val; - } - } - - barrier(CLK_LOCAL_MEM_FENCE); - - // actual top_diffs and scales - for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j) - { - int lcl_off_v = - (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_PAD + j) * MLO_LRN_LCL_DATA_WIDTH; - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++) - { - _FLOAT scale_ratio = - ratio_data[lcl_off_v + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD + i]; - prv_exp_scale[j][i] = exp(-beta * log(scale_ratio)); - } - } - - barrier(CLK_LOCAL_MEM_FENCE); - // read top and load ratio tile - int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE; - for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1) - { - int top_y_act = top_y + b_j - MLO_LRN_PAD; - - bool invisibleY = (top_y_act < 0) || (top_y_act >= MLO_LRN_TOP_HEIGHT); - - top_y_act = (invisibleY) ? 0 : top_y_act; - - int top_y_off = top_y_act * MLO_LRN_TOP_STRIDE; - - int lcl_off_v = b_j * MLO_LRN_LCL_DATA_WIDTH; - - for(int b_i = lcl_id0; b_i < MLO_LRN_LCL_DATA_WIDTH; b_i += MLO_LRN_GROUP_SZ0) - { - - int top_x_act = top_x + b_i - MLO_LRN_PAD; - - bool invisibleX = (top_x_act < 0) || (top_x_act >= MLO_LRN_TOP_WIDTH); - - top_x_act = (invisibleX) ? 0 : top_x_act; -#if DBG_RANGE - - if(top_off + top_y_off + top_x_act >= MLO_LRN_BATCH_SZ * MLO_LRN_TOP_BATCH_STRIDE) - { - printf("K:err:top-off_range\n"); - } -#endif - - _FLOAT top_val = top[top_off + top_y_off + top_x_act]; - - top_val = (invisibleX || invisibleY) ? 0 : top_val; - - _FLOAT top_df_val = top_df_data[lcl_off_v + b_i]; - - _FLOAT scale_val = ratio_data[lcl_off_v + b_i]; - - // scale val is not 0 - _FLOAT ratio_dta = (top_df_val * top_val) / scale_val; - // replacing scale with ratio - ratio_data[lcl_off_v + b_i] = ratio_dta; - } - } - - barrier(CLK_LOCAL_MEM_FENCE); - - // caculate bot diff - _FLOAT prv_bot_diff[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; - - for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j) - { - int v_off_v = (lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + j); - int hstart = y + v_off_v - MLO_LRN_PAD; - int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_HEIGHT + MLO_LRN_PRE_PAD); - - // accum offset, vertical - // int lcl_a_off_v = v_off_v * MLO_LRN_LCL_DATA_WIDTH; - // value offset, vertical - int lcl_v_off_v = (v_off_v + MLO_LRN_PAD) * MLO_LRN_LCL_DATA_WIDTH; - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++) - { - _FLOAT prv_ratio_accum = (_FLOAT)0; - int v_off_h = lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + i; - - int wstart = x + v_off_h - MLO_LRN_PAD; - int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_TOP_WIDTH + MLO_LRN_PRE_PAD); - - int adj_area_size = (hend - hstart) * (wend - wstart); - - // accum offset, horiz - int lcl_a_off_h = v_off_h; - // value offset, horiz - int lcl_v_off_h = lcl_a_off_h + MLO_LRN_PAD; - - for(int k = 0; k < MLO_LRN_KERNEL_SZ; k++) - { - for(int l = 0; l < MLO_LRN_KERNEL_SZ; l++) - { - prv_ratio_accum += - ratio_data[(v_off_v + k) * MLO_LRN_LCL_DATA_WIDTH + lcl_a_off_h + l]; - } - } - - _FLOAT top_df_val = top_df_data[lcl_v_off_v + lcl_v_off_h]; - - uint bot_off0 = MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * o + - MLO_LRN_BOT_STRIDE * (y + v_off_v) + x + v_off_h; - - uint bot_off = (y + v_off_v < MLO_LRN_BOT_HEIGHT && x + v_off_h < MLO_LRN_BOT_WIDTH && - b < MLO_LRN_BATCH_SZ && o < MLO_LRN_N_OUTPUTS) - ? bot_off0 - : MLO_LRN_BATCH_SZ * MLO_LRN_BOT_BATCH_STRIDE - 1; -#if DBG_RANGE - - if(bot_off >= MLO_LRN_BATCH_SZ * MLO_LRN_BOT_BATCH_STRIDE) - { - printf("K:err:bot-off_range\n"); - } -#endif - _FLOAT bot_dta = bot[bot_off]; - - bot_dta = (y + v_off_v < MLO_LRN_BOT_HEIGHT && x + v_off_h < MLO_LRN_BOT_WIDTH && - b < MLO_LRN_BATCH_SZ && o < MLO_LRN_N_OUTPUTS) - ? bot_dta - : 0; - - _FLOAT adj_ratio = (_FLOAT)2.f * alpha * beta / adj_area_size; - _FLOAT prv_accum_ratio = adj_ratio * bot_dta * prv_ratio_accum; - prv_bot_diff[j][i] = prv_exp_scale[j][i] * top_df_val - prv_accum_ratio; - } - } - - for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; j++) - { - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; i++) - { - if(bot_y + j < MLO_LRN_BOT_HEIGHT && bot_x + i < MLO_LRN_BOT_WIDTH && - b < MLO_LRN_BATCH_SZ && o < MLO_LRN_N_OUTPUTS) - { -#if DBG_RANGE - - if(MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o + - MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i >= - MLO_LRN_BATCH_SZ * MLO_LRN_BOTDF_BATCH_STRIDE) - { - printf("K:err:botdf-off_range\n"); - } -#endif - bot_df[MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * o + - MLO_LRN_BOTDF_STRIDE * (bot_y + j) + bot_x + i] = prv_bot_diff[j][i]; - } - } - } -} - -#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ) -#define MLO_LOW_CHNL_COUNT 1 -#else -#define MLO_LOW_CHNL_COUNT 0 -#endif - -__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2))) -__kernel void -MIOpenLRNAcrossChannelsBwd1(const __global _FLOAT* top, - const __global _FLOAT* bot, - const __global _FLOAT* top_df, - const __global _FLOAT* scale, - __global _FLOAT* bot_df, - _FLOAT ratio, - UNUSED _FLOAT alpha, - _FLOAT beta) -{ - int x = get_global_id(0); // channel x - int y = get_global_id(1); // channel y - int b = get_global_id(2); // batch - _FLOAT accum_ratio = 0; - _FLOAT top_df_in[MLO_LRN_KERNEL_SZ]; - _FLOAT scale_in[MLO_LRN_KERNEL_SZ]; - _FLOAT ratio_dta[MLO_LRN_KERNEL_SZ]; - int c_i = 0, c_o = 0; - int bot_df_off = 0; - - for(c_i = 0; c_i < MLO_LRN_PRE_PAD; c_i++) - { - - top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + - MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x]; - scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + - MLO_LRN_SCALE_STRIDE * y + x]; - _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + - MLO_LRN_TOP_STRIDE * y + x]; - - ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i]; - -#if MLO_LOW_CHNL_COUNT == 1 - ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0; -#endif - - accum_ratio = accum_ratio + ratio_dta[c_i]; - } - - for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++) - { - top_df_in[c_i] = top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + - MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + MLO_LRN_TOPDF_STRIDE * y + x]; - scale_in[c_i] = scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + - MLO_LRN_SCALE_STRIDE * y + x]; - _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + - MLO_LRN_TOP_STRIDE * y + x]; - ratio_dta[c_i] = (top_df_in[c_i] * top_dta) / scale_in[c_i]; -#if MLO_LOW_CHNL_COUNT == 1 - ratio_dta[c_i] = (c_i < MLO_LRN_N_OUTPUTS) ? ratio_dta[c_i] : 0; -#endif - - accum_ratio = accum_ratio + ratio_dta[c_i]; -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_INPUTS) -#endif - { - _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o + - MLO_LRN_BOT_STRIDE * y + x]; - - _FLOAT prv_scale = scale_in[c_o]; - - _FLOAT exp_scale = exp(-beta * log(prv_scale)); - // pow(prv_scale, -beta); - - _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio; - - _FLOAT out_val = top_df_in[c_o] * exp_scale - prv_accum_ratio; - - bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + - MLO_LRN_BOTDF_STRIDE * y + x; - - bot_df[bot_df_off] = out_val; - } - } - - for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++) - { - - _FLOAT prv_top_df_in = - top_df[MLO_LRN_TOPDF_BATCH_STRIDE * b + MLO_LRN_TOPDF_CHANNEL_STRIDE * c_i + - MLO_LRN_TOPDF_STRIDE * y + x]; - _FLOAT prv_scale_in = - scale[MLO_LRN_SCALE_BATCH_STRIDE * b + MLO_LRN_SCALE_CHANNEL_STRIDE * c_i + - MLO_LRN_SCALE_STRIDE * y + x]; - _FLOAT top_dta = top[MLO_LRN_TOP_BATCH_STRIDE * b + MLO_LRN_TOP_CHANNEL_STRIDE * c_i + - MLO_LRN_TOP_STRIDE * y + x]; - _FLOAT prv_ratio_dta = prv_top_df_in * top_dta / prv_scale_in; -#if MLO_LOW_CHNL_COUNT == 1 - prv_ratio_dta = (c_i < MLO_LRN_N_OUTPUTS) ? prv_ratio_dta : 0; -#endif - - accum_ratio = accum_ratio + prv_ratio_dta; - - accum_ratio = accum_ratio - ratio_dta[0]; - - for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++) - { - top_df_in[i] = top_df_in[i + 1]; - scale_in[i] = scale_in[i + 1]; - ratio_dta[i] = ratio_dta[i + 1]; - } - - top_df_in[MLO_LRN_KERNEL_SZ - 1] = prv_top_df_in; - scale_in[MLO_LRN_KERNEL_SZ - 1] = prv_scale_in; - ratio_dta[MLO_LRN_KERNEL_SZ - 1] = prv_ratio_dta; - -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_INPUTS) -#endif - { - _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o + - MLO_LRN_BOT_STRIDE * y + x]; - - _FLOAT prv_scale = scale_in[MLO_LRN_PAD]; - - _FLOAT exp_scale = exp(-beta * log(prv_scale)); - // pow(prv_scale,-beta); - - _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio; - - _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio; - - bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + - MLO_LRN_BOTDF_STRIDE * y + x; - - bot_df[bot_df_off] = out_val; - } - } - - for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PRE_PAD; c_i++, c_o++) - { - - accum_ratio = accum_ratio - ratio_dta[0]; - - for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++) - { - top_df_in[i] = top_df_in[i + 1]; - scale_in[i] = scale_in[i + 1]; - ratio_dta[i] = ratio_dta[i + 1]; - } - -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_INPUTS) -#endif - { - _FLOAT bot_dta = bot[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_o + - MLO_LRN_BOT_STRIDE * y + x]; - - _FLOAT prv_scale = scale_in[MLO_LRN_PAD]; - - _FLOAT exp_scale = exp(-beta * log(prv_scale)); - // pow(prv_scale,-beta); - - _FLOAT prv_accum_ratio = ratio * bot_dta * accum_ratio; - - _FLOAT out_val = top_df_in[MLO_LRN_PAD] * exp_scale - prv_accum_ratio; - - bot_df_off = MLO_LRN_BOTDF_BATCH_STRIDE * b + MLO_LRN_BOTDF_CHANNEL_STRIDE * c_o + - MLO_LRN_BOTDF_STRIDE * y + x; - - bot_df[bot_df_off] = out_val; - } - } -} diff --git a/projects/miopen/src/kernels/MIOpenLRNFwd.cl b/projects/miopen/src/kernels/MIOpenLRNFwd.cl deleted file mode 100644 index 093b86f6dd08..000000000000 --- a/projects/miopen/src/kernels/MIOpenLRNFwd.cl +++ /dev/null @@ -1,713 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#define PPCAT_NX(A, B) A##B -#define PPCAT(A, B) PPCAT_NX(A, B) -#define TWO 2 -#define THREE 3 -#define FOUR 4 -#define EIGHT 8 - -#if MIOPEN_USE_FP16 == 1 -#pragma OPENCL EXTENSION cl_khr_fp16 : enable -#define _FLOAT half -#endif -#if MIOPEN_USE_FP32 == 1 -#define _FLOAT float -#endif - -#define _FLOAT2 PPCAT(_FLOAT, TWO) -#define _FLOAT3 PPCAT(_FLOAT, THREE) -#define _FLOAT4 PPCAT(_FLOAT, FOUR) -#define _FLOAT8 PPCAT(_FLOAT, EIGHT) - -#define DBG_OUT 0 - -#define UNUSED __attribute__((__unused__)) - -#define MLO_LRN_GROUP_SZ2 1 -#define MLO_LRN_STRIDE 1 - -#define MLO_LRN_LEFT_PAD0 (((MLO_LRN_PRE_PAD0 + MLO_READ_UNIT - 1) / MLO_READ_UNIT) * MLO_READ_UNIT) -#define MLO_LRN_RIGHT_SIDE \ - (((MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / \ - MLO_READ_UNIT) * \ - MLO_READ_UNIT) -#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_LEFT_PAD0 + MLO_LRN_RIGHT_SIDE) -#define MLO_LCL_READ4 (MLO_LRN_LCL_DATA_WIDTH / MLO_READ_UNIT) -#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1) -#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0) -// #define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2 - -struct LRNForwardParam -{ - _FLOAT alphaoverarea; - _FLOAT alpha; - _FLOAT beta; - _FLOAT K; -}; - -#include "math_ops.h" -__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2))) -__kernel void -MIOpenLRNWithinChannel_PS(const __global _FLOAT* bot, - __global _FLOAT* top, -#if MLO_LRN_DO_SCALE - __global _FLOAT* scale, -#endif - _FLOAT alphaoverarea, - UNUSED _FLOAT alpha, - _FLOAT beta, - _FLOAT K) -{ - // IT's taken from POOLING AVE with stride = 1' - __local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT]; - int x = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX; - int y = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX; - int lcl_id0 = get_local_id(0); - int lcl_id1 = get_local_id(1); - int ob = get_global_id(2); // output * batch_sz - int o = iDiv_legacy(ob, MLO_LRN_BATCH_SZ); - int b = iMod(ob, o, MLO_LRN_BATCH_SZ); - int bot_x = x; - int bot_y = y; - int bot_off = b * MLO_LRN_BOT_BATCH_STRIDE + o * MLO_LRN_BOT_CHANNEL_STRIDE; - - // load tile - for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1) - { - int bot_y_act = bot_y + b_j - MLO_LRN_PRE_PAD1; - - bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT); - - int bot_y_off = bot_y_act * MLO_LRN_BOT_STRIDE; - - int lcl_off_v = mul24(b_j, (int)MLO_LRN_LCL_DATA_WIDTH); - - for(int b_i = lcl_id0; b_i < MLO_LCL_READ4; b_i += MLO_LRN_GROUP_SZ0) - { - - int bot_x_act = bot_x + (b_i * MLO_READ_UNIT) - MLO_LRN_LEFT_PAD0; - - bool invisibleX; - for(int i = 0; i < MLO_READ_UNIT; ++i) - { - - int bot_off_x = bot_off + bot_y_off + bot_x_act + i; - - invisibleX = (bot_x_act + i < 0) || (bot_x_act + i >= MLO_LRN_BOT_WIDTH); - - bot_off_x = (invisibleX || invisibleY) ? 0 : bot_off_x; - - _FLOAT bot_val = bot[bot_off_x]; - - bot_val = (invisibleX || invisibleY) ? 0 : bot_val; - - bot_data[lcl_off_v + (b_i * MLO_READ_UNIT) + i] = bot_val; - } - } - } - - barrier(CLK_LOCAL_MEM_FENCE); -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - _FLOAT partial_sum_x[MLO_LRN_N_HORIZ_OUT_PIX - 1]; // horizontal partial sum -#endif -#if MLO_LRN_N_VERT_OUT_PIX > 1 - _FLOAT partial_sum_xy[MLO_LRN_N_VERT_OUT_PIX - 1] - [MLO_LRN_N_HORIZ_OUT_PIX]; // horizontal-vertical partial sums. -#endif - _FLOAT accum[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; // accumulator - - int top_y = mad24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX, y); - int top_x = mad24(lcl_id0, (int)MLO_LRN_N_HORIZ_OUT_PIX, x); - - int lcl_y = mul24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX); - int lcl_x = - mad24(lcl_id0, (int)(MLO_LRN_N_HORIZ_OUT_PIX), (int)(MLO_LRN_LEFT_PAD0 - MLO_LRN_PRE_PAD0)); - int lcl_off = mad24(lcl_y, MLO_LRN_LCL_DATA_WIDTH, lcl_x); - - for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j) - { - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i) - { - accum[j][i] = 0; - } - } -#if MLO_LRN_N_VERT_OUT_PIX > 1 - for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX - 1; ++j) - { - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i) - { - partial_sum_xy[j][i] = 0; - } - } -#endif - - // running window summation - _FLOAT mov_accum; - int jj = 0; - int ii = 0; - - // first to get vertica partial sums - -#if MLO_LRN_N_VERT_OUT_PIX > 1 - for(; jj < (int)(MLO_LRN_N_VERT_OUT_PIX - 1); ++jj) - { - for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - - _FLOAT accum_tmp = bot_val * bot_val; - -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - // save horizontal partial sums - partial_sum_x[ii] = accum_tmp; -#endif - // accumulate in vert-horizontal(0) - partial_sum_xy[jj][0] += accum_tmp; - } - - for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - // accumulate in vert horizontal(0) - partial_sum_xy[jj][0] += accum_tmp; - } - - // running horizontal window - - for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - // calculate all vertical-horizontal partial sums - partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0 + 1] = - partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0] + - (accum_tmp -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - - partial_sum_x[ii - MLO_LRN_KERNEL_SZ0] -#endif - ); - } - - // put into accumulator[0][i] - // whatever has been accumulated so far - for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i) - { - accum[0][i] += partial_sum_xy[jj][i]; - } - } -#endif - - // calculate row 0 accumulators - for(; jj < (int)MLO_LRN_KERNEL_SZ1; ++jj) - { - mov_accum = 0; - - for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - partial_sum_x[ii] = accum_tmp; -#endif - mov_accum += accum_tmp; - } - - for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - mov_accum += accum_tmp; - } - - accum[0][0] += mov_accum; - // running horizontal window - - for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - // running horizontal window - mov_accum += (accum_tmp -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - - partial_sum_x[ii - MLO_LRN_KERNEL_SZ0] -#endif - ); - accum[0][ii - MLO_LRN_KERNEL_SZ0 + 1] += mov_accum; - } - } - - // accumulate all other rows besides 0 - for(; jj < (int)(MLO_LRN_KERNEL_SZ1 + MLO_LRN_N_VERT_OUT_PIX - 1); ++jj) - { - // first running horizontal winodw as before - mov_accum = 0; - for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - partial_sum_x[ii] = accum_tmp; -#endif - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp; - } - for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp; - } - // running horizontal window - - int ii1 = ii; - for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - _FLOAT bot_val = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii]; - _FLOAT accum_tmp = bot_val * bot_val; - // - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] = - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0] + accum_tmp; -#if MLO_LRN_N_HORIZ_OUT_PIX > 1 - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -= - partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]; -#endif - } - - // finally running vertical window - - for(ii = ii1; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii) - { - - // finish horizontal summation - // add/substarct vertical patial sum - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] += - accum[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1]; -#if MLO_LRN_N_VERT_OUT_PIX > 1 - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -= - partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1]; -#endif - } -#if MLO_LRN_N_VERT_OUT_PIX > 1 - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] -= partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][0]; -#endif - accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum[jj - MLO_LRN_KERNEL_SZ1][0]; - } - - // normalization - _FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; - _FLOAT adj_alphaoverarea = alphaoverarea; - for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++) - { - - // int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX + k - - // MLO_LRN_PAD1; - // int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT + - // MLO_LRN_PAD1); - - for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++) - { - - // int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l - - // MLO_LRN_PAD0; - // int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH - //+ - // MLO_LRN_PAD0); - // int adj_area_size = (hend - hstart) * (wend - wstart); - // adj_alphaoverarea = alpha / adj_area_size; - - prv_scale[k][l] = K + accum[k][l] * adj_alphaoverarea; - } - } - - int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE + - top_y * MLO_LRN_TOP_STRIDE + top_x; -#if MLO_LRN_DO_SCALE - int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE + - top_y * MLO_LRN_SCALE_STRIDE + top_x; -#endif - - // final output - - for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX -#if MLO_OUT_VERT_ALIGNED == 0 - && (top_y + k < MLO_LRN_TOP_HEIGHT) -#endif - ; - k++) - { - for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX -#if MLO_OUT_HORIZ_ALIGNED == 0 - && (top_x + l < MLO_LRN_TOP_WIDTH) -#endif - ; - l++) - { - _FLOAT s; - s = exp((_FLOAT)-beta * log(prv_scale[k][l])); - // s = pow(prv_scale[k][l], -beta); - _FLOAT bot_val = bot_data[lcl_off + mad24((k + MLO_LRN_PRE_PAD1), - (int)MLO_LRN_LCL_DATA_WIDTH, - (l + MLO_LRN_PRE_PAD0))]; -#if MLO_LRN_DO_SCALE - scale[scale_off + k * MLO_LRN_SCALE_STRIDE + l] = prv_scale[k][l]; -#endif - top[top_off + k * MLO_LRN_TOP_STRIDE + l] = bot_val * s; - } - } -} - -#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ) -#define MLO_LOW_CHNL_COUNT 1 -#else -#define MLO_LOW_CHNL_COUNT 0 -#endif -__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2))) -__kernel void -MIOpenLRNAcrossChannels4(const __global _FLOAT* bottom, - __global _FLOAT* top, -#if MLO_LRN_DO_SCALE - __global _FLOAT* scale, -#endif - _FLOAT alphaoverarea, - UNUSED _FLOAT alpha, - _FLOAT beta, - _FLOAT K) -{ - int pix_id = get_global_id(0); // - int b = get_global_id(2); // batch - MLO_READ_TYPE accum = 0; - MLO_READ_TYPE bot_in2[MLO_LRN_KERNEL_SZ]; - MLO_READ_TYPE bot_in[MLO_LRN_KERNEL_SZ]; - int c_i = 0, c_o = 0; - for(int i = 0; i < MLO_LRN_KERNEL_SZ; ++i) - { - bot_in2[i] = 0; - bot_in[i] = 0; - } - - int top_off = 0; -#if MLO_LRN_DO_SCALE - int scale_off; -#endif - - for(c_i = 0; c_i < MLO_LRN_PAD; c_i++) - { - MLO_READ_TYPE prv_in; - prv_in = 0; - -#if MLO_LOW_CHNL_COUNT == 1 - if(c_i < MLO_LRN_N_INPUTS) -#endif - { -#if MLO_C1x1_PIXLEFT > 0 - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - ((_FLOAT*)&prv_in)[j] = - bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT) + j]; - } - } - else -#endif - { - prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b + - MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT)]; - } - } - - bot_in2[c_i] = prv_in * prv_in; - bot_in[c_i] = prv_in; - accum = accum + bot_in2[c_i]; - // fma(bot_in2[c_i + MLO_LRN_PAD], bot_in2[c_i + MLO_LRN_PAD], - // accum); - } - - for(; c_i < MLO_LRN_KERNEL_SZ; c_i++, c_o++) - { - MLO_READ_TYPE prv_in; - prv_in = 0; - -#if MLO_LOW_CHNL_COUNT == 1 - if(c_i < MLO_LRN_N_INPUTS) -#endif - { - -#if MLO_C1x1_PIXLEFT > 0 - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - ((_FLOAT*)&prv_in)[j] = - bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT) + j]; - } - } - else -#endif - { - prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b + - MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT)]; - } - } - - bot_in2[c_i] = prv_in * prv_in; - bot_in[c_i] = prv_in; - accum = accum + bot_in2[c_i]; - - top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#if MLO_LRN_DO_SCALE - scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#endif - MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea); - // fma(accum,alphaoverarea, (_FLOAT)1.f); - - MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale)); - // pow(prv_scale,-beta); - // bug - // MLO_READ_TYPE prv_out = sqrt(bot_in2[c_o]); - MLO_READ_TYPE prv_out = bot_in[c_o]; - MLO_READ_TYPE out_val = prv_out * exp_scale; -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_OUTPUTS) -#endif - { - -#if MLO_C1x1_PIXLEFT > 0 - - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - top[top_off + j] = ((_FLOAT*)&out_val)[j]; -#if DBG_OUT - printf("K:o0: %d %f %f %f %f %f\n", - top_off + j, - top[top_off + j], - ((_FLOAT*)&prv_out)[j], - ((_FLOAT*)&exp_scale)[j], - ((_FLOAT*)&prv_scale)[j], - ((_FLOAT*)&accum)[j]); -#endif - -#if MLO_LRN_DO_SCALE - scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j]; -#endif - } - } - else -#endif - { - - *((__global MLO_READ_TYPE*)&top[top_off]) = out_val; -#if MLO_LRN_DO_SCALE - *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale; -#endif - } - } - } - - for(; c_i < MLO_LRN_N_INPUTS; c_i++, c_o++) - { - - MLO_READ_TYPE prv_in; - prv_in = 0; - -#if MLO_C1x1_PIXLEFT > 0 - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - ((_FLOAT*)&prv_in)[j] = - bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT) + j]; - } - } - else -#endif - { - prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b + - MLO_LRN_BOT_CHANNEL_STRIDE * c_i + - (pix_id * MLO_READ_UNIT)]; - } - - MLO_READ_TYPE prv_bot_in2 = prv_in * prv_in; - accum = accum + prv_bot_in2; - - accum = accum - bot_in2[0]; - // fma(-bot_in2[0], bot_in2[0], accum); - - for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++) - { - bot_in2[i] = bot_in2[i + 1]; - bot_in[i] = bot_in[i + 1]; - } - - bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2; - bot_in[MLO_LRN_KERNEL_SZ - 1] = prv_in; - - top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#if MLO_LRN_DO_SCALE - scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#endif - MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea); - // fma(accum,alphaoverarea, (_FLOAT)1.f); - - MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale)); - // pow(prv_scale,-beta); - // bug - // MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PRE_PAD]); - MLO_READ_TYPE prv_out = bot_in[MLO_LRN_PRE_PAD]; - MLO_READ_TYPE out_val = prv_out * exp_scale; - -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_OUTPUTS) -#endif - { - -#if MLO_C1x1_PIXLEFT > 0 - - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - top[top_off + j] = ((_FLOAT*)&out_val)[j]; -#if DBG_OUT - printf("K:o1: %d %f %f %f\n", - top_off + j, - top[top_off + j], - ((_FLOAT*)&prv_out)[j], - ((_FLOAT*)&exp_scale)[j]); -#endif - -#if MLO_LRN_DO_SCALE - scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j]; -#endif - } - } - else -#endif - { - - *((__global MLO_READ_TYPE*)&top[top_off]) = out_val; -#if MLO_LRN_DO_SCALE - *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale; -#endif - } - } - } - - for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++) - { - - accum = accum - bot_in2[0]; - // fma(-bot_in2[0], bot_in2[0], accum); - - for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++) - { - bot_in2[i] = bot_in2[i + 1]; - bot_in[i] = bot_in[i + 1]; - } - - top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#if MLO_LRN_DO_SCALE - scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE + - (pix_id * MLO_READ_UNIT); -#endif - MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea); - // fma(accum,alphaoverarea, (_FLOAT)1.f); - - MLO_READ_TYPE exp_scale = exp((MLO_READ_TYPE)-beta * log(prv_scale)); - // pow(prv_scale,-beta); - // bug - // MLO_READ_TYPE prv_out = sqrt(bot_in2[MLO_LRN_PRE_PAD]); - MLO_READ_TYPE prv_out = bot_in[MLO_LRN_PRE_PAD]; - - MLO_READ_TYPE out_val = prv_out * exp_scale; -#if MLO_LOW_CHNL_COUNT == 1 - if(c_o < MLO_LRN_N_OUTPUTS) -#endif - { - -#if MLO_C1x1_PIXLEFT > 0 - - // if the last one - if(pix_id == MLO_MAP_SZ4 - 1) - { - for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j) - { - top[top_off + j] = ((_FLOAT*)&out_val)[j]; -#if DBG_OUT - printf("K:o2: %d %f %f %f\n", - top_off + j, - top[top_off + j], - ((_FLOAT*)&prv_out)[j], - ((_FLOAT*)&exp_scale)[j]); -#endif - -#if MLO_LRN_DO_SCALE - scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j]; -#endif - } - } - else -#endif - { - - *((__global MLO_READ_TYPE*)&top[top_off]) = out_val; -#if MLO_LRN_DO_SCALE - *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale; -#endif - } - } - } -} diff --git a/projects/miopen/src/kernels/MIOpenPooling.cpp b/projects/miopen/src/kernels/MIOpenPooling.cpp index ae0f632708d5..d7e965679325 100644 --- a/projects/miopen/src/kernels/MIOpenPooling.cpp +++ b/projects/miopen/src/kernels/MIOpenPooling.cpp @@ -1,30 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#ifndef MIOPEN_HIP_RUNTIME_COMPILE #include +#endif + #include "miopen_cstdint.hpp" #include "pooling_functions.h" diff --git a/projects/miopen/src/kernels/MIOpenPoolingBwd.cpp b/projects/miopen/src/kernels/MIOpenPoolingBwd.cpp index 92fa283ca425..2a24443ddf5c 100644 --- a/projects/miopen/src/kernels/MIOpenPoolingBwd.cpp +++ b/projects/miopen/src/kernels/MIOpenPoolingBwd.cpp @@ -1,30 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#ifndef MIOPEN_HIP_RUNTIME_COMPILE #include +#endif + #include "miopen_cstdint.hpp" #include "miopen_limits.hpp" #include "float_types.h" diff --git a/projects/miopen/src/kernels/MIOpenPoolingBwdND.cpp b/projects/miopen/src/kernels/MIOpenPoolingBwdND.cpp index 1f9759dc98de..931f9fbf82ff 100644 --- a/projects/miopen/src/kernels/MIOpenPoolingBwdND.cpp +++ b/projects/miopen/src/kernels/MIOpenPoolingBwdND.cpp @@ -1,30 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#ifndef MIOPEN_HIP_RUNTIME_COMPILE #include +#endif + #include "miopen_cstdint.hpp" #include "miopen_limits.hpp" #include "float_types.h" diff --git a/projects/miopen/src/kernels/MIOpenPoolingForwardNaive.cpp b/projects/miopen/src/kernels/MIOpenPoolingForwardNaive.cpp index 929ad0429fb2..efcec448a069 100644 --- a/projects/miopen/src/kernels/MIOpenPoolingForwardNaive.cpp +++ b/projects/miopen/src/kernels/MIOpenPoolingForwardNaive.cpp @@ -1,30 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#ifndef MIOPEN_HIP_RUNTIME_COMPILE #include +#endif + #include "miopen_cstdint.hpp" #include "pooling_functions.h" diff --git a/projects/miopen/src/kernels/MIOpenPoolingND.cpp b/projects/miopen/src/kernels/MIOpenPoolingND.cpp index e4957f1c6111..8aeb0c30a26d 100644 --- a/projects/miopen/src/kernels/MIOpenPoolingND.cpp +++ b/projects/miopen/src/kernels/MIOpenPoolingND.cpp @@ -1,30 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019-2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#ifndef MIOPEN_HIP_RUNTIME_COMPILE #include +#endif + #include "miopen_cstdint.hpp" #include "pooling_functions.h" diff --git a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp index 76d4fd1aa651..528277f851dc 100644 --- a/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/projects/miopen/src/kernels/MIOpenTensorKernelsHip.cpp @@ -908,183 +908,3 @@ extern "C" __global__ void Op2dTensorLite(const MIOPEN_TYPE* a, } } #endif - -#ifdef USE_LEADING_ONES -extern "C" __global__ void OpTensorLeadingOnes(MIOPEN_TYPE* a, - MIOPEN_TYPE* b, - MIOPEN_TYPE* c, - const int c_c, - const int c_h, - const int c_w, - const int c_nstride, - const int c_cstride, - const int work_per_wg, - const MIOPEN_TYPE alpha0, - const MIOPEN_TYPE alpha1, - const MIOPEN_TYPE beta, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, - const int num_wg, - const unsigned int bitmap) -{ - /* Special case for leading ones where the total no. of threads is the - * inner_product of the tensor dims. Each thread just updates one value - */ - - MIOPEN_TYPE* a_off = a + Aoffset; - MIOPEN_TYPE* b_off = b + Boffset; - MIOPEN_TYPE* c_off = c + Coffset; - - int gid = (bitmap == 0xF) ? (blockIdx.x * blockDim.x + threadIdx.x) : blockIdx.x; - - // num_wg: the number of workgroups should be launched - // MAX_NUM_WG: the maximum number of workgroups actually launched - for(; gid < num_wg; gid += MAX_NUM_WG) - { - int lid = (bitmap == 0xF) ? 0 : threadIdx.x; - int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; - - int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; - int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; - int o_c = - (bitmap & (1 << 2)) - ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % - c_c) - : 0; - int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * - ((bitmap & (1 << 2)) ? c_c : 1)); - - MIOPEN_TYPE operand = b_off[gid] * alpha1; - while(lid < work_per_wg) - { - int index = o_n * c_nstride + o_c * c_cstride + o_h * c_w + o_w + lid; - c_off[index] = MIOPEN_TENSOR_OP(a_off[index] * alpha0, operand) + beta * c_off[index]; - lid += lcl_sz; - } - } -} -#endif - -#ifdef USE_LEADING_ONES_GENERIC -extern "C" __global__ void OpTensorLeadingOnesGeneric(MIOPEN_TYPE* a, - const int a_nstride, - const int a_cstride, - const int a_hstride, - MIOPEN_TYPE* b, - const int b_nstride, - const int b_cstride, - const int b_hstride, - MIOPEN_TYPE* c, - const int c_c, - const int c_h, - const int c_w, - const int c_nstride, - const int c_cstride, - const int c_hstride, - const MIOPEN_TYPE alpha0, - const MIOPEN_TYPE alpha1, - const MIOPEN_TYPE beta, - const int work_per_wg, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, - const int num_wg, - const unsigned int bitmap) -{ - /* Special case for leading ones where the total no. of threads is the - * inner_product of the tensor dims. Each thread just updates one value - */ - - MIOPEN_TYPE* a_off = a + Aoffset; - MIOPEN_TYPE* b_off = b + Boffset; - MIOPEN_TYPE* c_off = c + Coffset; - - int gid = (bitmap == 0xF) ? blockIdx.x * blockDim.x + threadIdx.x : blockIdx.x; - - // num_wg: the number of workgroups should be launched - // MAX_NUM_WG: the maximum number of workgroups actually launched -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" - if(beta == static_cast(0)) -#pragma clang diagnostic pop - { - for(; gid < num_wg; gid += MAX_NUM_WG) - { - int lid = (bitmap == 0xF) ? 0 : threadIdx.x; - int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; - - int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; - int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; - int o_c = - (bitmap & (1 << 2)) - ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % - c_c) - : 0; - int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * - ((bitmap & (1 << 2)) ? c_c : 1)); - - int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w; - MIOPEN_TYPE operand = b_off[bindex] * alpha1; - - while(lid < work_per_wg) - { - o_c = (bitmap & (1 << 2)) ? o_c : (lid % c_c); - o_h = (bitmap & (1 << 1)) - ? o_h - : ((bitmap & (1 << 2)) ? (lid / c_w) : ((lid / c_c) % c_h)); - o_w = (bitmap & (1 << 0)) - ? o_w - : ((bitmap & (1 << 1)) - ? lid - : ((bitmap & (1 << 2)) ? (lid % c_w) : ((lid / c_c) / c_h))); - int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w; - int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w; - c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand); - - lid += lcl_sz; - } - } - } - else - { - for(; gid < num_wg; gid += MAX_NUM_WG) - { - int lid = (bitmap == 0xF) ? 0 : threadIdx.x; - int lcl_sz = (bitmap == 0xF) ? work_per_wg : blockDim.x; - - int o_w = (bitmap & (1 << 0)) ? (gid % c_w) : 0; - int o_h = (bitmap & (1 << 1)) ? ((gid / ((bitmap & (1 << 0)) ? c_w : 1)) % c_h) : 0; - int o_c = - (bitmap & (1 << 2)) - ? ((gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1))) % - c_c) - : 0; - int o_n = gid / (((bitmap & (1 << 0)) ? c_w : 1) * ((bitmap & (1 << 1)) ? c_h : 1) * - ((bitmap & (1 << 2)) ? c_c : 1)); - - int bindex = o_n * b_nstride + o_c * b_cstride + o_h * b_hstride + o_w; - MIOPEN_TYPE operand = b_off[bindex] * alpha1; - - while(lid < work_per_wg) - { - o_c = (bitmap & (1 << 2)) ? o_c : (lid % c_c); - o_h = (bitmap & (1 << 1)) - ? o_h - : ((bitmap & (1 << 2)) ? (lid / c_w) : ((lid / c_c) % c_h)); - o_w = (bitmap & (1 << 0)) - ? o_w - : ((bitmap & (1 << 1)) - ? lid - : ((bitmap & (1 << 2)) ? (lid % c_w) : ((lid / c_c) / c_h))); - int aindex = o_n * a_nstride + o_c * a_cstride + o_h * a_hstride + o_w; - int cindex = o_n * c_nstride + o_c * c_cstride + o_h * c_hstride + o_w; - c_off[cindex] = - MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex]; - - lid += lcl_sz; - } - } - } -} -#endif diff --git a/projects/miopen/src/kernels/gfx942.kdb.bz2.dvc b/projects/miopen/src/kernels/gfx942.kdb.bz2.dvc index d72f40bdfac3..261fa6787c59 100644 --- a/projects/miopen/src/kernels/gfx942.kdb.bz2.dvc +++ b/projects/miopen/src/kernels/gfx942.kdb.bz2.dvc @@ -1,5 +1,5 @@ outs: -- md5: 6afe97d7bd441fe15280b28b6e3b1748 - size: 3359707 +- md5: 1e6548449c25d9701e4a6197ba746b31 + size: 3354077 hash: md5 path: gfx942.kdb.bz2 diff --git a/projects/miopen/src/kernels/hip_math_ops.hpp b/projects/miopen/src/kernels/hip_math_ops.hpp index b9864cda2d7d..7e51f7b59221 100644 --- a/projects/miopen/src/kernels/hip_math_ops.hpp +++ b/projects/miopen/src/kernels/hip_math_ops.hpp @@ -12,3 +12,8 @@ inline __device__ unsigned int iRemquo(unsigned int x, unsigned int y, unsigned k = x / y; return x - __mul24(k, y); } + +// Responsibility of caller to ensure that `u`, `d`, and `u*d` don't exceed the 24-bit range. +inline __device__ unsigned iMod(unsigned v, unsigned u, unsigned d) { return v - __mul24(u, d); } + +inline __device__ unsigned iDiv(unsigned v, unsigned d) { return v / d; } \ No newline at end of file diff --git a/projects/miopen/src/kernels/miopen_math.hpp b/projects/miopen/src/kernels/miopen_math.hpp index aebe7239485a..e4a9e3a65b01 100644 --- a/projects/miopen/src/kernels/miopen_math.hpp +++ b/projects/miopen/src/kernels/miopen_math.hpp @@ -147,7 +147,7 @@ __forceinline__ __device__ ushort cos(ushort x) } __forceinline__ __device__ ushort fabs(ushort x) { - return float_to_bfloat16(__habs(bfloat16_to_float(x))); + return float_to_bfloat16(fabsf(bfloat16_to_float(x))); } __forceinline__ __device__ ushort fmax(ushort x, ushort y) { @@ -529,4 +529,4 @@ __forceinline__ __device__ FpVecType fabs(FpVecType x) } } -} // namespace miopen \ No newline at end of file +} // namespace miopen diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s deleted file mode 100644 index 2da803df4d11..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_fp16_fp32acc_f2x3_stride1.s +++ /dev/null @@ -1,34 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -.include "Conv_Winograd_Rage_v4_6_0_metadata.inc" - -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) - KERNEL_PROLOG _fp16_fp32acc_f2x3_stride1 - .include "Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc" - KERNEL_EPILOG _fp16_fp32acc_f2x3_stride1 -.else - .error "Unsupported gfx version" -.endif diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc deleted file mode 100644 index e1c9de904526..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc +++ /dev/null @@ -1,4572 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -v_lshlrev_b32 v1, 7, v0 -s_getpc_b64 [s8, s9] -label_6: -s_mov_b32 s10, label_0 - label_6 -s_mov_b32 s11, 131072 -buffer_load_dword v2, v1, [s8, s9, s10, s11], 0 offen -s_waitcnt vmcnt(0) -s_getpc_b64 [s6, s7] -label_1: -label_2: -s_load_dwordx2 [s22, s23], [s2, s3], 0 -s_load_dwordx16 [s24, s25, s26, s27, s28, s29, s30, s31, s32, s33, s34, s35, s36, s37, s38, s39], [s2, s3], 8 -s_load_dwordx8 [s40, s41, s42, s43, s44, s45, s46, s47], [s2, s3], 72 -v_readfirstlane_b32 s1, v0 -v_and_b32 v6, 255, v0 -v_lshrrev_b32 v7, 1, v6 -v_and_b32 v8, 1, v0 -v_and_b32 v1, 15, v6 -v_mul_u32_u24 v3, 4160, v8 -v_lshlrev_b32 v1, 3, v1 -v_and_b32 v4, 31, v7 -v_bfe_u32 v2, v6, 4, 2 -v_lshl_add_u32 v3, v4, 3, v3 -v_lshl_add_u32 v1, v2, 10, v1 -v_bfe_u32 v4, v6, 6, 2 -v_bfe_u32 v2, v6, 6, 2 -v_bfe_u32 v5, v0, 8, 0 -v_lshl_add_u32 v1, v2, 8, v1 -v_lshlrev_b32 v4, v5, v4 -v_bfe_u32 v2, v0, 8, 1 -v_lshl_add_u32 v3, v4, 10, v3 -v_lshl_add_u32 v1, v2, 7, v1 -s_lshr_b32 s1, s1, 6 -s_cmp_ge_u32 s1, 8 -s_cbranch_scc0 label_3 -s_setprio 3 -label_3: -s_waitcnt lgkmcnt(0) -s_and_b32 s28, s28, 262143 -s_and_b32 s5, s28, 8192 -s_cbranch_scc0 label_7 -s_load_dwordx8 [s48, s49, s50, s51, s52, s53, s54, s55], [s2, s3], 104 -label_7: -s_and_b32 s5, s28, 512 -s_cbranch_scc0 label_8 -s_load_dwordx8 [s56, s57, s58, s59, s60, s61, s62, s63], [s2, s3], 136 -s_load_dwordx4 [s64, s65, s66, s67], [s2, s3], 168 -label_8: -s_and_b32 s5, s28, 1024 -s_cbranch_scc0 label_9 -s_load_dwordx4 [s68, s69, s70, s71], [s2, s3], 184 -label_9: -s_and_b32 s5, s28, 81920 -s_cbranch_scc0 label_10 -s_load_dword s72, [s2, s3], 200 -label_10: -s_and_b32 s5, s28, 65536 -s_cbranch_scc0 label_11 -s_load_dword s73, [s2, s3], 204 -s_load_dwordx2 [s74, s75], [s2, s3], 208 -label_11: -s_and_b32 s5, s28, 131072 -s_cbranch_scc0 label_12 -s_load_dwordx4 [s76, s77, s78, s79], [s2, s3], 216 -label_12: -.long 0xba7f0241 -.long 0x00000000 // encoding fixup: s_setreg_imm32_b32 hwreg(1, 9, 1), 0 -s_bitcmp1_b32 s28, 3 -s_cbranch_scc0 label_13 -.long 0xba7f1901 -.long 0x0000000f // encoding fixup: s_setreg_imm32_b32 hwreg(1, 4, 4), 15 -.long 0xba7f1801 -.long 0x00000000 // encoding fixup: s_setreg_imm32_b32 hwreg(1, 0, 4), 0 -label_13: -s_waitcnt lgkmcnt(0) -s_bitcmp1_b32 s28, 6 -s_cbranch_scc0 label_17 -s_and_b32 s31, s31, 65535 -s_and_b32 s33, s33, 65535 -s_and_b32 s35, s35, 65535 -s_and_b32 s45, s45, 65535 -s_and_b32 s77, s77, 65535 -s_and_b32 s75, s75, 65535 -s_load_dwordx2 [s30, s31], [s30, s31], 0 -s_load_dwordx2 [s32, s33], [s32, s33], 0 -s_load_dwordx2 [s34, s35], [s34, s35], 0 -s_bitcmp1_b32 s28, 7 -s_cbranch_scc0 label_14 -s_load_dwordx2 [s44, s45], [s44, s45], 0 -label_14: -s_bitcmp1_b32 s28, 17 -s_cbranch_scc0 label_15 -s_load_dwordx2 [s76, s77], [s76, s77], 0 -label_15: -s_bitcmp1_b32 s28, 16 -s_cbranch_scc0 label_16 -s_load_dwordx2 [s74, s75], [s74, s75], 0 -label_16: -label_17: -s_bitcmp1_b32 s28, 9 -s_cbranch_scc1 label_18 -s_mov_b32 s58, s25 -s_mul_i32 s57, s24, s25 -s_mul_i32 s56, s57, s23 -s_mov_b32 s66, s43 -s_mul_i32 s65, s42, s43 -s_mul_i32 s64, s65, s26 -s_mul_i32 s5, s38, s39 -s_bitcmp1_b32 s28, 2 -s_cselect_b32 s8, s26, s23 -s_mul_i32 s8, s5, s8 -s_mov_b32 s62, s39 -s_bitcmp1_b32 s28, 2 -s_cselect_b32 s61, s8, s5 -s_cselect_b32 s60, s5, s8 -label_18: -s_and_b32 s8, s72, -256 -s_bitcmp1_b32 s28, 8 -s_cselect_b32 s9, 1, 0 -s_bitcmp1_b32 s28, 14 -s_cselect_b32 s72, s72, s9 -s_or_b32 s72, s72, s8 -s_cmp_lt_u32 s60, s61 -s_cselect_b32 s5, 0, 2147483648 -s_or_b32 s28, s28, s5 -s_waitcnt lgkmcnt(0) -s_bitcmp1_b32 s28, 13 -s_cbranch_scc0 label_19 -s_add_u32 s30, s30, s48 -s_addc_u32 s31, s31, s49 -s_add_u32 s34, s34, s52 -s_addc_u32 s35, s35, s53 -s_add_u32 s32, s32, s50 -s_addc_u32 s33, s33, s51 -s_add_u32 s44, s44, s54 -s_addc_u32 s45, s45, s55 -s_add_u32 s76, s76, s78 -s_addc_u32 s77, s77, s79 -label_19: -s_bitcmp1_b32 s28, 10 -s_cbranch_scc0 label_20 -v_ffbh_u32 v6, s27 -v_lshlrev_b32 v7, v6, s27 -v_and_b32 v5, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v5, v5 -v_rcp_f32 v2, v5 -v_subb_co_u32 v4, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v5, v5, v2, -1.0 -v_fma_f32 v5, v6, v2, v5 -v_fmaak_f32 v5, v5, v2, 2667577344 -v_mul_f32 v5, 1602224128, v5 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v5, neg(v5) -v_lshl_add_u32 v2, v2, 9, v5 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v2, [v6, v7] -v_subb_co_u32 v2, [vcc_lo, vcc_hi], v2, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v5, s4, v2 -v_add_co_u32 v2, [vcc_lo, vcc_hi], v5, s4 -v_addc_co_u32 v5, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v4 -v_cndmask_b32 v2, v2, v5, [vcc_lo, vcc_hi] -v_alignbit_b32 v2, v5, v2, v4 -s_nop 0 -v_readfirstlane_b32 s5, v2 -s_mul_i32 s8, s5, s27 -s_sub_u32 s4, s4, s8 -s_mul_i32 s8, s69, s5 -s_mul_hi_u32 s9, s69, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s30, s30, s8 -s_addc_u32 s31, s31, s9 -s_mul_i32 s8, s70, s5 -s_mul_hi_u32 s9, s70, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s32, s32, s8 -s_addc_u32 s33, s33, s9 -s_mul_i32 s8, s71, s5 -s_mul_hi_u32 s9, s71, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s34, s34, s8 -s_addc_u32 s35, s35, s9 -s_mul_i32 s8, s26, s5 -s_lshl_b32 s8, s8, 1 -s_add_u32 s44, s44, s8 -s_addc_u32 s45, s45, 0 -label_20: -s_cmp_eq_u32 s27, 304 -s_cbranch_scc0 label_21 -s_and_b32 s5, s4, 7 -s_lshr_b32 s8, s4, 3 -s_mul_i32 s4, s5, 38 -s_add_u32 s4, s4, s8 -label_21: -s_add_u32 s9, s26, 31 -s_lshr_b32 s9, s9, 5 -v_ffbh_u32 v6, s9 -v_lshlrev_b32 v7, v6, s9 -v_and_b32 v8, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v8, v8 -v_rcp_f32 v2, v8 -v_subb_co_u32 v5, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v8, v8, v2, -1.0 -v_fma_f32 v8, v6, v2, v8 -v_fmaak_f32 v8, v8, v2, 2667577344 -v_mul_f32 v8, 1602224128, v8 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v8, neg(v8) -v_lshl_add_u32 v2, v2, 9, v8 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v2, [v6, v7] -v_subb_co_u32 v2, [vcc_lo, vcc_hi], v2, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v6, s27, v2 -v_add_co_u32 v2, [vcc_lo, vcc_hi], v6, s27 -v_addc_co_u32 v6, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v5 -v_cndmask_b32 v2, v2, v6, [vcc_lo, vcc_hi] -v_alignbit_b32 v2, v6, v2, v5 -v_ffbh_u32 v6, v2 -v_lshlrev_b32 v7, v6, v2 -v_and_b32 v8, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v8, v8 -v_rcp_f32 v4, v8 -v_subb_co_u32 v5, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v8, v8, v4, -1.0 -v_fma_f32 v8, v6, v4, v8 -v_fmaak_f32 v8, v8, v4, 2667577344 -v_mul_f32 v8, 1602224128, v8 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v8, neg(v8) -v_lshl_add_u32 v4, v4, 9, v8 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v4, [v6, v7] -v_subb_co_u32 v4, [vcc_lo, vcc_hi], v4, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v6, s4, v4 -v_add_co_u32 v4, [vcc_lo, vcc_hi], v6, s4 -v_addc_co_u32 v6, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v5 -v_cndmask_b32 v4, v4, v6, [vcc_lo, vcc_hi] -v_alignbit_b32 v4, v6, v4, v5 -v_readfirstlane_b32 s5, v2 -v_readfirstlane_b32 s10, v4 -s_cmp_ge_u32 s10, s9 -s_cbranch_scc1 label_0 -s_mul_i32 s27, s5, s9 -s_mul_i32 s8, s10, s5 -s_sub_u32 s8, s4, s8 -s_mul_i32 s11, s10, 32 -s_sub_u32 s26, s26, s11 -s_min_u32 s26, s26, 32 -s_lshl_b32 s11, s11, 1 -s_mul_i32 s12, s11, s60 -s_mul_hi_u32 s13, s11, s60 -s_add_u32 s32, s32, s12 -s_addc_u32 s33, s33, s13 -s_mul_i32 s12, s11, s65 -s_mul_hi_u32 s13, s11, s65 -s_add_u32 s34, s34, s12 -s_addc_u32 s35, s35, s13 -s_add_u32 s44, s44, s11 -s_addc_u32 s45, s45, 0 -s_cmp_ge_u32 s1, 8 -s_cbranch_scc1 label_5 -label_4: -v_and_b32 v19, 255, v0 -s_bitcmp1_b32 s28, 31 -s_cselect_b32 s48, 2, 0 -s_cselect_b32 s49, 0, 2 -v_bfe_u32 v17, v19, 2, s48 -v_bfe_u32 v20, v19, 6, s49 -v_add_u32 v20, v17, v20 -v_lshrrev_b32 v21, 2, v19 -v_lshrrev_b32 v21, s48, v21 -v_and_b32 v17, 3, v21 -v_bfe_u32 v4, v21, 2, 1 -v_lshl_add_u32 v17, v4, 2, v17 -v_bfe_u32 v4, v21, 3, 1 -v_and_b32 v18, 15, v19 -v_lshl_add_u32 v17, v4, 3, v17 -v_bfe_u32 v2, v19, 6, 2 -v_mul_u32_u24 v4, 24, v17 -v_lshl_add_u32 v18, v2, 4, v18 -v_xor_b32 v17, v19, v19 quad_perm:[0, 0, 0, 1] -v_mul_u32_u24 v2, 24, v18 -v_lshl_add_u32 v4, v17, 3, v4 -v_bfe_u32 v18, v19, 4, 2 -v_mov_b32 v22, 1568 -v_mad_u32_u16 v4, v20, v22, v4 op_sel:[0, 0, 0, 0] -v_mov_b32 v22, 1568 -v_mad_u32_u16 v2, v18, v22, v2 op_sel:[0, 0, 0, 0] -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s50, 1024, 0 -s_cselect_b32 s51, 4, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s50, 65536, s50 -s_cselect_b32 s51, 65536, s51 -s_and_b32 s53, s1, 3 -s_mul_i32 s52, s53, 2048 -v_and_b32 v17, 63, v0 -v_lshl_add_u32 v5, v17, 4, s50 -v_mov_b32 v7, s51 -v_lshl_add_u32 v6, v17, 4, s52 -s_barrier -ds_read_b32 v16, v7 offset:55808 -s_mov_b32 s11, 0 -s_mov_b32 s20, 0 -s_lshl_b32 s48, s57, 1 -s_lshl_b32 s49, s61, 1 -s_and_b32 s50, s1, 3 -s_mul_i32 s51, 1, s50 -s_sub_u32 s9, s23, s51 -s_addc_u32 s9, s9, 3 -s_lshl_b32 s51, s48, 0 -s_mul_i32 s51, s51, s50 -s_add_u32 s30, s30, s51 -s_addc_u32 s31, s31, 0 -s_lshl_b32 s48, s48, 2 -s_lshl_b32 s49, s49, 2 -s_and_b32 s31, s31, 65535 -s_add_u32 s31, s31, 131072 -s_and_b32 s33, s33, 65535 -s_add_u32 s33, s33, 131072 -s_mov_b64 [s12, s13], [s30, s31] -s_mov_b32 s14, 0 -s_mov_b32 s15, 131072 -s_mov_b64 [s16, s17], [s30, s31] -s_mov_b32 s18, 0 -s_mov_b32 s19, 131072 -s_mov_b32 s10, 2147483648 -s_cmp_ge_u32 s1, 4 -s_cbranch_scc0 label_22 -s_branch label_34 -.align 32 -label_50: -v_readfirstlane_b32 s21, v16 -ds_read_b32 v16, v7 offset:55808 -ds_read_b128 [v8, v9, v10, v11], v6 offset:45568 -ds_read_b128 [v12, v13, v14, v15], v6 offset:46592 -s_bitcmp1_b32 s21, 0 -s_cmov_b32 s10, 0 -s_addc_u32 s20, s20, 2 -s_sub_u32 s11, s23, 1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_mov_b32 s2, s9 -s_mov_b64 [s12, s13], [s32, s33] -s_mov_b32 s14, s10 -s_mov_b32 s3, s9 -s_mov_b64 [s16, s17], [s32, s33] -s_mov_b32 s18, s10 -s_setpc_b64 [s36, s37] -.align 32 -label_51: -v_readfirstlane_b32 s21, v16 -ds_read_b32 v16, v7 offset:55808 -ds_read_b128 [v8, v9, v10, v11], v5 offset:53760 -ds_read_b128 [v12, v13, v14, v15], v5 offset:54784 -s_bitcmp1_b32 s21, 0 -s_cmov_b32 s10, 0 -s_addc_u32 s20, s20, 2 -s_sub_u32 s11, s23, 1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_mov_b32 s2, s9 -s_mov_b64 [s12, s13], [s30, s31] -s_mov_b32 s14, s10 -s_mov_b32 s3, s9 -s_mov_b64 [s16, s17], [s30, s31] -s_mov_b32 s18, s10 -s_setpc_b64 [s36, s37] -.align 32 -label_22: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v17 -v_mov_b32 v155, v106 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v112 -v_mov_b32 v157, v111 -v_pk_add_f16 v112, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v111, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_53 -label_52: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v107, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_23 -s_mov_b32 s29, label_23 - label_1 -s_branch label_46 -.align 32 -label_53: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v107, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_23 -s_mov_b32 s29, label_23 - label_1 -s_branch label_46 -.align 32 -label_23: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v112, v109 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v111, v110 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v112 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_55 -label_54: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v111, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v135, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_24 -s_mov_b32 s29, label_24 - label_1 -s_branch label_47 -.align 32 -label_55: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v111, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v135, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_24 -s_mov_b32 s29, label_24 - label_1 -s_branch label_47 -.align 32 -label_24: -s_mov_b32 s29, label_56 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_56: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v113 -v_mov_b32 v155, v114 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v120 -v_mov_b32 v157, v119 -v_pk_add_f16 v120, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v119, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_58 -label_57: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v115, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_25 -s_mov_b32 s29, label_25 - label_1 -s_branch label_48 -.align 32 -label_58: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v115, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_25 -s_mov_b32 s29, label_25 - label_1 -s_branch label_48 -.align 32 -label_25: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v120, v117 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v119, v118 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v120 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_60 -label_59: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v119, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v143, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_26 -s_mov_b32 s29, label_26 - label_1 -s_branch label_49 -.align 32 -label_60: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v119, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v143, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_26 -s_mov_b32 s29, label_26 - label_1 -s_branch label_49 -.align 32 -label_26: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v122 -v_mov_b32 v155, v124 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v127 -v_mov_b32 v157, v125 -v_pk_add_f16 v127, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v125, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_62 -label_61: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v121, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_27 -s_mov_b32 s29, label_27 - label_1 -s_branch label_46 -.align 32 -label_62: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v121, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_27 -s_mov_b32 s29, label_27 - label_1 -s_branch label_46 -.align 32 -label_27: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v127, v126 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v125, v128 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v127 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_64 -label_63: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v125, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v149, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_28 -s_mov_b32 s29, label_28 - label_1 -s_branch label_47 -.align 32 -label_64: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v125, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v149, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_28 -s_mov_b32 s29, label_28 - label_1 -s_branch label_47 -.align 32 -label_28: -s_mov_b32 s29, label_65 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_65: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v17 -v_mov_b32 v155, v109 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v107 -v_mov_b32 v157, v111 -v_pk_add_f16 v107, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v111, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_67 -label_66: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v112, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_29 -s_mov_b32 s29, label_29 - label_1 -s_branch label_48 -.align 32 -label_67: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v112, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_29 -s_mov_b32 s29, label_29 - label_1 -s_branch label_48 -.align 32 -label_29: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v107, v106 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v111, v110 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v107 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_69 -label_68: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v111, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v135, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_30 -s_mov_b32 s29, label_30 - label_1 -s_branch label_49 -.align 32 -label_69: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v111, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v135, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_30 -s_mov_b32 s29, label_30 - label_1 -s_branch label_49 -.align 32 -label_30: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v113 -v_mov_b32 v155, v117 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v115 -v_mov_b32 v157, v119 -v_pk_add_f16 v115, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v119, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_71 -label_70: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v120, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_31 -s_mov_b32 s29, label_31 - label_1 -s_branch label_46 -.align 32 -label_71: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v120, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_31 -s_mov_b32 s29, label_31 - label_1 -s_branch label_46 -.align 32 -label_31: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v115, v114 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v119, v118 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v115 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_73 -label_72: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v119, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v143, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_32 -s_mov_b32 s29, label_32 - label_1 -s_branch label_47 -.align 32 -label_73: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v119, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v143, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_32 -s_mov_b32 s29, label_32 - label_1 -s_branch label_47 -.align 32 -label_32: -s_mov_b32 s29, label_74 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_74: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v122 -v_mov_b32 v155, v126 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v121 -v_mov_b32 v157, v125 -v_pk_add_f16 v121, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v125, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_76 -label_75: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v127, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_33 -s_mov_b32 s29, label_33 - label_1 -s_branch label_48 -.align 32 -label_76: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v127, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_33 -s_mov_b32 s29, label_33 - label_1 -s_branch label_48 -.align 32 -label_33: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v121, v124 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v125, v128 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v121 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_78 -label_77: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v125, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v149, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_22 -s_mov_b32 s29, label_22 - label_1 -s_branch label_49 -.align 32 -label_78: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v125, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v149, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_22 -s_mov_b32 s29, label_22 - label_1 -s_branch label_49 -.align 32 -label_34: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v108, v108, v132 op_sel:[0, 0, 0] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v108, -1.0, v17 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v107, -1.0, v106 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v108, 1.0, v109 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v107, 1.0, v110 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_79: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v108, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v107, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v132, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_35 -s_mov_b32 s29, label_35 - label_1 -s_branch label_46 -.align 32 -label_35: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v109, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v110, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v109, -1.0, v112 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v110, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_80: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v112, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v111, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v136, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v135, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_36 -s_mov_b32 s29, label_36 - label_1 -s_branch label_47 -.align 32 -label_36: -s_mov_b32 s29, label_81 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_81: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v116, v116, v140 op_sel:[0, 0, 0] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v116, -1.0, v113 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v115, -1.0, v114 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v116, 1.0, v117 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v115, 1.0, v118 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_82: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v116, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v115, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v140, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_37 -s_mov_b32 s29, label_37 - label_1 -s_branch label_48 -.align 32 -label_37: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v117, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v118, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v117, -1.0, v120 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v118, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_83: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v120, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v119, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v144, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v143, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_38 -s_mov_b32 s29, label_38 - label_1 -s_branch label_49 -.align 32 -label_38: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v123, v123, v147 op_sel:[0, 0, 0] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v123, -1.0, v122 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v121, -1.0, v124 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v123, 1.0, v126 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v121, 1.0, v128 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_84: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v123, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v121, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v147, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_39 -s_mov_b32 s29, label_39 - label_1 -s_branch label_46 -.align 32 -label_39: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v126, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v128, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v126, -1.0, v127 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v128, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_85: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v127, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v125, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v151, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v149, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_40 -s_mov_b32 s29, label_40 - label_1 -s_branch label_47 -.align 32 -label_40: -s_mov_b32 s29, label_86 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_86: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v108, v108, v132 op_sel:[0, 0, 0] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v108, -1.0, v17 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v112, -1.0, v109 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v108, 1.0, v106 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v112, 1.0, v110 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_87: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v108, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v112, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v132, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_41 -s_mov_b32 s29, label_41 - label_1 -s_branch label_48 -.align 32 -label_41: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v106, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v110, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v106, -1.0, v107 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v110, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_88: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v107, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v111, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v131, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v135, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_42 -s_mov_b32 s29, label_42 - label_1 -s_branch label_49 -.align 32 -label_42: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v116, v116, v140 op_sel:[0, 0, 0] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v116, -1.0, v113 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v120, -1.0, v117 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v116, 1.0, v114 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v120, 1.0, v118 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_89: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v116, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v120, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v140, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_43 -s_mov_b32 s29, label_43 - label_1 -s_branch label_46 -.align 32 -label_43: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v114, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v118, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v114, -1.0, v115 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v118, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_90: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v115, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v119, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v139, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v143, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_44 -s_mov_b32 s29, label_44 - label_1 -s_branch label_47 -.align 32 -label_44: -s_mov_b32 s29, label_91 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_91: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v123, v123, v147 op_sel:[0, 0, 0] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v123, -1.0, v122 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v127, -1.0, v126 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v123, 1.0, v124 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v127, 1.0, v128 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_92: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v123, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v127, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v147, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_45 -s_mov_b32 s29, label_45 - label_1 -s_branch label_48 -.align 32 -label_45: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v124, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v128, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v124, -1.0, v121 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v128, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_93: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v121, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v125, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v145, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v149, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_34 -s_mov_b32 s29, label_34 - label_1 -s_branch label_49 -.align 32 -label_46: -v_add_f32 v18, v18, v36 -v_add_f32 v19, v19, v37 -v_add_f32 v20, v20, v38 -v_add_f32 v21, v21, v39 -v_add_f32 v70, v70, neg(v52) -v_add_f32 v71, v71, neg(v53) -v_add_f32 v72, v72, neg(v54) -v_add_f32 v73, v73, neg(v55) -v_add_f32 v18, v18, v52 -v_add_f32 v19, v19, v53 -v_add_f32 v20, v20, v54 -v_add_f32 v21, v21, v55 -v_add_f32 v70, v70, v36 -v_add_f32 v71, v71, v37 -v_add_f32 v72, v72, v38 -v_add_f32 v73, v73, v39 -v_cvt_f16_f32 v160, v18 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v20 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v71 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v72 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v73 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v18, 0 -v_mov_b32 v19, 0 -v_mov_b32 v20, 0 -v_mov_b32 v21, 0 -v_mov_b32 v36, 0 -v_mov_b32 v37, 0 -v_mov_b32 v38, 0 -v_mov_b32 v39, 0 -v_mov_b32 v52, 0 -v_mov_b32 v53, 0 -v_mov_b32 v54, 0 -v_mov_b32 v55, 0 -v_mov_b32 v70, 0 -v_mov_b32 v71, 0 -v_mov_b32 v72, 0 -v_mov_b32 v73, 0 -ds_write_b64 v1, [v160, v161] offset:4160 -ds_write_b64 v1, [v158, v159] offset:8 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_47: -v_add_f32 v26, v26, v44 -v_add_f32 v27, v27, v45 -v_add_f32 v28, v28, v46 -v_add_f32 v29, v29, v47 -v_add_f32 v78, v78, neg(v60) -v_add_f32 v79, v79, neg(v61) -v_add_f32 v80, v80, neg(v62) -v_add_f32 v81, v81, neg(v63) -v_add_f32 v26, v26, v60 -v_add_f32 v27, v27, v61 -v_add_f32 v28, v28, v62 -v_add_f32 v29, v29, v63 -v_add_f32 v78, v78, v44 -v_add_f32 v79, v79, v45 -v_add_f32 v80, v80, v46 -v_add_f32 v81, v81, v47 -v_cvt_f16_f32 v160, v26 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v28 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v78 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v79 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v80 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v26, 0 -v_mov_b32 v27, 0 -v_mov_b32 v28, 0 -v_mov_b32 v29, 0 -v_mov_b32 v44, 0 -v_mov_b32 v45, 0 -v_mov_b32 v46, 0 -v_mov_b32 v47, 0 -v_mov_b32 v60, 0 -v_mov_b32 v61, 0 -v_mov_b32 v62, 0 -v_mov_b32 v63, 0 -v_mov_b32 v78, 0 -v_mov_b32 v79, 0 -v_mov_b32 v80, 0 -v_mov_b32 v81, 0 -ds_write_b64 v1, [v160, v161] offset:12416 -ds_write_b64 v1, [v158, v159] offset:8264 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_48: -v_add_f32 v22, v22, v40 -v_add_f32 v23, v23, v41 -v_add_f32 v24, v24, v42 -v_add_f32 v25, v25, v43 -v_add_f32 v74, v74, neg(v56) -v_add_f32 v75, v75, neg(v57) -v_add_f32 v76, v76, neg(v58) -v_add_f32 v77, v77, neg(v59) -v_add_f32 v22, v22, v56 -v_add_f32 v23, v23, v57 -v_add_f32 v24, v24, v58 -v_add_f32 v25, v25, v59 -v_add_f32 v74, v74, v40 -v_add_f32 v75, v75, v41 -v_add_f32 v76, v76, v42 -v_add_f32 v77, v77, v43 -v_cvt_f16_f32 v160, v22 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v24 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v25 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v74 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v76 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v22, 0 -v_mov_b32 v23, 0 -v_mov_b32 v24, 0 -v_mov_b32 v25, 0 -v_mov_b32 v40, 0 -v_mov_b32 v41, 0 -v_mov_b32 v42, 0 -v_mov_b32 v43, 0 -v_mov_b32 v56, 0 -v_mov_b32 v57, 0 -v_mov_b32 v58, 0 -v_mov_b32 v59, 0 -v_mov_b32 v74, 0 -v_mov_b32 v75, 0 -v_mov_b32 v76, 0 -v_mov_b32 v77, 0 -ds_write_b64 v1, [v160, v161] offset:4160 -ds_write_b64 v1, [v158, v159] offset:8 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_49: -v_add_f32 v30, v30, v48 -v_add_f32 v31, v31, v49 -v_add_f32 v32, v32, v50 -v_add_f32 v33, v33, v51 -v_add_f32 v82, v82, neg(v64) -v_add_f32 v83, v83, neg(v65) -v_add_f32 v84, v84, neg(v66) -v_add_f32 v85, v85, neg(v67) -v_add_f32 v30, v30, v64 -v_add_f32 v31, v31, v65 -v_add_f32 v32, v32, v66 -v_add_f32 v33, v33, v67 -v_add_f32 v82, v82, v48 -v_add_f32 v83, v83, v49 -v_add_f32 v84, v84, v50 -v_add_f32 v85, v85, v51 -v_cvt_f16_f32 v160, v30 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v32 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v82 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v83 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v84 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v30, 0 -v_mov_b32 v31, 0 -v_mov_b32 v32, 0 -v_mov_b32 v33, 0 -v_mov_b32 v48, 0 -v_mov_b32 v49, 0 -v_mov_b32 v50, 0 -v_mov_b32 v51, 0 -v_mov_b32 v64, 0 -v_mov_b32 v65, 0 -v_mov_b32 v66, 0 -v_mov_b32 v67, 0 -v_mov_b32 v82, 0 -v_mov_b32 v83, 0 -v_mov_b32 v84, 0 -v_mov_b32 v85, 0 -ds_write_b64 v1, [v160, v161] offset:12416 -ds_write_b64 v1, [v158, v159] offset:8264 -s_bitcmp1_b32 s20, 4 -s_cmov_b32 s29, label_0 - label_1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -label_5: -s_and_b32 s45, s45, 65535 -s_add_u32 s45, s45, 131072 -s_mov_b64 [s76, s77], [s44, s45] -s_mov_b32 s78, 2147483648 -s_mov_b32 s79, 131072 -s_bitcmp0_b32 s28, 7 -s_cselect_b32 s78, 0, 2147483648 -s_and_b32 s73, s1, 3 -v_and_b32 v85, v0, 63 -v_lshrrev_b32 v84, 1, v85 -v_and_b32 v81, v85, 1 -v_mad_u32_u24 v81, v84, 8, v81 -v_lshlrev_b32 v81, 1, v81 -v_mad_u32_u24 v81, 4, s73, v81 -v_add_u32 v84, v81, 1 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v81, s26 -v_cndmask_b32 v81, -1, v81, [vcc_lo, vcc_hi] -v_cmp_lt_u32 [vcc_lo, vcc_hi], v84, s26 -v_cndmask_b32 v84, -1, v84, [vcc_lo, vcc_hi] -buffer_load_short_d16 v75, v81, [s76, s77, s78, s79], 0 idxen -buffer_load_short_d16 v77, v84, [s76, s77, s78, s79], 0 idxen -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s44, 1536, 0 -s_cselect_b32 s45, 12, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s44, 65536, s44 -s_cselect_b32 s45, 65536, s45 -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s73, 1024, 0 -s_cselect_b32 s74, 4, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s73, 65536, s73 -s_cselect_b32 s74, 65536, s74 -s_and_b32 s76, s1, 3 -s_mul_i32 s75, s76, 2048 -v_and_b32 v81, 63, v0 -v_lshl_add_u32 v5, v81, 3, s44 -v_lshl_add_u32 v10, v81, 3, 0 -v_mov_b32 v54, s45 -v_mov_b32 v71, 0 -v_lshl_add_u32 v72, v81, 4, s73 -v_mov_b32 v74, s74 -v_lshl_add_u32 v73, v81, 4, s75 -s_add_u32 s45, s42, 1 -s_lshr_b32 s45, s45, 1 -s_lshl_b32 s73, s45, 1 -s_add_u32 s74, s43, 1 -s_lshr_b32 s74, s74, 1 -s_lshl1_add_u32 s74, s74, 2 -s_pack_ll_b32_b16 s3, s74, s45 -s_pack_ll_b32_b16 s10, s25, s24 -s_sub_u32 s11, s73, s40 -s_sub_u32 s44, s74, s41 -s_pack_ll_b32_b16 s11, s44, s11 -s_pack_ll_b32_b16 s13, s43, s42 -s_sub_u32 s44, s74, 1 -s_pack_ll_b32_b16 s14, s44, s73 -v_lshrrev_b32 v86, 16, s3 -v_mov_b32 v101, 65535 -v_bfi_b32 v87, v101, s3, 0 -v_and_b32 v89, 1, v0 -v_bfe_u32 v95, v0, 6, 1 -v_and_b32 v84, 63, v0 -v_mov_b32 v101, 124 -v_mad_u32_u16 v90, v101, s5, 0 op_sel:[0, 0, 0, 0] -v_mov_b32 v101, 2 -v_mad_u32_u16 v95, v101, s8, v95 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v88, v86, v87, 0 op_sel:[0, 0, 0, 0] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 0, v89 -v_cndmask_b32 v96, v88, v87, [vcc_lo, vcc_hi] -v_mad_u32_u16 v85, 62, v95, v84 op_sel:[0, 0, 0, 0] -v_cndmask_b32 v85, v90, v85, [vcc_lo, vcc_hi] -v_ffbh_u32 v102, v96 -v_lshlrev_b32 v103, v102, v96 -v_and_b32 v101, -256, v103 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v103 -v_cvt_f32_u32 v101, v101 -v_rcp_f32 v97, v101 -v_subb_co_u32 v98, [vcc_lo, vcc_hi], 32, v102, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v102, v103 -v_fma_f32 v101, v101, v97, -1.0 -v_fma_f32 v101, v102, v97, v101 -v_fmaak_f32 v101, v101, v97, 2667577344 -v_mul_f32 v101, 1602224128, v101 -v_mov_b32 v102, 0 -v_cvt_flr_i32_f32 v101, neg(v101) -v_lshl_add_u32 v97, v97, 9, v101 -v_mad_u64_u32 [v102, v103], [vcc_lo, vcc_hi], v103, v97, [v102, v103] -v_subb_co_u32 v97, [vcc_lo, vcc_hi], v97, -1, [vcc_lo, vcc_hi] -v_mov_b32 v100, v98 quad_perm:[1, 1, 1, 1] -v_mov_b32 v98, v98 quad_perm:[0, 0, 0, 0] -v_mov_b32 v99, v97 quad_perm:[1, 1, 1, 1] -v_mov_b32 v97, v97 quad_perm:[0, 0, 0, 0] -v_mul_hi_u32 v101, v85, v97 -v_add_co_u32 v81, [vcc_lo, vcc_hi], v101, v85 -v_addc_co_u32 v101, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v98 -v_cndmask_b32 v81, v81, v101, [vcc_lo, vcc_hi] -v_alignbit_b32 v81, v101, v81, v98 -v_mul_hi_u32 v101, v85, v99 -v_add_co_u32 v4, [vcc_lo, vcc_hi], v101, v85 -v_addc_co_u32 v101, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v100 -v_cndmask_b32 v4, v4, v101, [vcc_lo, vcc_hi] -v_alignbit_b32 v4, v101, v4, v100 -v_mad_u32_u16 v94, v81, v87, 0 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v93, v4, v86, 0 op_sel:[0, 0, 0, 0] -v_sub_u32 v94, v85, v94 -v_sub_u32 v93, v81, v93 -v_readlane_b32 s75, v94, 1 -v_sub_u32 v94, v94, v87 -v_readlane_b32 s9, v93, 1 -v_sub_u32 v93, v93, v86 -v_readlane_b32 s2, v4, 1 -v_sub_u32 v4, v4, s22 -s_lshl_b32 s9, s9, 16 -s_and_b32 s75, s75, lit(0xffff) -s_add_u32 s9, s9, s75 -v_mov_b32 v94, v94 quad_perm:[0, 0, 2, 2] -v_mov_b32 v93, v93 quad_perm:[0, 0, 2, 2] -v_mov_b32 v4, v4 quad_perm:[0, 0, 2, 2] -v_add_co_u32 v94, [vcc_lo, vcc_hi], v94, v89 -v_cndmask_b32 v92, 0, v87, [vcc_lo, vcc_hi] -v_addc_co_u32 v93, [vcc_lo, vcc_hi], v93, 0, [vcc_lo, vcc_hi] -v_cndmask_b32 v91, 0, v86, [vcc_lo, vcc_hi] -v_addc_co_u32 v4, [vcc_lo, vcc_hi], v4, 0, [vcc_lo, vcc_hi] -v_min_u32 v89, v84, 63 -v_sub_u32 v94, v94, v92 -v_sub_u32 v93, v93, v91 -v_cmp_eq_u32 [vcc_lo, vcc_hi], v84, v89 -v_lshlrev_b32 v1, 16, v93 -v_mov_b32 v104, 65535 -v_bfi_b32 v1, v104, v94, v1 -v_add_u32 v104, v4, s22 -v_med3_u32 v89, v84, 1, 62 -v_mul_lo_u32 v2, v104, s56 -v_mul_lo_u32 v11, v104, s64 -s_mul_i32 s12, s2, s56 -s_mul_i32 s15, s2, s64 -v_mov_b32 v105, 2147483648 -v_cndmask_b32 v2, v105, v2, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], v84, v89 -v_cndmask_b32 v11, v105, v11, [vcc_lo, vcc_hi] -v_cmp_ge_u32 [s20, s21], v104, s22 -v_cndmask_b32 v2, v2, v105, [s20, s21] -v_cndmask_b32 v11, v11, v105, [s20, s21] -v_mov_b32 v80, s20 -ds_write_b32 v74, v80 offset:55808 -s_and_b32 s24, s72, 255 -s_cmp_eq_u32 s24, 0 -s_cmov_b32 s46, lit(0x3c003c00) -s_cbranch_scc1 label_118 -s_cmp_eq_u32 s24, 4 -s_cbranch_scc1 label_120 -s_cmp_eq_u32 s24, 2 -s_cbranch_scc1 label_121 -v_cvt_f16_f32 v81, s46 -s_nop 0 -v_readfirstlane_b32 s46, v81 -v_cvt_f16_f32 v81, s47 -v_cmp_gt_f16 [vcc_lo, vcc_hi], s46, 1.0 -v_readfirstlane_b32 s47, v81 -s_pack_ll_b32_b16 s46, s46, s46 -s_pack_ll_b32_b16 s47, s47, s47 -s_cmp_eq_u32 s24, 3 -s_cbranch_scc1 label_122 -s_cbranch_vccnz label_119 -label_118: -s_mov_b32 s71, label_98 - label_1 -s_branch label_123 -label_119: -s_mov_b32 s71, label_102 - label_1 -s_branch label_123 -label_120: -s_mov_b32 s71, label_106 - label_1 -s_branch label_123 -label_121: -s_mov_b32 s71, label_110 - label_1 -s_branch label_123 -label_122: -s_mov_b32 s71, label_114 - label_1 -label_123: -s_lshl_b32 s54, s65, 1 -s_mul_i32 s52, s54, 16 -s_mul_hi_u32 s53, s54, 16 -s_and_b32 s24, s1, 3 -s_lshl_b32 s25, s54, 2 -s_mul_i32 s25, s25, s24 -s_add_u32 s34, s34, s25 -s_addc_u32 s35, s35, 0 -s_lshl_b32 s59, s24, 2 -s_sub_u32 s59, s26, s59 -s_cmov_b32 s59, 0 -s_and_b32 s35, s35, 65535 -s_add_u32 s35, s35, 131072 -s_mov_b64 [s48, s49], [s34, s35] -s_mov_b32 s50, 0 -s_mov_b32 s51, 131072 -s_mov_b32 s16, 65536 -s_mov_b32 s17, 196610 -s_mov_b32 s18, 65536 -s_waitcnt lgkmcnt(0) -s_barrier -s_sub_u32 s25, s39, 1 -s_bitcmp1_b32 s28, 1 -s_cselect_b32 s25, s25, 0 -s_cselect_b32 s24, -1, 1 -s_sub_u32 s41, s38, 1 -s_bitcmp1_b32 s28, 0 -s_cselect_b32 s41, s41, 0 -s_cselect_b32 s40, -1, 1 -s_bitcmp1_b32 s28, 31 -s_cselect_b32 s43, 0, 1 -s_cselect_b32 s42, 4, 0 -s_cselect_b32 s45, 2, 0 -s_cselect_b32 s73, 2, 4 -s_and_b32 s44, s1, 3 -v_bfe_u32 v85, v0, 2, 6 -v_lshlrev_b32 v86, 0, s44 -v_bfe_u32 v84, v85, 0, s45 -v_mad_u32_u24 v84, v86, s43, v84 -v_mov_b32 v81, s73 -v_bfe_u32 v81, v85, s45, v81 -v_mad_u32_u24 v81, v86, s42, v81 -v_mov_b32 v85, s41 -v_add_u32 v86, s40, v85 -v_add_u32 v87, s40, v86 -v_xor_b32 v88, v0, v0 quad_perm:[0, 0, 0, 1] -v_mov_b32 v89, s25 -v_mad_i32_i24 v89, s24, v88, v89 -v_mad_u32_u24 v89, s61, v84, v89 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v88, s39 -v_cmp_lt_u32 [s44, s45], v85, s38 -v_cmp_lt_u32 [s74, s75], v86, s38 -v_cmp_lt_u32 [s76, s77], v87, s38 -s_and_b64 [s44, s45], [vcc_lo, vcc_hi], [s44, s45] -s_and_b64 [s74, s75], [vcc_lo, vcc_hi], [s74, s75] -s_and_b64 [s76, s77], [vcc_lo, vcc_hi], [s76, s77] -s_add_u32 s73, s23, 15 -s_and_b32 s73, s73, -16 -s_sub_u32 s73, s73, 16 -s_sub_u32 s73, s23, s73 -v_sub_u32 v21, s73, v84 -v_add_u32 v17, v21, 3 -v_add_u32 v21, v21, 3 -v_mad_u32_u24 v14, s60, v81, v89 -v_mad_u32_u24 v16, s62, v87, v14 -v_mad_u32_u24 v15, s62, v86, v14 -v_mad_u32_u24 v14, s62, v85, v14 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v81, s26 -v_cndmask_b32 v14, -1, v14, [vcc_lo, vcc_hi] -v_cndmask_b32 v15, -1, v15, [vcc_lo, vcc_hi] -v_cndmask_b32 v16, -1, v16, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, -1, v14, [s44, s45] -v_cndmask_b32 v15, -1, v15, [s74, s75] -v_cndmask_b32 v16, -1, v16, [s76, s77] -v_add_u32 v90, v81, 16 -v_mad_u32_u24 v18, s60, v90, v89 -v_mad_u32_u24 v20, s62, v87, v18 -v_mad_u32_u24 v19, s62, v86, v18 -v_mad_u32_u24 v18, s62, v85, v18 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v90, s26 -v_cndmask_b32 v18, -1, v18, [vcc_lo, vcc_hi] -v_cndmask_b32 v19, -1, v19, [vcc_lo, vcc_hi] -v_cndmask_b32 v20, -1, v20, [vcc_lo, vcc_hi] -v_cndmask_b32 v18, -1, v18, [s44, s45] -v_cndmask_b32 v19, -1, v19, [s74, s75] -v_cndmask_b32 v20, -1, v20, [s76, s77] -ds_write_b128 v73, [v14, v15, v16, v17] offset:45568 -ds_write_b128 v73, [v18, v19, v20, v21] offset:46592 -s_waitcnt vmcnt(0) -v_pack_b32_f16 v75, v75, v77 op_sel:[0, 0, 0] -s_nop 0 -v_readlane_b32 s19, v75, 0 -v_readlane_b32 s29, v75, 1 -v_readlane_b32 s30, v75, 2 -v_readlane_b32 s31, v75, 3 -s_mov_b32 s63, 0 -s_mov_b32 s67, 0 -s_mov_b32 s32, 2 -s_lshl_b32 s33, s32, 9 -s_lshl_b32 s36, s32, 2 -s_branch label_97 -.align 32 -label_94: -s_bitcmp1_b32 s20, 0 -s_addc_u32 s63, s63, 2 -s_sub_u32 s67, s23, 1 -s_add_u32 s68, s6, s70 -s_addc_u32 s69, s7, 0 -s_setprio 3 -v_mov_b32 v84, v12 -v_mov_b32 v85, v13 -v_mov_b32 v86, v80 -v_mov_b32 v95, 2147483648 -v_mov_b32 v92, s56 -v_mov_b32 v89, s64 -v_mov_b32 v94, s2 -v_add_co_u32 v81, [vcc_lo, vcc_hi], v1, s9 -v_mov_b32 v96, 131073 -v_pk_mad_u16 v87, v1, v96, s11 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mad_u16 v88, v1, v96, s14 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_min_u16 v90, v81, v1 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_cndmask_b32 v91, 0, v92, [vcc_lo, vcc_hi] -v_cndmask_b32 v93, 0, v89, [vcc_lo, vcc_hi] -v_mad_u32_u16 v6, v87, 1, v2 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v12, v88, 1, v11 op_sel:[0, 0, 0, 0] -v_add3_u32 v2, v2, s12, v91 -v_add3_u32 v11, v11, s15, v93 -v_pk_sub_u16 v90, v1, v90 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_addc_co_u32 v4, [s20, s21], v4, v94, [vcc_lo, vcc_hi] -v_cndmask_b32 v2, v2, v95, [s20, s21] -v_cndmask_b32 v11, v11, v95, [s20, s21] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v87, s10 -v_cndmask_b32 v6, v95, v6, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v88, s13 -v_cndmask_b32 v12, v95, v12, [vcc_lo, vcc_hi] -v_pk_ashrrev_i16 v90, 15, v90 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_u16 v89, v87, s17 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_u16 v92, v87, s16 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_mad_u32_u16 v9, v89, s58, v6 op_sel:[1, 0, 0, 0] -v_mad_u32_u16 v7, v92, s58, v6 op_sel:[1, 0, 0, 0] -v_pk_add_u16 v94, v88, s18 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v89, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v9, v95, v9, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v92, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v7, v95, v7, [vcc_lo, vcc_hi] -v_mad_u32_u16 v13, v94, s66, v12 op_sel:[1, 0, 0, 0] -v_mad_u32_u16 v8, v89, s58, v6 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v6, v92, s58, v6 op_sel:[0, 0, 0, 0] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v94, s13 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v13, v95, v13, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v89, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v8, v95, v8, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v92, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v6, v95, v6, [vcc_lo, vcc_hi] -v_mad_u32_u16 v12, v94, s66, v12 op_sel:[0, 0, 0, 0] -v_pk_mad_u16 v1, v90, s3, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v94, s13 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v12, v95, v12, [vcc_lo, vcc_hi] -v_mov_b32 v80, s20 -v_add_u32 v75, s33, v5 -v_add_u32 v77, s36, v54 -ds_write_b64 v75, [v84, v85] offset:55816 -ds_write_b32 v77, v86 offset:58888 -ds_write_b128 v72, [v6, v7, v8, v9] offset:53760 -ds_write_b32 v74, v80 offset:55808 -s_setpc_b64 [s68, s69] -label_97: -label_124: -s_barrier -s_lshl_b32 s63, s63, 2 -s_mov_b32 s70, label_126 - label_1 -s_sub_u32 s67, s67, 16 -s_cbranch_scc1 label_94 -label_126: -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_125 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_sub_u32 s32, s32, 1 -s_cselect_b32 s32, 2, s32 -s_lshl_b32 s33, s32, 9 -s_lshl_b32 s36, s32, 2 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_125: -s_bitcmp1_b32 s63, 5 -s_cbranch_scc1 label_95 -label_96: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_127 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_127: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_128 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_128: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s63, 5 -s_cselect_b32 s68, s71, label_129 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_129: -s_branch label_124 -label_130: -s_barrier -s_lshl_b32 s63, s63, 2 -s_mov_b32 s70, label_132 - label_1 -s_sub_u32 s67, s67, 16 -s_cbranch_scc1 label_94 -label_132: -s_setprio 0 -ds_read_b64 [v44, v45], v3 offset:8256 -ds_read_b64 [v42, v43], v3 offset:8512 -ds_read_b64 [v52, v53], v3 offset:8768 -ds_read_b64 [v50, v51], v3 offset:9024 -s_setprio 3 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_131 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_sub_u32 s32, s32, 1 -s_cselect_b32 s32, 2, s32 -s_lshl_b32 s33, s32, 9 -s_lshl_b32 s36, s32, 2 -s_waitcnt lgkmcnt(0) -v_fma_mix_f32 v67, v44, 1.0, v42 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v44, v44, 1.0, v42 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v68, v45, 1.0, v43 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v45, v45, 1.0, v43 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v67, v67, 1.0, v52 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v67, v44, 1.0, v52 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v68, v68, 1.0, v53 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v68, v45, 1.0, v53 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v69, v52, -1.0, v42 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v52, v52, -1.0, v42 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v70, v53, -1.0, v43 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v53, v53, -1.0, v43 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v69, v69, 1.0, v50 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v69, v52, 1.0, v50 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v70, v70, 1.0, v51 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v70, v53, 1.0, v51 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_swappc_b64 [s68, s69], [s68, s69] -label_131: -s_bitcmp1_b32 s63, 5 -s_cbranch_scc0 label_96 -label_95: -s_barrier -s_setprio 0 -ds_read_b64 [v24, v25], v3 offset:0 -ds_read_b64 [v22, v23], v3 offset:256 -ds_read_b64 [v32, v33], v3 offset:512 -ds_read_b64 [v30, v31], v3 offset:768 -s_setprio 3 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_133 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_133: -v_fma_mix_f32 v55, v24, 1.0, v22 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v24, v24, 1.0, v22 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v56, v25, 1.0, v23 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v25, v25, 1.0, v23 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v55, v55, 1.0, v32 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v55, v24, 1.0, v32 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v56, v56, 1.0, v33 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v56, v25, 1.0, v33 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v57, v32, -1.0, v22 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v32, v32, -1.0, v22 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v58, v33, -1.0, v23 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v33, v33, -1.0, v23 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v57, v57, 1.0, v30 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v57, v32, 1.0, v30 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v58, v58, 1.0, v31 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v58, v33, 1.0, v31 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_barrier -s_setprio 0 -ds_read_b64 [v40, v41], v3 offset:8256 -ds_read_b64 [v38, v39], v3 offset:8512 -ds_read_b64 [v48, v49], v3 offset:8768 -ds_read_b64 [v46, v47], v3 offset:9024 -s_setprio 3 -s_bitcmp1_b32 s63, 7 -s_cselect_b32 s68, s71, label_134 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s68, s69], [s68, s69] -label_134: -v_fma_mix_f32 v63, v40, 1.0, v38 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v40, v40, 1.0, v38 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v64, v41, 1.0, v39 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v41, v41, 1.0, v39 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v63, v63, 1.0, v48 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v63, v40, 1.0, v48 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v64, v64, 1.0, v49 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v64, v41, 1.0, v49 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v65, v48, -1.0, v38 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v48, v48, -1.0, v38 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v66, v49, -1.0, v39 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v49, v49, -1.0, v39 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v65, v65, 1.0, v46 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v65, v48, 1.0, v46 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v66, v66, 1.0, v47 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v66, v49, 1.0, v47 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_add_u32 v83, s33, v10 -v_add_u32 v76, s36, v71 -s_barrier -s_setprio 0 -ds_read_b64 [v78, v79], v83 offset:55816 -ds_read_b64 [v82, v83], v83 offset:57352 -ds_read_b32 v76, v76 offset:58888 -ds_read_b64 [v28, v29], v3 offset:0 -ds_read_b64 [v26, v27], v3 offset:256 -ds_read_b64 [v36, v37], v3 offset:512 -ds_read_b64 [v34, v35], v3 offset:768 -s_setprio 3 -s_bitcmp1_b32 s63, 5 -s_cselect_b32 s68, s71, label_135 - label_1 -s_add_u32 s68, s6, s68 -s_addc_u32 s69, s7, 0 -s_waitcnt lgkmcnt(0) -v_fma_mix_f32 v59, v28, 1.0, v26 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v28, v28, 1.0, v26 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v60, v29, 1.0, v27 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v29, v29, 1.0, v27 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v59, v59, 1.0, v36 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v59, v28, 1.0, v36 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v60, v60, 1.0, v37 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v60, v29, 1.0, v37 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v61, v36, -1.0, v26 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v36, v36, -1.0, v26 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v62, v37, -1.0, v27 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v37, v37, -1.0, v27 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v61, v61, 1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v61, v36, 1.0, v34 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v62, v62, 1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v62, v37, 1.0, v35 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_swappc_b64 [s68, s69], [s68, s69] -label_135: -s_branch label_130 -label_98: -s_setprio 3 -v_readfirstlane_b32 s37, v76 -v_pk_add_f16 v55, v55, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v55, v55, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v59, v59, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v57, v57, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v61, v61, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_99 - label_1 -s_setpc_b64 [s68, s69] -label_99: -s_setprio 3 -v_pk_add_f16 v56, v56, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v56, v56, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v60, v60, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v58, v58, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v62, v62, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_100 - label_1 -s_sub_u32 s55, s59, 17 -s_cselect_b32 s50, 0, s50 -s_add_u32 s48, s34, s52 -s_addc_u32 s49, s35, s53 -s_setpc_b64 [s68, s69] -label_100: -s_setprio 3 -v_pk_add_f16 v63, v63, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v63, v63, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v67, v67, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v65, v65, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v69, v69, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_101 - label_1 -s_setpc_b64 [s68, s69] -label_101: -s_setprio 3 -v_pk_add_f16 v64, v64, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v64, v64, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v68, v68, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v66, v66, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v70, v70, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_98 - label_1 -s_bitcmp1_b32 s37, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s55, s59, 1 -s_cselect_b32 s50, 0, 2147483648 -s_mov_b64 [s48, s49], [s34, s35] -s_setpc_b64 [s68, s69] -label_102: -s_setprio 3 -v_readfirstlane_b32 s37, v76 -v_pk_add_f16 v55, v55, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v55, v55, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v59, v59, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v57, v57, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v61, v61, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_103 - label_1 -s_setpc_b64 [s68, s69] -label_103: -s_setprio 3 -v_pk_add_f16 v56, v56, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v56, v56, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v60, v60, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v58, v58, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v62, v62, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_104 - label_1 -s_sub_u32 s55, s59, 17 -s_cselect_b32 s50, 0, s50 -s_add_u32 s48, s34, s52 -s_addc_u32 s49, s35, s53 -s_setpc_b64 [s68, s69] -label_104: -s_setprio 3 -v_pk_add_f16 v63, v63, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v63, v63, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v67, v67, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v75, s46, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v65, v65, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v69, v69, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_105 - label_1 -s_setpc_b64 [s68, s69] -label_105: -s_setprio 3 -v_pk_add_f16 v64, v64, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v64, v64, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v68, v68, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v81, s46, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v66, v66, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v70, v70, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_102 - label_1 -s_bitcmp1_b32 s37, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s55, s59, 1 -s_cselect_b32 s50, 0, 2147483648 -s_mov_b64 [s48, s49], [s34, s35] -s_setpc_b64 [s68, s69] -label_106: -s_setprio 3 -v_readfirstlane_b32 s37, v76 -v_pk_add_f16 v55, v55, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v55, 0, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v59, 0, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v57, 0, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v61, 0, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_107 - label_1 -s_setpc_b64 [s68, s69] -label_107: -s_setprio 3 -v_pk_add_f16 v56, v56, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v56, 0, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v60, 0, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v58, 0, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v62, 0, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_108 - label_1 -s_sub_u32 s55, s59, 17 -s_cselect_b32 s50, 0, s50 -s_add_u32 s48, s34, s52 -s_addc_u32 s49, s35, s53 -s_setpc_b64 [s68, s69] -label_108: -s_setprio 3 -v_pk_add_f16 v63, v63, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v63, 0, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v67, 0, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v65, 0, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v69, 0, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_109 - label_1 -s_setpc_b64 [s68, s69] -label_109: -s_setprio 3 -v_pk_add_f16 v64, v64, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v64, 0, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v68, 0, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v66, 0, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v70, 0, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_106 - label_1 -s_bitcmp1_b32 s37, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s55, s59, 1 -s_cselect_b32 s50, 0, 2147483648 -s_mov_b64 [s48, s49], [s34, s35] -s_setpc_b64 [s68, s69] -label_110: -s_setprio 3 -v_readfirstlane_b32 s37, v76 -v_pk_add_f16 v55, v55, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v55, v89, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v59, v89, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v55, v55 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v59, v59 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v55, v55 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v59, v59 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v55, 1.0, v55 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v59, 1.0, v59 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v55, v55 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v59, v59 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v55, v55 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v59, v59 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v57, v89, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v61, v89, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v57, v57 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v61, v61 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v57, v57 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v61, v61 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v57, 1.0, v57 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v61, 1.0, v61 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v57, v57 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v61, v61 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v57, v57 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v61, v61 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_111 - label_1 -s_setpc_b64 [s68, s69] -label_111: -s_setprio 3 -v_pk_add_f16 v56, v56, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v56, v89, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v60, v89, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v56, v56 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v60, v60 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v56, v56 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v60, v60 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v56, 1.0, v56 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v60, 1.0, v60 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v56, v56 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v60, v60 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v56, v56 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v60, v60 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v58, v89, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v62, v89, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v58, v58 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v62, v62 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v58, v58 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v62, v62 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v58, 1.0, v58 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v62, 1.0, v62 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v58, v58 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v62, v62 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v58, v58 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v62, v62 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_112 - label_1 -s_sub_u32 s55, s59, 17 -s_cselect_b32 s50, 0, s50 -s_add_u32 s48, s34, s52 -s_addc_u32 s49, s35, s53 -s_setpc_b64 [s68, s69] -label_112: -s_setprio 3 -v_pk_add_f16 v63, v63, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v63, v89, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v67, v89, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v63, v63 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v67, v67 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v63, v63 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v67, v67 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v63, 1.0, v63 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v67, 1.0, v67 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v63, v63 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v67, v67 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v63, v63 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v67, v67 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v65, v89, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v69, v89, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v65, v65 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v69, v69 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v65, v65 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v69, v69 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v65, 1.0, v65 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v69, 1.0, v69 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v65, v65 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v69, v69 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v65, v65 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v69, v69 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_113 - label_1 -s_setpc_b64 [s68, s69] -label_113: -s_setprio 3 -v_pk_add_f16 v64, v64, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v64, v89, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v68, v89, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v64, v64 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v68, v68 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v64, v64 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v68, v68 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v64, 1.0, v64 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v68, 1.0, v68 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v64, v64 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v68, v68 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v64, v64 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v68, v68 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v66, v89, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 3183852997 -v_pk_mul_f16 v70, v89, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v66, v66 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v70, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v66, v66 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v70, v70 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v66, 1.0, v66 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v70, 1.0, v70 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v66, v66 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v70, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v66, v66 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v70, v70 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_110 - label_1 -s_bitcmp1_b32 s37, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s55, s59, 1 -s_cselect_b32 s50, 0, 2147483648 -s_mov_b64 [s48, s49], [s34, s35] -s_setpc_b64 [s68, s69] -label_114: -s_setprio 3 -v_readfirstlane_b32 s37, v76 -v_pk_add_f16 v55, v55, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s19 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v55, v55, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v59, v59, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v75, 2147450879, v55 -v_and_b32 v85, 2147450879, v59 -v_mov_b32 v77, 3056514606 -v_mov_b32 v86, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v77, v75, v89, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v86, v85, v89, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v77, v75, v77, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v86, v85, v86, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v77, v75, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v77, v75, v77, v75 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v75, v89, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v85, v89, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v75, 1.0, v75 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v75, v75, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v55), v89 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v59), v89 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v55), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v59), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v75, 2147450879 -v_bfi_b32 v55, v75, v77, v55 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v59, v85, v86, v59 -v_pk_mul_f16 v55, v55, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v59, v59, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v57, v57, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v61, v61, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v75, 2147450879, v57 -v_and_b32 v85, 2147450879, v61 -v_mov_b32 v77, 3056514606 -v_mov_b32 v86, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v77, v75, v89, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v86, v85, v89, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v77, v75, v77, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v86, v85, v86, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v77, v75, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v77, v75, v77, v75 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v75, v89, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v85, v89, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v75, 1.0, v75 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v75, v75, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v57), v89 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v61), v89 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v57), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v61), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v75, 2147450879 -v_bfi_b32 v57, v75, v77, v57 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v61, v85, v86, v61 -v_pk_mul_f16 v57, v57, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v61, v61, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v55, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v59, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v57, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v61, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_115 - label_1 -s_setpc_b64 [s68, s69] -label_115: -s_setprio 3 -v_pk_add_f16 v56, v56, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s29 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v56, v56, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v60, v60, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v81, 2147450879, v56 -v_and_b32 v87, 2147450879, v60 -v_mov_b32 v84, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v84, v81, v89, v84 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v88, v87, v89, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v84, v81, v84, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v88, v87, v88, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v84, v81, v84 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v84, v81, v84, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v81, v89, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v87, v89, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v81, 1.0, v81 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v81, v81, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v56), v89 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v60), v89 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v56), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v60), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v81, 2147450879 -v_bfi_b32 v56, v81, v84, v56 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v60, v87, v88, v60 -v_pk_mul_f16 v56, v56, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v60, v60, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v58, v58, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v62, v62, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v81, 2147450879, v58 -v_and_b32 v87, 2147450879, v62 -v_mov_b32 v84, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v84, v81, v89, v84 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v88, v87, v89, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v84, v81, v84, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v88, v87, v88, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v84, v81, v84 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v84, v81, v84, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v81, v89, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v87, v89, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v81, 1.0, v81 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v81, v81, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v58), v89 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v62), v89 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v58), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v62), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v81, 2147450879 -v_bfi_b32 v58, v81, v84, v58 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v62, v87, v88, v62 -v_pk_mul_f16 v58, v58, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v62, v62, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v56, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v60, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v58, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v62, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_116 - label_1 -s_sub_u32 s55, s59, 17 -s_cselect_b32 s50, 0, s50 -s_add_u32 s48, s34, s52 -s_addc_u32 s49, s35, s53 -s_setpc_b64 [s68, s69] -label_116: -s_setprio 3 -v_pk_add_f16 v63, v63, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s30 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v63, v63, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v67, v67, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v75, 2147450879, v63 -v_and_b32 v85, 2147450879, v67 -v_mov_b32 v77, 3056514606 -v_mov_b32 v86, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v77, v75, v89, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v86, v85, v89, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v77, v75, v77, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v86, v85, v86, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v77, v75, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v77, v75, v77, v75 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v75, v89, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v85, v89, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v75, 1.0, v75 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v75, v75, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v63), v89 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v67), v89 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v63), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v67), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v75, 2147450879 -v_bfi_b32 v63, v75, v77, v63 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v67, v85, v86, v67 -v_pk_mul_f16 v63, v63, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v67, v67, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v65, v65, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v69, v69, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v75, 2147450879, v65 -v_and_b32 v85, 2147450879, v69 -v_mov_b32 v77, 3056514606 -v_mov_b32 v86, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v77, v75, v89, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v86, v85, v89, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v77, v75, v77, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v86, v85, v86, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v77, v75, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v77, v75, v77, v75 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v75, v89, v75 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v85, v89, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v75, 1.0, v75 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v75, v75 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v75, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v75, v75, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v65), v89 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v69), v89 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v65), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v77, v77, v75, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v69), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v75, 2147450879 -v_bfi_b32 v65, v75, v77, v65 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v69, v85, v86, v69 -v_pk_mul_f16 v65, v65, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v69, v69, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v63, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v67, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v65, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v69, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_117 - label_1 -s_setpc_b64 [s68, s69] -label_117: -s_setprio 3 -v_pk_add_f16 v64, v64, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s31 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v64, v64, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v68, v68, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v81, 2147450879, v64 -v_and_b32 v87, 2147450879, v68 -v_mov_b32 v84, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v84, v81, v89, v84 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v88, v87, v89, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v84, v81, v84, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v88, v87, v88, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v84, v81, v84 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v84, v81, v84, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v81, v89, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v87, v89, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v81, 1.0, v81 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v81, v81, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v64), v89 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v68), v89 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v64), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v68), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v81, 2147450879 -v_bfi_b32 v64, v81, v84, v64 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v68, v87, v88, v68 -v_pk_mul_f16 v64, v64, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v68, v68, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v66, v66, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v70, v70, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v81, 2147450879, v66 -v_and_b32 v87, 2147450879, v70 -v_mov_b32 v84, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v84, v81, v89, v84 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 814231688 -v_pk_fma_f16 v88, v87, v89, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v84, v81, v84, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 506666547 -v_pk_fma_f16 v88, v87, v88, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v84, v81, v84 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v84, v81, v84, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v81, v89, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v89, 1103446469 -v_pk_mul_f16 v87, v89, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v81, 1.0, v81 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v81, v81 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v81, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v81, v81, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v66), v89 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v70), v89 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v66), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v84, v84, v81, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v89, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v70), v89 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v81, 2147450879 -v_bfi_b32 v66, v81, v84, v66 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v70, v87, v88, v70 -v_pk_mul_f16 v66, v66, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v70, v70, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -buffer_store_short_d16_hi v64, v78, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v68, v82, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v66, v79, [s48, s49, s50, s51], 0 idxen -buffer_store_short_d16_hi v70, v83, [s48, s49, s50, s51], 0 idxen -s_add_u32 s48, s48, s54 -s_addc_u32 s49, s49, 0 -s_sub_u32 s55, s55, 1 -s_cselect_b32 s50, 0, s50 -s_setprio 2 -s_mov_b32 s71, label_114 - label_1 -s_bitcmp1_b32 s37, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s55, s59, 1 -s_cselect_b32 s50, 0, 2147483648 -s_mov_b64 [s48, s49], [s34, s35] -s_setpc_b64 [s68, s69] -label_0: -s_endpgm -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_metadata.inc b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_metadata.inc deleted file mode 100644 index 62bcfede48a4..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_6_0_metadata.inc +++ /dev/null @@ -1,212 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -.macro PROLOG_KERNEL_DESCRIPTOR kernel_name:req -.text -.globl \kernel_name -.p2align 8 -.type \kernel_name,@function -\kernel_name: -.endm - -.macro METADATA sc:req, wc:req, wg_x:req, kernel_name:req -.amdgpu_metadata ---- -amdhsa.version: [ 1, 2 ] -amdhsa.kernels: - - .name: \kernel_name - .symbol: \kernel_name\().kd - .sgpr_count: \sc - .vgpr_count: \wc - .group_segment_fixed_size: 65536 - .private_segment_fixed_size: 0 - .kernarg_segment_size: 232 - .kernarg_segment_align: 8 - .reqd_workgroup_size: [ \wg_x, 1, 1 ] - .max_flat_workgroup_size: \wg_x - .wavefront_size: 64 - .workgroup_processor_mode: 0 - .args: - - { .size: 4, .offset: 0, .value_kind: by_value, .name: N_ } - - { .size: 4, .offset: 4, .value_kind: by_value, .name: C } - - { .size: 4, .offset: 8, .value_kind: by_value, .name: H } - - { .size: 4, .offset: 12, .value_kind: by_value, .name: W } - - - { .size: 4, .offset: 16, .value_kind: by_value, .name: K } - - { .size: 4, .offset: 20, .value_kind: by_value, .name: n_groups } - - - { .size: 8, .offset: 24, .value_kind: by_value, .name: flags64 } - - - { .size: 8, .offset: 32, .value_kind: global_buffer, .name: data_addr, .address_space: global, .is_const: true } - - { .size: 8, .offset: 40, .value_kind: global_buffer, .name: filter_addr, .address_space: global, .is_const: true } - - { .size: 8, .offset: 48, .value_kind: global_buffer, .name: output_addr, .address_space: global, .is_const: false } - - { .size: 8, .offset: 56, .value_kind: by_value } - - - { .size: 4, .offset: 64, .value_kind: by_value, .name: R } - - { .size: 4, .offset: 68, .value_kind: by_value, .name: S } - - { .size: 4, .offset: 72, .value_kind: by_value, .name: pad_h } - - { .size: 4, .offset: 76, .value_kind: by_value, .name: pad_w } - - { .size: 4, .offset: 80, .value_kind: by_value, .name: out_h } - - { .size: 4, .offset: 84, .value_kind: by_value, .name: out_w } - - - { .size: 8, .offset: 88, .value_kind: global_buffer, .name: bias_addr, .address_space: global, .is_const: true } - - { .size: 4, .offset: 96, .value_kind: by_value, .name: alpha } - - { .size: 4, .offset: 100, .value_kind: by_value, .name: beta } - - - { .size: 8, .offset: 104, .value_kind: by_value, .name: d_offset } - - { .size: 8, .offset: 112, .value_kind: by_value, .name: f_offset } - - { .size: 8, .offset: 120, .value_kind: by_value, .name: o_offset } - - { .size: 8, .offset: 128, .value_kind: by_value, .name: b_offset } - - - { .size: 4, .offset: 136, .value_kind: by_value, .name: d_N_stride } - - { .size: 4, .offset: 140, .value_kind: by_value, .name: d_C_stride } - - { .size: 4, .offset: 144, .value_kind: by_value, .name: d_H_stride } - - { .size: 4, .offset: 148, .value_kind: by_value } - - - { .size: 4, .offset: 152, .value_kind: by_value, .name: f_K_stride } - - { .size: 4, .offset: 156, .value_kind: by_value, .name: f_C_stride } - - { .size: 4, .offset: 160, .value_kind: by_value, .name: f_R_stride } - - { .size: 4, .offset: 164, .value_kind: by_value } - - - { .size: 4, .offset: 168, .value_kind: by_value, .name: o_N_stride } - - { .size: 4, .offset: 172, .value_kind: by_value, .name: o_K_stride } - - { .size: 4, .offset: 176, .value_kind: by_value, .name: o_H_stride } - - { .size: 4, .offset: 180, .value_kind: by_value } - - - { .size: 4, .offset: 184, .value_kind: by_value, .name: G } - - { .size: 4, .offset: 188, .value_kind: by_value, .name: d_G_stride } - - { .size: 4, .offset: 192, .value_kind: by_value, .name: f_G_stride } - - { .size: 4, .offset: 196, .value_kind: by_value, .name: o_G_stride } - - - { .size: 1, .offset: 200, .value_kind: by_value, .name: activation_mode } - - { .size: 1, .offset: 201, .value_kind: by_value, .name: sync_limit } - - { .size: 1, .offset: 202, .value_kind: by_value, .name: sync_period } - - { .size: 1, .offset: 203, .value_kind: by_value } - - - { .size: 4, .offset: 204, .value_kind: by_value } - - { .size: 8, .offset: 208, .value_kind: global_buffer, .name: sync_addr, .address_space: global, .is_const: false } - - - { .size: 8, .offset: 216, .value_kind: global_buffer, .name: acc_addr, .address_space: global, .is_const: false } - - { .size: 8, .offset: 224, .value_kind: by_value, .name: a_offset } -... -.end_amdgpu_metadata -.endm // METADATA - -.altmacro -.macro METADATA_WRAPPER sc:req, wc:req, wg_x:req, kernel_name:req - METADATA %\sc, %\wc, %\wg_x, \kernel_name -.endm - -.macro kernel_end kernel_name:req -.align 256 -.Lfunc_end0: - .size \kernel_name, .Lfunc_end0 - \kernel_name -.endm - -.macro EPILOG_KERNEL_DESCRIPTOR kernel_name:req - -kernel_end \kernel_name - -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) - sgpr_cnt = 80 - vgpr_cnt = 164 - workgroup_size_x = 768 -.endif - -.amdgcn.next_free_sgpr = sgpr_cnt -.amdgcn.next_free_vgpr = vgpr_cnt - -__group_segment_fixed_size = 65536 -__sgpr_dispatch_ptr = 1 -__sgpr_kernarg_segment_ptr = 1 -__sgpr_workgroup_id_x = 1 -__sgpr_workgroup_id_y = 0 -__sgpr_workgroup_id_z = 0 -__sgpr_workgroup_info = 0 -__vgpr_workitem_id = 1 -__reserve_vcc = 1 -__float_round_mode_32 = 0 -__float_round_mode_16_64 = 0 -__float_denorm_mode_32 = 0 -__float_denorm_mode_16_64 = 0 -__dx10_clamp = 0 -__ieee_mode = 0 -__fp16_overflow = 0 - -.rodata -.p2align 6 -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) -.amdhsa_kernel \kernel_name - .amdhsa_group_segment_fixed_size __group_segment_fixed_size - .amdhsa_user_sgpr_dispatch_ptr __sgpr_dispatch_ptr // s[0:1] - .amdhsa_user_sgpr_kernarg_segment_ptr __sgpr_kernarg_segment_ptr // s[2:3] - .amdhsa_system_sgpr_workgroup_id_x __sgpr_workgroup_id_x - .amdhsa_system_sgpr_workgroup_id_y __sgpr_workgroup_id_y - .amdhsa_system_sgpr_workgroup_id_z __sgpr_workgroup_id_z - .amdhsa_system_sgpr_workgroup_info __sgpr_workgroup_info - .amdhsa_system_vgpr_workitem_id __vgpr_workitem_id - .amdhsa_reserve_vcc __reserve_vcc - .amdhsa_next_free_sgpr .amdgcn.next_free_sgpr - .amdhsa_next_free_vgpr .amdgcn.next_free_vgpr - .amdhsa_accum_offset .amdgcn.next_free_vgpr - .amdhsa_float_round_mode_32 __float_round_mode_32 - .amdhsa_float_round_mode_16_64 __float_round_mode_16_64 - .amdhsa_float_denorm_mode_32 __float_denorm_mode_32 - .amdhsa_float_denorm_mode_16_64 __float_denorm_mode_16_64 - .amdhsa_dx10_clamp __dx10_clamp - .amdhsa_ieee_mode __ieee_mode - .amdhsa_fp16_overflow __fp16_overflow -/* \todo add gfx12 support with these directives - .amdhsa_wavefront_size32 0 - .amdhsa_workgroup_processor_mode 0 -*/ -.end_amdhsa_kernel -.endif - -METADATA_WRAPPER sgpr_cnt, vgpr_cnt, workgroup_size_x, <\kernel_name> - -.endm - -.macro PROLOG_KERNEL_DESCRIPTOR_WRAPPER machine_version:req, kernel_name_postfix:req - PROLOG_KERNEL_DESCRIPTOR miopenSp3AsmConvRage_v4_6_0_gfx\machine_version\()\kernel_name_postfix -.endm - -.macro EPILOG_KERNEL_DESCRIPTOR_WRAPPER machine_version:req, kernel_name_postfix:req - EPILOG_KERNEL_DESCRIPTOR miopenSp3AsmConvRage_v4_6_0_gfx\machine_version\()\kernel_name_postfix -.endm - -.macro KERNEL_PROLOG kernel_name_postfix:req - PROLOG_KERNEL_DESCRIPTOR_WRAPPER %.amdgcn.gfx_generation_number, \kernel_name_postfix -.endm - -.macro KERNEL_EPILOG kernel_name_postfix:req - EPILOG_KERNEL_DESCRIPTOR_WRAPPER %.amdgcn.gfx_generation_number, \kernel_name_postfix -.endm - -.if (.amdgcn.gfx_generation_number != 9 || .amdgcn.gfx_generation_minor != 4 && .amdgcn.gfx_generation_stepping != 2) - .error "Unsupported gfx version" - .end -.endif diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s deleted file mode 100644 index 77b53662a9bb..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_fp16_fp32acc_f2x3_stride1.s +++ /dev/null @@ -1,34 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -.include "Conv_Winograd_Rage_v4_7_0_metadata.inc" - -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) - KERNEL_PROLOG _fp16_fp32acc_f2x3_stride1 - .include "Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc" - KERNEL_EPILOG _fp16_fp32acc_f2x3_stride1 -.else - .error "Unsupported gfx version" -.endif diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc deleted file mode 100644 index 01c1a41d3d38..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_gfx94x_fp16_fp32acc_f2x3_stride1.inc +++ /dev/null @@ -1,4620 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -v_lshlrev_b32 v1, 7, v0 -s_getpc_b64 [s8, s9] -label_6: -s_mov_b32 s10, label_0 - label_6 -s_mov_b32 s11, 131072 -buffer_load_dword v2, v1, [s8, s9, s10, s11], 0 offen -s_waitcnt vmcnt(0) -s_getpc_b64 [s6, s7] -label_1: -label_2: -s_load_dwordx2 [s22, s23], [s2, s3], 0 -s_load_dwordx16 [s24, s25, s26, s27, s28, s29, s30, s31, s32, s33, s34, s35, s36, s37, s38, s39], [s2, s3], 8 -s_load_dwordx8 [s40, s41, s42, s43, s44, s45, s46, s47], [s2, s3], 72 -v_readfirstlane_b32 s1, v0 -v_and_b32 v6, 255, v0 -v_lshrrev_b32 v7, 1, v6 -v_and_b32 v8, 1, v0 -v_and_b32 v1, 15, v6 -v_mul_u32_u24 v3, 4160, v8 -v_lshlrev_b32 v1, 3, v1 -v_and_b32 v4, 31, v7 -v_bfe_u32 v2, v6, 4, 2 -v_lshl_add_u32 v3, v4, 3, v3 -v_lshl_add_u32 v1, v2, 10, v1 -v_bfe_u32 v4, v6, 6, 2 -v_bfe_u32 v2, v6, 6, 2 -v_bfe_u32 v5, v0, 8, 0 -v_lshl_add_u32 v1, v2, 8, v1 -v_lshlrev_b32 v4, v5, v4 -v_bfe_u32 v2, v0, 8, 1 -v_lshl_add_u32 v3, v4, 10, v3 -v_lshl_add_u32 v1, v2, 7, v1 -s_lshr_b32 s1, s1, 6 -s_cmp_ge_u32 s1, 8 -s_cbranch_scc0 label_3 -s_setprio 3 -label_3: -s_waitcnt lgkmcnt(0) -s_and_b32 s28, s28, 262143 -s_and_b32 s5, s28, 8192 -s_cbranch_scc0 label_7 -s_load_dwordx8 [s48, s49, s50, s51, s52, s53, s54, s55], [s2, s3], 104 -label_7: -s_and_b32 s5, s28, 512 -s_cbranch_scc0 label_8 -s_load_dwordx8 [s56, s57, s58, s59, s60, s61, s62, s63], [s2, s3], 136 -s_load_dwordx4 [s64, s65, s66, s67], [s2, s3], 168 -label_8: -s_and_b32 s5, s28, 1024 -s_cbranch_scc0 label_9 -s_load_dwordx4 [s68, s69, s70, s71], [s2, s3], 184 -label_9: -s_and_b32 s5, s28, 81920 -s_cbranch_scc0 label_10 -s_load_dword s72, [s2, s3], 200 -label_10: -s_and_b32 s5, s28, 65536 -s_cbranch_scc0 label_11 -s_load_dword s73, [s2, s3], 204 -s_load_dwordx2 [s74, s75], [s2, s3], 208 -label_11: -s_and_b32 s5, s28, 131072 -s_cbranch_scc0 label_12 -s_load_dwordx4 [s76, s77, s78, s79], [s2, s3], 216 -label_12: -.long 0xba7f0241 -.long 0x00000000 // encoding fixup: s_setreg_imm32_b32 hwreg(1, 9, 1), 0 -s_bitcmp1_b32 s28, 3 -s_cbranch_scc0 label_13 -.long 0xba7f1901 -.long 0x0000000f // encoding fixup: s_setreg_imm32_b32 hwreg(1, 4, 4), 15 -.long 0xba7f1801 -.long 0x00000000 // encoding fixup: s_setreg_imm32_b32 hwreg(1, 0, 4), 0 -label_13: -s_waitcnt lgkmcnt(0) -s_bitcmp1_b32 s28, 6 -s_cbranch_scc0 label_17 -s_and_b32 s31, s31, 65535 -s_and_b32 s33, s33, 65535 -s_and_b32 s35, s35, 65535 -s_and_b32 s45, s45, 65535 -s_and_b32 s77, s77, 65535 -s_and_b32 s75, s75, 65535 -s_load_dwordx2 [s30, s31], [s30, s31], 0 -s_load_dwordx2 [s32, s33], [s32, s33], 0 -s_load_dwordx2 [s34, s35], [s34, s35], 0 -s_bitcmp1_b32 s28, 7 -s_cbranch_scc0 label_14 -s_load_dwordx2 [s44, s45], [s44, s45], 0 -label_14: -s_bitcmp1_b32 s28, 17 -s_cbranch_scc0 label_15 -s_load_dwordx2 [s76, s77], [s76, s77], 0 -label_15: -s_bitcmp1_b32 s28, 16 -s_cbranch_scc0 label_16 -s_load_dwordx2 [s74, s75], [s74, s75], 0 -label_16: -label_17: -s_bitcmp1_b32 s28, 9 -s_cbranch_scc1 label_18 -s_mov_b32 s58, s25 -s_mul_i32 s57, s24, s25 -s_mul_i32 s56, s57, s23 -s_mov_b32 s66, s43 -s_mul_i32 s65, s42, s43 -s_mul_i32 s64, s65, s26 -s_mul_i32 s5, s38, s39 -s_bitcmp1_b32 s28, 2 -s_cselect_b32 s8, s26, s23 -s_mul_i32 s8, s5, s8 -s_mov_b32 s62, s39 -s_bitcmp1_b32 s28, 2 -s_cselect_b32 s61, s8, s5 -s_cselect_b32 s60, s5, s8 -label_18: -s_and_b32 s8, s72, -256 -s_bitcmp1_b32 s28, 8 -s_cselect_b32 s9, 1, 0 -s_bitcmp1_b32 s28, 14 -s_cselect_b32 s72, s72, s9 -s_or_b32 s72, s72, s8 -s_cmp_lt_u32 s60, s61 -s_cselect_b32 s5, 0, 2147483648 -s_or_b32 s28, s28, s5 -s_waitcnt lgkmcnt(0) -s_bitcmp1_b32 s28, 13 -s_cbranch_scc0 label_19 -s_add_u32 s30, s30, s48 -s_addc_u32 s31, s31, s49 -s_add_u32 s34, s34, s52 -s_addc_u32 s35, s35, s53 -s_add_u32 s32, s32, s50 -s_addc_u32 s33, s33, s51 -s_add_u32 s44, s44, s54 -s_addc_u32 s45, s45, s55 -s_add_u32 s76, s76, s78 -s_addc_u32 s77, s77, s79 -label_19: -s_bitcmp1_b32 s28, 10 -s_cbranch_scc0 label_20 -v_ffbh_u32 v6, s27 -v_lshlrev_b32 v7, v6, s27 -v_and_b32 v5, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v5, v5 -v_rcp_f32 v2, v5 -v_subb_co_u32 v4, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v5, v5, v2, -1.0 -v_fma_f32 v5, v6, v2, v5 -v_fmaak_f32 v5, v5, v2, 2667577344 -v_mul_f32 v5, 1602224128, v5 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v5, neg(v5) -v_lshl_add_u32 v2, v2, 9, v5 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v2, [v6, v7] -v_subb_co_u32 v2, [vcc_lo, vcc_hi], v2, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v5, s4, v2 -v_add_co_u32 v2, [vcc_lo, vcc_hi], v5, s4 -v_addc_co_u32 v5, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v4 -v_cndmask_b32 v2, v2, v5, [vcc_lo, vcc_hi] -v_alignbit_b32 v2, v5, v2, v4 -s_nop 0 -v_readfirstlane_b32 s5, v2 -s_mul_i32 s8, s5, s27 -s_sub_u32 s4, s4, s8 -s_mul_i32 s8, s69, s5 -s_mul_hi_u32 s9, s69, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s30, s30, s8 -s_addc_u32 s31, s31, s9 -s_mul_i32 s8, s70, s5 -s_mul_hi_u32 s9, s70, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s32, s32, s8 -s_addc_u32 s33, s33, s9 -s_mul_i32 s8, s71, s5 -s_mul_hi_u32 s9, s71, s5 -s_lshl_b64 [s8, s9], [s8, s9], 1 -s_add_u32 s34, s34, s8 -s_addc_u32 s35, s35, s9 -s_mul_i32 s8, s26, s5 -s_lshl_b32 s8, s8, 1 -s_add_u32 s44, s44, s8 -s_addc_u32 s45, s45, 0 -label_20: -s_cmp_eq_u32 s27, 304 -s_cbranch_scc0 label_21 -s_and_b32 s5, s4, 7 -s_lshr_b32 s8, s4, 3 -s_mul_i32 s4, s5, 38 -s_add_u32 s4, s4, s8 -label_21: -s_add_u32 s9, s26, 31 -s_lshr_b32 s9, s9, 5 -v_ffbh_u32 v6, s9 -v_lshlrev_b32 v7, v6, s9 -v_and_b32 v8, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v8, v8 -v_rcp_f32 v2, v8 -v_subb_co_u32 v5, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v8, v8, v2, -1.0 -v_fma_f32 v8, v6, v2, v8 -v_fmaak_f32 v8, v8, v2, 2667577344 -v_mul_f32 v8, 1602224128, v8 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v8, neg(v8) -v_lshl_add_u32 v2, v2, 9, v8 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v2, [v6, v7] -v_subb_co_u32 v2, [vcc_lo, vcc_hi], v2, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v6, s27, v2 -v_add_co_u32 v2, [vcc_lo, vcc_hi], v6, s27 -v_addc_co_u32 v6, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v5 -v_cndmask_b32 v2, v2, v6, [vcc_lo, vcc_hi] -v_alignbit_b32 v2, v6, v2, v5 -v_ffbh_u32 v6, v2 -v_lshlrev_b32 v7, v6, v2 -v_and_b32 v8, -256, v7 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v7 -v_cvt_f32_u32 v8, v8 -v_rcp_f32 v4, v8 -v_subb_co_u32 v5, [vcc_lo, vcc_hi], 32, v6, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v6, v7 -v_fma_f32 v8, v8, v4, -1.0 -v_fma_f32 v8, v6, v4, v8 -v_fmaak_f32 v8, v8, v4, 2667577344 -v_mul_f32 v8, 1602224128, v8 -v_mov_b32 v6, 0 -v_cvt_flr_i32_f32 v8, neg(v8) -v_lshl_add_u32 v4, v4, 9, v8 -v_mad_u64_u32 [v6, v7], [vcc_lo, vcc_hi], v7, v4, [v6, v7] -v_subb_co_u32 v4, [vcc_lo, vcc_hi], v4, -1, [vcc_lo, vcc_hi] -v_mul_hi_u32 v6, s4, v4 -v_add_co_u32 v4, [vcc_lo, vcc_hi], v6, s4 -v_addc_co_u32 v6, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v5 -v_cndmask_b32 v4, v4, v6, [vcc_lo, vcc_hi] -v_alignbit_b32 v4, v6, v4, v5 -v_readfirstlane_b32 s5, v2 -v_readfirstlane_b32 s10, v4 -s_cmp_ge_u32 s10, s9 -s_cbranch_scc1 label_0 -s_mul_i32 s27, s5, s9 -s_mul_i32 s8, s10, s5 -s_sub_u32 s8, s4, s8 -s_mul_i32 s11, s10, 32 -s_sub_u32 s26, s26, s11 -s_min_u32 s26, s26, 32 -s_lshl_b32 s11, s11, 1 -s_mul_i32 s12, s11, s60 -s_mul_hi_u32 s13, s11, s60 -s_add_u32 s32, s32, s12 -s_addc_u32 s33, s33, s13 -s_mul_i32 s12, s11, s65 -s_mul_hi_u32 s13, s11, s65 -s_add_u32 s34, s34, s12 -s_addc_u32 s35, s35, s13 -s_add_u32 s44, s44, s11 -s_addc_u32 s45, s45, 0 -s_cmp_ge_u32 s1, 8 -s_cbranch_scc1 label_5 -label_4: -v_and_b32 v19, 255, v0 -s_bitcmp1_b32 s28, 31 -s_cselect_b32 s48, 2, 0 -s_cselect_b32 s49, 0, 2 -v_bfe_u32 v17, v19, 2, s48 -v_bfe_u32 v20, v19, 6, s49 -v_add_u32 v20, v17, v20 -v_lshrrev_b32 v21, 2, v19 -v_lshrrev_b32 v21, s48, v21 -v_and_b32 v17, 3, v21 -v_bfe_u32 v4, v21, 2, 1 -v_lshl_add_u32 v17, v4, 2, v17 -v_bfe_u32 v4, v21, 3, 1 -v_and_b32 v18, 15, v19 -v_lshl_add_u32 v17, v4, 3, v17 -v_bfe_u32 v2, v19, 6, 2 -v_mul_u32_u24 v4, 24, v17 -v_lshl_add_u32 v18, v2, 4, v18 -v_xor_b32 v17, v19, v19 quad_perm:[0, 0, 0, 1] -v_mul_u32_u24 v2, 24, v18 -v_lshl_add_u32 v4, v17, 3, v4 -v_bfe_u32 v18, v19, 4, 2 -v_mov_b32 v22, 1568 -v_mad_u32_u16 v4, v20, v22, v4 op_sel:[0, 0, 0, 0] -v_mov_b32 v22, 1568 -v_mad_u32_u16 v2, v18, v22, v2 op_sel:[0, 0, 0, 0] -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s50, 1024, 0 -s_cselect_b32 s51, 4, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s50, 65536, s50 -s_cselect_b32 s51, 65536, s51 -s_and_b32 s53, s1, 3 -s_mul_i32 s52, s53, 2048 -v_and_b32 v17, 63, v0 -v_lshl_add_u32 v5, v17, 4, s50 -v_mov_b32 v7, s51 -v_lshl_add_u32 v6, v17, 4, s52 -s_mov_b32 s11, 0 -s_mov_b32 s20, 0 -s_lshl_b32 s48, s57, 1 -s_lshl_b32 s49, s61, 1 -s_and_b32 s50, s1, 3 -s_mul_i32 s51, 1, s50 -s_sub_u32 s9, s23, s51 -s_addc_u32 s9, s9, 3 -s_lshl_b32 s51, s48, 0 -s_mul_i32 s51, s51, s50 -s_add_u32 s30, s30, s51 -s_addc_u32 s31, s31, 0 -s_lshl_b32 s48, s48, 2 -s_lshl_b32 s49, s49, 2 -s_and_b32 s31, s31, 65535 -s_add_u32 s31, s31, 131072 -s_and_b32 s33, s33, 65535 -s_add_u32 s33, s33, 131072 -s_mov_b64 [s12, s13], [s30, s31] -s_mov_b32 s14, 0 -s_mov_b32 s15, 131072 -s_mov_b64 [s16, s17], [s30, s31] -s_mov_b32 s18, 0 -s_mov_b32 s19, 131072 -s_mov_b32 s10, 2147483648 -s_cmp_ge_u32 s1, 4 -s_cbranch_scc0 label_22 -s_branch label_34 -.align 32 -label_50: -ds_read_b32 v16, v7 offset:55808 -ds_read_b128 [v8, v9, v10, v11], v6 offset:45568 -ds_read_b128 [v12, v13, v14, v15], v6 offset:46592 -s_sub_u32 s11, s23, 1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_waitcnt lgkmcnt(2) -v_readfirstlane_b32 s21, v16 -s_bitcmp1_b32 s21, 0 -s_cmov_b32 s10, 0 -s_add_u32 s20, s20, s21 -s_mov_b32 s2, s9 -s_mov_b64 [s12, s13], [s32, s33] -s_mov_b32 s14, s10 -s_mov_b32 s3, s9 -s_mov_b64 [s16, s17], [s32, s33] -s_mov_b32 s18, s10 -s_setpc_b64 [s36, s37] -.align 32 -label_51: -ds_read_b32 v16, v7 offset:55808 -ds_read_b128 [v8, v9, v10, v11], v5 offset:53760 -ds_read_b128 [v12, v13, v14, v15], v5 offset:54784 -s_sub_u32 s11, s23, 1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_waitcnt lgkmcnt(2) -v_readfirstlane_b32 s21, v16 -s_bitcmp1_b32 s21, 0 -s_cmov_b32 s10, 0 -s_add_u32 s20, s20, s21 -s_mov_b32 s2, s9 -s_mov_b64 [s12, s13], [s30, s31] -s_mov_b32 s14, s10 -s_mov_b32 s3, s9 -s_mov_b64 [s16, s17], [s30, s31] -s_mov_b32 s18, s10 -s_setpc_b64 [s36, s37] -.align 32 -label_22: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v17 -v_mov_b32 v155, v106 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v112 -v_mov_b32 v157, v111 -v_pk_add_f16 v112, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v111, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_53 -label_52: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v107, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_23 -s_mov_b32 s29, label_23 - label_1 -s_branch label_46 -.align 32 -label_53: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v107, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_23 -s_mov_b32 s29, label_23 - label_1 -s_branch label_46 -.align 32 -label_23: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v112, v109 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v111, v110 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v112 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_55 -label_54: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v111, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v135, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_24 -s_mov_b32 s29, label_24 - label_1 -s_branch label_47 -.align 32 -label_55: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v111, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v135, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_24 -s_mov_b32 s29, label_24 - label_1 -s_branch label_47 -.align 32 -label_24: -s_mov_b32 s29, label_56 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_56: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v113 -v_mov_b32 v155, v114 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v120 -v_mov_b32 v157, v119 -v_pk_add_f16 v120, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v119, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_58 -label_57: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v115, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_25 -s_mov_b32 s29, label_25 - label_1 -s_branch label_48 -.align 32 -label_58: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v115, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_25 -s_mov_b32 s29, label_25 - label_1 -s_branch label_48 -.align 32 -label_25: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v120, v117 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v119, v118 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v120 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_60 -label_59: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v119, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v143, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_26 -s_mov_b32 s29, label_26 - label_1 -s_branch label_49 -.align 32 -label_60: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v119, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v143, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_26 -s_mov_b32 s29, label_26 - label_1 -s_branch label_49 -.align 32 -label_26: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v122 -v_mov_b32 v155, v124 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v127 -v_mov_b32 v157, v125 -v_pk_add_f16 v127, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v125, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_62 -label_61: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v121, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_27 -s_mov_b32 s29, label_27 - label_1 -s_branch label_46 -.align 32 -label_62: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v121, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_27 -s_mov_b32 s29, label_27 - label_1 -s_branch label_46 -.align 32 -label_27: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v127, v126 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v125, v128 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v127 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_64 -label_63: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v125, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v149, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_28 -s_mov_b32 s29, label_28 - label_1 -s_branch label_47 -.align 32 -label_64: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v125, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v149, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_28 -s_mov_b32 s29, label_28 - label_1 -s_branch label_47 -.align 32 -label_28: -s_mov_b32 s29, label_65 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_65: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v17 -v_mov_b32 v155, v109 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v107 -v_mov_b32 v157, v111 -v_pk_add_f16 v107, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v111, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_67 -label_66: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v112, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_29 -s_mov_b32 s29, label_29 - label_1 -s_branch label_48 -.align 32 -label_67: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v112, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_29 -s_mov_b32 s29, label_29 - label_1 -s_branch label_48 -.align 32 -label_29: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v107, v106 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v111, v110 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v107 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_69 -label_68: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v111, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v135, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_30 -s_mov_b32 s29, label_30 - label_1 -s_branch label_49 -.align 32 -label_69: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v111, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v135, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_30 -s_mov_b32 s29, label_30 - label_1 -s_branch label_49 -.align 32 -label_30: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v113 -v_mov_b32 v155, v117 -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_mov_b32 v156, v115 -v_mov_b32 v157, v119 -v_pk_add_f16 v115, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v119, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_71 -label_70: -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v120, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_31 -s_mov_b32 s29, label_31 - label_1 -s_branch label_46 -.align 32 -label_71: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33024 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v120, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:34176 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_31 -s_mov_b32 s29, label_31 - label_1 -s_branch label_46 -.align 32 -label_31: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v115, v114 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v119, v118 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v115 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_73 -label_72: -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v119, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v143, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_32 -s_mov_b32 s29, label_32 - label_1 -s_branch label_47 -.align 32 -label_73: -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v154, v155] offset:33408 -buffer_load_short_d16 v119, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -v_sub_co_u32 v15, [vcc_lo, vcc_hi], v15, 4 -v_cndmask_b32 v12, v12, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v13, v13, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v14, v14, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v4, [v156, v157] offset:33792 -buffer_load_short_d16 v143, v14, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s49 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_32 -s_mov_b32 s29, label_32 - label_1 -s_branch label_47 -.align 32 -label_32: -s_mov_b32 s29, label_74 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_50 -label_74: -s_setprio 1 -s_waitcnt vmcnt(24) -s_waitcnt lgkmcnt(0) -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mov_b32 v154, v122 -v_mov_b32 v155, v126 -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_mov_b32 v156, v121 -v_mov_b32 v157, v125 -v_pk_add_f16 v121, v156, v154 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v125, v157, v155 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_76 -label_75: -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v127, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_33 -s_mov_b32 s29, label_33 - label_1 -s_branch label_48 -.align 32 -label_76: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39296 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v127, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_write_b64 v4, [v156, v157] offset:40448 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_33 -s_mov_b32 s29, label_33 - label_1 -s_branch label_48 -.align 32 -label_33: -s_setprio 1 -s_waitcnt lgkmcnt(0) -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_add_f16 v154, v121, v124 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v155, v125, v128 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_mul_f16 v154, v154, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v155, v155, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v154, -1.0, v121 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v155, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_barrier -s_setprio 2 -s_cmp_lt_u32 s11, 15 -s_cbranch_scc1 label_78 -label_77: -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v125, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v149, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_22 -s_mov_b32 s29, label_22 - label_1 -s_branch label_49 -.align 32 -label_78: -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v154, v155] offset:39680 -buffer_load_short_d16 v125, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -v_sub_co_u32 v11, [vcc_lo, vcc_hi], v11, 4 -v_cndmask_b32 v8, v8, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v9, v9, -1, [vcc_lo, vcc_hi] -v_cndmask_b32 v10, v10, -1, [vcc_lo, vcc_hi] -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v4, [v156, v157] offset:40064 -buffer_load_short_d16 v149, v10, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s49 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_22 -s_mov_b32 s29, label_22 - label_1 -s_branch label_49 -.align 32 -label_34: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v108, v108, v132 op_sel:[0, 0, 0] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v108, -1.0, v17 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v107, -1.0, v106 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v108, 1.0, v109 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v107, 1.0, v110 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_79: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v17, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v108, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v106, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v107, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v129, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v132, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v130, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v131, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_35 -s_mov_b32 s29, label_35 - label_1 -s_branch label_46 -.align 32 -label_35: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v109, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v110, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v109, -1.0, v112 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v110, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_80: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v109, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v112, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v110, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v111, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v133, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v136, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v134, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v135, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_36 -s_mov_b32 s29, label_36 - label_1 -s_branch label_47 -.align 32 -label_36: -s_mov_b32 s29, label_81 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_81: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v116, v116, v140 op_sel:[0, 0, 0] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v116, -1.0, v113 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v115, -1.0, v114 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v116, 1.0, v117 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v115, 1.0, v118 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_82: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v113, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v116, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v114, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v115, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v137, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v140, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v138, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v139, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_37 -s_mov_b32 s29, label_37 - label_1 -s_branch label_48 -.align 32 -label_37: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v117, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v118, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v117, -1.0, v120 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v118, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_83: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v117, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v120, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v118, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v119, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v141, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v144, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v142, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v143, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_38 -s_mov_b32 s29, label_38 - label_1 -s_branch label_49 -.align 32 -label_38: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v123, v123, v147 op_sel:[0, 0, 0] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v123, -1.0, v122 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v121, -1.0, v124 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v123, 1.0, v126 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v121, 1.0, v128 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_84: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v122, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v123, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v124, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v121, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v146, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v147, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v148, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v145, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_39 -s_mov_b32 s29, label_39 - label_1 -s_branch label_46 -.align 32 -label_39: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v126, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v128, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v126, -1.0, v127 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v128, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_85: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v126, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v127, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v128, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v125, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v150, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v151, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v152, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v149, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_40 -s_mov_b32 s29, label_40 - label_1 -s_branch label_47 -.align 32 -label_40: -s_mov_b32 s29, label_86 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_86: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v17, v17, v129 op_sel:[0, 0, 0] -v_pack_b32_f16 v109, v109, v133 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v106, v106, v130 op_sel:[0, 0, 0] -v_pack_b32_f16 v110, v110, v134 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v108, v108, v132 op_sel:[0, 0, 0] -v_pack_b32_f16 v112, v112, v136 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v107, v107, v131 op_sel:[0, 0, 0] -v_pack_b32_f16 v111, v111, v135 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v108, -1.0, v17 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v112, -1.0, v109 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v108, 1.0, v106 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v112, 1.0, v110 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_87: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v17, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v108, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v109, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v112, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v129, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v132, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v133, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v136, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_41 -s_mov_b32 s29, label_41 - label_1 -s_branch label_48 -.align 32 -label_41: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v106, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v110, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v106, -1.0, v107 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v110, -1.0, v111 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_88: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v106, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v107, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v110, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v111, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v130, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v131, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v134, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v135, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_42 -s_mov_b32 s29, label_42 - label_1 -s_branch label_49 -.align 32 -label_42: -s_lshl_b32 s20, s20, 2 -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v113, v113, v137 op_sel:[0, 0, 0] -v_pack_b32_f16 v117, v117, v141 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v18, v19, v20, v21], [v90, v91], [v34, v35], [v18, v19, v20, v21] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v114, v114, v138 op_sel:[0, 0, 0] -v_pack_b32_f16 v118, v118, v142 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v70, v71, v72, v73], [v102, v103], [v88, v89], [v70, v71, v72, v73] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v116, v116, v140 op_sel:[0, 0, 0] -v_pack_b32_f16 v120, v120, v144 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v52, v53, v54, v55], [v98, v99], [v86, v87], [v52, v53, v54, v55] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v115, v115, v139 op_sel:[0, 0, 0] -v_pack_b32_f16 v119, v119, v143 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v36, v37, v38, v39], [v94, v95], [v68, v69], [v36, v37, v38, v39] -v_pk_fma_f16 v154, v116, -1.0, v113 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v120, -1.0, v117 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v116, 1.0, v114 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v120, 1.0, v118 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_89: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v92, v93], v2 offset:39296 -buffer_load_short_d16 v113, v12, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:16512 -buffer_load_short_d16 v116, v14, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v96, v97], v2 offset:39304 -buffer_load_short_d16 v117, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v120, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_write_b64 v3, [v156, v157] offset:16768 -buffer_load_short_d16 v137, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v104, v105], v2 offset:39312 -buffer_load_short_d16 v140, v14, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v141, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v144, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_43 -s_mov_b32 s29, label_43 - label_1 -s_branch label_46 -.align 32 -label_43: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v100, v92, v104 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v101, v93, v105 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v26, v27, v28, v29], [v92, v93], [v34, v35], [v26, v27, v28, v29] -v_pk_mul_f16 v100, v100, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v101, v101, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v78, v79, v80, v81], [v104, v105], [v88, v89], [v78, v79, v80, v81] -v_pk_fma_f16 v100, v96, -0.5, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v101, v97, -0.5, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v114, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v118, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v60, v61, v62, v63], [v100, v101], [v86, v87], [v60, v61, v62, v63] -v_pk_fma_f16 v96, v96, 1.0, v100 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v97, v97, 1.0, v101 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v114, -1.0, v115 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v118, -1.0, v119 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v44, v45, v46, v47], [v96, v97], [v68, v69], [v44, v45, v46, v47] -s_setprio 2 -label_90: -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v34, v35], v1 offset:24768 -buffer_load_short_d16 v114, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v68, v69], v1 offset:28928 -buffer_load_short_d16 v115, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v154, v155] offset:17024 -buffer_load_short_d16 v118, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v119, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_sub_u32 s3, s3, 4 -s_cselect_b32 s18, 0, s18 -ds_read_b64 [v86, v87], v1 offset:24776 -buffer_load_short_d16 v138, v12, [s16, s17, s18, s19], 0 idxen -ds_read_b64 [v88, v89], v1 offset:28936 -buffer_load_short_d16 v139, v14, [s16, s17, s18, s19], 0 idxen -ds_write_b64 v3, [v156, v157] offset:17280 -buffer_load_short_d16 v142, v13, [s16, s17, s18, s19], 0 idxen -buffer_load_short_d16 v143, v15, [s16, s17, s18, s19], 0 idxen -s_add_u32 s16, s16, s48 -s_addc_u32 s17, s17, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_44 -s_mov_b32 s29, label_44 - label_1 -s_branch label_47 -.align 32 -label_44: -s_mov_b32 s29, label_91 - label_1 -s_sub_u32 s11, s11, 16 -s_cbranch_scc1 label_51 -label_91: -s_setprio 1 -s_waitcnt vmcnt(32) -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_fma_f16 v34, v86, -1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v35, v87, -1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v122, v122, v146 op_sel:[0, 0, 0] -v_pack_b32_f16 v126, v126, v150 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v22, v23, v24, v25], [v90, v91], [v34, v35], [v22, v23, v24, v25] -v_pk_fma_f16 v88, v68, -1.0, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v89, v69, -1.0, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v124, v124, v148 op_sel:[0, 0, 0] -v_pack_b32_f16 v128, v128, v152 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v74, v75, v76, v77], [v102, v103], [v88, v89], [v74, v75, v76, v77] -v_pk_fma_f16 v86, v68, -1.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v87, v69, -1.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v123, v123, v147 op_sel:[0, 0, 0] -v_pack_b32_f16 v127, v127, v151 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v56, v57, v58, v59], [v98, v99], [v86, v87], [v56, v57, v58, v59] -v_pk_fma_f16 v68, v68, 2.0, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v69, v69, 2.0, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pack_b32_f16 v121, v121, v145 op_sel:[0, 0, 0] -v_pack_b32_f16 v125, v125, v149 op_sel:[0, 0, 0] -v_mfma_f32_16x16x16_f16 [v40, v41, v42, v43], [v94, v95], [v68, v69], [v40, v41, v42, v43] -v_pk_fma_f16 v154, v123, -1.0, v122 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v127, -1.0, v126 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v123, 1.0, v124 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v127, 1.0, v128 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -s_setprio 2 -label_92: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v90, v91], v2 offset:33024 -buffer_load_short_d16 v122, v8, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:24768 -buffer_load_short_d16 v123, v10, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v94, v95], v2 offset:33032 -buffer_load_short_d16 v126, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v127, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_write_b64 v3, [v156, v157] offset:25024 -buffer_load_short_d16 v146, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v102, v103], v2 offset:33040 -buffer_load_short_d16 v147, v10, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v150, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v151, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_45 -s_mov_b32 s29, label_45 - label_1 -s_branch label_48 -.align 32 -label_45: -s_setprio 1 -s_waitcnt lgkmcnt(0) -s_barrier -v_pk_add_f16 v98, v90, v102 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v99, v91, v103 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mfma_f32_16x16x16_f16 [v30, v31, v32, v33], [v92, v93], [v34, v35], [v30, v31, v32, v33] -v_pk_mul_f16 v98, v98, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_pk_mul_f16 v99, v99, 0.5 op_sel:[0, 0, 0] op_sel_hi:[1, 0, 1] -v_mfma_f32_16x16x16_f16 [v82, v83, v84, v85], [v104, v105], [v88, v89], [v82, v83, v84, v85] -v_pk_fma_f16 v98, v94, -0.5, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v99, v95, -0.5, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v154, v124, -2.0, v156 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v155, v128, -2.0, v157 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v64, v65, v66, v67], [v100, v101], [v86, v87], [v64, v65, v66, v67] -v_pk_fma_f16 v94, v94, 1.0, v98 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v95, v95, 1.0, v99 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v156, v124, -1.0, v121 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_pk_fma_f16 v157, v128, -1.0, v125 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 1] -v_mfma_f32_16x16x16_f16 [v48, v49, v50, v51], [v96, v97], [v68, v69], [v48, v49, v50, v51] -s_setprio 2 -label_93: -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v34, v35], v1 offset:16512 -buffer_load_short_d16 v124, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v68, v69], v1 offset:20672 -buffer_load_short_d16 v121, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v154, v155] offset:25280 -buffer_load_short_d16 v128, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v125, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_sub_u32 s2, s2, 4 -s_cselect_b32 s14, 0, s14 -ds_read_b64 [v86, v87], v1 offset:16520 -buffer_load_short_d16 v148, v8, [s12, s13, s14, s15], 0 idxen -ds_read_b64 [v88, v89], v1 offset:20680 -buffer_load_short_d16 v145, v10, [s12, s13, s14, s15], 0 idxen -ds_write_b64 v3, [v156, v157] offset:25536 -buffer_load_short_d16 v152, v9, [s12, s13, s14, s15], 0 idxen -buffer_load_short_d16 v149, v11, [s12, s13, s14, s15], 0 idxen -s_add_u32 s12, s12, s48 -s_addc_u32 s13, s13, 0 -s_bitcmp1_b32 s20, 5 -s_cbranch_scc0 label_34 -s_mov_b32 s29, label_34 - label_1 -s_branch label_49 -.align 32 -label_46: -v_add_f32 v18, v18, v36 -v_add_f32 v19, v19, v37 -v_add_f32 v20, v20, v38 -v_add_f32 v21, v21, v39 -v_add_f32 v70, v70, neg(v52) -v_add_f32 v71, v71, neg(v53) -v_add_f32 v72, v72, neg(v54) -v_add_f32 v73, v73, neg(v55) -v_add_f32 v18, v18, v52 -v_add_f32 v19, v19, v53 -v_add_f32 v20, v20, v54 -v_add_f32 v21, v21, v55 -v_add_f32 v70, v70, v36 -v_add_f32 v71, v71, v37 -v_add_f32 v72, v72, v38 -v_add_f32 v73, v73, v39 -v_cvt_f16_f32 v160, v18 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v19 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v20 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v21 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v71 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v72 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v73 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v18, 0 -v_mov_b32 v19, 0 -v_mov_b32 v20, 0 -v_mov_b32 v21, 0 -v_mov_b32 v36, 0 -v_mov_b32 v37, 0 -v_mov_b32 v38, 0 -v_mov_b32 v39, 0 -v_mov_b32 v52, 0 -v_mov_b32 v53, 0 -v_mov_b32 v54, 0 -v_mov_b32 v55, 0 -v_mov_b32 v70, 0 -v_mov_b32 v71, 0 -v_mov_b32 v72, 0 -v_mov_b32 v73, 0 -ds_write_b64 v1, [v160, v161] offset:4160 -ds_write_b64 v1, [v158, v159] offset:8 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_47: -v_add_f32 v26, v26, v44 -v_add_f32 v27, v27, v45 -v_add_f32 v28, v28, v46 -v_add_f32 v29, v29, v47 -v_add_f32 v78, v78, neg(v60) -v_add_f32 v79, v79, neg(v61) -v_add_f32 v80, v80, neg(v62) -v_add_f32 v81, v81, neg(v63) -v_add_f32 v26, v26, v60 -v_add_f32 v27, v27, v61 -v_add_f32 v28, v28, v62 -v_add_f32 v29, v29, v63 -v_add_f32 v78, v78, v44 -v_add_f32 v79, v79, v45 -v_add_f32 v80, v80, v46 -v_add_f32 v81, v81, v47 -v_cvt_f16_f32 v160, v26 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v27 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v28 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v29 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v78 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v79 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v80 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v81 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v26, 0 -v_mov_b32 v27, 0 -v_mov_b32 v28, 0 -v_mov_b32 v29, 0 -v_mov_b32 v44, 0 -v_mov_b32 v45, 0 -v_mov_b32 v46, 0 -v_mov_b32 v47, 0 -v_mov_b32 v60, 0 -v_mov_b32 v61, 0 -v_mov_b32 v62, 0 -v_mov_b32 v63, 0 -v_mov_b32 v78, 0 -v_mov_b32 v79, 0 -v_mov_b32 v80, 0 -v_mov_b32 v81, 0 -ds_write_b64 v1, [v160, v161] offset:12416 -ds_write_b64 v1, [v158, v159] offset:8264 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_48: -v_add_f32 v22, v22, v40 -v_add_f32 v23, v23, v41 -v_add_f32 v24, v24, v42 -v_add_f32 v25, v25, v43 -v_add_f32 v74, v74, neg(v56) -v_add_f32 v75, v75, neg(v57) -v_add_f32 v76, v76, neg(v58) -v_add_f32 v77, v77, neg(v59) -v_add_f32 v22, v22, v56 -v_add_f32 v23, v23, v57 -v_add_f32 v24, v24, v58 -v_add_f32 v25, v25, v59 -v_add_f32 v74, v74, v40 -v_add_f32 v75, v75, v41 -v_add_f32 v76, v76, v42 -v_add_f32 v77, v77, v43 -v_cvt_f16_f32 v160, v22 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v23 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v24 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v25 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v74 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v75 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v76 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v22, 0 -v_mov_b32 v23, 0 -v_mov_b32 v24, 0 -v_mov_b32 v25, 0 -v_mov_b32 v40, 0 -v_mov_b32 v41, 0 -v_mov_b32 v42, 0 -v_mov_b32 v43, 0 -v_mov_b32 v56, 0 -v_mov_b32 v57, 0 -v_mov_b32 v58, 0 -v_mov_b32 v59, 0 -v_mov_b32 v74, 0 -v_mov_b32 v75, 0 -v_mov_b32 v76, 0 -v_mov_b32 v77, 0 -ds_write_b64 v1, [v160, v161] offset:4160 -ds_write_b64 v1, [v158, v159] offset:8 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -.align 32 -label_49: -v_add_f32 v30, v30, v48 -v_add_f32 v31, v31, v49 -v_add_f32 v32, v32, v50 -v_add_f32 v33, v33, v51 -v_add_f32 v82, v82, neg(v64) -v_add_f32 v83, v83, neg(v65) -v_add_f32 v84, v84, neg(v66) -v_add_f32 v85, v85, neg(v67) -v_add_f32 v30, v30, v64 -v_add_f32 v31, v31, v65 -v_add_f32 v32, v32, v66 -v_add_f32 v33, v33, v67 -v_add_f32 v82, v82, v48 -v_add_f32 v83, v83, v49 -v_add_f32 v84, v84, v50 -v_add_f32 v85, v85, v51 -v_cvt_f16_f32 v160, v30 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v160, v31 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v32 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v161, v33 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v82 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v158, v83 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v84 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE -v_cvt_f16_f32 v159, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE -v_mov_b32 v30, 0 -v_mov_b32 v31, 0 -v_mov_b32 v32, 0 -v_mov_b32 v33, 0 -v_mov_b32 v48, 0 -v_mov_b32 v49, 0 -v_mov_b32 v50, 0 -v_mov_b32 v51, 0 -v_mov_b32 v64, 0 -v_mov_b32 v65, 0 -v_mov_b32 v66, 0 -v_mov_b32 v67, 0 -v_mov_b32 v82, 0 -v_mov_b32 v83, 0 -v_mov_b32 v84, 0 -v_mov_b32 v85, 0 -ds_write_b64 v1, [v160, v161] offset:12416 -ds_write_b64 v1, [v158, v159] offset:8264 -s_bitcmp1_b32 s20, 4 -s_cmov_b32 s29, label_0 - label_1 -s_add_u32 s36, s6, s29 -s_addc_u32 s37, s7, 0 -s_setpc_b64 [s36, s37] -label_5: -s_and_b32 s45, s45, 65535 -s_add_u32 s45, s45, 131072 -s_mov_b64 [s80, s81], [s44, s45] -s_mov_b32 s82, 2147483648 -s_mov_b32 s83, 131072 -s_bitcmp0_b32 s28, 7 -s_cselect_b32 s82, 0, 2147483648 -s_and_b32 s79, s1, 3 -v_and_b32 v87, v0, 63 -v_lshrrev_b32 v86, 1, v87 -v_and_b32 v85, v87, 1 -v_mad_u32_u24 v85, v86, 8, v85 -v_lshlrev_b32 v85, 1, v85 -v_mad_u32_u24 v85, 4, s79, v85 -v_add_u32 v86, v85, 1 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v85, s26 -v_cndmask_b32 v85, -1, v85, [vcc_lo, vcc_hi] -v_cmp_lt_u32 [vcc_lo, vcc_hi], v86, s26 -v_cndmask_b32 v86, -1, v86, [vcc_lo, vcc_hi] -buffer_load_short_d16 v77, v85, [s80, s81, s82, s83], 0 idxen -buffer_load_short_d16 v81, v86, [s80, s81, s82, s83], 0 idxen -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s44, 1536, 0 -s_cselect_b32 s45, 12, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s44, 65536, s44 -s_cselect_b32 s45, 65536, s45 -s_cmp_eq_u32 s1, 9 -s_cselect_b32 s79, 1024, 0 -s_cselect_b32 s80, 4, 0 -s_cmp_ge_u32 s1, 10 -s_cselect_b32 s79, 65536, s79 -s_cselect_b32 s80, 65536, s80 -s_and_b32 s82, s1, 3 -s_mul_i32 s81, s82, 2048 -v_and_b32 v85, 63, v0 -v_lshl_add_u32 v54, v85, 3, s44 -v_lshl_add_u32 v71, v85, 3, 0 -v_mov_b32 v72, s45 -v_mov_b32 v73, 0 -v_lshl_add_u32 v74, v85, 4, s79 -v_mov_b32 v76, s80 -v_lshl_add_u32 v75, v85, 4, s81 -s_add_u32 s80, s23, 15 -s_and_b32 s80, s80, -16 -s_sub_u32 s80, s80, 16 -s_sub_u32 s80, s23, s80 -s_bitcmp1_b32 s28, 1 -s_cselect_b32 s81, s39, 1 -s_cselect_b32 s29, -1, 1 -s_sub_u32 s79, s81, 1 -s_bitcmp1_b32 s28, 0 -s_cselect_b32 s81, s38, 1 -s_cselect_b32 s19, -1, 1 -s_sub_u32 s81, s81, 1 -s_mul_i32 s19, s19, s62 -s_mul_i32 s81, s81, s62 -s_add_u32 s79, s79, s81 -s_bitcmp1_b32 s28, 31 -s_cselect_b32 s45, 0, 1 -s_cselect_b32 s44, 4, 0 -s_cselect_b32 s81, 2, 0 -s_cselect_b32 s82, 2, 4 -v_bfe_u32 v86, v0, 2, 6 -v_and_b32 v87, s1, 3 -v_bfe_u32 v85, v86, 0, s81 -v_mad_u32_u24 v85, v87, s45, v85 -v_mov_b32 v10, s82 -v_bfe_u32 v10, v86, s81, v10 -v_mad_u32_u24 v10, v87, s44, v10 -v_mov_b32 v86, s79 -v_mad_u32_u24 v5, s61, v85, v86 -v_sub_u32 v21, s80, v85 -v_add_u32 v17, v21, 3 -v_add_u32 v21, v21, 3 -s_add_u32 s45, s42, 1 -s_lshr_b32 s45, s45, 1 -s_lshl_b32 s61, s45, 1 -s_add_u32 s62, s43, 1 -s_lshr_b32 s62, s62, 1 -s_lshl1_add_u32 s62, s62, 2 -s_pack_ll_b32_b16 s3, s62, s45 -s_pack_ll_b32_b16 s10, s25, s24 -s_sub_u32 s11, s61, s40 -s_sub_u32 s44, s62, s41 -s_pack_ll_b32_b16 s11, s44, s11 -s_pack_ll_b32_b16 s13, s43, s42 -s_sub_u32 s44, s62, 1 -s_pack_ll_b32_b16 s14, s44, s61 -v_lshrrev_b32 v88, 16, s3 -v_mov_b32 v103, 65535 -v_bfi_b32 v89, v103, s3, 0 -v_and_b32 v91, 1, v0 -v_bfe_u32 v97, v0, 6, 1 -v_and_b32 v86, 63, v0 -v_mov_b32 v103, 124 -v_mad_u32_u16 v92, v103, s5, 0 op_sel:[0, 0, 0, 0] -v_mov_b32 v103, 2 -v_mad_u32_u16 v97, v103, s8, v97 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v90, v88, v89, 0 op_sel:[0, 0, 0, 0] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 0, v91 -v_cndmask_b32 v98, v90, v89, [vcc_lo, vcc_hi] -v_mad_u32_u16 v87, 62, v97, v86 op_sel:[0, 0, 0, 0] -v_cndmask_b32 v87, v92, v87, [vcc_lo, vcc_hi] -v_ffbh_u32 v104, v98 -v_lshlrev_b32 v105, v104, v98 -v_and_b32 v103, -256, v105 -v_cmp_eq_u32 [vcc_lo, vcc_hi], 2147483648, v105 -v_cvt_f32_u32 v103, v103 -v_rcp_f32 v99, v103 -v_subb_co_u32 v100, [vcc_lo, vcc_hi], 32, v104, [vcc_lo, vcc_hi] -v_cvt_f32_ubyte0 v104, v105 -v_fma_f32 v103, v103, v99, -1.0 -v_fma_f32 v103, v104, v99, v103 -v_fmaak_f32 v103, v103, v99, 2667577344 -v_mul_f32 v103, 1602224128, v103 -v_mov_b32 v104, 0 -v_cvt_flr_i32_f32 v103, neg(v103) -v_lshl_add_u32 v99, v99, 9, v103 -v_mad_u64_u32 [v104, v105], [vcc_lo, vcc_hi], v105, v99, [v104, v105] -v_subb_co_u32 v99, [vcc_lo, vcc_hi], v99, -1, [vcc_lo, vcc_hi] -v_mov_b32 v102, v100 quad_perm:[1, 1, 1, 1] -v_mov_b32 v100, v100 quad_perm:[0, 0, 0, 0] -v_mov_b32 v101, v99 quad_perm:[1, 1, 1, 1] -v_mov_b32 v99, v99 quad_perm:[0, 0, 0, 0] -v_mul_hi_u32 v103, v87, v99 -v_add_co_u32 v85, [vcc_lo, vcc_hi], v103, v87 -v_addc_co_u32 v103, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v100 -v_cndmask_b32 v85, v85, v103, [vcc_lo, vcc_hi] -v_alignbit_b32 v85, v103, v85, v100 -v_mul_hi_u32 v103, v87, v101 -v_add_co_u32 v4, [vcc_lo, vcc_hi], v103, v87 -v_addc_co_u32 v103, [vcc_lo, vcc_hi], 0, 0, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], 32, v102 -v_cndmask_b32 v4, v4, v103, [vcc_lo, vcc_hi] -v_alignbit_b32 v4, v103, v4, v102 -v_mad_u32_u16 v96, v85, v89, 0 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v95, v4, v88, 0 op_sel:[0, 0, 0, 0] -v_sub_u32 v96, v87, v96 -v_sub_u32 v95, v85, v95 -v_readlane_b32 s79, v96, 1 -v_sub_u32 v96, v96, v89 -v_readlane_b32 s9, v95, 1 -v_sub_u32 v95, v95, v88 -v_readlane_b32 s2, v4, 1 -v_sub_u32 v4, v4, s22 -s_lshl_b32 s9, s9, 16 -s_and_b32 s79, s79, lit(0xffff) -s_add_u32 s9, s9, s79 -v_mov_b32 v96, v96 quad_perm:[0, 0, 2, 2] -v_mov_b32 v95, v95 quad_perm:[0, 0, 2, 2] -v_mov_b32 v4, v4 quad_perm:[0, 0, 2, 2] -v_add_co_u32 v96, [vcc_lo, vcc_hi], v96, v91 -v_cndmask_b32 v94, 0, v89, [vcc_lo, vcc_hi] -v_addc_co_u32 v95, [vcc_lo, vcc_hi], v95, 0, [vcc_lo, vcc_hi] -v_cndmask_b32 v93, 0, v88, [vcc_lo, vcc_hi] -v_addc_co_u32 v4, [vcc_lo, vcc_hi], v4, 0, [vcc_lo, vcc_hi] -v_min_u32 v91, v86, 63 -v_sub_u32 v96, v96, v94 -v_sub_u32 v95, v95, v93 -v_cmp_eq_u32 [vcc_lo, vcc_hi], v86, v91 -v_lshlrev_b32 v1, 16, v95 -v_mov_b32 v106, 65535 -v_bfi_b32 v1, v106, v96, v1 -v_add_u32 v106, v4, s22 -v_med3_u32 v91, v86, 1, 62 -v_mul_lo_u32 v2, v106, s56 -v_mul_lo_u32 v11, v106, s64 -s_mul_i32 s12, s2, s56 -s_mul_i32 s15, s2, s64 -v_mov_b32 v107, 2147483648 -v_cndmask_b32 v2, v107, v2, [vcc_lo, vcc_hi] -v_cmp_eq_u32 [vcc_lo, vcc_hi], v86, v91 -v_cndmask_b32 v11, v107, v11, [vcc_lo, vcc_hi] -v_cmp_ge_u32 [s20, s21], v106, s22 -v_cndmask_b32 v2, v2, v107, [s20, s21] -v_cndmask_b32 v11, v11, v107, [s20, s21] -s_and_b32 s24, s72, 255 -s_cmp_eq_u32 s24, 0 -s_cmov_b32 s46, lit(0x3c003c00) -s_cbranch_scc1 label_118 -s_cmp_eq_u32 s24, 4 -s_cbranch_scc1 label_120 -s_cmp_eq_u32 s24, 2 -s_cbranch_scc1 label_121 -v_cvt_f16_f32 v85, s46 -s_nop 0 -v_readfirstlane_b32 s46, v85 -v_cvt_f16_f32 v85, s47 -v_cmp_gt_f16 [vcc_lo, vcc_hi], s46, 1.0 -v_readfirstlane_b32 s47, v85 -s_pack_ll_b32_b16 s46, s46, s46 -s_pack_ll_b32_b16 s47, s47, s47 -s_cmp_eq_u32 s24, 3 -s_cbranch_scc1 label_122 -s_cbranch_vccnz label_119 -label_118: -s_mov_b32 s78, label_98 - label_1 -s_branch label_123 -label_119: -s_mov_b32 s78, label_102 - label_1 -s_branch label_123 -label_120: -s_mov_b32 s78, label_106 - label_1 -s_branch label_123 -label_121: -s_mov_b32 s78, label_110 - label_1 -s_branch label_123 -label_122: -s_mov_b32 s78, label_114 - label_1 -label_123: -s_lshl_b32 s59, s65, 1 -s_mul_i32 s54, s59, 16 -s_mul_hi_u32 s55, s59, 16 -s_and_b32 s24, s1, 3 -s_lshl_b32 s25, s59, 2 -s_mul_i32 s25, s25, s24 -s_add_u32 s34, s34, s25 -s_addc_u32 s35, s35, 0 -s_lshl_b32 s67, s24, 2 -s_sub_u32 s67, s26, s67 -s_cmov_b32 s67, 0 -s_and_b32 s35, s35, 65535 -s_add_u32 s35, s35, 131072 -s_mov_b64 [s68, s69], [s34, s35] -s_mov_b32 s70, 0 -s_mov_b32 s71, 131072 -s_mov_b32 s16, 65536 -s_mov_b32 s17, 196610 -s_mov_b32 s18, 65536 -s_add_u32 s30, s38, 2 -s_add_u32 s31, s39, 2 -s_mul_hi_u32 s25, s30, 1431655766 -s_add_u32 s30, s25, s30 -s_addc_u32 s25, 0, 0 -s_mov_b32 s24, s30 -s_lshr_b64 [s24, s25], [s24, s25], 2 -s_mov_b32 s30, s24 -s_mul_hi_u32 s25, s31, 1431655766 -s_add_u32 s31, s25, s31 -s_addc_u32 s25, 0, 0 -s_mov_b32 s24, s31 -s_lshr_b64 [s24, s25], [s24, s25], 2 -s_mov_b32 s31, s24 -s_sub_u32 s30, 0, s30 -s_sub_u32 s31, 0, s31 -s_mov_b32 s32, -1 -s_mov_b32 s33, -1 -s_waitcnt vmcnt(0) -v_pack_b32_f16 v77, v77, v81 op_sel:[0, 0, 0] -s_nop 0 -v_readlane_b32 s36, v77, 0 -v_readlane_b32 s37, v77, 1 -v_readlane_b32 s48, v77, 2 -v_readlane_b32 s49, v77, 3 -s_mov_b32 s73, 0 -s_mov_b32 s74, 0 -s_mov_b32 s50, 2 -s_lshl_b32 s51, s50, 9 -s_lshl_b32 s52, s50, 2 -s_branch label_97 -.align 32 -label_94: -s_sub_u32 s74, s23, 1 -s_add_u32 s76, s6, s75 -s_addc_u32 s77, s7, 0 -s_setprio 3 -s_add_u32 s33, s33, 1 -s_cselect_b32 s33, s31, s33 -s_addc_u32 s32, s32, 0 -s_cselect_b32 s32, s30, s32 -s_cbranch_scc0 label_124 -s_bitcmp1_b32 s20, 0 -s_addc_u32 s73, s73, 2 -v_add_u32 v77, s51, v54 -v_add_u32 v81, s52, v72 -v_and_b32 v84, s73, 3 -ds_write_b64 v77, [v12, v13] offset:55816 -ds_write_b32 v81, v84 offset:58888 -label_124: -s_sub_u32 s24, -1, s32 -s_sub_u32 s25, -1, s33 -s_mul_i32 s24, 3, s24 -s_mul_i32 s25, 3, s25 -s_pack_ll_b32_b16 s40, s25, s24 -s_cmp_eq_u32 s40, 0 -s_cbranch_scc1 label_126 -label_125: -v_mov_b32 v81, 2147483648 -v_mov_b32 v87, 131073 -v_pk_mad_u16 v87, v1, v87, s11 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_add_u16 v87, s40, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mad_u32_u16 v6, v87, 1, v2 op_sel:[0, 0, 0, 0] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v87, s10 -v_cndmask_b32 v6, v81, v6, [vcc_lo, vcc_hi] -v_pk_add_u16 v77, v87, s17 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_u16 v88, v87, s16 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_mad_u32_u16 v9, v77, s58, v6 op_sel:[1, 0, 0, 0] -v_mad_u32_u16 v7, v88, s58, v6 op_sel:[1, 0, 0, 0] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v77, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v9, v81, v9, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v88, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v7, v81, v7, [vcc_lo, vcc_hi] -v_mad_u32_u16 v8, v77, s58, v6 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v6, v88, s58, v6 op_sel:[0, 0, 0, 0] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v77, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v8, v81, v8, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v88, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v6, v81, v6, [vcc_lo, vcc_hi] -s_branch label_127 -label_126: -v_mov_b32 v91, 2147483648 -v_mov_b32 v92, s56 -v_mov_b32 v81, s64 -v_mov_b32 v90, s2 -v_add_co_u32 v77, [vcc_lo, vcc_hi], v1, s9 -v_mov_b32 v93, 131073 -v_pk_mad_u16 v87, v1, v93, s11 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mad_u16 v88, v1, v93, s14 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_min_u16 v86, v77, v1 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_cndmask_b32 v85, 0, v92, [vcc_lo, vcc_hi] -v_cndmask_b32 v89, 0, v81, [vcc_lo, vcc_hi] -v_mad_u32_u16 v6, v87, 1, v2 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v12, v88, 1, v11 op_sel:[0, 0, 0, 0] -v_add3_u32 v2, v2, s12, v85 -v_add3_u32 v11, v11, s15, v89 -v_pk_sub_u16 v86, v1, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_addc_co_u32 v4, [s20, s21], v4, v90, [vcc_lo, vcc_hi] -v_cndmask_b32 v2, v2, v91, [s20, s21] -v_cndmask_b32 v11, v11, v91, [s20, s21] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v87, s10 -v_cndmask_b32 v6, v91, v6, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v88, s13 -v_cndmask_b32 v12, v91, v12, [vcc_lo, vcc_hi] -v_pk_ashrrev_i16 v86, 15, v86 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_u16 v81, v87, s17 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_u16 v92, v87, s16 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_mad_u32_u16 v9, v81, s58, v6 op_sel:[1, 0, 0, 0] -v_mad_u32_u16 v7, v92, s58, v6 op_sel:[1, 0, 0, 0] -v_pk_add_u16 v90, v88, s18 op_sel:[1, 0, 0] op_sel_hi:[1, 1, 1] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v81, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v9, v91, v9, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v92, s10 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v7, v91, v7, [vcc_lo, vcc_hi] -v_mad_u32_u16 v13, v90, s66, v12 op_sel:[1, 0, 0, 0] -v_mad_u32_u16 v8, v81, s58, v6 op_sel:[0, 0, 0, 0] -v_mad_u32_u16 v6, v92, s58, v6 op_sel:[0, 0, 0, 0] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v90, s13 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v13, v91, v13, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v81, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v8, v91, v8, [vcc_lo, vcc_hi] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v92, s10 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v6, v91, v6, [vcc_lo, vcc_hi] -v_mad_u32_u16 v12, v90, s66, v12 op_sel:[0, 0, 0, 0] -v_pk_mad_u16 v1, v86, s3, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_cmp_lt_u16 [vcc_lo, vcc_hi], v90, s13 src0_sel:WORD_0 src1_sel:WORD_1 -v_cndmask_b32 v12, v91, v12, [vcc_lo, vcc_hi] -label_127: -v_and_b32 v84, s73, 3 -v_xor_b32 v77, v0, v0 quad_perm:[0, 0, 0, 1] -v_add_u32 v77, s25, v77 -v_cmp_lt_u32 [s44, s45], v77, s39 -v_mul_i32_i24 v77, s29, v77 -s_mul_i32 s41, s19, s24 -s_sub_u32 s42, s38, 1 -s_sub_u32 s42, s42, s24 -s_cselect_b64 [s44, s45], 0, [s44, s45] -s_sub_u32 s42, s42, 1 -s_cselect_b64 [s80, s81], 0, [s44, s45] -s_sub_u32 s42, s42, 1 -s_cselect_b64 [s82, s83], 0, [s80, s81] -v_mov_b32 v81, 3221225472 -v_add3_u32 v85, v77, s41, v5 -v_mad_u32_u24 v14, s60, v10, v85 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v10, s26 -v_cndmask_b32 v14, v81, v14, [vcc_lo, vcc_hi] -v_add_u32 v15, s19, v14 -v_add_u32 v16, s19, v15 -v_cndmask_b32 v14, v81, v14, [s44, s45] -v_cndmask_b32 v15, v81, v15, [s80, s81] -v_cndmask_b32 v16, v81, v16, [s82, s83] -v_add_u32 v86, v10, 16 -v_mad_u32_u24 v18, s60, v86, v85 -v_cmp_lt_u32 [vcc_lo, vcc_hi], v86, s26 -v_cndmask_b32 v18, v81, v18, [vcc_lo, vcc_hi] -v_add_u32 v19, s19, v18 -v_add_u32 v20, s19, v19 -v_cndmask_b32 v18, v81, v18, [s44, s45] -v_cndmask_b32 v19, v81, v19, [s80, s81] -v_cndmask_b32 v20, v81, v20, [s82, s83] -ds_write_b128 v75, [v14, v15, v16, v17] offset:45568 -ds_write_b128 v75, [v18, v19, v20, v21] offset:46592 -ds_write_b128 v74, [v6, v7, v8, v9] offset:53760 -ds_write_b32 v76, v84 offset:55808 -s_setpc_b64 [s76, s77] -label_97: -label_128: -s_barrier -s_lshl_b32 s73, s73, 2 -s_mov_b32 s75, label_130 - label_1 -s_sub_u32 s74, s74, 16 -s_cbranch_scc1 label_94 -label_130: -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_129 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_sub_u32 s50, s50, 1 -s_cselect_b32 s50, 2, s50 -s_lshl_b32 s51, s50, 9 -s_lshl_b32 s52, s50, 2 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_129: -s_bitcmp1_b32 s73, 5 -s_cbranch_scc1 label_95 -label_96: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_131 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_131: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_132 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_132: -s_barrier -s_setprio 0 -s_setprio 2 -s_bitcmp1_b32 s73, 5 -s_cselect_b32 s76, s78, label_133 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_133: -s_branch label_128 -label_134: -s_barrier -s_lshl_b32 s73, s73, 2 -s_mov_b32 s75, label_136 - label_1 -s_sub_u32 s74, s74, 16 -s_cbranch_scc1 label_94 -label_136: -s_setprio 0 -ds_read_b64 [v44, v45], v3 offset:8256 -ds_read_b64 [v42, v43], v3 offset:8512 -ds_read_b64 [v52, v53], v3 offset:8768 -ds_read_b64 [v50, v51], v3 offset:9024 -s_setprio 3 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_135 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_sub_u32 s50, s50, 1 -s_cselect_b32 s50, 2, s50 -s_lshl_b32 s51, s50, 9 -s_lshl_b32 s52, s50, 2 -s_waitcnt lgkmcnt(0) -v_fma_mix_f32 v67, v44, 1.0, v42 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v44, v44, 1.0, v42 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v68, v45, 1.0, v43 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v45, v45, 1.0, v43 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v67, v67, 1.0, v52 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v67, v44, 1.0, v52 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v68, v68, 1.0, v53 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v68, v45, 1.0, v53 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v69, v52, -1.0, v42 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v52, v52, -1.0, v42 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v70, v53, -1.0, v43 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v53, v53, -1.0, v43 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v69, v69, 1.0, v50 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v69, v52, 1.0, v50 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v70, v70, 1.0, v51 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v70, v53, 1.0, v51 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_swappc_b64 [s76, s77], [s76, s77] -label_135: -s_bitcmp1_b32 s73, 5 -s_cbranch_scc0 label_96 -label_95: -s_barrier -s_setprio 0 -ds_read_b64 [v24, v25], v3 offset:0 -ds_read_b64 [v22, v23], v3 offset:256 -ds_read_b64 [v32, v33], v3 offset:512 -ds_read_b64 [v30, v31], v3 offset:768 -s_setprio 3 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_137 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_137: -v_fma_mix_f32 v55, v24, 1.0, v22 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v24, v24, 1.0, v22 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v56, v25, 1.0, v23 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v25, v25, 1.0, v23 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v55, v55, 1.0, v32 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v55, v24, 1.0, v32 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v56, v56, 1.0, v33 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v56, v25, 1.0, v33 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v57, v32, -1.0, v22 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v32, v32, -1.0, v22 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v58, v33, -1.0, v23 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v33, v33, -1.0, v23 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v57, v57, 1.0, v30 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v57, v32, 1.0, v30 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v58, v58, 1.0, v31 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v58, v33, 1.0, v31 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_barrier -s_setprio 0 -ds_read_b64 [v40, v41], v3 offset:8256 -ds_read_b64 [v38, v39], v3 offset:8512 -ds_read_b64 [v48, v49], v3 offset:8768 -ds_read_b64 [v46, v47], v3 offset:9024 -s_setprio 3 -s_bitcmp1_b32 s73, 7 -s_cselect_b32 s76, s78, label_138 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -s_swappc_b64 [s76, s77], [s76, s77] -label_138: -v_fma_mix_f32 v63, v40, 1.0, v38 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v40, v40, 1.0, v38 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v64, v41, 1.0, v39 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v41, v41, 1.0, v39 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v63, v63, 1.0, v48 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v63, v40, 1.0, v48 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v64, v64, 1.0, v49 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v64, v41, 1.0, v49 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v65, v48, -1.0, v38 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v48, v48, -1.0, v38 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v66, v49, -1.0, v39 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v49, v49, -1.0, v39 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v65, v65, 1.0, v46 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v65, v48, 1.0, v46 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v66, v66, 1.0, v47 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v66, v49, 1.0, v47 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_add_u32 v83, s51, v71 -v_add_u32 v80, s52, v73 -s_barrier -s_setprio 0 -ds_read_b64 [v78, v79], v83 offset:55816 -ds_read_b64 [v82, v83], v83 offset:57352 -ds_read_b32 v80, v80 offset:58888 -ds_read_b64 [v28, v29], v3 offset:0 -ds_read_b64 [v26, v27], v3 offset:256 -ds_read_b64 [v36, v37], v3 offset:512 -ds_read_b64 [v34, v35], v3 offset:768 -s_setprio 3 -s_bitcmp1_b32 s73, 5 -s_cselect_b32 s76, s78, label_139 - label_1 -s_add_u32 s76, s6, s76 -s_addc_u32 s77, s7, 0 -s_waitcnt lgkmcnt(0) -v_fma_mix_f32 v59, v28, 1.0, v26 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v28, v28, 1.0, v26 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v60, v29, 1.0, v27 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v29, v29, 1.0, v27 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v59, v59, 1.0, v36 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v59, v28, 1.0, v36 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v60, v60, 1.0, v37 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v60, v29, 1.0, v37 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mix_f32 v61, v36, -1.0, v26 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v36, v36, -1.0, v26 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v62, v37, -1.0, v27 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mix_f32 v37, v37, -1.0, v27 op_sel:[1, 0, 1, 0] op_sel_hi:[1, 0, 1, 0] -v_fma_mixlo_f16 v61, v61, 1.0, v34 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v61, v36, 1.0, v34 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixlo_f16 v62, v62, 1.0, v35 op_sel:[0, 0, 0, 0] op_sel_hi:[0, 0, 1, 0] -v_fma_mixhi_f16 v62, v37, 1.0, v35 op_sel:[0, 0, 1, 0] op_sel_hi:[0, 0, 1, 0] -s_swappc_b64 [s76, s77], [s76, s77] -label_139: -s_branch label_134 -label_98: -s_setprio 3 -v_readfirstlane_b32 s53, v80 -v_pk_add_f16 v55, v55, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v55, v55, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v59, v59, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v57, v57, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v61, v61, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_99 - label_1 -s_setpc_b64 [s76, s77] -label_99: -s_setprio 3 -v_pk_add_f16 v56, v56, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v56, v56, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v60, v60, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v58, v58, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v62, v62, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_100 - label_1 -s_sub_u32 s63, s67, 17 -s_cselect_b32 s70, 0, s70 -s_add_u32 s68, s34, s54 -s_addc_u32 s69, s35, s55 -s_setpc_b64 [s76, s77] -label_100: -s_setprio 3 -v_pk_add_f16 v63, v63, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v63, v63, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v67, v67, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v65, v65, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v69, v69, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_101 - label_1 -s_setpc_b64 [s76, s77] -label_101: -s_setprio 3 -v_pk_add_f16 v64, v64, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v64, v64, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v68, v68, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v66, v66, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v70, v70, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_98 - label_1 -s_bitcmp1_b32 s53, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s63, s67, 1 -s_cselect_b32 s70, 0, 2147483648 -s_mov_b64 [s68, s69], [s34, s35] -s_setpc_b64 [s76, s77] -label_102: -s_setprio 3 -v_readfirstlane_b32 s53, v80 -v_pk_add_f16 v55, v55, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v55, v55, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v59, v59, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v57, v57, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v61, v61, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_103 - label_1 -s_setpc_b64 [s76, s77] -label_103: -s_setprio 3 -v_pk_add_f16 v56, v56, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v56, v56, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v60, v60, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v58, v58, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v62, v62, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_104 - label_1 -s_sub_u32 s63, s67, 17 -s_cselect_b32 s70, 0, s70 -s_add_u32 s68, s34, s54 -s_addc_u32 s69, s35, s55 -s_setpc_b64 [s76, s77] -label_104: -s_setprio 3 -v_pk_add_f16 v63, v63, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v63, v63, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v67, v67, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v77, s46, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v87, s46, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v65, v65, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v69, v69, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_105 - label_1 -s_setpc_b64 [s76, s77] -label_105: -s_setprio 3 -v_pk_add_f16 v64, v64, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v64, v64, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v68, v68, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v85, s46, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v89, s46, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v66, v66, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_min_f16 v70, v70, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_102 - label_1 -s_bitcmp1_b32 s53, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s63, s67, 1 -s_cselect_b32 s70, 0, 2147483648 -s_mov_b64 [s68, s69], [s34, s35] -s_setpc_b64 [s76, s77] -label_106: -s_setprio 3 -v_readfirstlane_b32 s53, v80 -v_pk_add_f16 v55, v55, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v55, 0, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v59, 0, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v57, 0, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v61, 0, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_107 - label_1 -s_setpc_b64 [s76, s77] -label_107: -s_setprio 3 -v_pk_add_f16 v56, v56, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v56, 0, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v60, 0, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v58, 0, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v62, 0, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_108 - label_1 -s_sub_u32 s63, s67, 17 -s_cselect_b32 s70, 0, s70 -s_add_u32 s68, s34, s54 -s_addc_u32 s69, s35, s55 -s_setpc_b64 [s76, s77] -label_108: -s_setprio 3 -v_pk_add_f16 v63, v63, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v63, 0, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v67, 0, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v65, 0, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v69, 0, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_109 - label_1 -s_setpc_b64 [s76, s77] -label_109: -s_setprio 3 -v_pk_add_f16 v64, v64, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v64, 0, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v68, 0, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v66, 0, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_max_f16 v70, 0, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_106 - label_1 -s_bitcmp1_b32 s53, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s63, s67, 1 -s_cselect_b32 s70, 0, 2147483648 -s_mov_b64 [s68, s69], [s34, s35] -s_setpc_b64 [s76, s77] -label_110: -s_setprio 3 -v_readfirstlane_b32 s53, v80 -v_pk_add_f16 v55, v55, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v55, v91, v55 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v59, v91, v59 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v55, v55 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v59, v59 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v55, v55 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v59, v59 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v55, 1.0, v55 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v59, 1.0, v59 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v55, v55 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v59, v59 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v55, v55 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v59, v59 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v57, v91, v57 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v61, v91, v61 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v57, v57 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v61, v61 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v57, v57 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v61, v61 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v57, 1.0, v57 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v61, 1.0, v61 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v57, v57 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v61, v61 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v57, v57 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v61, v61 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_111 - label_1 -s_setpc_b64 [s76, s77] -label_111: -s_setprio 3 -v_pk_add_f16 v56, v56, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v56, v91, v56 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v60, v91, v60 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v56, v56 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v60, v60 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v56, v56 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v60, v60 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v56, 1.0, v56 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v60, 1.0, v60 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v56, v56 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v60, v60 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v56, v56 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v60, v60 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v58, v91, v58 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v62, v91, v62 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v58, v58 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v62, v62 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v58, v58 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v62, v62 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v58, 1.0, v58 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v62, 1.0, v62 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v58, v58 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v62, v62 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v58, v58 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v62, v62 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_112 - label_1 -s_sub_u32 s63, s67, 17 -s_cselect_b32 s70, 0, s70 -s_add_u32 s68, s34, s54 -s_addc_u32 s69, s35, s55 -s_setpc_b64 [s76, s77] -label_112: -s_setprio 3 -v_pk_add_f16 v63, v63, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v63, v91, v63 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v67, v91, v67 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v63, v63 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v67, v67 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v63, v63 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v67, v67 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v63, 1.0, v63 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v67, 1.0, v67 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v63, v63 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v67, v67 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v63, v63 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v67, v67 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v65, v91, v65 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v69, v91, v69 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v65, v65 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v69, v69 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v65, v65 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v69, v69 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v65, 1.0, v65 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v69, 1.0, v69 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v65, v65 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v69, v69 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v65, v65 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v69, v69 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_113 - label_1 -s_setpc_b64 [s76, s77] -label_113: -s_setprio 3 -v_pk_add_f16 v64, v64, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v64, v91, v64 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v68, v91, v68 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v64, v64 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v68, v68 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v64, v64 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v68, v68 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v64, 1.0, v64 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v68, 1.0, v68 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v64, v64 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v68, v68 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v64, v64 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v68, v68 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v66, v91, v66 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 3183852997 -v_pk_mul_f16 v70, v91, v70 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v66, v66 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v70, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v66, v66 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v70, v70 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v66, 1.0, v66 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v70, 1.0, v70 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v66, v66 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v70, v70 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v66, v66 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v70, v70 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -s_setprio 0 -buffer_store_short v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_110 - label_1 -s_bitcmp1_b32 s53, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s63, s67, 1 -s_cselect_b32 s70, 0, 2147483648 -s_mov_b64 [s68, s69], [s34, s35] -s_setpc_b64 [s76, s77] -label_114: -s_setprio 3 -v_readfirstlane_b32 s53, v80 -v_pk_add_f16 v55, v55, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v59, v59, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v57, v57, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v61, v61, s36 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v55, v55, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v59, v59, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v77, 2147450879, v55 -v_and_b32 v87, 2147450879, v59 -v_mov_b32 v81, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v81, v77, v91, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v88, v87, v91, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v81, v77, v81, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v88, v87, v88, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v81, v77, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v81, v77, v81, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v77, v91, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v87, v91, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v77, 1.0, v77 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v77, v77, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v55), v91 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v59), v91 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v55), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v59), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v77, 2147450879 -v_bfi_b32 v55, v77, v81, v55 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v59, v87, v88, v59 -v_pk_mul_f16 v55, v55, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v59, v59, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v57, v57, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v61, v61, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v77, 2147450879, v57 -v_and_b32 v87, 2147450879, v61 -v_mov_b32 v81, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v81, v77, v91, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v88, v87, v91, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v81, v77, v81, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v88, v87, v88, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v81, v77, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v81, v77, v81, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v77, v91, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v87, v91, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v77, 1.0, v77 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v77, v77, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v57), v91 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v61), v91 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v57), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v61), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v77, 2147450879 -v_bfi_b32 v57, v77, v81, v57 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v61, v87, v88, v61 -v_pk_mul_f16 v57, v57, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v61, v61, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v55, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v59, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v57, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v61, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_115 - label_1 -s_setpc_b64 [s76, s77] -label_115: -s_setprio 3 -v_pk_add_f16 v56, v56, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v60, v60, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v58, v58, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v62, v62, s37 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v56, v56, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v60, v60, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v85, 2147450879, v56 -v_and_b32 v89, 2147450879, v60 -v_mov_b32 v86, 3056514606 -v_mov_b32 v90, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v86, v85, v91, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v90, v89, v91, v90 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v86, v85, v86, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v90, v89, v90, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v90, v89, v90 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v90, v89, v90, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v85, v91, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v89, v91, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v89, 1.0, v89 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v89, v89, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v56), v91 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v60), v91 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v56), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v60), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v56, v85, v86, v56 -v_mov_b32 v89, 2147450879 -v_bfi_b32 v60, v89, v90, v60 -v_pk_mul_f16 v56, v56, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v60, v60, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v58, v58, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v62, v62, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v85, 2147450879, v58 -v_and_b32 v89, 2147450879, v62 -v_mov_b32 v86, 3056514606 -v_mov_b32 v90, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v86, v85, v91, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v90, v89, v91, v90 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v86, v85, v86, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v90, v89, v90, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v90, v89, v90 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v90, v89, v90, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v85, v91, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v89, v91, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v89, 1.0, v89 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v89, v89, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v58), v91 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v62), v91 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v58), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v62), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v58, v85, v86, v58 -v_mov_b32 v89, 2147450879 -v_bfi_b32 v62, v89, v90, v62 -v_pk_mul_f16 v58, v58, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v62, v62, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v56, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v60, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v58, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v62, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_116 - label_1 -s_sub_u32 s63, s67, 17 -s_cselect_b32 s70, 0, s70 -s_add_u32 s68, s34, s54 -s_addc_u32 s69, s35, s55 -s_setpc_b64 [s76, s77] -label_116: -s_setprio 3 -v_pk_add_f16 v63, v63, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v67, v67, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v65, v65, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v69, v69, s48 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v63, v63, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v67, v67, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v77, 2147450879, v63 -v_and_b32 v87, 2147450879, v67 -v_mov_b32 v81, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v81, v77, v91, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v88, v87, v91, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v81, v77, v81, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v88, v87, v88, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v81, v77, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v81, v77, v81, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v77, v91, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v87, v91, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v77, 1.0, v77 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v77, v77, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v63), v91 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v67), v91 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v63), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v67), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v77, 2147450879 -v_bfi_b32 v63, v77, v81, v63 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v67, v87, v88, v67 -v_pk_mul_f16 v63, v63, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v67, v67, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v65, v65, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v69, v69, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v77, 2147450879, v65 -v_and_b32 v87, 2147450879, v69 -v_mov_b32 v81, 3056514606 -v_mov_b32 v88, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v81, v77, v91, v81 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v88, v87, v91, v88 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v81, v77, v81, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v88, v87, v88, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v81, v77, v81 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v88, v87, v88 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v81, v77, v81, v77 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v88, v87, v88, v87 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v77, v91, v77 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v87, v91, v87 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v77, 1.0, v77 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v87, 1.0, v87 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v77, v77 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v87, v87 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v77, v77 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v87, v87 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v77, v77, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v87, v87, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v65), v91 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v69), v91 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v65), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v81, v81, v77, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v69), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v88, v88, v87, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v77, 2147450879 -v_bfi_b32 v65, v77, v81, v65 -v_mov_b32 v87, 2147450879 -v_bfi_b32 v69, v87, v88, v69 -v_pk_mul_f16 v65, v65, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v69, v69, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v63, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v67, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v65, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v69, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_117 - label_1 -s_setpc_b64 [s76, s77] -label_117: -s_setprio 3 -v_pk_add_f16 v64, v64, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v68, v68, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v66, v66, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_add_f16 v70, v70, s49 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v64, v64, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v68, v68, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v85, 2147450879, v64 -v_and_b32 v89, 2147450879, v68 -v_mov_b32 v86, 3056514606 -v_mov_b32 v90, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v86, v85, v91, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v90, v89, v91, v90 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v86, v85, v86, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v90, v89, v90, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v90, v89, v90 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v90, v89, v90, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v85, v91, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v89, v91, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v89, 1.0, v89 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v89, v89, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v64), v91 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v68), v91 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v64), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v68), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v64, v85, v86, v64 -v_mov_b32 v89, 2147450879 -v_bfi_b32 v68, v89, v90, v68 -v_pk_mul_f16 v64, v64, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v68, v68, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v66, v66, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v70, v70, s47 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_and_b32 v85, 2147450879, v66 -v_and_b32 v89, 2147450879, v70 -v_mov_b32 v86, 3056514606 -v_mov_b32 v90, 3056514606 -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v86, v85, v91, v86 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 814231688 -v_pk_fma_f16 v90, v89, v91, v90 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v86, v85, v86, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 506666547 -v_pk_fma_f16 v90, v89, v90, v91 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_mul_f16 v86, v85, v86 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v90, v89, v90 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_fma_f16 v86, v85, v86, v85 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_pk_fma_f16 v90, v89, v90, v89 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v85, v91, v85 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_mov_b32 v91, 1103446469 -v_pk_mul_f16 v89, v91, v89 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_exp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_exp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_exp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_exp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_add_f16 v85, 1.0, v85 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_pk_add_f16 v89, 1.0, v89 op_sel:[0, 0, 0] op_sel_hi:[0, 1, 1] -v_rcp_f16 v85, v85 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_rcp_f16 v89, v89 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 -v_nop -v_nop -v_rcp_f16 v85, v85 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_rcp_f16 v89, v89 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 -v_nop -v_nop -v_pk_fma_f16 v85, v85, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_pk_fma_f16 v89, v89, -2.0, 1.0 op_sel:[0, 0, 0, 0] op_sel_hi:[1, 0, 0, 1] -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v66), v91 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v70), v91 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_0 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v66), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v86, v86, v85, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v91, 1002585026 -v_cmp_gt_f16 [vcc_lo, vcc_hi], abs(v70), v91 src0_sel:WORD_1 src1_sel:WORD_1 -v_cndmask_b32 v90, v90, v89, [vcc_lo, vcc_hi] dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_1 src1_sel:WORD_1 -v_mov_b32 v85, 2147450879 -v_bfi_b32 v66, v85, v86, v66 -v_mov_b32 v89, 2147450879 -v_bfi_b32 v70, v89, v90, v70 -v_pk_mul_f16 v66, v66, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -v_pk_mul_f16 v70, v70, s46 op_sel:[0, 0, 0] op_sel_hi:[1, 1, 1] -s_setprio 0 -buffer_store_short v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -buffer_store_short_d16_hi v64, v78, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v68, v82, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v66, v79, [s68, s69, s70, s71], 0 idxen -buffer_store_short_d16_hi v70, v83, [s68, s69, s70, s71], 0 idxen -s_add_u32 s68, s68, s59 -s_addc_u32 s69, s69, 0 -s_sub_u32 s63, s63, 1 -s_cselect_b32 s70, 0, s70 -s_setprio 2 -s_mov_b32 s78, label_114 - label_1 -s_bitcmp1_b32 s53, 0 -s_cbranch_scc1 label_0 -s_sub_u32 s63, s67, 1 -s_cselect_b32 s70, 0, 2147483648 -s_mov_b64 [s68, s69], [s34, s35] -s_setpc_b64 [s76, s77] -label_0: -s_endpgm -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 diff --git a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_metadata.inc b/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_metadata.inc deleted file mode 100644 index dcb11dd3cf89..000000000000 --- a/projects/miopen/src/kernels/winograd/Conv_Winograd_Rage_v4_7_0_metadata.inc +++ /dev/null @@ -1,212 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -.macro PROLOG_KERNEL_DESCRIPTOR kernel_name:req -.text -.globl \kernel_name -.p2align 8 -.type \kernel_name,@function -\kernel_name: -.endm - -.macro METADATA sc:req, wc:req, wg_x:req, kernel_name:req -.amdgpu_metadata ---- -amdhsa.version: [ 1, 2 ] -amdhsa.kernels: - - .name: \kernel_name - .symbol: \kernel_name\().kd - .sgpr_count: \sc - .vgpr_count: \wc - .group_segment_fixed_size: 65536 - .private_segment_fixed_size: 0 - .kernarg_segment_size: 232 - .kernarg_segment_align: 8 - .reqd_workgroup_size: [ \wg_x, 1, 1 ] - .max_flat_workgroup_size: \wg_x - .wavefront_size: 64 - .workgroup_processor_mode: 0 - .args: - - { .size: 4, .offset: 0, .value_kind: by_value, .name: N_ } - - { .size: 4, .offset: 4, .value_kind: by_value, .name: C } - - { .size: 4, .offset: 8, .value_kind: by_value, .name: H } - - { .size: 4, .offset: 12, .value_kind: by_value, .name: W } - - - { .size: 4, .offset: 16, .value_kind: by_value, .name: K } - - { .size: 4, .offset: 20, .value_kind: by_value, .name: n_groups } - - - { .size: 8, .offset: 24, .value_kind: by_value, .name: flags64 } - - - { .size: 8, .offset: 32, .value_kind: global_buffer, .name: data_addr, .address_space: global, .is_const: true } - - { .size: 8, .offset: 40, .value_kind: global_buffer, .name: filter_addr, .address_space: global, .is_const: true } - - { .size: 8, .offset: 48, .value_kind: global_buffer, .name: output_addr, .address_space: global, .is_const: false } - - { .size: 8, .offset: 56, .value_kind: by_value } - - - { .size: 4, .offset: 64, .value_kind: by_value, .name: R } - - { .size: 4, .offset: 68, .value_kind: by_value, .name: S } - - { .size: 4, .offset: 72, .value_kind: by_value, .name: pad_h } - - { .size: 4, .offset: 76, .value_kind: by_value, .name: pad_w } - - { .size: 4, .offset: 80, .value_kind: by_value, .name: out_h } - - { .size: 4, .offset: 84, .value_kind: by_value, .name: out_w } - - - { .size: 8, .offset: 88, .value_kind: global_buffer, .name: bias_addr, .address_space: global, .is_const: true } - - { .size: 4, .offset: 96, .value_kind: by_value, .name: alpha } - - { .size: 4, .offset: 100, .value_kind: by_value, .name: beta } - - - { .size: 8, .offset: 104, .value_kind: by_value, .name: d_offset } - - { .size: 8, .offset: 112, .value_kind: by_value, .name: f_offset } - - { .size: 8, .offset: 120, .value_kind: by_value, .name: o_offset } - - { .size: 8, .offset: 128, .value_kind: by_value, .name: b_offset } - - - { .size: 4, .offset: 136, .value_kind: by_value, .name: d_N_stride } - - { .size: 4, .offset: 140, .value_kind: by_value, .name: d_C_stride } - - { .size: 4, .offset: 144, .value_kind: by_value, .name: d_H_stride } - - { .size: 4, .offset: 148, .value_kind: by_value } - - - { .size: 4, .offset: 152, .value_kind: by_value, .name: f_K_stride } - - { .size: 4, .offset: 156, .value_kind: by_value, .name: f_C_stride } - - { .size: 4, .offset: 160, .value_kind: by_value, .name: f_R_stride } - - { .size: 4, .offset: 164, .value_kind: by_value } - - - { .size: 4, .offset: 168, .value_kind: by_value, .name: o_N_stride } - - { .size: 4, .offset: 172, .value_kind: by_value, .name: o_K_stride } - - { .size: 4, .offset: 176, .value_kind: by_value, .name: o_H_stride } - - { .size: 4, .offset: 180, .value_kind: by_value } - - - { .size: 4, .offset: 184, .value_kind: by_value, .name: G } - - { .size: 4, .offset: 188, .value_kind: by_value, .name: d_G_stride } - - { .size: 4, .offset: 192, .value_kind: by_value, .name: f_G_stride } - - { .size: 4, .offset: 196, .value_kind: by_value, .name: o_G_stride } - - - { .size: 1, .offset: 200, .value_kind: by_value, .name: activation_mode } - - { .size: 1, .offset: 201, .value_kind: by_value, .name: sync_limit } - - { .size: 1, .offset: 202, .value_kind: by_value, .name: sync_period } - - { .size: 1, .offset: 203, .value_kind: by_value } - - - { .size: 4, .offset: 204, .value_kind: by_value } - - { .size: 8, .offset: 208, .value_kind: global_buffer, .name: sync_addr, .address_space: global, .is_const: false } - - - { .size: 8, .offset: 216, .value_kind: global_buffer, .name: acc_addr, .address_space: global, .is_const: false } - - { .size: 8, .offset: 224, .value_kind: by_value, .name: a_offset } -... -.end_amdgpu_metadata -.endm // METADATA - -.altmacro -.macro METADATA_WRAPPER sc:req, wc:req, wg_x:req, kernel_name:req - METADATA %\sc, %\wc, %\wg_x, \kernel_name -.endm - -.macro kernel_end kernel_name:req -.align 256 -.Lfunc_end0: - .size \kernel_name, .Lfunc_end0 - \kernel_name -.endm - -.macro EPILOG_KERNEL_DESCRIPTOR kernel_name:req - -kernel_end \kernel_name - -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) - sgpr_cnt = 84 - vgpr_cnt = 164 - workgroup_size_x = 768 -.endif - -.amdgcn.next_free_sgpr = sgpr_cnt -.amdgcn.next_free_vgpr = vgpr_cnt - -__group_segment_fixed_size = 65536 -__sgpr_dispatch_ptr = 1 -__sgpr_kernarg_segment_ptr = 1 -__sgpr_workgroup_id_x = 1 -__sgpr_workgroup_id_y = 0 -__sgpr_workgroup_id_z = 0 -__sgpr_workgroup_info = 0 -__vgpr_workitem_id = 1 -__reserve_vcc = 1 -__float_round_mode_32 = 0 -__float_round_mode_16_64 = 0 -__float_denorm_mode_32 = 0 -__float_denorm_mode_16_64 = 0 -__dx10_clamp = 0 -__ieee_mode = 0 -__fp16_overflow = 0 - -.rodata -.p2align 6 -.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 2) -.amdhsa_kernel \kernel_name - .amdhsa_group_segment_fixed_size __group_segment_fixed_size - .amdhsa_user_sgpr_dispatch_ptr __sgpr_dispatch_ptr // s[0:1] - .amdhsa_user_sgpr_kernarg_segment_ptr __sgpr_kernarg_segment_ptr // s[2:3] - .amdhsa_system_sgpr_workgroup_id_x __sgpr_workgroup_id_x - .amdhsa_system_sgpr_workgroup_id_y __sgpr_workgroup_id_y - .amdhsa_system_sgpr_workgroup_id_z __sgpr_workgroup_id_z - .amdhsa_system_sgpr_workgroup_info __sgpr_workgroup_info - .amdhsa_system_vgpr_workitem_id __vgpr_workitem_id - .amdhsa_reserve_vcc __reserve_vcc - .amdhsa_next_free_sgpr .amdgcn.next_free_sgpr - .amdhsa_next_free_vgpr .amdgcn.next_free_vgpr - .amdhsa_accum_offset .amdgcn.next_free_vgpr - .amdhsa_float_round_mode_32 __float_round_mode_32 - .amdhsa_float_round_mode_16_64 __float_round_mode_16_64 - .amdhsa_float_denorm_mode_32 __float_denorm_mode_32 - .amdhsa_float_denorm_mode_16_64 __float_denorm_mode_16_64 - .amdhsa_dx10_clamp __dx10_clamp - .amdhsa_ieee_mode __ieee_mode - .amdhsa_fp16_overflow __fp16_overflow -/* \todo add gfx12 support with these directives - .amdhsa_wavefront_size32 0 - .amdhsa_workgroup_processor_mode 0 -*/ -.end_amdhsa_kernel -.endif - -METADATA_WRAPPER sgpr_cnt, vgpr_cnt, workgroup_size_x, <\kernel_name> - -.endm - -.macro PROLOG_KERNEL_DESCRIPTOR_WRAPPER machine_version:req, kernel_name_postfix:req - PROLOG_KERNEL_DESCRIPTOR miopenSp3AsmConvRage_v4_7_0_gfx\machine_version\()\kernel_name_postfix -.endm - -.macro EPILOG_KERNEL_DESCRIPTOR_WRAPPER machine_version:req, kernel_name_postfix:req - EPILOG_KERNEL_DESCRIPTOR miopenSp3AsmConvRage_v4_7_0_gfx\machine_version\()\kernel_name_postfix -.endm - -.macro KERNEL_PROLOG kernel_name_postfix:req - PROLOG_KERNEL_DESCRIPTOR_WRAPPER %.amdgcn.gfx_generation_number, \kernel_name_postfix -.endm - -.macro KERNEL_EPILOG kernel_name_postfix:req - EPILOG_KERNEL_DESCRIPTOR_WRAPPER %.amdgcn.gfx_generation_number, \kernel_name_postfix -.endm - -.if (.amdgcn.gfx_generation_number != 9 || .amdgcn.gfx_generation_minor != 4 && .amdgcn.gfx_generation_stepping != 2) - .error "Unsupported gfx version" - .end -.endif diff --git a/projects/miopen/src/logger.cpp b/projects/miopen/src/logger.cpp index 7a8fd5eaa655..a08e59c73a2b 100644 --- a/projects/miopen/src/logger.cpp +++ b/projects/miopen/src/logger.cpp @@ -70,37 +70,61 @@ MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_ENABLE_LOGGING_ROCTX) /// Disable logging quieting. MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_LOGGING_QUIETING_DISABLE) -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_LOG_BUFFER_SIZE, 128); +MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_LOG_BUFFER_SIZE, 0); namespace miopen { -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) -static thread_local size_t log_buffer_size = env::value(MIOPEN_LOG_BUFFER_SIZE); -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) -static thread_local size_t log_buffer_i = 0; -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) -static thread_local std::vector log_buffer(log_buffer_size, ""); +size_t GetBufferSize() { return env::value(MIOPEN_LOG_BUFFER_SIZE); } -void ClearBufferLog() +size_t& GetBufferIdx() { - log_buffer_i = 0; - log_buffer = std::vector(miopen::log_buffer_size, ""); + static thread_local size_t log_buffer_i = 0; + return log_buffer_i; +} + +std::vector& GetLogBuffer() +{ + auto log_buffer_size = GetBufferSize(); + static thread_local std::vector log_buffer(log_buffer_size, ""); + if(log_buffer_size != log_buffer.size()) + { + log_buffer.resize(log_buffer_size); + auto& log_buffer_i = GetBufferIdx(); + if(log_buffer_i > 0 && log_buffer_i >= log_buffer_size) + log_buffer_i = 0; + } + return log_buffer; +} + +bool IsLogBufferOn() { return GetBufferSize() != 0; } + +void ClearLogBuffer() +{ + auto& log_buffer = GetLogBuffer(); + auto& log_buffer_i = GetBufferIdx(); + log_buffer = std::vector(GetBufferSize(), ""); + log_buffer_i = 0; } void BufferLog(std::string line) { + auto& log_buffer = GetLogBuffer(); + auto& log_buffer_i = GetBufferIdx(); log_buffer[log_buffer_i] = line; - log_buffer_i = (log_buffer_i + 1) % log_buffer_size; + log_buffer_i = (log_buffer_i + 1) % GetBufferSize(); } void OutputBufferedLogs() { - auto buffer_size = (log_buffer[log_buffer_size - 1] == "") ? log_buffer_i : log_buffer_size; + auto& log_buffer = GetLogBuffer(); + auto& log_buffer_i = GetBufferIdx(); + auto log_buffer_size = GetBufferSize(); + auto buffer_size = (log_buffer[log_buffer_size - 1] == "") ? log_buffer_i : log_buffer_size; auto filename = fs::temp_directory_path() / ("miopen_error_" + std::to_string(::getpid()) + ".log"); std::cerr << "Buffered " << buffer_size << " messages to file: " << sysinfo::GetSystemHostname() << ":" << filename.string() << std::endl; - auto err_file = std::ofstream{filename}; + auto err_file = std::ofstream{filename, std::ofstream::app}; size_t i = log_buffer_i; do { @@ -110,6 +134,8 @@ void OutputBufferedLogs() } i = (i + 1) % log_buffer_size; } while(i != log_buffer_i); + err_file << std::endl; + ClearLogBuffer(); } namespace debug { diff --git a/projects/miopen/src/mlo_dir_conv.cpp b/projects/miopen/src/mlo_dir_conv.cpp index 47f8959e5876..53f3d2c74aa8 100644 --- a/projects/miopen/src/mlo_dir_conv.cpp +++ b/projects/miopen/src/mlo_dir_conv.cpp @@ -141,7 +141,25 @@ static auto GetWindogradSolvers() miopen::solver::conv::ConvMPBidirectWinograd_xdlops<5, 3>, miopen::solver::conv::ConvMPBidirectWinograd_xdlops<6, 3>, miopen::solver::conv::ConvWinoFuryRxS<2, 3>, - miopen::solver::conv::ConvWinoRageRxS<2, 3>>{}; + miopen::solver::conv::ConvWinoRageRxS<2, 3>, + // Transposed Winograd solvers for NHWC layout support + miopen::solver::conv::TransposedConvBinWinograd3x3U, + miopen::solver::conv::TransposedConvBinWinogradRxS, + miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1, + miopen::solver::conv::TransposedConvWinoFuryRxS<2, 3>, + miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd<2, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd<3, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd<4, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd<5, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd<6, 3>, + miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>, + miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>, + miopen::solver::conv::TransposedConvMPBidirectWinograd_xdlops<2, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd_xdlops<3, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd_xdlops<4, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd_xdlops<5, 3>, + miopen::solver::conv::TransposedConvMPBidirectWinograd_xdlops<6, 3>>{}; } static auto GetImplicitGemmWrWSolvers() @@ -183,7 +201,28 @@ static auto GetWindogradWrWSolvers() miopen::solver::conv::ConvWinograd3x3MultipassWrW<5, 3>, miopen::solver::conv::ConvWinograd3x3MultipassWrW<5, 4>, miopen::solver::conv::ConvWinoFuryRxS<2, 3>, - miopen::solver::conv::ConvWinoRageRxS<2, 3>>{}; + miopen::solver::conv::ConvWinoRageRxS<2, 3>, + // Transposed Winograd solvers for NHWC layout support + miopen::solver::conv::TransposedConvBinWinograd3x3U, + miopen::solver::conv::TransposedConvBinWinogradRxS, + miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1, + miopen::solver::conv::TransposedConvWinoFuryRxS<2, 3>, + miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 2>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 3>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 4>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 5>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 6>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 3>, + miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 4>, + miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>, + miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>>{}; } static auto GetBwdWrW2DSolvers() diff --git a/projects/miopen/src/nogpu/handle.cpp b/projects/miopen/src/nogpu/handle.cpp index 417d098cd77e..6497313125c9 100644 --- a/projects/miopen/src/nogpu/handle.cpp +++ b/projects/miopen/src/nogpu/handle.cpp @@ -69,6 +69,8 @@ Handle::WriteTo(const void* /* data */, Allocator::ManageDataPtr& ddata, std::si return ddata; } +void Handle::WriteTo(const void*, Data_t, std::size_t) const {} + void Handle::ReadTo(void* /* data */, const Allocator::ManageDataPtr& /* ddata */, std::size_t /* sz */) const diff --git a/projects/miopen/src/ocl/convolutionocl.cpp b/projects/miopen/src/ocl/convolutionocl.cpp index a0875e02ecaa..527585dc8fe6 100644 --- a/projects/miopen/src/ocl/convolutionocl.cpp +++ b/projects/miopen/src/ocl/convolutionocl.cpp @@ -1052,10 +1052,14 @@ ConvolutionDescriptor::GetSolutions(const ExecutionContext& ctx, { if(fallbackPathTaken != nullptr) *fallbackPathTaken = FallbackPath::None; - return solutions; + } + else + { + solutions = + GetSolutionsFallback(ctx, problem, maxSolutionCount, fallbackPathTaken, invokeParams); } - return GetSolutionsFallback(ctx, problem, maxSolutionCount, fallbackPathTaken, invokeParams); + return solutions; } std::size_t ConvolutionDescriptor::GetForwardSolutionWorkspaceSize(const Handle& handle, diff --git a/projects/miopen/src/ocl/lrn_ocl.cpp b/projects/miopen/src/ocl/lrn_ocl.cpp index 07fad30e3ce5..7fbcc7ea8aa3 100644 --- a/projects/miopen/src/ocl/lrn_ocl.cpp +++ b/projects/miopen/src/ocl/lrn_ocl.cpp @@ -126,7 +126,6 @@ miopenStatus_t LRNDescriptor::Forward(const Handle& handle, y, workSpace, as_float(f_norm_alphaoverarea), - as_float(f_norm_alpha), as_float(f_norm_beta), as_float(f_norm_K)); } @@ -135,7 +134,6 @@ miopenStatus_t LRNDescriptor::Forward(const Handle& handle, kernels.front()(x, y, as_float(f_norm_alphaoverarea), - as_float(f_norm_alpha), as_float(f_norm_beta), as_float(f_norm_K)); } @@ -143,7 +141,7 @@ miopenStatus_t LRNDescriptor::Forward(const Handle& handle, } else { - const std::string program_name = construct_params.getKernelFile(); // CL kernel filename + const std::string program_name = construct_params.getKernelFile(); // kernel filename const std::string kernel_name = construct_params.getKernelName(); // kernel name const std::string& compiler_parms = construct_params.getCompilerOptions(); // kernel parameters @@ -159,7 +157,6 @@ miopenStatus_t LRNDescriptor::Forward(const Handle& handle, y, workSpace, as_float(f_norm_alphaoverarea), - as_float(f_norm_alpha), as_float(f_norm_beta), as_float(f_norm_K)); } @@ -168,7 +165,6 @@ miopenStatus_t LRNDescriptor::Forward(const Handle& handle, obj(x, y, as_float(f_norm_alphaoverarea), - as_float(f_norm_alpha), as_float(f_norm_beta), as_float(f_norm_K)); } diff --git a/projects/miopen/src/ocl/mloNorm.cpp b/projects/miopen/src/ocl/mloNorm.cpp index fc58e1b4ccc7..946a2c04357d 100644 --- a/projects/miopen/src/ocl/mloNorm.cpp +++ b/projects/miopen/src/ocl/mloNorm.cpp @@ -49,10 +49,8 @@ inline bool is_tensor_packed(int c, int h, int w, int b_str, int c_str, int h_st return h_str == w && c_str == h * h_str && b_str == c * c_str; } -int mlo_construct_norm::mloConstructFwd() +void mlo_construct_norm::mloConstructFwd() { - int ret = 0; - size_t maxComputeUnits = _ctx.GetStream().GetMaxComputeUnits(); _hw_wave_sz = 64; @@ -63,14 +61,6 @@ int mlo_construct_norm::mloConstructFwd() if(pre_pad < 0 || pad < 0) MIOPEN_THROW("Wrong LRN kernel size"); - int top_df_stride = 1; - int top_df_channel_stride = 1; - int top_df_batch_stride = 1; - - int bot_df_stride = 1; - int bot_df_channel_stride = 1; - int bot_df_batch_stride = 1; - _grp_tile0 = (_problem.GetOutWidth() <= 16) ? 8 : 16; _grp_tile1 = 8; _out_pix_tile0 = 1; @@ -83,14 +73,14 @@ int mlo_construct_norm::mloConstructFwd() _problem.GetInChannelStride(), _problem.GetInStride()); - int MAP_SZ4 = _problem.GetInWidth() * (is_in_packed ? _problem.GetInHeight() : 1); + int map_size_4 = _problem.GetInWidth() * (is_in_packed ? _problem.GetInHeight() : 1); int read_unit; if(_norm_region == MLO_LRN_ACROSS_CHANNELS) { _grp_tile0 = (_problem.GetOutWidth() <= 8) ? 8 : 16; _grp_tile1 = (_problem.GetOutHeight() <= 8) ? 8 : 16; - read_unit = (MAP_SZ4 % 4 == 0) ? 4 : (MAP_SZ4 % 2 == 0) ? 2 : 1; - MAP_SZ4 /= read_unit; + read_unit = (map_size_4 % 4 == 0) ? 4 : (map_size_4 % 2 == 0) ? 2 : 1; + map_size_4 /= read_unit; } else { @@ -98,31 +88,26 @@ int mlo_construct_norm::mloConstructFwd() _out_pix_tile0 = (_problem.GetOutWidth() <= 8) ? 1 : 2; _out_pix_tile1 = (_problem.GetOutHeight() <= 8) ? 1 : 2; read_unit = 4; - MAP_SZ4 = (MAP_SZ4 + 3) / 4; + map_size_4 = (map_size_4 + 3) / 4; } - MAP_SZ4 *= (is_in_packed ? 1 : _problem.GetInHeight()); + map_size_4 *= (is_in_packed ? 1 : _problem.GetInHeight()); assert(_out_pix_tile0 - 1 <= _norm_area && _out_pix_tile1 - 1 <= _norm_area); - auto ocl_group_lg2sz0 = - static_cast(ceil(log(static_cast(_out_pix_tile0)) / std::numbers::ln2)); - auto ocl_group_lg2sz1 = - static_cast(ceil(log(static_cast(_out_pix_tile1)) / std::numbers::ln2)); - - _kernel_file = "MIOpenLRNFwd.cl"; + _kernel_file = "MIOpenLRNFwd.cpp"; _kernel_name = (_norm_region == MLO_LRN_ACROSS_CHANNELS) ? "MIOpenLRNAcrossChannels4" : "MIOpenLRNWithinChannel_PS"; if(_norm_region == MLO_LRN_ACROSS_CHANNELS) { _grp_tile0 = 8 * 8; _grp_tile1 = 1; - int n_waves = (_problem.GetBatchSize() * MAP_SZ4 + _hw_wave_sz - 1) / _hw_wave_sz; + int n_waves = (_problem.GetBatchSize() * map_size_4 + _hw_wave_sz - 1) / _hw_wave_sz; if(n_waves <= maxComputeUnits * 8) { - MAP_SZ4 = _problem.GetInWidth() * (is_in_packed ? _problem.GetInHeight() : 1); - read_unit = (MAP_SZ4 % 2 == 0) ? 2 : 1; - MAP_SZ4 /= read_unit; - MAP_SZ4 *= (is_in_packed ? 1 : _problem.GetInHeight()); + map_size_4 = _problem.GetInWidth() * (is_in_packed ? _problem.GetInHeight() : 1); + read_unit = (map_size_4 % 2 == 0) ? 2 : 1; + map_size_4 /= read_unit; + map_size_4 *= (is_in_packed ? 1 : _problem.GetInHeight()); } } @@ -135,8 +120,8 @@ int mlo_construct_norm::mloConstructFwd() { MIOPEN_LOG_I("Workaround for #1057: " << name << ',' << miopen::GetDataTypeName(_problem.GetInDataType()) << ',' - << MAP_SZ4 << ',' << read_unit); - MAP_SZ4 *= read_unit; + << map_size_4 << ',' << read_unit); + map_size_4 *= read_unit; read_unit = 1; } } @@ -151,96 +136,65 @@ int mlo_construct_norm::mloConstructFwd() auto g_wk_height = static_cast((_problem.GetOutHeight() + _grp_tile1 * _out_pix_tile1 - 1) / (_grp_tile1 * _out_pix_tile1)); - int OUT_VERT_ALIGNED = + int out_vert_aligned = (g_wk_height * (_grp_tile1 * _out_pix_tile1) == _problem.GetOutHeight()) ? 1 : 0; - int OUT_HORIZ_ALIGNED = + int out_horiz_aligned = (g_wk_width * (_grp_tile0 * _out_pix_tile0) == _problem.GetOutWidth()) ? 1 : 0; // currently always 1 - int DIVBY4 = (MAP_SZ4 * read_unit == _problem.GetInWidth() * _problem.GetInHeight()) ? 1 : 0; - int C1x1_PIXLEFT = - (DIVBY4 == 1) ? 0 - : _problem.GetInWidth() * _problem.GetInHeight() - (MAP_SZ4 - 1) * read_unit; - - std::string READ_TYPE = - (read_unit == 1) ? "_FLOAT" : "_FLOAT" + std::to_string(static_cast(read_unit)); - + bool div_by_4 = (map_size_4 * read_unit == _problem.GetInWidth() * _problem.GetInHeight()); + int c1x1_pixleft = + div_by_4 ? 0 + : _problem.GetInWidth() * _problem.GetInHeight() - (map_size_4 - 1) * read_unit; _comp_options = - std::string(" -DMLO_LRN_KERNEL_SZ=") + std::to_string(static_cast(_norm_area)) + - std::string(" -DMLO_LRN_PAD=") + std::to_string(static_cast(pad)) + - std::string(" -DMLO_LRN_KERNEL_SZ1=") + std::to_string(static_cast(_norm_area)) + - std::string(" -DMLO_LRN_PAD1=") + std::to_string(static_cast(pad)) + - std::string(" -DMLO_LRN_KERNEL_SZ0=") + std::to_string(static_cast(_norm_area)) + - std::string(" -DMLO_LRN_PAD0=") + std::to_string(static_cast(pad)) + - std::string(" -DMLO_LRN_PRE_PAD=") + std::to_string(static_cast(pre_pad)) + - std::string(" -DMLO_LRN_PRE_PAD1=") + std::to_string(static_cast(pre_pad)) + - std::string(" -DMLO_LRN_PRE_PAD0=") + std::to_string(static_cast(pre_pad)) + - std::string(" -DMLO_LRN_N_OUTPUTS=") + + std::string(" -DKERNEL_SIZE=") + std::to_string(static_cast(_norm_area)) + + std::string(" -DPAD=") + std::to_string(static_cast(pad)) + + std::string(" -DKERNEL_SIZE1=") + std::to_string(static_cast(_norm_area)) + + std::string(" -DPAD0=") + std::to_string(static_cast(pad)) + + std::string(" -DPRE_PAD=") + std::to_string(static_cast(pre_pad)) + + std::string(" -DPRE_PAD1=") + std::to_string(static_cast(pre_pad)) + + std::string(" -DKERNEL_SIZE0=") + std::to_string(static_cast(_norm_area)) + + std::string(" -DPRE_PAD0=") + std::to_string(static_cast(pre_pad)) + + std::string(" -DN_OUTPUTS=") + std::to_string(static_cast(_problem.GetOutChannels())) + - std::string(" -DMLO_LRN_N_INPUTS=") + + std::string(" -DN_INPUTS=") + std::to_string(static_cast(_problem.GetInChannels())) + - std::string(" -DMLO_LRN_N_HORIZ_OUT_PIX=") + - std::to_string(static_cast(_out_pix_tile0)) + - std::string(" -DMLO_LRN_N_VERT_OUT_PIX=") + - std::to_string(static_cast(_out_pix_tile1)) + - std::string(" -DMLO_LRN_GROUP_SZ0=") + std::to_string(static_cast(_grp_tile0)) + - std::string(" -DMLO_LRN_GROUP_SZ1=") + std::to_string(static_cast(_grp_tile1)) + - std::string(" -DMLO_LRN_GROUP_LG2SZ0=") + - std::to_string(static_cast(ocl_group_lg2sz0)) + - std::string(" -DMLO_LRN_GROUP_LG2SZ1=") + - std::to_string(static_cast(ocl_group_lg2sz1)) + - std::string(" -DMLO_LRN_BOT_BATCH_STRIDE=") + + std::string(" -DHORIZ_OUT_PIX=") + std::to_string(static_cast(_out_pix_tile0)) + + std::string(" -DVERT_OUT_PIX=") + std::to_string(static_cast(_out_pix_tile1)) + + std::string(" -DGROUP_SIZE_X=") + std::to_string(static_cast(_grp_tile0)) + + std::string(" -DGROUP_SIZE_Y=") + std::to_string(static_cast(_grp_tile1)) + + std::string(" -DBOT_BATCH_STRIDE=") + std::to_string(static_cast(_problem.GetInBatchStride())) + - std::string(" -DMLO_LRN_BOT_CHANNEL_STRIDE=") + + std::string(" -DBOT_CHANNEL_STRIDE=") + std::to_string(static_cast(_problem.GetInChannelStride())) + - std::string(" -DMLO_LRN_BOT_STRIDE=") + + std::string(" -DBOT_STRIDE=") + std::to_string(static_cast(_problem.GetInStride())) + - std::string(" -DMLO_LRN_TOP_BATCH_STRIDE=") + + std::string(" -DTOP_BATCH_STRIDE=") + std::to_string(static_cast(_problem.GetOutBatchStride())) + - std::string(" -DMLO_LRN_TOP_CHANNEL_STRIDE=") + + std::string(" -DTOP_CHANNEL_STRIDE=") + std::to_string(static_cast(_problem.GetOutChannelStride())) + - std::string(" -DMLO_LRN_TOP_STRIDE=") + + std::string(" -DTOP_STRIDE=") + std::to_string(static_cast(_problem.GetOutStride())) + - std::string(" -DMLO_LRN_BOT_WIDTH=") + + std::string(" -DBOT_WIDTH=") + std::to_string(static_cast(_problem.GetOutWidth())) + - std::string(" -DMLO_LRN_BOT_HEIGHT=") + + std::string(" -DBOT_HEIGHT=") + std::to_string(static_cast(_problem.GetOutHeight())) + - std::string(" -DMLO_LRN_TOP_WIDTH=") + + std::string(" -DTOP_WIDTH=") + std::to_string(static_cast(_problem.GetOutWidth())) + - std::string(" -DMLO_LRN_TOP_HEIGHT=") + + std::string(" -DTOP_HEIGHT=") + std::to_string(static_cast(_problem.GetOutHeight())) + - std::string(" -DMLO_LRN_SCALE_BATCH_STRIDE=") + + std::string(" -DSCALE_BATCH_STRIDE=") + std::to_string(static_cast(scale_batch_stride)) + - std::string(" -DMLO_LRN_SCALE_CHANNEL_STRIDE=") + + std::string(" -DSCALE_CHANNEL_STRIDE=") + std::to_string(static_cast(scale_channel_stride)) + - std::string(" -DMLO_LRN_SCALE_STRIDE=") + - std::to_string(static_cast(scale_stride)) + - std::string(" -DMLO_LRN_TOPDF_BATCH_STRIDE=") + - std::to_string(static_cast(top_df_batch_stride)) + - std::string(" -DMLO_LRN_TOPDF_CHANNEL_STRIDE=") + - std::to_string(static_cast(top_df_channel_stride)) + - std::string(" -DMLO_LRN_TOPDF_STRIDE=") + - std::to_string(static_cast(top_df_stride)) + - std::string(" -DMLO_LRN_BOTDF_BATCH_STRIDE=") + - std::to_string(static_cast(bot_df_batch_stride)) + - std::string(" -DMLO_LRN_BOTDF_CHANNEL_STRIDE=") + - std::to_string(static_cast(bot_df_channel_stride)) + - std::string(" -DMLO_LRN_BOTDF_STRIDE=") + - std::to_string(static_cast(bot_df_stride)) + - std::string(" -DMLO_LRN_BATCH_SZ=") + + std::string(" -DSCALE_STRIDE=") + std::to_string(static_cast(scale_stride)) + + std::string(" -DBATCH_SIZE=") + std::to_string(static_cast(_problem.GetBatchSize())) + - std::string(" -DMLO_LRN_N_INPUTS=") + - std::to_string(static_cast(_problem.GetInChannels())) + - std::string(" -DMLO_LRN_N_OUTPUTS=") + - std::to_string(static_cast(_problem.GetOutChannels())) + - std::string(" -DMLO_LRN_DO_SCALE=") + std::to_string(static_cast(scale)) + - std::string(" -DMLO_OUT_VERT_ALIGNED=") + - std::to_string(static_cast(OUT_VERT_ALIGNED)) + - std::string(" -DMLO_OUT_HORIZ_ALIGNED=") + - std::to_string(static_cast(OUT_HORIZ_ALIGNED)) + std::string(" -DMLO_MAP_SZ4=") + - std::to_string(static_cast(MAP_SZ4)) + std::string(" -DMLO_C1x1_PIXLEFT=") + - std::to_string(static_cast(C1x1_PIXLEFT)) + std::string(" -DMLO_DIVBY4=") + - std::to_string(static_cast(DIVBY4)) + std::string(" -DMLO_READ_TYPE=") + - READ_TYPE + std::string(" -DMLO_READ_UNIT=") + + std::string(" -DDO_SCALE=") + std::to_string(static_cast(scale)) + + std::string(" -DVERT_ALIGNED=") + std::to_string(static_cast(out_vert_aligned)) + + std::string(" -DHORIZ_ALIGNED=") + + std::to_string(static_cast(out_horiz_aligned)) + std::string(" -DMAP_SZ_4=") + + std::to_string(static_cast(map_size_4)) + std::string(" -DC1x1_PIXLEFT=") + + std::to_string(static_cast(c1x1_pixleft)) + std::string(" -DREAD_UNIT=") + std::to_string(static_cast(read_unit)) + getGeneralCompOptions(); _l_wk.clear(); @@ -252,7 +206,7 @@ int mlo_construct_norm::mloConstructFwd() if(_norm_region == MLO_LRN_ACROSS_CHANNELS) { - _g_wk.push_back(MAP_SZ4); + _g_wk.push_back(map_size_4); _g_wk.push_back(1); _g_wk.push_back(_problem.GetBatchSize()); } @@ -268,14 +222,10 @@ int mlo_construct_norm::mloConstructFwd() // calculate workspace size_t scale_sz = static_cast(_problem.GetBatchSize()) * scale_batch_stride * data_len; _workspace_sz = (doBackward()) ? scale_sz : 0; - - return (ret); } -int mlo_construct_norm::mloConstructBwd() +void mlo_construct_norm::mloConstructBwd() { - int ret = 0; - _out_pix_tile0 = 1; _out_pix_tile1 = 1; _grp_tile0 = 8; @@ -290,10 +240,6 @@ int mlo_construct_norm::mloConstructBwd() _out_pix_tile0 = (_in_df_width <= 8) ? 1 : (_in_df_width <= 16) ? 2 : 4; _out_pix_tile1 = (_in_df_height <= 8) ? 1 : (_in_df_height <= 16) ? 2 : 4; } - auto ocl_group_lg2sz0 = - static_cast(ceil(log(static_cast(_grp_tile0)) / std::numbers::ln2)); - auto ocl_group_lg2sz1 = - static_cast(ceil(log(static_cast(_grp_tile1)) / std::numbers::ln2)); int pre_pad = (_norm_area - 1) / 2; int pad = _norm_area - pre_pad - 1; @@ -305,69 +251,55 @@ int mlo_construct_norm::mloConstructBwd() MIOPEN_THROW("Wrong LRN kernel size"); _comp_options = - std::string(" -DMLO_LRN_KERNEL_SZ=") + std::to_string(static_cast(_norm_area)) + - std::string(" -DMLO_LRN_N_OUTPUTS=") + - std::to_string(static_cast(_problem.GetOutChannels())) + - std::string(" -DMLO_LRN_N_CHANNELS=") + - std::to_string(static_cast(_problem.GetInChannels())) + - std::string(" -DMLO_LRN_PAD=") + std::to_string(static_cast(pad)) + - std::string(" -DMLO_LRN_PRE_PAD=") + std::to_string(static_cast(pre_pad)) + - std::string(" -DMLO_LRN_N_HORIZ_OUT_PIX=") + - std::to_string(static_cast(_out_pix_tile0)) + - std::string(" -DMLO_LRN_N_VERT_OUT_PIX=") + - std::to_string(static_cast(_out_pix_tile1)) + - std::string(" -DMLO_LRN_GROUP_SZ0=") + std::to_string(static_cast(_grp_tile0)) + - std::string(" -DMLO_LRN_GROUP_SZ1=") + std::to_string(static_cast(_grp_tile1)) + - std::string(" -DMLO_LRN_GROUP_LG2SZ0=") + - std::to_string(static_cast(ocl_group_lg2sz0)) + - std::string(" -DMLO_LRN_GROUP_LG2SZ1=") + - std::to_string(static_cast(ocl_group_lg2sz1)) + - std::string(" -DMLO_LRN_BOT_BATCH_STRIDE=") + + std::string(" -DKERNEL_SIZE=") + std::to_string(static_cast(_norm_area)) + + std::string(" -DOUT_CHANNELS=") + + std::to_string(static_cast(_problem.GetOutChannels())) + std::string(" -DPAD=") + + std::to_string(static_cast(pad)) + std::string(" -DPRE_PAD=") + + std::to_string(static_cast(pre_pad)) + std::string(" -DHORIZ_OUT_PIX=") + + std::to_string(static_cast(_out_pix_tile0)) + std::string(" -DVERT_OUT_PIX=") + + std::to_string(static_cast(_out_pix_tile1)) + std::string(" -DGROUP_SIZE_X=") + + std::to_string(static_cast(_grp_tile0)) + std::string(" -DGROUP_SIZE_Y=") + + std::to_string(static_cast(_grp_tile1)) + std::string(" -DBOT_BATCH_STRIDE=") + std::to_string(static_cast(_problem.GetInBatchStride())) + - std::string(" -DMLO_LRN_BOT_CHANNEL_STRIDE=") + + std::string(" -DBOT_CHANNEL_STRIDE=") + std::to_string(static_cast(_problem.GetInChannelStride())) + - std::string(" -DMLO_LRN_BOT_STRIDE=") + + std::string(" -DBOT_STRIDE=") + std::to_string(static_cast(_problem.GetInStride())) + - std::string(" -DMLO_LRN_TOP_BATCH_STRIDE=") + + std::string(" -DTOP_BATCH_STRIDE=") + std::to_string(static_cast(_problem.GetOutBatchStride())) + - std::string(" -DMLO_LRN_TOP_CHANNEL_STRIDE=") + + std::string(" -DTOP_CHANNEL_STRIDE=") + std::to_string(static_cast(_problem.GetOutChannelStride())) + - std::string(" -DMLO_LRN_TOP_STRIDE=") + + std::string(" -DTOP_STRIDE=") + std::to_string(static_cast(_problem.GetOutStride())) + - std::string(" -DMLO_LRN_BOT_WIDTH=") + + std::string(" -DBOT_WIDTH=") + std::to_string(static_cast(_problem.GetInWidth())) + - std::string(" -DMLO_LRN_BOT_HEIGHT=") + + std::string(" -DBOT_HEIGHT=") + std::to_string(static_cast(_problem.GetInHeight())) + - std::string(" -DMLO_LRN_TOP_WIDTH=") + + std::string(" -DTOP_WIDTH=") + std::to_string(static_cast(_problem.GetOutWidth())) + - std::string(" -DMLO_LRN_TOP_HEIGHT=") + + std::string(" -DTOP_HEIGHT=") + std::to_string(static_cast(_problem.GetOutHeight())) + - std::string(" -DMLO_LRN_SCALE_BATCH_STRIDE=") + + std::string(" -DSCALE_BATCH_STRIDE=") + std::to_string(static_cast(scale_batch_stride)) + - std::string(" -DMLO_LRN_SCALE_CHANNEL_STRIDE=") + + std::string(" -DSCALE_CHANNEL_STRIDE=") + std::to_string(static_cast(scale_channel_stride)) + - std::string(" -DMLO_LRN_SCALE_STRIDE=") + - std::to_string(static_cast(scale_stride)) + - std::string(" -DMLO_LRN_TOPDF_BATCH_STRIDE=") + + std::string(" -DSCALE_STRIDE=") + std::to_string(static_cast(scale_stride)) + + std::string(" -DTOPDF_BATCH_STRIDE=") + std::to_string(static_cast(_out_df_batch_stride)) + - std::string(" -DMLO_LRN_TOPDF_CHANNEL_STRIDE=") + + std::string(" -DTOPDF_CHANNEL_STRIDE=") + std::to_string(static_cast(_out_df_channel_stride)) + - std::string(" -DMLO_LRN_TOPDF_STRIDE=") + - std::to_string(static_cast(_out_df_stride)) + - std::string(" -DMLO_LRN_BOTDF_BATCH_STRIDE=") + + std::string(" -DTOPDF_STRIDE=") + std::to_string(static_cast(_out_df_stride)) + + std::string(" -DBOTDF_BATCH_STRIDE=") + std::to_string(static_cast(_in_df_batch_stride)) + - std::string(" -DMLO_LRN_BOTDF_CHANNEL_STRIDE=") + + std::string(" -DBOTDF_CHANNEL_STRIDE=") + std::to_string(static_cast(_in_df_channel_stride)) + - std::string(" -DMLO_LRN_BOTDF_STRIDE=") + - std::to_string(static_cast(_in_df_stride)) + - std::string(" -DMLO_LRN_BATCH_SZ=") + + std::string(" -DBOTDF_STRIDE=") + std::to_string(static_cast(_in_df_stride)) + + std::string(" -DBATCH_SIZE=") + std::to_string(static_cast(_problem.GetBatchSize())) + - std::string(" -DMLO_LRN_N_INPUTS=") + - std::to_string(static_cast(_problem.GetInChannels())) + - std::string(" -DMLO_LRN_N_OUTPUTS=") + - std::to_string(static_cast(_problem.GetOutChannels())) + getGeneralCompOptions(); + std::string(" -DN_INPUTS=") + + std::to_string(static_cast(_problem.GetInChannels())) + getGeneralCompOptions(); - _kernel_file = "MIOpenLRNBwd.cl"; + _kernel_file = "MIOpenLRNBwd.cpp"; _l_wk.clear(); _g_wk.clear(); @@ -394,6 +326,4 @@ int mlo_construct_norm::mloConstructBwd() _g_wk.push_back(static_cast(_problem.GetInChannels()) * _problem.GetBatchSize()); _kernel_name = "MIOpenLRNWithinChannelBwd"; } - - return (ret); } diff --git a/projects/miopen/src/ocl/rnnocl.cpp b/projects/miopen/src/ocl/rnnocl.cpp index 898eaf3cee73..e68e2a9efedb 100644 --- a/projects/miopen/src/ocl/rnnocl.cpp +++ b/projects/miopen/src/ocl/rnnocl.cpp @@ -929,31 +929,31 @@ void RNNDescriptor::RNNForwardMS(const Handle& handle, call_hx_next_gemm(layer_id); call_next_hidden_state_update(layer_id); } - ms_controller.RecordEvent(layer_chunk_end_event[layer_id][chunk_id].get(), stream_id); + (void)ms_controller.RecordEvent(layer_chunk_end_event[layer_id][chunk_id].get(), stream_id); }; auto sync_next_chunk_across_time = [&layer_chunk_end_event, &ms_controller](int stream_id, int layer_id, int chunk_id) { if(chunk_id > 0) { - ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id][chunk_id - 1].get(), - stream_id); + (void)ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id][chunk_id - 1].get(), + stream_id); } }; - auto sync_next_chunk_across_layers = - [&layer_chunk_end_event, &ms_controller](int stream_id, int layer_id, int chunk_id) { - if(layer_id > 0) - { - ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id - 1][chunk_id].get(), - stream_id); - } - }; + auto sync_next_chunk_across_layers = [&layer_chunk_end_event, &ms_controller]( + int stream_id, int layer_id, int chunk_id) { + if(layer_id > 0) + { + (void)ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id - 1][chunk_id].get(), + stream_id); + } + }; { // extra_space clean set 0 const int fill_val = 0; // if(biasMode == 0u) req - hipMemsetAsync(extra_space, fill_val, extra_space_size, handle.GetStream()); + (void)hipMemsetAsync(extra_space, fill_val, extra_space_size, handle.GetStream()); } // stage 0 bias and input preload @@ -983,8 +983,8 @@ void RNNDescriptor::RNNForwardMS(const Handle& handle, // sync first to second stream const miopen::HipEventPtr next_chunk_inx = make_hip_fast_event(); - ms_controller.RecordEvent(next_chunk_inx.get(), extra_stream_id); - ms_controller.SetWaitEvent(next_chunk_inx.get(), stream_id); + (void)ms_controller.RecordEvent(next_chunk_inx.get(), extra_stream_id); + (void)ms_controller.SetWaitEvent(next_chunk_inx.get(), stream_id); } auto spiral_dispatch = [&](int first_stream, int last_stream) { @@ -1131,8 +1131,8 @@ void RNNDescriptor::RNNForwardMS(const Handle& handle, ms_controller.ChangeActiveStream(root_stream_id); - ms_controller.SetWaitEvent(layer_chunk_end_event[nLayers - 1][chunks_cnt - 1].get(), - root_stream_id); + (void)ms_controller.SetWaitEvent(layer_chunk_end_event[nLayers - 1][chunks_cnt - 1].get(), + root_stream_id); // output tensor copy { diff --git a/projects/miopen/src/ocl/utilocl.cpp b/projects/miopen/src/ocl/utilocl.cpp index 713de16c6039..733a59a2cc76 100644 --- a/projects/miopen/src/ocl/utilocl.cpp +++ b/projects/miopen/src/ocl/utilocl.cpp @@ -623,6 +623,142 @@ float Col2Im3dGPU(const Handle& handle, return handle.GetKernelTime(); } +float Col2Im3dGPUBatched(const Handle& handle, + ConstData_t col, + uint32_t out_d, + uint32_t out_h, + uint32_t out_w, + uint32_t wei_d, + uint32_t wei_h, + uint32_t wei_w, + uint32_t pad_d, + uint32_t pad_h, + uint32_t pad_w, + uint32_t stride_d, + uint32_t stride_h, + uint32_t stride_w, + uint32_t dilation_d, + uint32_t dilation_h, + uint32_t dilation_w, + uint32_t in_c, + uint32_t in_d, + uint32_t in_h, + uint32_t in_w, + uint32_t batch_count, + uint64_t col_batch_stride, + Data_t im, + uint64_t im_batch_stride, + uint64_t im_offset, + miopenDataType_t type) +{ + std::string program_name = "MIOpenCol2Im3d.cpp"; + std::string kernel_name = "Col2Im3dUBatched"; + + // clang-format off + std::string network_config = + "c" + std::to_string(in_c) + + "i" + std::to_string(in_d) + + "_" + std::to_string(in_h) + + "_" + std::to_string(in_w) + + "w" + std::to_string(wei_d) + + "_" + std::to_string(wei_h) + + "_" + std::to_string(wei_w) + + "p" + std::to_string(pad_d) + + "_" + std::to_string(pad_h) + + "_" + std::to_string(pad_w) + + "s" + std::to_string(stride_d) + + "_" + std::to_string(stride_h) + + "_" + std::to_string(stride_w) + + "d" + std::to_string(dilation_d) + + "_" + std::to_string(dilation_h) + + "_" + std::to_string(dilation_w) + + "t" + std::to_string(type); + // clang-format on + + auto&& kernels = handle.GetKernels("miopenCol2Im3dBatched", network_config); + + if(!kernels.empty()) + { + auto kernel = kernels.front(); + kernel(col, + out_d, + out_h, + out_w, + wei_d, + wei_h, + wei_w, + pad_d, + pad_h, + pad_w, + stride_d, + stride_h, + stride_w, + dilation_d, + dilation_h, + dilation_w, + in_c, + in_d, + in_h, + in_w, + batch_count, + col_batch_stride, + im, + im_batch_stride, + im_offset); + } + else + { + std::size_t index_size = static_cast(batch_count) * in_c * out_d * out_h * out_w * + wei_d * wei_w * wei_h * sizeof(ConstData_t); + + const bool use_64_bit_index = index_size > 0xffffffffULL; + + std::string params = GetDataTypeKernelParams(type); + params += use_64_bit_index ? " -DMIOPEN_USE_64BIT_INDEX=1" : " -DMIOPEN_USE_64BIT_INDEX=0"; + // Force a distinct code-object cache key for the batched entry point. + params += " -DMIOPEN_COL2IM3D_BATCHED_VARIANT=1"; + + size_t global_threads = + static_cast(batch_count) * static_cast(in_c) * in_d * in_h * in_w; + size_t local_threads = std::min(WG_SIZE, global_threads); + if(global_threads % local_threads != 0) + { + global_threads = ((global_threads / local_threads) + 1) * local_threads; + } + const std::vector vgd{global_threads, 1, 1}; + const std::vector vld{local_threads, 1, 1}; + + handle.AddKernel( + "miopenCol2Im3dBatched", network_config, program_name, kernel_name, vld, vgd, params)( + col, + out_d, + out_h, + out_w, + wei_d, + wei_h, + wei_w, + pad_d, + pad_h, + pad_w, + stride_d, + stride_h, + stride_w, + dilation_d, + dilation_h, + dilation_w, + in_c, + in_d, + in_h, + in_w, + batch_count, + col_batch_stride, + im, + im_batch_stride, + im_offset); + } + return handle.GetKernelTime(); +} + float Im2ColGPU(const Handle& handle, std::size_t spatial_dim, ConstData_t im, diff --git a/projects/miopen/src/problem.cpp b/projects/miopen/src/problem.cpp index f82db9d6ea8d..3ca996a1c020 100644 --- a/projects/miopen/src/problem.cpp +++ b/projects/miopen/src/problem.cpp @@ -1,28 +1,5 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #include @@ -33,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -50,8 +28,6 @@ #include -#include - namespace miopen::debug { /// \todo: This should be updated when a separate driver command is implemented void LogCmdFindConvolution(const miopen::TensorDescriptor& x, @@ -196,7 +172,7 @@ std::vector Problem::FindSolutions(const Handle& handle, allocate(pair.first, pair.second); auto ret = std::visit( - boost::hof::match( + miopen::hof_match( [&](const ConvolutionDescriptor& op_desc) { if(op_desc.mode == miopenTranspose) return MakeTransposed().FindSolutionsImpl( @@ -272,7 +248,7 @@ Problem Problem::MakeTransposed() const for(const auto& descriptor : tensor_descriptors) transposed.tensor_descriptors.emplace(descriptor.first, descriptor.second); - const auto transpose_tensors = boost::hof::match( + const auto transpose_tensors = miopen::hof_match( [&](const ConvolutionDescriptor& op_desc) { return transposed.TransposeImpl(op_desc); }, [](auto&&) { MIOPEN_THROW(miopenStatusNotImplemented); }); @@ -587,39 +563,36 @@ Problem::FindSolutionsImpl(const Handle& handle, static solver::softmax::AttnSoftmax attnSoftmaxSolver; static solver::softmax::Softmax regularSoftmaxSolver; - std::vector solvers; - - solvers.push_back(&attnSoftmaxSolver); - solvers.push_back(®ularSoftmaxSolver); + auto check_solver = [&](const Solver* solver) { + if(ret.size() >= max_solutions) + { + return; + } - for(auto solver : solvers) - { if(!solver->IsApplicable(ctx, problem_description)) { MIOPEN_LOG_I2(solver->SolverDbId() << ": Not applicable"); - continue; + return; } auto solution = Solution(); /// \todo time measurement will be done later. For now we set less time for attention /// softmax and slightly bigger for regular - solution.SetTime(solver == &attnSoftmaxSolver ? 1.0f : 2.0f); + solution.SetTime(std::is_same_v ? 1.0f : 2.0f); solution.SetWorkspaceSize(solver->GetWorkspaceSize(ctx, problem_description)); solution.SetSolver(solver->SolverDbId()); solution.SetProblem({*this}); - MIOPEN_LOG_I("Found solution: " << solution.GetSolver().ToString() << " , " + MIOPEN_LOG_I("Found solution: " << solution.GetSolver().ToString() << ", " << solution.GetWorkspaceSize() << ", " << solution.GetTime()); ret.emplace_back(std::move(solution)); + }; - if(ret.size() >= max_solutions) - { - break; - } - } + check_solver(&attnSoftmaxSolver); + check_solver(®ularSoftmaxSolver); return ret; } @@ -892,7 +865,7 @@ void Problem::CalculateOutput() return; std::visit( - boost::hof::match( + miopen::hof_match( [&](const ConvolutionDescriptor& conv) { const auto& in = GetInput(); conv.GetForwardOutputTensor(in, @@ -915,7 +888,7 @@ void Problem::CalculateOutput() miopenTensorArgumentId_t Problem::GetInputId() const { - return std::visit(boost::hof::match( + return std::visit(miopen::hof_match( [&](const ConvolutionDescriptor&) { return direction == miopenProblemDirectionForward ? miopenTensorConvolutionX @@ -942,7 +915,7 @@ miopenTensorArgumentId_t Problem::GetInputId() const miopenTensorArgumentId_t Problem::GetOutputId() const { - return std::visit(boost::hof::match( + return std::visit(miopen::hof_match( [&](const ConvolutionDescriptor&) { return direction == miopenProblemDirectionForward ? miopenTensorConvolutionY @@ -1025,7 +998,7 @@ std::vector FusedProblem::FindSolutions(const Handle& handle, void FusedProblem::AddProblemToPlan(FusionPlanDescriptor& plan, const Problem& problem) { std::visit( - boost::hof::match( + miopen::hof_match( [&](const ConvolutionDescriptor& conv_desc) { plan.AddOp(std::make_shared( conv_desc, @@ -1116,7 +1089,7 @@ fusion::FusionInvokeParams FusedProblem::MakeInvokeParams( get_buffer(pair.first, pair.second); std::visit( - boost::hof::match( + miopen::hof_match( [&](const ConvolutionDescriptor& conv_desc) { gfx90aaltimpl = conv_desc.attribute.gfx90aFp16alt.GetFwd(); diff --git a/projects/miopen/src/reducetensor.cpp b/projects/miopen/src/reducetensor.cpp index eddee1eb0506..8becd7fa005a 100644 --- a/projects/miopen/src/reducetensor.cpp +++ b/projects/miopen/src/reducetensor.cpp @@ -37,8 +37,8 @@ #include // headers from composable kernel, to get consistent ID mapping -#include "legacy_composable_kernel/composable_kernel/include/utility/data_type_enum.hpp" -#include "legacy_composable_kernel/composable_kernel/include/utility/reduction_enums.hpp" +#include <../legacy_composable_kernel/composable_kernel/include/utility/data_type_enum.hpp> +#include <../legacy_composable_kernel/composable_kernel/include/utility/reduction_enums.hpp> namespace miopen { diff --git a/projects/miopen/src/reducetensor_api.cpp b/projects/miopen/src/reducetensor_api.cpp index a6eea0764e1f..45b1edd19b9a 100644 --- a/projects/miopen/src/reducetensor_api.cpp +++ b/projects/miopen/src/reducetensor_api.cpp @@ -146,7 +146,7 @@ miopenGetReduceTensorDescriptor(const miopenReduceTensorDescriptor_t reduceTenso }; extern "C" miopenStatus_t -miopenGetReductionIndicesSize(miopenHandle_t handle, +miopenGetReductionIndicesSize([[maybe_unused]] miopenHandle_t handle, const miopenReduceTensorDescriptor_t reduceTensorDesc, const miopenTensorDescriptor_t aDesc, const miopenTensorDescriptor_t cDesc, diff --git a/projects/miopen/src/rnn/Solutions/Base/bw_data_modular.cpp b/projects/miopen/src/rnn/Solutions/Base/bw_data_modular.cpp index 506e31065412..6490677ab23f 100644 --- a/projects/miopen/src/rnn/Solutions/Base/bw_data_modular.cpp +++ b/projects/miopen/src/rnn/Solutions/Base/bw_data_modular.cpp @@ -342,7 +342,7 @@ void RNNBackwardDataModularAlgo::PropDy(const Handle& handle, ws_ht_layer_size[0] = 1; - return ws_ht_layer_size; + return std::move(ws_ht_layer_size); }(workspaceInfo.hStateSizes); auto ws_dy_dst_desc = miopen::TensorDescriptor(rnn_data_type, ws_dst_size, ws_dst_strides); diff --git a/projects/miopen/src/rnn/Solutions/Base/fw_data_modular.cpp b/projects/miopen/src/rnn/Solutions/Base/fw_data_modular.cpp index 5b642915f232..7476ebf7fcbb 100644 --- a/projects/miopen/src/rnn/Solutions/Base/fw_data_modular.cpp +++ b/projects/miopen/src/rnn/Solutions/Base/fw_data_modular.cpp @@ -502,7 +502,7 @@ void RNNForwardDataModularAlgo::PropY(const Handle& handle, const runtimeArgsFwd ws_ht_layer_size[0] = 1; - return ws_ht_layer_size; + return std::move(ws_ht_layer_size); }(reservLayout.hStateSizes); auto tmp_y_desc = miopen::TensorDescriptor(rnn_data_type, tmp_y_size, tmp_y_strides); diff --git a/projects/miopen/src/rnn/Solutions/bwd_multi_stream.cpp b/projects/miopen/src/rnn/Solutions/bwd_multi_stream.cpp index 7c0674cb86bc..60a5efcb00e0 100644 --- a/projects/miopen/src/rnn/Solutions/bwd_multi_stream.cpp +++ b/projects/miopen/src/rnn/Solutions/bwd_multi_stream.cpp @@ -82,18 +82,20 @@ class SpiralDispatch { if(chunk_id > 0) { - msController.SetWaitEvent(layerChunkEndEvent[layer_id][chunk_id - 1].get(), stream_id); + (void)msController.SetWaitEvent(layerChunkEndEvent[layer_id][chunk_id - 1].get(), + stream_id); } if(layer_id > 0) { - msController.SetWaitEvent(layerChunkEndEvent[layer_id - 1][chunk_id].get(), stream_id); + (void)msController.SetWaitEvent(layerChunkEndEvent[layer_id - 1][chunk_id].get(), + stream_id); } } inline void PostDispatchUpdate(size_t layer_id, size_t chunk_id, int stream_id) { layerNewTimeState[layer_id] += timeChunkSz; - msController.RecordEvent(layerChunkEndEvent[layer_id][chunk_id].get(), stream_id); + (void)msController.RecordEvent(layerChunkEndEvent[layer_id][chunk_id].get(), stream_id); } inline bool IsDispatchable(size_t layer_id, size_t dispatch_chunk_time) const diff --git a/projects/miopen/src/softmax.cpp b/projects/miopen/src/softmax.cpp index b5dd190bedaa..a4ba67dc0c4e 100644 --- a/projects/miopen/src/softmax.cpp +++ b/projects/miopen/src/softmax.cpp @@ -1,28 +1,7 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +#include "miopen/execution_context.hpp" +#include "miopen/softmax/problem_description.hpp" #include #include #include @@ -37,6 +16,15 @@ namespace miopen { +namespace softmax { +miopen::PerformanceDb GetDb(const miopen::ExecutionContext& context, + const miopen::softmax::ProblemDescriptionTag&) +{ + return { + DbKinds::PerfDb, context.GetPerfDbPath("softmax"), context.GetUserPerfDbPath("softmax")}; +} +} // namespace softmax + extern "C" miopenStatus_t miopenCreateSoftmaxDescriptor(miopenSoftmaxDescriptor_t* softmaxDesc) { MIOPEN_LOG_FUNCTION(softmaxDesc); diff --git a/projects/miopen/src/solution.cpp b/projects/miopen/src/solution.cpp index 1d4357f2d823..b215e75987ec 100644 --- a/projects/miopen/src/solution.cpp +++ b/projects/miopen/src/solution.cpp @@ -1,28 +1,5 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #include @@ -30,6 +7,7 @@ #include #include #include +#include #include #include @@ -41,7 +19,7 @@ #include -#include +#include "miopen/conv/db_getter.hpp" #include "miopen/fusion/problem_description.hpp" #include "miopen/fusion/context.hpp" @@ -70,10 +48,10 @@ void Solution::Run(const Handle& handle, std::to_string(workspace_size) + " was provided"); } - std::visit(boost::hof::match( + std::visit(miopen::hof_match( [&](const Problem& problem_) { std::visit( - boost::hof::match( + miopen::hof_match( [&](const ConvolutionDescriptor& op_desc) { RunImpl(handle, inputs, workspace, workspace_size, op_desc); }, @@ -134,7 +112,7 @@ void Solution::LogDriverCommand(const BatchnormDescriptor& desc) const void Solution::LogDriverCommand(const Problem& problem_) const { - std::visit(boost::hof::match( + std::visit(miopen::hof_match( [&](const BiasDescriptor&) { /* \todo: think on how to log bias */ }, [&](const MhaDescriptor&) { /* \todo: think on how to log mha */ }, [&](const SoftmaxDescriptor&) { /* \todo: think on how to log softmax */ }, @@ -166,7 +144,7 @@ void Solution::RunImpl(const Handle& handle, auto ret = found->second; if(!ret.descriptor.has_value()) ret.descriptor = problem_casted.GetTensorDescriptorChecked(name, name_str); - return ret; + return std::move(ret); }; auto x = get_input_checked(miopenTensorConvolutionX, "miopenTensorConvolutionX"); @@ -293,7 +271,7 @@ void Solution::RunImpl(const Handle& handle, auto ret = found->second; if(!ret.descriptor.has_value()) ret.descriptor = problem_casted.GetTensorDescriptorChecked(name, name_str); - return ret; + return std::move(ret); }; const mha::ProblemDescription problem_description = problem_casted.AsMha(); @@ -489,7 +467,7 @@ void Solution::RunImpl(const Handle& handle, auto ret = found->second; if(!ret.descriptor.has_value()) ret.descriptor = problem_casted.GetTensorDescriptorChecked(name, name_str); - return ret; + return std::move(ret); }; const softmax::ProblemDescription problem_description = problem_casted.AsSoftmax(); @@ -541,11 +519,14 @@ void Solution::RunImpl(const Handle& handle, if(!kernels.empty()) { - const auto ctx = ExecutionContext{&handle}; - const auto softmax_solution = GetSolver() == regularSoftmax.SolverDbId() - ? regularSoftmax.GetSolution(ctx, problem_description) - : attnSoftmax.GetSolution(ctx, problem_description); - auto kernel_handles = std::vector{std::begin(kernels), std::end(kernels)}; + const auto ctx = ExecutionContext{&handle}; + auto db_getter = MakeConvDbGetter(ctx); + const auto softmax_solution = + GetSolver() == regularSoftmax.SolverDbId() + ? solver::FindSolution( + regularSoftmax, ctx, problem_description, db_getter, invoke_ctx) + : attnSoftmax.GetSolution(ctx, problem_description); + auto kernel_handles = std::vector{std::begin(kernels), std::end(kernels)}; if(softmax_solution.invoker_factory.has_value()) { @@ -569,11 +550,12 @@ void Solution::RunImpl(const Handle& handle, return; } - auto ctx = ExecutionContext{&handle}; - - const auto softmax_solution = GetSolver() == regularSoftmax.SolverDbId() - ? regularSoftmax.GetSolution(ctx, problem_description) - : attnSoftmax.GetSolution(ctx, problem_description); + auto ctx = ExecutionContext{&handle}; + auto db_getter = MakeConvDbGetter(ctx); + const auto softmax_solution = + GetSolver() == regularSoftmax.SolverDbId() + ? solver::FindSolution(regularSoftmax, ctx, problem_description, db_getter, invoke_ctx) + : attnSoftmax.GetSolution(ctx, problem_description); if(softmax_solution.invoker_factory.has_value()) { diff --git a/projects/miopen/src/solver.cpp b/projects/miopen/src/solver.cpp index e23186cc9601..5dd6368efca1 100644 --- a/projects/miopen/src/solver.cpp +++ b/projects/miopen/src/solver.cpp @@ -684,11 +684,7 @@ inline SolverRegistrar::SolverRegistrar(IdRegistryData& registry) RegisterWithSolver( registry, ++id, conv::ConvWinoRageRxS<2, 3>{}, miopenConvolutionAlgoWinograd); - Register(registry, - ++id, - Primitive::Fusion, - fusion::ConvWinoRageRxSFused<2, 3>{}.SolverDbId(), - miopenConvolutionAlgoWinograd); + ++id; // Removed ConvWinoRageRxSFused solver Register(registry, ++id, Primitive::Fusion, @@ -703,8 +699,121 @@ inline SolverRegistrar::SolverRegistrar(IdRegistryData& registry) Register(registry, ++id, Primitive::Normalization, layernorm::LayernormBackward().SolverDbId()); RegisterWithSolver(registry, ++id, conv::ConvDepthwiseFwd2D{}, miopenConvolutionAlgoDirect); - // IMPORTANT: New solvers should be added to the end of the function, and don't leave a white - // space between this comment and the newly registered solver(s)! + + // Transposed Winograd solvers for NHWC layout support + RegisterWithSolver( + registry, ++id, conv::TransposedConvBinWinograd3x3U{}, miopenConvolutionAlgoWinograd); + MIOPEN_LOG_I("Registering TransposedConvBinWinograd3x3U with id=" << id); + RegisterWithSolver( + registry, ++id, conv::TransposedConvBinWinogradRxS{}, miopenConvolutionAlgoWinograd); + MIOPEN_LOG_I("Registering TransposedConvBinWinogradRxS with id=" << id); + RegisterWithSolver( + registry, ++id, conv::TransposedConvBinWinogradRxSf2x3g1{}, miopenConvolutionAlgoWinograd); + MIOPEN_LOG_I("Registering TransposedConvBinWinogradRxSf2x3g1 with id=" << id); + // Transposed non-tunable Winograd solvers + RegisterWithSolver( + registry, ++id, conv::TransposedConvWinoFuryRxS<2, 3>{}, miopenConvolutionAlgoWinograd); + RegisterWithSolver( + registry, ++id, conv::TransposedConvWinoRageRxS<2, 3>{}, miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd<2, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd<3, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd<4, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd<5, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd<6, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<3, 2>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<3, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<3, 4>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<3, 5>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<3, 6>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<7, 2>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<7, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<5, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvWinograd3x3MultipassWrW<5, 4>{}, + miopenConvolutionAlgoWinograd); + // Transposed tunable Winograd solvers + RegisterWithSolver( + registry, ++id, conv::TransposedConvBinWinoRxS<2, 3>{}, miopenConvolutionAlgoWinograd); + RegisterWithSolver( + registry, ++id, conv::TransposedConvBinWinoRxS<3, 2>{}, miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd_xdlops<2, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd_xdlops<3, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd_xdlops<4, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd_xdlops<5, 3>{}, + miopenConvolutionAlgoWinograd); + RegisterWithSolver(registry, + ++id, + conv::TransposedConvMPBidirectWinograd_xdlops<6, 3>{}, + miopenConvolutionAlgoWinograd); + // IMPORTANT: New solvers should be added to the end of the function, and don't leave a white + // space between this comment and the newly registered solver(s)! } } // namespace solver diff --git a/projects/miopen/src/solver/activ/fwd_0.cpp b/projects/miopen/src/solver/activ/fwd_0.cpp index 0d389c022414..03972d5151b7 100644 --- a/projects/miopen/src/solver/activ/fwd_0.cpp +++ b/projects/miopen/src/solver/activ/fwd_0.cpp @@ -130,7 +130,8 @@ ConvSolution ActivFwdSolver0::GetSolution(const ExecutionContext&, MIOPEN_LOG_E("Unsupported data types configuration: " << miopen::GetDataTypeName(problem.GetXDesc().GetType()) << "x" << miopen::GetDataTypeName(problem.GetYDesc().GetType())); - return {miopenStatusInternalError}; + result = ConvSolution{miopenStatusInternalError}; + return result; } { diff --git a/projects/miopen/src/solver/conv/conv_MP_bidirectional_winograd.cpp b/projects/miopen/src/solver/conv/conv_MP_bidirectional_winograd.cpp index ec27ce8910d0..b909363a6341 100644 --- a/projects/miopen/src/solver/conv/conv_MP_bidirectional_winograd.cpp +++ b/projects/miopen/src/solver/conv/conv_MP_bidirectional_winograd.cpp @@ -227,7 +227,12 @@ static bool IsApplicableTransform(const ExecutionContext& ctx, const ProblemDesc } } - if(!problem.IsLayoutDefault()) + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) return false; { @@ -326,8 +331,15 @@ bool ConvMPBidirectWinograd::IsA if(!problem.AllTensorsDimsFitIntoInt()) return false; - if(!problem.IsLayoutDefault()) - return false; + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + { + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) + return false; + } if(problem.IsTensorsCasted()) return false; @@ -735,6 +747,12 @@ template struct MIOPEN_INTERNALS_EXPORT ConvMPBidirectWinograd<4, 3>; template struct MIOPEN_INTERNALS_EXPORT ConvMPBidirectWinograd<5, 3>; template struct MIOPEN_INTERNALS_EXPORT ConvMPBidirectWinograd<6, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvMPBidirectWinograd<2, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvMPBidirectWinograd<3, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvMPBidirectWinograd<4, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvMPBidirectWinograd<5, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvMPBidirectWinograd<6, 3>; + // ExecutionContext and ProblemDescription transformation // for winograd buffers calculation using xdlops_convolution template @@ -962,6 +980,12 @@ template struct ConvMPBidirectWinograd_xdlops<4, 3>; template struct ConvMPBidirectWinograd_xdlops<5, 3>; template struct ConvMPBidirectWinograd_xdlops<6, 3>; +template struct TransposedConvMPBidirectWinograd_xdlops<2, 3>; +template struct TransposedConvMPBidirectWinograd_xdlops<3, 3>; +template struct TransposedConvMPBidirectWinograd_xdlops<4, 3>; +template struct TransposedConvMPBidirectWinograd_xdlops<5, 3>; +template struct TransposedConvMPBidirectWinograd_xdlops<6, 3>; + } // namespace conv } // namespace solver } // namespace miopen diff --git a/projects/miopen/src/solver/conv/conv_bin_wino3x3U.cpp b/projects/miopen/src/solver/conv/conv_bin_wino3x3U.cpp index 1f1034994220..561a2f7c98cf 100644 --- a/projects/miopen/src/solver/conv/conv_bin_wino3x3U.cpp +++ b/projects/miopen/src/solver/conv/conv_bin_wino3x3U.cpp @@ -71,7 +71,12 @@ bool ConvBinWinograd3x3U::IsApplicable(const ExecutionContext& ctx, if(!problem.AllTensorsDimsFitIntoInt()) return false; - if(!problem.IsLayoutDefault()) + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) return false; if(problem.IsTensorsCasted()) @@ -100,7 +105,7 @@ bool ConvBinWinograd3x3U::IsApplicable(const ExecutionContext& ctx, && problem.GetInChannels() >= (device_is_gfx8 ? 16 : 18) && problem.IsFp32() && problem.GetGroupCount() == 1 - && problem.GetInLayout() == "NCHW"; + && problem.GetIn().IsPossibleLayout4D5D("NCHW", strict); /// && (isForwardDirection() ? _weights_layout == "KCHW" : _weights_layout == "CKHW" ) /// Actually, K<->C flpping is controlled by separate flag, so we can support either /// layout in both directions. diff --git a/projects/miopen/src/solver/conv/conv_bin_winoRxS.cpp b/projects/miopen/src/solver/conv/conv_bin_winoRxS.cpp index ea82c84c1d98..587541d2a954 100644 --- a/projects/miopen/src/solver/conv/conv_bin_winoRxS.cpp +++ b/projects/miopen/src/solver/conv/conv_bin_winoRxS.cpp @@ -32,6 +32,7 @@ #include #include #include +#include /// Global switch MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RXS) @@ -183,7 +184,14 @@ static inline bool IsShaderContraintsMet(const miopen::ExecutionContext& ctx, return false; } const auto grid_workgroup_count_x = ctx.GetStream().GetMaxComputeUnits(); - if(!problem.IsLayoutDefault()) + + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + static const auto strict = + miopen::TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) return false; // clang-format off @@ -272,6 +280,10 @@ bool ConvBinWinogradRxS::IsApplicable(const ExecutionContext& ctx, } } + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + // clang-format off if (! (problem.GetKernelStrideW() <= 2 // -u inp_u 1 or 2 && problem.GetKernelStrideW() == problem.GetKernelStrideH() @@ -279,7 +291,7 @@ bool ConvBinWinogradRxS::IsApplicable(const ExecutionContext& ctx, && problem.GetDilationH() == 1 && problem.GetBias() == 0 && problem.GetGroupCount() == 1 - && problem.GetInLayout() == "NCHW")) + && problem.GetIn().IsPossibleLayout4D5D("NCHW", strict))) return false; // clang-format on diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp index 29ae7dedad22..4d9769bb6075 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp @@ -31,6 +31,7 @@ MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_BWD_XDLOPS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_BWD_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) #include #include @@ -465,6 +466,63 @@ void PerformanceConfigHipImplicitGemm3DGroupBwdXdlops::InitValidKernels( } #endif +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_3d_grp_bwd = { +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 64, 16, 16, Filter1x1Stride1Pad0, 32, 32, 1, 1, 16, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 32, 8, 8, Filter1x1Stride1Pad0, 32, 32, 1, 1, 8, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 32, 8, 8, Filter1x1Stride1Pad0, 32, 32, 1, 1, 8, 2, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 16, 4, 4, Filter1x1Stride1Pad0, 32, 32, 1, 1, 4, 2, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 1, 4, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 16, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 1, 8, 1, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 1, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 64, 64, 32, 8, 8, Filter1x1Stride1Pad0, 32, 32, 2, 2, 1, 1, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 64, 16, 16, Default, 32, 32, 1, 1, 16, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 32, 8, 8, Default, 32, 32, 1, 1, 8, 2, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 16, 32, 8, 8, Default, 16, 16, 1, 1, 8, 1, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 128, 32, 8, 8, Default, 32, 32, 1, 2, 1, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 64, 64, 32, 8, 8, Default, 32, 32, 2, 2, 1, 1, 1, 1>" +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_3d_grp_bwd_navi = { +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 8, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<256, 128, 256, 64, Default, 8, 8, 8>", +"DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<32, 64, 32, 64, Default, 8, 8, 8>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 8, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 8, 1, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<128, 64, 64, 32, Default, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 64, 32, 8, 8, Default, 16, 16, 4, 2, 1, 1, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 1, 8, 1, 1>", +"DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 1, 8, 1, 1>" +}; +// clang-format on + +void PerformanceConfigHipImplicitGemm3DGroupBwdXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_3d_grp_bwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_3d_grp_bwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel_str : ranked_1st_applicable) + { + auto it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + kernel_id = valid_kernels[index]; + return; + } + } +} + void PerformanceConfigHipImplicitGemm3DGroupBwdXdlops::HeuristicInit( const miopen::ExecutionContext& ctx, const ::miopen::conv::ProblemDescription& problem) { @@ -564,14 +622,16 @@ void PerformanceConfigHipImplicitGemm3DGroupBwdXdlops::HeuristicInit( #endif // 2. Default: index remains 0, first valid_kernel will be used - MIOPEN_LOG_I2("Step 2: Using default initialization (index=0)"); InitValidKernels(problem); if(!valid_kernels.empty()) { index = 0; kernel_id = valid_kernels[index]; + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); + MIOPEN_LOG_I("Step 2: Default initialization selected kernel: " << kernel_id - << " at index: 0"); + << " at index: " << index); } else { diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp index 6da6b35db30f..6e11bd1cd50c 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp @@ -32,6 +32,7 @@ MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS) MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS_IDX_OVERRIDE); MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) #include #include @@ -465,6 +466,77 @@ void PerformanceConfigHipImplicitGemm3DGroupFwdXdlops::InitValidKernels( } #endif +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_3d_grp_fwd = { +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 16, 16, Filter3x3, 16, 16, 4, 1, 4, 1, 1, 1, 1, 8>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 64, 64, Filter1x1Pad0, 32, 32, 2, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 128, 32, Filter1x1Stride1Pad0, 32, 32, 2, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 16, 32, 64, Filter1x1Stride1Pad0, 16, 16, 1, 1, 4, 4, 4, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 64, 16, Filter1x1Stride1Pad0, 32, 32, 2, 1, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 32, 16, 64, Filter1x1Stride1Pad0, 16, 16, 1, 1, 4, 4, 2, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 16, Filter1x1Stride1Pad0, 32, 32, 2, 1, 4, 4, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 128, 32, Filter1x1Pad0, 32, 32, 1, 2, 8, 8, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Filter1x1Pad0, 16, 16, 2, 2, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Filter1x1Pad0, 16, 16, 2, 2, 2, 1, 2, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 128, 32, Filter1x1Stride1Pad0, 32, 32, 2, 2, 1, 1, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 32, Filter1x1Stride1Pad0, 32, 32, 2, 1, 8, 8, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Filter1x1Pad0, 16, 16, 2, 2, 1, 2, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 16, Filter1x1Pad0, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 32, Filter1x1Stride1Pad0, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 64, 128, 64, Default, 32, 32, 1, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 128, 64, Default, 32, 32, 2, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor<256, 256, 128, 32, Default, 32, 32, 4, 2, 8, 8, 8, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 128, 32, Default, 32, 32, 2, 2, 1, 1, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 32, Default, 32, 32, 2, 1, 8, 8, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor<64, 64, 64, 32, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 16, 32, 64, Default, 16, 16, 1, 1, 4, 4, 4, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 128, 16, Default, 32, 32, 2, 2, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 16, Default, 32, 32, 2, 1, 4, 4, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 16, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor<256, 256, 128, 16, Default, 32, 32, 4, 2, 4, 4, 4, 1, 1>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor<64, 64, 64, 16, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1>" +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_3d_grp_fwd_navi = { +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 128, 256, 32, Default, 16, 16, 4, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 128, 128, 32, Default, 16, 16, 4, 4, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 128, 32, 32, Default, 16, 16, 4, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle_V3_Large_Tensor<128, 128, 128, Default, 16, 16, 4, 4, 8, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 128, 96, 64, Default, 16, 16, 4, 3, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 128, 256, 64, Default, 16, 16, 4, 4, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_WmmaPorted<256, 256, 128, 32, Default, 32, 32, 4, 2, 8, 8, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_WmmaPorted<256, 64, 64, 32, Default, 16, 16, 2, 2, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor_WmmaPorted<64, 64, 64, 32, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1>" +}; +// clang-format on + +void PerformanceConfigHipImplicitGemm3DGroupFwdXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_3d_grp_fwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_3d_grp_fwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel_str : ranked_1st_applicable) + { + auto it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + kernel_id = valid_kernels[index]; + return; + } + } +} + void PerformanceConfigHipImplicitGemm3DGroupFwdXdlops::HeuristicInit( const miopen::ExecutionContext& ctx, const ::miopen::conv::ProblemDescription& problem) { @@ -744,14 +816,16 @@ void PerformanceConfigHipImplicitGemm3DGroupFwdXdlops::HeuristicInit( #endif // 4. Default: index remains 0, first valid_kernel will be used - MIOPEN_LOG_I2("Step 4: Using default initialization (index=0)"); InitValidKernels(problem); if(!valid_kernels.empty()) { index = 0; kernel_id = valid_kernels[index]; + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); + MIOPEN_LOG_I("Step 4: Default initialization selected kernel: " << kernel_id - << " at index: 0"); + << " at index: " << index); } else { diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp index 7e770e496d62..1eb8213e0420 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp @@ -31,6 +31,7 @@ MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_WRW_XDLOPS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_3D_CONV_IMPLICIT_GEMM_HIP_WRW_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) #include #include @@ -424,6 +425,87 @@ void PerformanceConfigHipImplicitGemm3DGroupWrwXdlops::InitValidKernels( } #endif +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_3d_grp_wrw = { +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 128x128x64, WaveTile: 32x32, WaveMap: 2x2, VmemReadVec: 8x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3, BlkGemmPipelinePrefetchStages: 2>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 128x128x64, WaveTile: 32x32, WaveMap: 2x2, VmemReadVec: 4x4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4, BlkGemmPipelinePrefetchStages: 3>", 64), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 16, 256, 32, Default, 8, 1, 16, 8, 8, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 8>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 16, 128, 32, Default, 8, 1, 8, 4, 8, 4, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 4>", 64), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 32, 64, 32, Default, 8, 1, 2, 2, 2, 2, 2, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, 2>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 64x16x64, WaveTile: 16x16, WaveMap: 2x1, VmemReadVec: 8x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 64x16x64, WaveTile: 16x16, WaveMap: 2x1, VmemReadVec: 8x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 64, BlkTile: 16x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 128x128x64, WaveTile: 32x32, WaveMap: 2x2, VmemReadVec: 8x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 3>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 3>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x32x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 2x4, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 3>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 2x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 1x2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 1x2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x32x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 2x1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 3>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 2x1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 2, 8, 2, 8, 1, 1, 2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 1, 8, 4, 8, 1, 1, 4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 4, 8, 1, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 1, 8, 1, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<256, 64, 64, 8, Filter1x1Stride1Pad0, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<256, 64, 64, 8, Default, 8, 1, 1, 4, 4, 4, 4, 1, 1, 4>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<64, 32, 64, 4, Default, 4, 1, 2, 1, 2, 4, 4, 1, 1, 4>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<256, 64, 64, 8, Default, 8, 1, 1, 4, 4, 1, 4, 1, 1, 1>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<64, 64, 64, 4, Default, 4, 2, 2, 1, 4, 1, 4, 1, 1, 1>", 32) +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_3d_grp_wrw_navi = { +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<96, 96, 96, 48, Default, 8, 6, 2, 6, 8, 6, 8, 1, 1, 8, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 64, BlkTile: 32x32x128, WaveTile: 16x16, WaveMap: 2x1, VmemReadVec: 8x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<64, 64, 64, 64, Default, 8, 4, 2, 8, 8, 8, 8, 1, 1, 8, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 4), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<64, 32, 32, 32, Default, 8, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 32), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 16), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 128), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 16x256x64, WaveTile: 16x16, WaveMap: 1x4, VmemReadVec: 1x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 8) +}; +// clang-format on + +void PerformanceConfigHipImplicitGemm3DGroupWrwXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_3d_grp_wrw; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_3d_grp_wrw_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel : ranked_1st_applicable) + { + const auto& kernel_str = std::get<0>(kernel); + const auto& it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + split_k = 1; + kernel_id = valid_kernels[index] + "+" + std::to_string(split_k); + return; + } + } +} + void PerformanceConfigHipImplicitGemm3DGroupWrwXdlops::HeuristicInit( [[maybe_unused]] const miopen::ExecutionContext& ctx, const ::miopen::conv::ProblemDescription& problem) @@ -549,15 +631,17 @@ void PerformanceConfigHipImplicitGemm3DGroupWrwXdlops::HeuristicInit( MIOPEN_LOG_I2("Step 1: AI heuristics not available (MIOPEN_ENABLE_AI_KERNEL_TUNING disabled)"); #endif - MIOPEN_LOG_I2("Step 2: Using default initialization (index=0)"); InitValidKernels(problem); if(!valid_kernels.empty()) { index = 0; split_k = 1; kernel_id = valid_kernels[index] + "+" + std::to_string(split_k); + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); + MIOPEN_LOG_I("Step 2: Default initialization selected kernel: " << kernel_id - << " at index: 0"); + << " at index: " << index); } else { diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_bwd_data_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_bwd_data_xdlops.cpp index 23992a2cb21a..af0c0c06721f 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_bwd_data_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_bwd_data_xdlops.cpp @@ -39,6 +39,7 @@ #include #include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM_HIP_BWD_XDLOPS) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) namespace miopen { namespace solver { @@ -172,7 +173,59 @@ bool ConvHipImplicitGemmBwdXdlops::CheckCKApplicability(const ProblemDescription } #endif +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_bwd = { +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<256, 64, 128, 4, 4, 1, 2, 4, 4, 2, 4> Filter1x1Stride1Pad0", +"DeviceConv2dBwdDataXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, 4, 1, 2, 4, 4, 2, 4>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 32, 64, 4, 4, 1, 2, 4, 4, 4, 4> Filter1x1Stride1Pad0", +"DeviceConv2dBwdDataXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<64, 32, 64, 4, 4, 1, 2, 4, 4, 4, 4>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<128, 128, 32, 4, 4, 2, 1, 4, 4, 1, 4> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 64, 32, 4, 4, 2, 1, 4, 4, 2, 4> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 64, 32, 4, 4, 2, 1, 4, 4, 2, 4>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<256, 64, 128, 4, 8, 1, 2, 8, 8, 2, 8> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<256, 128, 64, 4, 8, 2, 1, 8, 8, 1, 8> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<128, 128, 32, 4, 8, 2, 1, 8, 8, 1, 8> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 64, 32, 4, 8, 2, 1, 8, 8, 2, 8> Filter1x1Stride1Pad0", +"DeviceConv2dBwdDataXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 128, 4, 8, 2, 2, 8, 8, 2, 8>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<256, 64, 128, 4, 8, 1, 2, 8, 8, 2, 8>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<256, 128, 64, 4, 8, 2, 1, 8, 8, 1, 8>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<128, 128, 32, 4, 8, 2, 1, 8, 8, 1, 8>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 64, 32, 4, 8, 2, 1, 8, 8, 2, 8>", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 32, 64, 4, 8, 1, 2, 8, 8, 2, 8> Filter1x1Stride1Pad0", +"DeviceConvNdBwdDataNwcKxcNwk_Xdl<64, 32, 64, 4, 8, 1, 2, 8, 8, 2, 8>" +}; + +//no results for navi +static const std::vector ranked_gemm_bwd_navi = {}; +// clang-format on + +void PerformanceConfigHipImplicitGemmBwdXdlops::DefaultKernelFromList(const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_bwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_bwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel_str : ranked_1st_applicable) + { + auto it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + kernel_id = valid_kernels[index]; + return; + } + } +} + void PerformanceConfigHipImplicitGemmBwdXdlops::HeuristicInit( + [[maybe_unused]] const ExecutionContext& ctx, [[maybe_unused]] const ProblemDescription& problem) { index = 0; @@ -191,14 +244,30 @@ void PerformanceConfigHipImplicitGemmBwdXdlops::HeuristicInit( case miopenInt64: case miopenDouble: break; } + + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); #endif } bool PerformanceConfigHipImplicitGemmBwdXdlops::SetNextValue(const ProblemDescription& problem) { +#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL if(valid_kernels.empty()) { - HeuristicInit(problem); + switch(problem.GetInDataType()) + { + case miopenHalf: Init(problem); break; + case miopenFloat: Init(problem); break; + case miopenBFloat16: Init(problem); break; + case miopenFloat8_fnuz: + case miopenBFloat8_fnuz: + case miopenInt8: + case miopenInt32: + case miopenInt64: + case miopenDouble: break; + } + assert(!valid_kernels.empty()); return true; } @@ -209,6 +278,7 @@ bool PerformanceConfigHipImplicitGemmBwdXdlops::SetNextValue(const ProblemDescri return true; } else +#endif return false; } @@ -244,11 +314,11 @@ bool PerformanceConfigHipImplicitGemmBwdXdlops::operator==( } PerformanceConfigHipImplicitGemmBwdXdlops -ConvHipImplicitGemmBwdXdlops::GetDefaultPerformanceConfig(const ExecutionContext&, +ConvHipImplicitGemmBwdXdlops::GetDefaultPerformanceConfig(const ExecutionContext& ctx, const ProblemDescription& problem) const { PerformanceConfigHipImplicitGemmBwdXdlops pp; - pp.HeuristicInit(problem); + pp.HeuristicInit(ctx, problem); return pp; } diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_fwd_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_fwd_xdlops.cpp index 0022eb8f8685..966722454000 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_fwd_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_fwd_xdlops.cpp @@ -40,6 +40,7 @@ #include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) namespace miopen { namespace solver { @@ -173,7 +174,61 @@ bool ConvHipImplicitGemmFwdXdlops::CheckCKApplicability(const ProblemDescription } #endif +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_fwd = { +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, Filter1x1Pad0, 4, 32, 32, 1, 2, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, Default, 4, 32, 32, 1, 2, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<64, 32, 64, 4, Filter1x1Stride1Pad0, 4, 32, 32, 1, 2, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 64, 4, Filter1x1Pad0, 4, 32, 32, 2, 1, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<64, 64, 32, 4, Filter1x1Stride1Pad0, 4, 32, 32, 2, 1, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 64, 4, Default, 4, 32, 32, 2, 1, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<64, 64, 32, 4, Default, 4, 32, 32, 2, 1, 4, 4, 4, 4>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 256, 4, Default, 8, 2, 4, 8, 8, 8, 8, 1, 1, 8>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, Filter1x1Stride1Pad0, 8, 1, 2, 8, 8, 8, 8, 1, 1, 8>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<128, 64, 128, 4, Filter1x1Pad0, 8, 2, 2, 8, 8, 8, 8, 1, 1, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, Filter1x1Stride1Pad0, 8, 32, 32, 1, 2, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, Filter1x1Pad0, 8, 32, 32, 1, 2, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<128, 64, 128, 4, Default, 8, 32, 32, 2, 2, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 64, 128, 4, OddC, 8, 1, 2, 1, 1, 1, 1, 1, 1, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 64, 4, Filter1x1Pad0, 8, 32, 32, 2, 1, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<128, 128, 32, 4, Filter1x1Stride1Pad0, 8, 32, 32, 2, 1, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 64, 4, Default, 8, 32, 32, 2, 1, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<128, 128, 32, 4, Default, 8, 32, 32, 2, 1, 8, 8, 8, 8>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<256, 128, 64, 4, OddC, 8, 2, 1, 1, 1, 1, 1, 1, 1, 8>", +"DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<64, 64, 32, 4, OddC, 8, 2, 1, 1, 1, 1, 1, 1, 1, 8>" +}; + +//no results for navi +static const std::vector ranked_gemm_fwd_navi = {}; +// clang-format on + +void PerformanceConfigHipImplicitGemmFwdXdlops::DefaultKernelFromList(const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_fwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_fwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel_str : ranked_1st_applicable) + { + auto it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + kernel_id = valid_kernels[index]; + return; + } + } +} + void PerformanceConfigHipImplicitGemmFwdXdlops::HeuristicInit( + [[maybe_unused]] const ExecutionContext& ctx, [[maybe_unused]] const ProblemDescription& problem) { index = 0; @@ -192,14 +247,30 @@ void PerformanceConfigHipImplicitGemmFwdXdlops::HeuristicInit( case miopenInt32: case miopenDouble: break; } + + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); #endif } bool PerformanceConfigHipImplicitGemmFwdXdlops::SetNextValue(const ProblemDescription& problem) { +#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL if(valid_kernels.empty()) { - HeuristicInit(problem); + switch(problem.GetInDataType()) + { + case miopenInt8: Init(problem); break; + case miopenHalf: Init(problem); break; + case miopenFloat: Init(problem); break; + case miopenBFloat16: Init(problem); break; + case miopenFloat8_fnuz: + case miopenBFloat8_fnuz: + case miopenInt64: + case miopenInt32: + case miopenDouble: break; + } + assert(!valid_kernels.empty()); return true; } @@ -211,6 +282,7 @@ bool PerformanceConfigHipImplicitGemmFwdXdlops::SetNextValue(const ProblemDescri return true; } else +#endif return false; } @@ -246,11 +318,11 @@ bool PerformanceConfigHipImplicitGemmFwdXdlops::operator==( } PerformanceConfigHipImplicitGemmFwdXdlops -ConvHipImplicitGemmFwdXdlops::GetDefaultPerformanceConfig(const ExecutionContext&, +ConvHipImplicitGemmFwdXdlops::GetDefaultPerformanceConfig(const ExecutionContext& ctx, const ProblemDescription& problem) const { PerformanceConfigHipImplicitGemmFwdXdlops pp; - pp.HeuristicInit(problem); + pp.HeuristicInit(ctx, problem); return pp; } diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_bwd_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_bwd_xdlops.cpp index 4988ddad9e40..8a66d2e7c0f5 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_bwd_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_bwd_xdlops.cpp @@ -40,6 +40,7 @@ #include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM_HIP_GROUP_BWD_XDLOPS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_GROUP_CONV_IMPLICIT_GEMM_HIP_BWD_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) namespace miopen { namespace solver { @@ -447,6 +448,80 @@ bool PerformanceConfigHipImplicitGemmGroupBwdXdlops::IsModelApplicable( return true; } +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_grp_bwd = { +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 64, 16, 16, Filter1x1Stride1Pad0, 32, 32, 1, 1, 16, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 16, 4, 4, Filter1x1Stride1Pad0, 32, 32, 1, 1, 4, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 16, 4, 4, Filter1x1Stride1Pad0, 32, 32, 1, 1, 4, 2, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 16, 4, 4, Filter1x1Stride1Pad0, 32, 32, 1, 1, 4, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 128, 32, 8, 8, Filter1x1Stride1Pad0, 32, 32, 1, 2, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 64, 16, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 4, 1, 1, 4, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 64, 64, 32, 8, 8, Filter1x1Stride1Pad0, 32, 32, 2, 2, 1, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 64, 16, 16, Default, 32, 32, 1, 1, 16, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 16, 64, 16, 16, Default, 16, 16, 1, 1, 16, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<128, 128, 32, 32, 8, 8, Default, 32, 32, 2, 1, 8, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 16, 32, 8, 8, Default, 16, 16, 1, 1, 8, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 32, 8, 8, Default, 32, 32, 1, 1, 8, 2, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 64, 16, 32, 8, 8, Default, 16, 16, 1, 1, 8, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<256, 128, 32, 16, 4, 4, Default, 32, 32, 1, 1, 4, 2, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 64, 64, 32, 8, 8, Default, 32, 32, 2, 2, 1, 1, 1, 1>", 1) +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_grp_bwd_navi = { +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<128, 128, 64, 64, Filter1x1Stride1Pad0, 8, 8, 2>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 8, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<256, 128, 64, 64, Filter1x1Stride1Pad0, 8, 8, 2>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 8, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 32, 64, 8, 8, Default, 16, 16, 4, 1, 8, 4, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<128, 64, 64, 32, Default, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 64, 32, 8, 8, Default, 16, 16, 4, 2, 1, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<256, 128, 64, 64, Default, 8, 8, 2>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<64, 64, 64, 64, Default, 8, 8, 8>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<128, 64, 64, 32, Filter1x1Stride1Pad0, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 64, 32, 8, 8, Default, 16, 16, 4, 2, 4, 4, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<64, 32, 64, 64, Default, 8, 8, 8>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<128, 128, 64, 64, Default, 8, 8, 2>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<128, 128, 128, 32, 8, 8, Default, 16, 16, 8, 2, 8, 4, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<128, 128, 64, 64, 8, 8, Default, 16, 16, 8, 1, 8, 4, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle<256, 128, 256, 64, Default, 8, 8, 8>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Default, 16, 16, 1, 4, 8, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 4, 2, 1, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 8, 1, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1<64, 16, 64, 32, 8, 8, Filter1x1Stride1Pad0, 16, 16, 1, 4, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffleV3<64, 64, 32, 32, 8, 8, Default, 16, 16, 4, 1, 8, 1, 1, 1>", 1) +}; +// clang-format on + +void PerformanceConfigHipImplicitGemmGroupBwdXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_grp_bwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_grp_bwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel : ranked_1st_applicable) + { + const auto& kernel_str = std::get<0>(kernel); + const auto& it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + split_k = 1; + kernel_id = valid_kernels[index] + "+" + std::to_string(split_k); + return; + } + } +} + void PerformanceConfigHipImplicitGemmGroupBwdXdlops::HeuristicInit( [[maybe_unused]] const ExecutionContext& ctx, [[maybe_unused]] const ProblemDescription& problem) @@ -507,6 +582,9 @@ void PerformanceConfigHipImplicitGemmGroupBwdXdlops::HeuristicInit( case miopenDouble: break; } + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); + // Invariant: split_k must always be 1 in deterministic mode assert(!is_deterministic || split_k == 1); #endif diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_fwd_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_fwd_xdlops.cpp index b2f9fb1997ef..2a444a968152 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_fwd_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_fwd_xdlops.cpp @@ -43,6 +43,7 @@ #include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_GROUP_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_GROUP_CONV_IMPLICIT_GEMM_HIP_FWD_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) namespace miopen { namespace solver { @@ -425,6 +426,99 @@ bool PerformanceConfigHipImplicitGemmGroupFwdXdlops::IsModelApplicable( return true; } +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_grp_fwd = { +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 16, 32, Filter3x3, 16, 16, 4, 1, 4, 1, 1, 1, 1, 16>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 16, 32, Default, 16, 16, 4, 1, 4, 1, 1, 1, 1, 8>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 16, 16, Filter3x3, 16, 16, 4, 1, 4, 1, 1, 1, 1, 16>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 16, 16, Default, 16, 16, 4, 1, 4, 1, 1, 1, 1, 8>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 64, 128, 64, Filter1x1Pad0, 32, 32, 1, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 128, 32, Filter1x1Stride1Pad0, 32, 32, 2, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 64, 64, 64, Default, 32, 32, 1, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 256, 128, 32, Filter1x1Stride1Pad0, 32, 32, 4, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 32, 16, 64, Filter1x1Stride1Pad0, 16, 16, 1, 1, 4, 4, 2, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 16, 32, 64, Filter1x1Pad0, 16, 16, 1, 1, 4, 4, 4, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 16, Filter1x1Pad0, 32, 32, 2, 1, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 16, Filter1x1Pad0, 32, 32, 2, 1, 4, 4, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 16, Filter1x1Pad0, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 128, 64, OddC, 32, 32, 2, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_DirectLoad<256, 64, 64, 64, Default, 16, 16, 2, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Filter1x1Pad0, 16, 16, 2, 2, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 32, Filter1x1Stride1Pad0, 32, 32, 2, 1, 8, 8, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Filter1x1Pad0, 16, 16, 2, 2, 2, 1, 2, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 32, Filter1x1Stride1Pad0, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_DirectLoad<128, 16, 32, 64, Default, 16, 16, 1, 1, 8, 8, 4, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v4>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<128, 32, 16, 64, Default, 16, 16, 1, 1, 4, 4, 2, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 32, 16, Default, 32, 32, 2, 1, 4, 4, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Default, 16, 16, 2, 2, 4, 4, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 128, 32, Default, 32, 32, 2, 2, 1, 1, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 128, 16, OddC, 32, 32, 2, 2, 1, 1, 4, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor<64, 64, 64, 16, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Default, 16, 16, 2, 2, 2, 1, 2, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<64, 64, 64, 32, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1, 1>" +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector ranked_gemm_grp_fwd_navi = { +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 128, 128, 64, Filter1x1Stride1Pad0, 16, 16, 2, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 128, 256, 64, Filter1x1Stride1Pad0, 16, 16, 4, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 64, 256, 32, Filter1x1Stride1Pad0, 16, 16, 4, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 128, 64, 64, Filter1x1Stride1Pad0, 16, 16, 2, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 32, 64, 64, Filter1x1Stride1Pad0, 32, 32, 1, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 32, 64, 64, Default, 32, 32, 1, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 128, 128, 32, OddC, 16, 16, 4, 2, 1, 1, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 64, 16, 64, Filter1x1Pad0, 16, 16, 2, 1, 8, 8, 2, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 64, 16, 64, OddC, 16, 16, 2, 1, 8, 8, 2, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 64, 64, 64, Filter1x1Pad0, 16, 16, 2, 2, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_WmmaPorted<64, 64, 16, 16, Filter3x3, 16, 16, 4, 1, 4, 1, 1, 1, 1, 8>", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle<32, 16, 16, 32, Filter1x1Pad0, 8, 16, 16, 1, 1> AEnableLds: 1, BEnableLds: 1, ABlockTransferSrcScalarPerVector: 8, BBlockTransferSrcScalarPerVector: 8", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle<64, 64, 32, 32, Filter1x1Pad0, 8, 16, 16, 2, 2> AEnableLds: 1, BEnableLds: 1, ABlockTransferSrcScalarPerVector: 8, BBlockTransferSrcScalarPerVector: 8", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 64, 16, 64, Default, 16, 16, 2, 1, 8, 8, 2, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 64, 16, 64, Filter1x1Stride1Pad0, 16, 16, 2, 1, 8, 8, 2, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 32, 64, 64, Default, 32, 32, 1, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 32, 128, 64, Default, 32, 32, 1, 2, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle_V3_Large_Tensor<128, 64, 64, Default, 16, 16, 2, 2, 8, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 128, 48, 64, Default, 16, 16, 2, 3, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 128, 32, 64, Default, 32, 32, 2, 1, 8, 8, 4, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_WmmaPorted<128, 128, 32, 32, Default, 32, 32, 2, 1, 8, 8, 8, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 64, 192, 32, Filter1x1Stride1Pad0, 16, 16, 2, 6, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<128, 64, 64, 64, Default, 16, 16, 2, 2, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 64, 64, 64, Default, 16, 16, 2, 1, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Wmma_CShuffle_V3<256, 64, 64, 64, OddC, 16, 16, 2, 1, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_WmmaPorted<64, 64, 16, 16, Default, 16, 16, 4, 1, 4, 1, 1, 1, 1, 8>", +"DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor_WmmaPorted<64, 64, 64, 32, Default, 32, 32, 2, 2, 1, 1, 1, 1, 1>", +"DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_WmmaPorted<128, 64, 16, 64, Filter1x1Pad0, 16, 16, 2, 1, 8, 8, 2, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2>", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle<64, 32, 32, 128, OddC, 16, 16, 16, 1, 2> AEnableLds: 1, BEnableLds: 1, ABlockTransferSrcScalarPerVector: 16, BBlockTransferSrcScalarPerVector: 16", +"DeviceGroupedConvFwdMultipleD_Wmma_CShuffle<64, 64, 32, 64, OddC, 16, 16, 16, 2, 2> AEnableLds: 1, BEnableLds: 1, ABlockTransferSrcScalarPerVector: 16, BBlockTransferSrcScalarPerVector: 16" +}; +// clang-format on + +void PerformanceConfigHipImplicitGemmGroupFwdXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_grp_fwd; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_grp_fwd_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel_str : ranked_1st_applicable) + { + auto it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + kernel_id = valid_kernels[index]; + return; + } + } +} + void PerformanceConfigHipImplicitGemmGroupFwdXdlops::HeuristicInit( [[maybe_unused]] const ExecutionContext& ctx, [[maybe_unused]] const ProblemDescription& problem) @@ -465,6 +559,9 @@ void PerformanceConfigHipImplicitGemmGroupFwdXdlops::HeuristicInit( case miopenBFloat8_fnuz: case miopenDouble: break; } + + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); #endif } diff --git a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_wrw_xdlops.cpp b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_wrw_xdlops.cpp index 95933569c5d5..2fbccc8b47cd 100644 --- a/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_wrw_xdlops.cpp +++ b/projects/miopen/src/solver/conv/conv_hip_implicit_gemm_grouped_wrw_xdlops.cpp @@ -40,6 +40,7 @@ #include MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_GROUP_CONV_IMPLICIT_GEMM_HIP_WRW_XDLOPS) MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_GROUP_CONV_IMPLICIT_GEMM_HIP_WRW_XDLOPS_AI_HEUR) +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CK_DEFAULT_KERNELS) namespace miopen { namespace solver { @@ -488,6 +489,85 @@ bool PerformanceConfigHipImplicitGemmGroupWrwXdlops::IsModelApplicable( return true; } +// clang-format off +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_grp_wrw = { +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 32, 128, 32, Default, 8, 1, 4, 8, 8, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 8>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 128x128x64, WaveTile: 32x32, WaveMap: 2x2, VmemReadVec: 8x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3, BlkGemmPipelinePrefetchStages: 2>", 128), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 64x16x64, WaveTile: 16x16, WaveMap: 2x1, VmemReadVec: 8x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 4), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 128x128x64, WaveTile: 32x32, WaveMap: 2x2, VmemReadVec: 8x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v5, BlkGemmPipelinePrefetchStages: 3>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 4x2, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 32), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 2, 8, 2, 8, 1, 1, 2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x32x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 1x4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 32), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x32x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 1x1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 3>", 32), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<256, 64, 64, 64, Default, 8, 1, 1, 4, 8, 1, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<256, 64, 64, 8, Filter1x1Stride1Pad0, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1>", 32), +std::make_tuple("DeviceGroupedConvBwdWeight_Xdl_CShuffle<256, 64, 64, 8, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 32x16x64, WaveTile: 16x16, WaveMap: 1x1, VmemReadVec: 1x2, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1) +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, cert-err58-cpp) +static const std::vector> ranked_gemm_grp_wrw_navi = { +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 48, 64, 128, Default, 8, 3, 1, 6, 8, 8, 8, 1, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 64), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 48, 64, 128, Default, 8, 3, 1, 6, 8, 8, 8, 1, 1, 8, 1, 1>", 4), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 8), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 4), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 64, BlkTile: 32x32x128, WaveTile: 16x16, WaveMap: 2x1, VmemReadVec: 4x4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", -1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 128), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3<32, 16, 16, 32, Default, 8, 1, 1, 1, 4, 1, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 64, 64, 128, Default, 8, 4, 1, 8, 8, 8, 8, 1, 1, 8, 1, 1>", 2), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 16x256x64, WaveTile: 16x16, WaveMap: 1x4, VmemReadVec: 2x1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<64, 64, 64, 64, Default, 8, 4, 2, 8, 8, 8, 8, 1, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 48x64x128, WaveTile: 16x16, WaveMap: 3x1, VmemReadVec: 6x4, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 128, BlkTile: 16x64x64, WaveTile: 16x16, WaveMap: 1x2, VmemReadVec: 2x8, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 128, 128, 32, Default, 8, 8, 2, 4, 8, 4, 8, 1, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 48, 64, 128, Default, 8, 3, 1, 6, 8, 8, 8, 1, 1, 8, 1, 1>", 16), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 48, 64, 128, Default, 8, 3, 1, 6, 8, 8, 8, 1, 1, 8, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<64, 64, 64, 64, Default, 8, 4, 2, 8, 8, 8, 8, 1, 1, 8, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 16, 128, 32, Default, 8, 1, 8, 4, 8, 4, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 4>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<96, 96, 96, 48, Default, 8, 6, 2, 6, 8, 6, 8, 1, 1, 8, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<128, 96, 128, 64, Default, 8, 6, 2, 6, 8, 8, 8, 1, 1, 8, 1, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Wmma_CShuffleV3<64, 32, 32, 32, Default, 8, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 1>", 8), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 16, 256, 32, Default, 8, 1, 16, 8, 8, 8, 8, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, 8>", 1), +std::make_tuple("DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle<64, 48, 64, 32, Default, 8, 3, 4, 3, 4, 4, 4, 1, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v5, 1>", 1), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 16x256x64, WaveTile: 16x16, WaveMap: 1x4, VmemReadVec: 1x8, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v2, BlkGemmPipelinePrefetchStages: 2>", 8), +std::make_tuple("DeviceGroupedConvBwdWeight_Explicit_Xdl BlkSize: 256, BlkTile: 256x96x64, WaveTile: 16x16, WaveMap: 2x6, VmemReadVec: 1x1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1, BlkGemmPipelinePrefetchStages: 1>", 2) +}; +// clang-format on + +void PerformanceConfigHipImplicitGemmGroupWrwXdlops::DefaultKernelFromList( + const ExecutionContext& ctx) +{ + const auto dev_name = ctx.GetStream().GetDeviceName(); + const bool is_gfx11 = StartsWith(dev_name, "gfx11"); + const bool is_gfx12 = StartsWith(dev_name, "gfx12"); + + auto* ranked_p = &ranked_gemm_grp_wrw; + if(is_gfx11 || is_gfx12) + ranked_p = &ranked_gemm_grp_wrw_navi; + + const auto ranked_1st_applicable = *ranked_p; + + for(const auto& kernel : ranked_1st_applicable) + { + const auto& kernel_str = std::get<0>(kernel); + const auto& it = std::find(valid_kernels.begin(), valid_kernels.end(), kernel_str); + if(it != valid_kernels.end()) + { + index = it - valid_kernels.begin(); + split_k = 1; + kernel_id = valid_kernels[index] + "+" + std::to_string(split_k); + return; + } + } +} + void PerformanceConfigHipImplicitGemmGroupWrwXdlops::HeuristicInit( [[maybe_unused]] const ExecutionContext& ctx, [[maybe_unused]] const ProblemDescription& problem) @@ -547,6 +627,9 @@ void PerformanceConfigHipImplicitGemmGroupWrwXdlops::HeuristicInit( case miopenDouble: break; } + if(!env::disabled(MIOPEN_DEBUG_CK_DEFAULT_KERNELS)) + DefaultKernelFromList(ctx); + // Invariant: split_k must always be 1 in deterministic mode assert(!is_deterministic || split_k == 1); #endif diff --git a/projects/miopen/src/solver/conv/conv_multipass_wino3x3WrW.cpp b/projects/miopen/src/solver/conv/conv_multipass_wino3x3WrW.cpp index 31adbb3cd1d3..9210ad85a802 100644 --- a/projects/miopen/src/solver/conv/conv_multipass_wino3x3WrW.cpp +++ b/projects/miopen/src/solver/conv/conv_multipass_wino3x3WrW.cpp @@ -474,8 +474,16 @@ bool ConvWinograd3x3MultipassWrW return false; if(problem.IsTensorsCasted()) return false; - if(!problem.IsLayoutDefault()) - return false; + + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + { + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) + return false; + } const auto& target = ctx.GetStream().GetTargetProperties(); if(target.isXnackEnabled()) @@ -524,8 +532,15 @@ bool ConvWinograd3x3MultipassWrW { return false; } - if(!problem.IsLayoutDefault()) - return false; + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + { + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) + return false; + } // clang-format off { @@ -812,6 +827,20 @@ template struct MIOPEN_INTERNALS_EXPORT ConvWinograd3x3MultipassWrW<7, 3, 1, 1>; template struct MIOPEN_INTERNALS_EXPORT ConvWinograd3x3MultipassWrW<5, 3>; template struct MIOPEN_INTERNALS_EXPORT ConvWinograd3x3MultipassWrW<5, 4>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<3, 2>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<3, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<3, 4>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<3, 5>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<3, 6>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<7, 2>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<7, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<5, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinograd3x3MultipassWrW<5, 4>; + } // namespace conv } // namespace solver } // namespace miopen diff --git a/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp b/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp index 2b8c560973d3..f97ea68664b7 100644 --- a/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp +++ b/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp @@ -772,11 +772,11 @@ template struct PerformanceConfigConvOclBwdWrw2<4>; template struct PerformanceConfigConvOclBwdWrw2<8>; template struct PerformanceConfigConvOclBwdWrw2<16>; -template struct ConvOclBwdWrW2<1>; -template struct ConvOclBwdWrW2<2>; -template struct ConvOclBwdWrW2<4>; -template struct ConvOclBwdWrW2<8>; -template struct ConvOclBwdWrW2<16>; +template struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2<1>; +template struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2<2>; +template struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2<4>; +template struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2<8>; +template struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2<16>; } // namespace conv } // namespace solver diff --git a/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp b/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp index dbaebbbd8145..b80b14a08e91 100644 --- a/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp +++ b/projects/miopen/src/solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp @@ -416,7 +416,8 @@ ConvSolution ConvOclBwdWrW53::GetSolution(const ExecutionContext& ctx, GRP_SZ) != miopenStatusSuccess || out_n_vert_reads <= 0) { - return {miopenStatusNotInitialized}; + result = ConvSolution{miopenStatusNotInitialized}; + return result; } int out_n_vert_read_loops = static_cast(std::ceil( @@ -440,12 +441,14 @@ ConvSolution ConvOclBwdWrW53::GetSolution(const ExecutionContext& ctx, if(out_n_horizon_read_loops > 2 && problem.GetPadW() != 0) { MIOPEN_LOG_I2("Padding where split is more than 2 ways is not supported."); - return {miopenStatusNotInitialized}; + result = ConvSolution{miopenStatusNotInitialized}; + return result; } if(out_n_horizon_read_loops > 1 && problem.GetGroupCount() > 1) { MIOPEN_LOG_I2("For large images, group support is missing."); - return {miopenStatusNotInitialized}; + result = ConvSolution{miopenStatusNotInitialized}; + return result; } int out_horizon_last_chunk_valid_read_units = std::ceil( diff --git a/projects/miopen/src/solver/conv/conv_ocl_dir2Dfwd.cpp b/projects/miopen/src/solver/conv/conv_ocl_dir2Dfwd.cpp index 5a894126722e..a90b5444eed2 100644 --- a/projects/miopen/src/solver/conv/conv_ocl_dir2Dfwd.cpp +++ b/projects/miopen/src/solver/conv/conv_ocl_dir2Dfwd.cpp @@ -319,7 +319,8 @@ ConvSolution ConvOclDirectFwd::BaseGetSolution(const ExecutionContext& ctx, if(result.out_pix_tile1 == 0 || result.out_pix_tile0 == 0 /* DIV/0 */) { MIOPEN_LOG_E("result.out_pix_tile1 == 0 || result.out_pix_tile0 == 0"); - return {miopenStatusInternalError}; + result = ConvSolution{miopenStatusInternalError}; + return result; } result.grp_tile0 = std::max(8, (result.in_tile0 / result.out_pix_tile0)); result.grp_tile1 = std::max(8, (result.in_tile1 / result.out_pix_tile1)); @@ -332,7 +333,8 @@ ConvSolution ConvOclDirectFwd::BaseGetSolution(const ExecutionContext& ctx, if(alu_tiles_sz > 256 || alu_tiles_sz == 0 /* DIV/0 */) { MIOPEN_LOG_E("need out pix size ajustments (alu_tiles_sz > 256 || alu_tiles_sz == 0)"); - return {miopenStatusInternalError}; + result = ConvSolution{miopenStatusInternalError}; + return result; } int n_alus_total = (result.grp_tile0 * result.grp_tile1); @@ -343,7 +345,8 @@ ConvSolution ConvOclDirectFwd::BaseGetSolution(const ExecutionContext& ctx, if(result.n_stacks == 0 /* DIV/0 */) { MIOPEN_LOG_E("result.n_stacks == 0"); - return {miopenStatusInternalError}; + result = ConvSolution{miopenStatusInternalError}; + return result; } int n_alus_perstack = (n_alus_total + result.n_stacks - 1) / result.n_stacks; @@ -464,7 +467,8 @@ ConvSolution ConvOclDirectFwd::BaseGetSolution(const ExecutionContext& ctx, if(n_out_tiles_perstack == 0 /* DIV/0 */) { MIOPEN_LOG_E("n_out_tiles_perstack == 0"); - return {miopenStatusInternalError}; + result = ConvSolution{miopenStatusInternalError}; + return result; } size_t gbl_wk1 = group_counts >= 2 diff --git a/projects/miopen/src/solver/conv/conv_winoRxS.cpp b/projects/miopen/src/solver/conv/conv_winoRxS.cpp index 2cf8c5f9e289..f85a2c1be74c 100644 --- a/projects/miopen/src/solver/conv/conv_winoRxS.cpp +++ b/projects/miopen/src/solver/conv/conv_winoRxS.cpp @@ -24,6 +24,8 @@ * *******************************************************************************/ +#define CONV_BIN_WINO_RXS_CPP + #include #include @@ -310,8 +312,15 @@ inline bool IsShaderConstraintsMet(const ProblemDescription& problem, return false; } - if(!problem.IsLayoutDefault()) - return false; + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + { + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && + problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) + return false; + } return IsWinogradV21Preferred(asic, problem) ? IsShaderConstraintsMetV21(problem, R, S, C, K, H, W, OH, OW, N) @@ -684,13 +693,17 @@ static bool IsApplicableBase(const ExecutionContext& ctx, const ProblemDescripti if(name == "gfx90a" && problem.IsGfx90aFp16altRequired()) return false; + // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string + // This allows transposed solvers to work correctly when they modify tensor strides + static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; + // clang-format off if (!((problem.GetKernelStrideW() == 1 || problem.GetKernelStrideW() == 2) && problem.GetKernelStrideW() == problem.GetKernelStrideH() && problem.GetDilationW() == 1 && problem.GetDilationH() == 1 && problem.GetBias() == 0 - && problem.GetInLayout() == "NCHW")) + && problem.GetIn().IsPossibleLayout4D5D("NCHW", strict))) return false; // clang-format on @@ -1148,6 +1161,7 @@ bool ConvBinWinogradRxSf2x3g1::IsApplicable(const ExecutionContext& ctx, { if(env::disabled(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1)) return false; + return IsApplicableBase<2, 3>(ctx, problem) && problem.GetGroupCount() == 1; } @@ -1163,9 +1177,11 @@ ConvSolution ConvBinWinogradRxSf2x3g1::GetSolution(const ExecutionContext& ctx, const auto tunable = ConvBinWinoRxS<2, 3>{}; return tunable.GetSolution(ctx, problem, tunable.GetDefaultPerformanceConfig(ctx, problem)); } +template struct MIOPEN_INTERNALS_EXPORT ConvBinWinoRxS<2, 3>; +template struct MIOPEN_INTERNALS_EXPORT ConvBinWinoRxS<3, 2>; -template struct ConvBinWinoRxS<2, 3>; -template struct ConvBinWinoRxS<3, 2>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinoRxS<2, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinoRxS<3, 2>; } // namespace conv } // namespace solver diff --git a/projects/miopen/src/solver/conv/conv_wino_fury_RxS.cpp b/projects/miopen/src/solver/conv/conv_wino_fury_RxS.cpp index ee9098ad93ee..bd9c9a77bff6 100644 --- a/projects/miopen/src/solver/conv/conv_wino_fury_RxS.cpp +++ b/projects/miopen/src/solver/conv/conv_wino_fury_RxS.cpp @@ -621,6 +621,7 @@ ConvWinoFuryRxS::GetSolution(const ExecutionContext& ctx, template struct MIOPEN_INTERNALS_EXPORT ConvWinoFuryRxS<2, 3>; // template struct MIOPEN_INTERNALS_EXPORT ConvWinoFuryRxS<3, 2>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinoFuryRxS<2, 3>; } // namespace conv diff --git a/projects/miopen/src/solver/conv/conv_wino_rage_RxS.cpp b/projects/miopen/src/solver/conv/conv_wino_rage_RxS.cpp index 8dafa97d8478..29d005721778 100644 --- a/projects/miopen/src/solver/conv/conv_wino_rage_RxS.cpp +++ b/projects/miopen/src/solver/conv/conv_wino_rage_RxS.cpp @@ -1,28 +1,5 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2025 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #define CONV_WINO_RAGE_RXS_CPP @@ -32,8 +9,12 @@ #include #include #include +#include #include #include +#include + +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3) namespace miopen { @@ -44,10 +25,259 @@ using WinoShaderArgs = miopen::WinoShaderArgsV2; namespace { -// Divide two non-negative integers and return ceil of the quotient -constexpr uint64_t DivCeil(uint64_t numer, uint64_t denom) { return (numer + denom - 1) / denom; } +struct PerfModelResult +{ + float wti; + float granularity_loss; + uint32_t n_groups; + uint64_t predicted_clk; +}; + +struct PerfModelCost +{ + uint64_t const_cost; + uint64_t fe_cost; + uint64_t ph_cost; + uint64_t be_cost; +}; + +struct PerfModelParams +{ + uint64_t mac_rate; // MAC operations per CU per clock + PerfModelCost cost; // Cost of internal shader operations +}; + +class ShaderModel +{ +protected: + const WinoShaderArgsV2 args; + const uint32_t cu_count; + PerfModelParams model_params; + +public: + ShaderModel(const WinoShaderArgsV2& shader_args, uint32_t cu_cnt) + : args(shader_args), cu_count(cu_cnt) + { + } + + virtual ~ShaderModel() = default; + + PerfModelResult ComputeWti(bool is_fp32) const + { + PerfModelResult out{}; + constexpr uint32_t max_dispatches = 8; + + for(uint32_t n_dispatches = 1; n_dispatches <= max_dispatches; n_dispatches++) + { + const uint64_t n_groups = static_cast(cu_count) * n_dispatches; + if(!IsShaderConstraintsMet(n_groups)) + continue; + + PerfModelResult prediction = PerfPrediction(n_groups, is_fp32); + + if(prediction.wti > out.wti) + out = prediction; + } + + return out; + } + + virtual bool IsShaderConstraintsMet(const uint64_t n_groups) const + { + // clang-format off + return args.dimsFit16bit() + && args.batchTensorSizesFit31bits() + && args.paddedSizesFit16bits() + && n_groups < WinoShaderArgsV2::PowOf2<16>() + && DivCeil(args.Kg, 32) <= n_groups; + // clang-format on + } + +protected: + // Divide two non-negative integers and return ceil of the quotient + uint64_t DivCeil(uint64_t numer, uint64_t denom) const { return (numer + denom - 1) / denom; } + uint64_t RoundUpToMultiple(uint64_t val, uint64_t mul) const { return DivCeil(val, mul) * mul; } + PerfModelResult PerfPrediction(const uint64_t n_groups, bool is_fp32) const + { + PerfModelResult out{}; + + constexpr uint64_t t_R = 3; + constexpr uint64_t t_S = 3; + constexpr uint64_t t_oH = 2; + constexpr uint64_t t_oW = 2; + + constexpr uint64_t nhw_factor = 62; + constexpr uint64_t k_factor = 32; + const uint64_t c_factor = is_fp32 ? 8 : 16; + const uint64_t nhw_factor_g = RoundUpToMultiple(nhw_factor, 32); + const uint64_t nkhw_per_work = k_factor * nhw_factor_g * t_oH * t_oW; + + const uint64_t Rg = RoundUpToMultiple(args.R, t_R); + const uint64_t Sg = RoundUpToMultiple(args.S, t_S); + const uint64_t Cg = RoundUpToMultiple(args.Cg, c_factor); + const uint64_t Kg = RoundUpToMultiple(args.Kg, k_factor); + const uint64_t oHg = RoundUpToMultiple(args.out_h, t_oH); + const uint64_t oWg = RoundUpToMultiple(args.out_w, t_oW) + t_oW; + + const uint64_t s_loops = Sg / t_S; + const uint64_t r_loops = Rg / t_R; + const uint64_t c_loops = Cg / c_factor; + const uint64_t k_tiles = Kg / k_factor; + + const uint64_t nhw_tiles = args.N * DivCeil(oHg, t_oH) * DivCeil(oWg, t_oW); + + const uint64_t n_groups_e = k_tiles * (n_groups / k_tiles); + const uint64_t n_dispatches = DivCeil(args.G * n_groups_e, cu_count); + + const uint64_t n_works = k_tiles * DivCeil(nhw_tiles, nhw_factor); + const uint64_t n_works_per_cu = DivCeil(n_works, n_groups_e); -constexpr uint64_t maxNGroups = WinoShaderArgs::PowOf2<16>() - 1; + const uint64_t macsg = + n_dispatches * n_works_per_cu * cu_count * nkhw_per_work * nhw_factor_g * Cg * Rg * Sg; + const uint64_t macs = static_cast(args.N) * args.G * Kg * Cg * args.out_h * + args.R * args.out_w * args.S; + + out.granularity_loss = static_cast(macsg - macs) / macsg; + + const uint64_t n_consts = n_dispatches; + const uint64_t fe_calls = n_dispatches * n_works_per_cu * s_loops * r_loops; + const uint64_t ph_calls = n_dispatches * n_works_per_cu * s_loops * r_loops * c_loops; + const uint64_t be_calls = n_dispatches * n_works_per_cu; + + const uint64_t const_cost = model_params.cost.const_cost; + const uint64_t fe_cost = model_params.cost.fe_cost; + const uint64_t ph_cost = model_params.cost.ph_cost; + const uint64_t be_cost = model_params.cost.be_cost; + + out.predicted_clk = + n_consts * const_cost + fe_calls * fe_cost + ph_calls * ph_cost + be_calls * be_cost; + + const float ideal_direct_clk = static_cast(macs) / model_params.mac_rate / cu_count; + + out.wti = ideal_direct_clk / out.predicted_clk; + out.n_groups = n_groups; + + return out; + } +}; + +class ShaderModelV4_6 : public ShaderModel +{ +public: + ShaderModelV4_6(const WinoShaderArgsV2& shader_args, + uint32_t cu_cnt, + const PerfModelParams& perf_params) + : ShaderModel(shader_args, cu_cnt) + { + model_params = perf_params; + } + + bool IsShaderConstraintsMet(const uint64_t n_groups) const override + { + return ShaderModel::IsShaderConstraintsMet(n_groups) && args.R_S_fit3x3(); + } +}; + +class ShaderModelV4_9 : public ShaderModel +{ +public: + ShaderModelV4_9(const WinoShaderArgsV2& shader_args, + uint32_t cu_cnt, + const PerfModelParams& perf_params) + : ShaderModel(shader_args, cu_cnt) + { + model_params = perf_params; + } +}; + +class ShaderModelFactory +{ +public: + // We keep two kernel versions because V4_6 delivers better performance + // on both gfx12x(+4%) and gfx942(+6%) for cases where input channels <= 16. + enum class KernelVersion + { + V4_6, // 3x3 filters, FP16 only + V4_9 // supports 3x3 and other filter sizes, FP16/FP32/BF16 + }; + + // Performance model parameters for different hardware architectures and data types + // Format: {mac_rate, {const_cost, fe_cost, ph_cost, be_cost}} + struct PerfParams + { + // clang-format off + static constexpr PerfModelParams GFX942_V4_9_fp16{1024, {22724, 512, 1372, 2244}}; + static constexpr PerfModelParams GFX942_V4_9_bf16{1024, {22724, 512, 1660, 2656}}; + static constexpr PerfModelParams GFX942_V4_9_fp32{128, {26044, 512, 2468, 2504}}; + static constexpr PerfModelParams GFX942_V4_6{1024, {22850, 244, 1396, 2244}}; + + static constexpr PerfModelParams GFX12_V4_9{512, {9740, 182, 1506, 1533}}; + static constexpr PerfModelParams GFX12_V4_6{512, {9505, 79, 1522, 1533}}; + // clang-format on + }; + + static KernelVersion DetermineKernelVersion(const WinoShaderArgsV2& args, bool is_fp16) + { + if(args.R_S_fit3x3() && is_fp16) + { + return KernelVersion::V4_6; + } + else + { + return KernelVersion::V4_9; + } + } + + // Get performance parameters for specific hardware + static PerfModelParams GetPerfParams(const std::string& dev_name, + KernelVersion kernel_version, + const ProblemDescription& problem) + { + if(StartsWith(dev_name, "gfx942")) + { + return (kernel_version == KernelVersion::V4_6) ? PerfParams::GFX942_V4_6 + : (problem.IsFp16()) ? PerfParams::GFX942_V4_9_fp16 + : (problem.IsBfp16()) ? PerfParams::GFX942_V4_9_bf16 + : PerfParams::GFX942_V4_9_fp32; + } + else if(StartsWith(dev_name, "gfx12")) + { + return (kernel_version == KernelVersion::V4_6) ? PerfParams::GFX12_V4_6 + : PerfParams::GFX12_V4_9; + } + else + { + MIOPEN_THROW(miopenStatusInternalError, "Unsupported device architecture: " + dev_name); + } + } + + static std::unique_ptr Create(const std::string& dev_name, + const WinoShaderArgsV2& args, + uint32_t cu_count, + const ProblemDescription& problem, + KernelVersion kernel_version) + { + auto perf_params = GetPerfParams(dev_name, kernel_version, problem); + + if(kernel_version == KernelVersion::V4_6) + { + return std::make_unique(args, cu_count, perf_params); + } + else // V4_9 + { + return std::make_unique(args, cu_count, perf_params); + } + } + + static std::unique_ptr Create(const std::string& dev_name, + const WinoShaderArgsV2& args, + uint32_t cu_count, + const ProblemDescription& problem) + { + auto kernel_version = DetermineKernelVersion(args, problem.IsFp16()); + return Create(dev_name, args, cu_count, problem, kernel_version); + } +}; template struct ConvWinoRageRxSCommon @@ -61,9 +291,11 @@ struct ConvWinoRageRxSCommon miopenActivationMode_t activ_mode = miopenActivationPASTHRU); private: - static int64_t getNGroups(const ExecutionContext& ctx) + static int64_t getMaxNGroups(const ExecutionContext& ctx) { - return std::min(ctx.GetStream().GetMaxHardwareComputeUnits(), maxNGroups); + // return max number of groups that ShaderModel considers + // see ShaderMode::ComputeWti() + return ctx.GetStream().GetMaxHardwareComputeUnits() * 8; } }; @@ -71,18 +303,34 @@ template bool ConvWinoRageRxSCommon::IsApplicable(const ExecutionContext& ctx, const ProblemDescription& problem) { + if(env::disabled(MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3)) + return false; if(!ctx.use_asm_kernels) return false; if(problem.IsTensorsCasted()) return false; - if(!problem.IsFp16()) - return false; if(problem.HasNonPackedTensors()) return false; + WinoShaderArgs args; + if(!args.SetConvParams(problem)) + return false; + const auto devName = ctx.GetStream().GetDeviceName(); - if(!(devName == "gfx942")) + if(devName == "gfx942") + { + if(!(problem.IsFp16() || problem.IsFp32() || problem.IsBfp16())) + return false; + } + else if(StartsWith(devName, "gfx12")) + { + if(!(problem.IsFp16())) + return false; + } + else + { return false; + } const auto& target = ctx.GetStream().GetTargetProperties(); if(target.isXnackEnabled()) @@ -93,19 +341,11 @@ bool ConvWinoRageRxSCommon::IsApplicable(const ExecutionCo if(!(problem.GetDilationH() == 1 && problem.GetDilationW() == 1)) return false; - WinoShaderArgs args; - if(!args.SetConvParams(problem)) - return false; - - args.n_groups = getNGroups(ctx); + args.n_groups = getMaxNGroups(ctx); + auto shader_model = ShaderModelFactory::Create( + devName, args, ctx.GetStream().GetMaxHardwareComputeUnits(), problem); - // clang-format off - return args.dimsFit16bit() - && args.R_S_fit16bit() - && args.batchTensorSizesFit31bits() - && args.paddedSizesFit16bits() - && DivCeil(args.Kg, 32) <= args.n_groups; - // clang-format on + return shader_model->IsShaderConstraintsMet(args.n_groups); } template @@ -114,7 +354,22 @@ float ConvWinoRageRxSCommon::GetWti(const ExecutionContext { std::ignore = ctx; std::ignore = problem; - return -2.0f; + + const auto dev_name = ctx.GetStream().GetDeviceName(); + const auto cu_count = ctx.GetStream().GetMaxHardwareComputeUnits(); + WinoShaderArgsV2 args; + + // Main convolution parameters + if(!args.SetConvParams(problem)) + { + MIOPEN_THROW(miopenStatusInternalError); + } + + auto shader_model = ShaderModelFactory::Create(dev_name, args, cu_count, problem); + + auto result = shader_model->ComputeWti(problem.IsFp32()); + + return result.wti; } template @@ -126,7 +381,6 @@ ConvWinoRageRxSCommon::GetSolution(const ExecutionContext& miopenActivationMode_t activ_mode) { // Kernel args - WinoShaderArgsV2 args; if(!args.SetConvParams(problem)) { @@ -146,25 +400,40 @@ ConvWinoRageRxSCommon::GetSolution(const ExecutionContext& if(do_bias) flags |= WinoShaderFlagsV2::F_BIAS; - auto nGroups = getNGroups(ctx); + const auto devName = ctx.GetStream().GetDeviceName(); + auto kernel_version = ShaderModelFactory::DetermineKernelVersion(args, problem.IsFp16()); + auto shader_model = ShaderModelFactory::Create( + devName, args, ctx.GetStream().GetMaxHardwareComputeUnits(), problem, kernel_version); + auto perfmodel_result = shader_model->ComputeWti(problem.IsFp32()); + auto nGroups = perfmodel_result.n_groups; + args.SetShaderParams(nGroups, flags, 0, 0); // Kernel name and file - std::string kernelVersion; - if(args.R_S_fit3x3()) + if(kernel_version == ShaderModelFactory::KernelVersion::V4_6) { - kernelVersion = "_v4_6_0"; + kernelVersion = "_v4_6_1"; } - else + else // V4_9 { - kernelVersion = "_v4_7_0"; + kernelVersion = "_v4_9_0"; } std::string kernelName = "miopenSp3AsmConvRage" + kernelVersion; std::string kernelFile = "Conv_Winograd_Rage" + kernelVersion; - /// \todo Add case for gfx12 kernels - kernelName += "_gfx9"; + if(devName == "gfx942") + { + kernelName += "_gfx9"; + } + else if(StartsWith(devName, "gfx12")) + { + kernelName += "_gfx12"; + } + else + { + MIOPEN_THROW(miopenStatusInternalError); + } std::string kernelPostfix; @@ -172,6 +441,14 @@ ConvWinoRageRxSCommon::GetSolution(const ExecutionContext& { kernelPostfix += "_fp16_fp32acc"; } + else if(problem.IsFp32()) + { + kernelPostfix += "_fp32_fp32acc"; + } + else if(problem.IsBfp16()) + { + kernelPostfix += "_bf16_fp32acc"; + } else { MIOPEN_THROW(miopenStatusInternalError); @@ -203,10 +480,13 @@ ConvWinoRageRxSCommon::GetSolution(const ExecutionContext& {"ROCM_METADATA_VERSION", 5}, }; kernelInfo.comp_options = options.GenerateFor(kbp::GcnAsm{}); - kernelInfo.comp_options += std::string(" -mcumode -mwavefrontsize64"); + kernelInfo.comp_options += std::string(" -mcumode"); - /// \todo Add case for gfx12 wgp size of 384U - uint64_t wgSize = 768U; + uint64_t wgSize = 768U; // value for gfx942 + if(StartsWith(devName, "gfx12")) + { + wgSize = 384U; + } kernelInfo.l_wk.push_back(wgSize); kernelInfo.l_wk.push_back(1); @@ -220,7 +500,6 @@ ConvWinoRageRxSCommon::GetSolution(const ExecutionContext& kernelInfo.kernel_name = kernelName; // Solution - ConvSolution result; result.construction_params.push_back(kernelInfo); result.invoker_factory = @@ -257,94 +536,10 @@ ConvWinoRageRxS::GetSolution(const ExecutionContext& ctx, } template struct MIOPEN_INTERNALS_EXPORT ConvWinoRageRxS<2, 3>; +template struct MIOPEN_INTERNALS_EXPORT TransposedConvWinoRageRxS<2, 3>; } // namespace conv -namespace fusion { - -template -bool ConvWinoRageRxSFused::IsApplicable( - const FusionContext& ctx, const FusionDescription& problem) const -{ - const auto& desc = *problem.fusion_plan_desc; - - if(desc.op_map.empty()) - { - MIOPEN_THROW(miopenStatusInternalError); - } - - if(desc.op_map.size() > 3) - return false; - if(desc.op_map[0]->kind() != miopenFusionOpConvForward) - return false; - if(desc.op_map.size() == 2) - { - const auto prim = desc.op_map[1]->kind(); - if(!(prim == miopenFusionOpBiasForward || prim == miopenFusionOpActivForward)) - return false; - } - if(desc.op_map.size() == 3) - { - if(desc.op_map[1]->kind() != miopenFusionOpBiasForward) - return false; - if(desc.op_map[2]->kind() != miopenFusionOpActivForward) - return false; - } - - const int activ_idx = GetOpIdx(desc.op_map, miopenFusionOpActivForward); - if(activ_idx != -1) - { - const auto& activ_op = dynamic_cast(*desc.op_map[activ_idx]); - switch(activ_op.activMode) - { - case miopenActivationPASTHRU: - case miopenActivationLOGISTIC: - case miopenActivationTANH: - case miopenActivationRELU: - case miopenActivationLEAKYRELU: break; - default: return false; - } - } - - const auto conv_problem = problem.GetConvProblem(0, miopen::conv::Direction::Forward); - return ConvWinoRageRxSCommon::IsApplicable(ctx, conv_problem); -} - -template -float ConvWinoRageRxSFused::GetWti(const FusionContext& ctx, - const FusionDescription& problem) const -{ - const auto conv_problem = problem.GetConvProblem(0, miopen::conv::Direction::Forward); - return ConvWinoRageRxSCommon::GetWti(ctx, conv_problem); -} - -template -ConvSolution -ConvWinoRageRxSFused::GetSolution(const FusionContext& ctx, - const FusionDescription& problem) const -{ - const auto& desc = *problem.fusion_plan_desc; - const int bias_idx = GetOpIdx(desc.op_map, miopenFusionOpBiasForward); - const int activ_idx = GetOpIdx(desc.op_map, miopenFusionOpActivForward); - - const auto conv_problem = problem.GetConvProblem(0, miopen::conv::Direction::Forward); - - const bool do_bias = (bias_idx != -1); - auto activ_mode = miopenActivationPASTHRU; - if(activ_idx != -1) - { - const auto& activ_op = dynamic_cast(*desc.op_map[activ_idx]); - activ_mode = activ_op.activMode; - } - - return ConvWinoRageRxSCommon::GetSolution( - ctx, conv_problem, true, do_bias, activ_mode); -} - -template struct MIOPEN_INTERNALS_EXPORT ConvWinoRageRxSFused<2, 3>; - -} // namespace fusion - } // namespace solver } // namespace miopen diff --git a/projects/miopen/src/solver/conv/gemm.cpp b/projects/miopen/src/solver/conv/gemm.cpp index 997fac659e05..4e403b9593f9 100644 --- a/projects/miopen/src/solver/conv/gemm.cpp +++ b/projects/miopen/src/solver/conv/gemm.cpp @@ -141,6 +141,13 @@ float GemmFwdBase::GetWti(const ExecutionContext&, const ProblemDescription& pro if(wDesc.GetType() == miopenInt8 && yDesc.GetType() != miopenInt32) n_CastTensor = 1; } + // 3D point-output fwd path with stride==filter can run one strided-batched GEMM. + else if(miopen::conv::IsFwdDataPointOutput3dStrideEqFilter(problem) && + wDesc.GetType() != miopenInt8) + { + n_gemm_runs = 1; + n_gemm_strided_batched = static_cast(in_n); + } else // not 1x1 { n_Im2ColGPU = in_n; @@ -809,6 +816,9 @@ size_t GemmFwdRest::GetWorkspaceSize(const ExecutionContext& context, decltype(auto) wDesc = problem.GetWeights(); decltype(auto) yDesc = problem.GetOut(); + if(miopen::conv::IsFwdDataPointOutput3dStrideEqFilter(problem) && wDesc.GetType() != miopenInt8) + return 0; + const auto spatial_dim = conv.GetSpatialDimension(); const auto wei_spatial = wDesc.GetLengths() | std::views::drop(2) | std::views::take(spatial_dim); @@ -857,6 +867,10 @@ bool GemmFwdRest::IsApplicable(const ExecutionContext& context, if(GemmFwd1x1_0_2{}.IsApplicable(context, problem)) return false; + if(miopen::conv::IsFwdDataPointOutput3dStrideEqFilter(problem) && + problem.GetWeights().GetType() != miopenInt8) + return true; + return GetWorkspaceSize(context, problem) > 0; #else std::ignore = context; @@ -881,10 +895,71 @@ ConvSolution GemmFwdRest::GetSolution(const ExecutionContext& context, const auto spatial_dim = conv.GetSpatialDimension(); const auto workspace_req = GetWorkspaceSize(context, problem); + const auto use_batched_fwd_point_output = + miopen::conv::IsFwdDataPointOutput3dStrideEqFilter(problem) && + wDesc.GetType() != miopenInt8; auto solution = ConvSolution{miopenStatusSuccess}; solution.workspace_sz = workspace_req; + if(use_batched_fwd_point_output) + { + const auto tmp_gemm_desc = [&]() { + auto tmp = CreateGemmDescriptorConvFwd(wDesc, xDesc, yDesc); + tmp.deterministic = problem.GetConv().attribute.deterministic; + tmp.conv_attributes = problem.GetConv().attribute; + if(problem.IsTensorsCasted()) + { + if(xDesc.GetCastType()) + tmp.a_cast_type = *wDesc.GetCastType(); + if(wDesc.GetCastType()) + tmp.b_cast_type = *xDesc.GetCastType(); + } + return tmp; + }(); + + solution.invoker_factory = [=](const std::vector&) { + MIOPEN_LOG_FUNCTION("convolution, non 1x1 point-output 3d"); + return [=](const Handle& handle, const AnyInvokeParams& primitive_params) { + const auto& conv_params = primitive_params.CastTo(); + const auto x = conv_params.tensors.in; + const auto w = conv_params.tensors.w; + const auto y = conv_params.tensors.out; + + auto gemm_desc = [&]() { + auto tmp = tmp_gemm_desc; + tmp.gfx90a_alt_impl = conv_params.gfx90aFp16alt; + return tmp; + }(); + + constexpr auto batched_backend = +#if MIOPEN_USE_HIPBLASLT + GemmBackend_t::hipblaslt; +#else + GemmBackend_t::rocblas; +#endif + + // C[N, K] = A[N, CZYX] * B^T[CZYX, K], where B stores W[K, CZYX]. + gemm_desc.batch_count = 1; + gemm_desc.strideA = 0; + gemm_desc.strideB = 0; + gemm_desc.strideC = 0; + gemm_desc.m = static_cast(in_n); + gemm_desc.n = static_cast(wei_k); + gemm_desc.transA = false; + gemm_desc.transB = true; + gemm_desc.lda = gemm_desc.k; + gemm_desc.ldb = gemm_desc.k; + gemm_desc.ldc = gemm_desc.n; + const auto gemm_status = + CallGemm(handle, gemm_desc, x, 0, w, 0, y, 0, batched_backend); + if(gemm_status != miopenStatusSuccess) + MIOPEN_THROW("GEMM execution failure"); + }; + }; + return solution; + } + solution.invoker_factory = [=](const std::vector&) { const auto tmp_gemm_desc = [&]() { auto tmp = conv.group_count > 1 diff --git a/projects/miopen/src/solver/conv/gemm_bwd.cpp b/projects/miopen/src/solver/conv/gemm_bwd.cpp index 03c05680b57a..371b3ceac637 100644 --- a/projects/miopen/src/solver/conv/gemm_bwd.cpp +++ b/projects/miopen/src/solver/conv/gemm_bwd.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,9 @@ float GemmBwdBase::GetWti(const ExecutionContext&, const ProblemDescription& pro const auto& wDesc = problem.GetWeights(); const auto& dxDesc = problem.GetOut(); + const auto prefer_point_output_shape = + miopen::conv::IsBwdDataPointOutput3dStrideEqFilter(problem); + int n_SetTensor = 0; int n_transpose_NCHW2CNHW = 0; int n_transpose_CNHW2NCHW = 0; @@ -118,9 +122,21 @@ float GemmBwdBase::GetWti(const ExecutionContext&, const ProblemDescription& pro // if not 1x1 else { - n_gemm_strided_batched = conv.group_count; - n_gemm_runs = in_n; - n_Col2ImGPU = in_n; + if(prefer_point_output_shape) + { + // Keep the WTI model aligned with the 3D execution path: + // one strided-batched GEMM + one batched Col2Im launch. + n_gemm_strided_batched = in_n; + n_gemm_runs = 1; + n_Col2ImGPU = 1; + } + else + { + // Preserve the previous non-3D behavior in WTI estimation. + n_gemm_strided_batched = conv.group_count; + n_gemm_runs = in_n; + n_Col2ImGPU = in_n; + } } auto wti = 1.0; @@ -130,6 +146,8 @@ float GemmBwdBase::GetWti(const ExecutionContext&, const ProblemDescription& pro wti *= gemm::SlowdownFactor(n_gemm_runs, 0.9, 0.9); wti *= gemm::SlowdownFactor(n_gemm_strided_batched, 1.0, 0.95); wti *= gemm::SlowdownFactor(n_Col2ImGPU, 0.4, 0.8); + if(prefer_point_output_shape && wti < 10.0) + wti = 10.0; return wti; } @@ -540,17 +558,23 @@ size_t GemmBwdRest::GetWorkspaceSize(const ExecutionContext& context, dyDesc.GetLengths() | std::views::drop(2) | std::views::take(spatial_dim); const auto wei_c = wDesc.GetLengths()[1]; - - const auto gemm_size = wei_c * - std::accumulate(wei_spatial.begin(), - wei_spatial.end(), - std::size_t(1), - std::multiplies()) * - std::accumulate(out_spatial.begin(), - out_spatial.end(), - std::size_t(1), - std::multiplies()) * - GetTypeSize(dyDesc.GetType()) * conv.group_count; + const auto in_n = dyDesc.GetLengths()[0]; + + auto gemm_size = wei_c * + std::accumulate(wei_spatial.begin(), + wei_spatial.end(), + std::size_t(1), + std::multiplies()) * + std::accumulate(out_spatial.begin(), + out_spatial.end(), + std::size_t(1), + std::multiplies()) * + GetTypeSize(dyDesc.GetType()) * conv.group_count; + + // 3D regular convolution uses one strided-batched GEMM over N, so workspace must hold + // all N GEMM outputs before batched Col2Im. + if(miopen::conv::IsBwdDataPointOutput3dStrideEqFilter(problem)) + gemm_size *= in_n; if(gemm_size > handle.GetMaxMemoryAllocSize()) { @@ -636,6 +660,8 @@ ConvSolution GemmBwdRest::GetSolution(const ExecutionContext& context, const auto in_spatial_size = std::accumulate( in_spatial.begin(), in_spatial.end(), std::size_t(1), std::multiplies()); + const auto wei_spatial_size = std::accumulate( + wei_spatial.begin(), wei_spatial.end(), std::size_t(1), std::multiplies()); const auto workspace_req = GetWorkspaceSize(context, problem); @@ -675,6 +701,76 @@ ConvSolution GemmBwdRest::GetSolution(const ExecutionContext& context, }(); float time_gemm = 0; + + // Specialized 3D path for non-grouped convolutions: + // launch one regular GEMM over N, then one batched Col2Im. + if(miopen::conv::IsBwdDataPointOutput3dStrideEqFilter(problem)) + { + auto single_gemm_desc = gemm_desc; + single_gemm_desc.batch_count = 1; + single_gemm_desc.strideA = 0; + single_gemm_desc.strideB = 0; + single_gemm_desc.strideC = 0; + // C[N, C*Z*Y*X] = DY[N, K] * W[K, C*Z*Y*X] + single_gemm_desc.m = static_cast(in_n); + single_gemm_desc.n = static_cast(in_c * wei_spatial_size * out_spatial_size); + single_gemm_desc.transA = false; + single_gemm_desc.transB = false; + single_gemm_desc.lda = static_cast(wei_k); + single_gemm_desc.ldb = single_gemm_desc.n; + single_gemm_desc.ldc = single_gemm_desc.n; + + constexpr auto batched_backend = +#if MIOPEN_USE_HIPBLASLT + GemmBackend_t::hipblaslt; +#else + GemmBackend_t::rocblas; +#endif + + const auto gemm_status = + CallGemm(handle, single_gemm_desc, dy, 0, w, 0, workspace, 0, batched_backend); + if(gemm_status != miopenStatusSuccess) + MIOPEN_THROW("GemmBwdRest single GEMM execution failure."); + + if(handle.IsProfilingEnabled()) + time_gemm += handle.GetKernelTime(); + + time_gemm += Col2Im3dGPUBatched(handle, + workspace, + out_spatial[0], + out_spatial[1], + out_spatial[2], + wei_spatial[0], + wei_spatial[1], + wei_spatial[2], + static_cast(pads[0]), + static_cast(pads[1]), + static_cast(pads[2]), + static_cast(strides[0]), + static_cast(strides[1]), + static_cast(strides[2]), + static_cast(dilations[0]), + static_cast(dilations[1]), + static_cast(dilations[2]), + static_cast(in_c), + static_cast(in_spatial[0]), + static_cast(in_spatial[1]), + static_cast(in_spatial[2]), + static_cast(in_n), + static_cast(in_c) * wei_spatial_size * + out_spatial_size, + dx, + static_cast(in_c) * in_spatial_size, + 0, + dyDesc_.GetType()); + if(handle.IsProfilingEnabled()) + { + handle.ResetKernelTime(); + handle.AccumKernelTime(time_gemm); + } + return; + } + for(std::size_t i = 0; i < in_n; i++) { std::size_t out_offset = i * wei_k * out_spatial_size; diff --git a/projects/miopen/src/solver/getitem/backward_getitem.cpp b/projects/miopen/src/solver/getitem/backward_getitem.cpp index 2e5477ffb1d7..fbf11f2785da 100644 --- a/projects/miopen/src/solver/getitem/backward_getitem.cpp +++ b/projects/miopen/src/solver/getitem/backward_getitem.cpp @@ -207,7 +207,7 @@ ConvSolution GetitemBackward::GetSolution(const ExecutionContext& /*context*/, reset_profiling_state = true; start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } build_index_kernel(params.indexs[i], @@ -227,7 +227,7 @@ ConvSolution GetitemBackward::GetSolution(const ExecutionContext& /*context*/, reset_profiling_state = true; start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } decltype(auto) kernel = handle_.Run(kernels[indexCount]); @@ -244,14 +244,14 @@ ConvSolution GetitemBackward::GetSolution(const ExecutionContext& /*context*/, if(reset_profiling_state) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.EnableProfiling(true); }; }; diff --git a/projects/miopen/src/solver/layernorm/backward_layernorm.cpp b/projects/miopen/src/solver/layernorm/backward_layernorm.cpp index 282bfa45d178..6a0a63ee8b8c 100644 --- a/projects/miopen/src/solver/layernorm/backward_layernorm.cpp +++ b/projects/miopen/src/solver/layernorm/backward_layernorm.cpp @@ -261,7 +261,7 @@ ConvSolution LayernormBackward::GetSolution(const ExecutionContext& context, { start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } kernel(params.dy, @@ -283,9 +283,9 @@ ConvSolution LayernormBackward::GetSolution(const ExecutionContext& context, if(handle_.IsProfilingEnabled()) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; @@ -308,7 +308,7 @@ ConvSolution LayernormBackward::GetSolution(const ExecutionContext& context, { start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } kernel(params.dy, @@ -324,9 +324,9 @@ ConvSolution LayernormBackward::GetSolution(const ExecutionContext& context, if(handle_.IsProfilingEnabled()) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/layernorm/backward_t5layernorm.cpp b/projects/miopen/src/solver/layernorm/backward_t5layernorm.cpp index dc777b950c13..2fe0917e350b 100644 --- a/projects/miopen/src/solver/layernorm/backward_t5layernorm.cpp +++ b/projects/miopen/src/solver/layernorm/backward_t5layernorm.cpp @@ -258,7 +258,7 @@ T5LayernormBackward::GetSolution(const ExecutionContext& context, { start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } kernel(params.dy, @@ -281,9 +281,9 @@ T5LayernormBackward::GetSolution(const ExecutionContext& context, if(handle_.IsProfilingEnabled()) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; @@ -306,7 +306,7 @@ T5LayernormBackward::GetSolution(const ExecutionContext& context, { start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } kernel(params.dy, @@ -326,9 +326,9 @@ T5LayernormBackward::GetSolution(const ExecutionContext& context, if(handle_.IsProfilingEnabled()) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/mha/mha_solver_backward.cpp b/projects/miopen/src/solver/mha/mha_solver_backward.cpp index dcd171a267fd..9a7c68362616 100644 --- a/projects/miopen/src/solver/mha/mha_solver_backward.cpp +++ b/projects/miopen/src/solver/mha/mha_solver_backward.cpp @@ -74,7 +74,7 @@ MultiBufferWorkspaceTraits SplitBufferToWorkspace(const std::vector& len miopen::HipEventPtr make_hip_fast_event() { hipEvent_t result = nullptr; - hipEventCreateWithFlags(&result, hipEventDisableTiming); + (void)hipEventCreateWithFlags(&result, hipEventDisableTiming); return miopen::HipEventPtr{result}; } } // namespace @@ -223,13 +223,13 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, auto recordSyncEvent = [&handle_]() { auto event = make_hip_fast_event(); - hipEventRecord(event.get(), handle_.GetStream()); + (void)hipEventRecord(event.get(), handle_.GetStream()); return event; }; auto waitSyncEvent = [&handle_](HipEventPtr&& event) { auto tmp_for_deletion(std::move(event)); - hipStreamWaitEvent(handle_.GetStream(), tmp_for_deletion.get(), 0); + (void)hipStreamWaitEvent(handle_.GetStream(), tmp_for_deletion.get(), 0); }; if(profiling) @@ -237,7 +237,7 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, start = make_hip_event(); stop = make_hip_event(); handle_.EnableProfiling(false); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } void* fp32_QxK_S_ws = getBuffPart(params.GetWorkspace(), 0); @@ -257,8 +257,8 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, dataBwd.dropoutProbabilityData, emb_dim, nhs); - hipMemsetAsync(dataBwd.amaxDSData, 0, sizeof(float), handle_.GetStream()); - hipMemsetAsync(dataBwd.amaxDVData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataBwd.amaxDSData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataBwd.amaxDVData, 0, sizeof(float), handle_.GetStream()); handle_.SetStreamFromPool(1); gemm(handle_, @@ -283,7 +283,7 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, true); HipEventPtr event_QxK = recordSyncEvent(); - hipMemsetAsync(dataBwd.amaxDQData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataBwd.amaxDQData, 0, sizeof(float), handle_.GetStream()); handle_.SetStreamFromPool(2); gemm(handle_, @@ -308,7 +308,7 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, true); HipEventPtr event_dOxV = recordSyncEvent(); - hipMemsetAsync(dataBwd.amaxDKData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataBwd.amaxDKData, 0, sizeof(float), handle_.GetStream()); handle_.SetStreamFromPool(0); waitSyncEvent(std::move(event_QxK)); @@ -439,11 +439,11 @@ ConvSolution MhaBackward::GetSolution(const ExecutionContext& context, if(profiling) { - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); + (void)hipEventSynchronize(stop.get()); float mS = 0; - hipEventElapsedTime(&mS, start.get(), stop.get()); + (void)hipEventElapsedTime(&mS, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(mS); } diff --git a/projects/miopen/src/solver/mha/mha_solver_forward.cpp b/projects/miopen/src/solver/mha/mha_solver_forward.cpp index 2b2a866df597..636bea055602 100644 --- a/projects/miopen/src/solver/mha/mha_solver_forward.cpp +++ b/projects/miopen/src/solver/mha/mha_solver_forward.cpp @@ -179,12 +179,12 @@ ConvSolution MhaForward::GetSolution(const ExecutionContext& context, start = make_hip_event(); stop = make_hip_event(); handle_.EnableProfiling(false); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } // zero amax output data to use atomics - hipMemsetAsync(dataFwd.amaxSData, 0, sizeof(float), handle_.GetStream()); - hipMemsetAsync(dataFwd.amaxOData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataFwd.amaxSData, 0, sizeof(float), handle_.GetStream()); + (void)hipMemsetAsync(dataFwd.amaxOData, 0, sizeof(float), handle_.GetStream()); void* fp32_ws = getBuffPart(params.GetWorkspace(), 0); void* fp8_ws = getBuffPart(params.GetWorkspace(), 1); @@ -258,11 +258,11 @@ ConvSolution MhaForward::GetSolution(const ExecutionContext& context, if(profiling) { - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); + (void)hipEventSynchronize(stop.get()); float mS = 0; - hipEventElapsedTime(&mS, start.get(), stop.get()); + (void)hipEventElapsedTime(&mS, start.get(), stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(mS); } diff --git a/projects/miopen/src/solver/multimarginloss/forward_multimarginloss.cpp b/projects/miopen/src/solver/multimarginloss/forward_multimarginloss.cpp index c475ae6d05e9..0389773b5a10 100644 --- a/projects/miopen/src/solver/multimarginloss/forward_multimarginloss.cpp +++ b/projects/miopen/src/solver/multimarginloss/forward_multimarginloss.cpp @@ -241,7 +241,7 @@ ConvSolution MultiMarginLossForward::GetSolution( handle_.EnableProfiling(false); start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } /* Phase 1: Calc loss for each element. */ { @@ -269,7 +269,7 @@ ConvSolution MultiMarginLossForward::GetSolution( wt.GetOffset(1)); int kernelCnt = 1; - for(kernelCnt; kernelCnt < kernels.size() - 1; kernelCnt++) + for(; kernelCnt < kernels.size() - 1; kernelCnt++) { decltype(auto) kernel = handle_.Run(kernels[kernelCnt]); kernel(reduce_in, reduce_out, size); @@ -282,13 +282,13 @@ ConvSolution MultiMarginLossForward::GetSolution( if(profiling) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); // Clean up - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); diff --git a/projects/miopen/src/solver/prelu/backward_prelu_multi_weights.cpp b/projects/miopen/src/solver/prelu/backward_prelu_multi_weights.cpp index c8137bee2e5b..e5fbac978ce4 100644 --- a/projects/miopen/src/solver/prelu/backward_prelu_multi_weights.cpp +++ b/projects/miopen/src/solver/prelu/backward_prelu_multi_weights.cpp @@ -106,10 +106,10 @@ MultiWeightsBackward::GetSolution(const ExecutionContext& context, if(profiling) { handle_.EnableProfiling(false); - hipStreamSynchronize(handle_.GetStream()); + (void)hipStreamSynchronize(handle_.GetStream()); start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } int kernelCnt = 0; @@ -152,13 +152,13 @@ MultiWeightsBackward::GetSolution(const ExecutionContext& context, if(profiling) { float elapsed = 0.0f; - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); // Clean up - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/prelu/backward_prelu_single_weight.cpp b/projects/miopen/src/solver/prelu/backward_prelu_single_weight.cpp index 5d9b0bb00ac7..87e994759295 100644 --- a/projects/miopen/src/solver/prelu/backward_prelu_single_weight.cpp +++ b/projects/miopen/src/solver/prelu/backward_prelu_single_weight.cpp @@ -128,10 +128,10 @@ SingleWeightBackward::GetSolution(const ExecutionContext& /*context*/, if(profiling) { handle_.EnableProfiling(false); - hipStreamSynchronize(handle_.GetStream()); + (void)hipStreamSynchronize(handle_.GetStream()); start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } auto work_a = getBuffPart(params.GetWorkspace(), 0); @@ -175,13 +175,13 @@ SingleWeightBackward::GetSolution(const ExecutionContext& /*context*/, if(profiling) { float elapsed = 0.0f; - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); // Clean up - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/reduce/forward_prod.cpp b/projects/miopen/src/solver/reduce/forward_prod.cpp index 1ae4145818f0..dee8653597a9 100644 --- a/projects/miopen/src/solver/reduce/forward_prod.cpp +++ b/projects/miopen/src/solver/reduce/forward_prod.cpp @@ -188,7 +188,7 @@ ProdForward::GetSolution(const ExecutionContext& context, reset_profiling_state = true; start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } parallel_kernel(params.x, @@ -208,12 +208,12 @@ ProdForward::GetSolution(const ExecutionContext& context, if(reset_profiling_state) { - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/reduce/forward_sum.cpp b/projects/miopen/src/solver/reduce/forward_sum.cpp index de99296dcf6b..19575cc291ab 100644 --- a/projects/miopen/src/solver/reduce/forward_sum.cpp +++ b/projects/miopen/src/solver/reduce/forward_sum.cpp @@ -188,7 +188,7 @@ SumForward::GetSolution(const ExecutionContext& context, reset_profiling_state = true; start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } parallel_kernel(params.x, @@ -208,12 +208,12 @@ SumForward::GetSolution(const ExecutionContext& context, if(reset_profiling_state) { - hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); handle_.EnableProfiling(true); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); }; diff --git a/projects/miopen/src/solver/softmarginloss/forward_softmarginloss.cpp b/projects/miopen/src/solver/softmarginloss/forward_softmarginloss.cpp index cf0d72e9b850..8c464f1b99d8 100644 --- a/projects/miopen/src/solver/softmarginloss/forward_softmarginloss.cpp +++ b/projects/miopen/src/solver/softmarginloss/forward_softmarginloss.cpp @@ -202,7 +202,7 @@ ConvSolution SoftMarginLossForward::GetSolution( handle_.EnableProfiling(false); start = miopen::make_hip_event(); stop = miopen::make_hip_event(); - hipEventRecord(start.get(), handle_.GetStream()); + (void)hipEventRecord(start.get(), handle_.GetStream()); } /* Phase 1: Calc loss for each element. */ { @@ -227,7 +227,7 @@ ConvSolution SoftMarginLossForward::GetSolution( wt.GetOffset(1)); int kernelCnt = 1; - for(kernelCnt; kernelCnt < kernels.size() - 1; kernelCnt++) + for(; kernelCnt < kernels.size() - 1; kernelCnt++) { decltype(auto) kernel = handle_.Run(kernels[kernelCnt]); kernel(reduce_in, reduce_out, size); @@ -240,13 +240,13 @@ ConvSolution SoftMarginLossForward::GetSolution( if(profiling) { - hipEventRecord(stop.get(), handle_.GetStream()); - hipEventSynchronize(stop.get()); - hipEventElapsedTime(&elapsed, start.get(), stop.get()); + (void)hipEventRecord(stop.get(), handle_.GetStream()); + (void)hipEventSynchronize(stop.get()); + (void)hipEventElapsedTime(&elapsed, start.get(), stop.get()); // Clean up - hipEventDestroy(start.get()); - hipEventDestroy(stop.get()); + (void)hipEventDestroy(start.get()); + (void)hipEventDestroy(stop.get()); handle_.ResetKernelTime(); handle_.AccumKernelTime(elapsed); diff --git a/projects/miopen/src/solver/softmax/softmax.cpp b/projects/miopen/src/solver/softmax/softmax.cpp index 1e1774e1ab0f..86bd0f2f313c 100644 --- a/projects/miopen/src/solver/softmax/softmax.cpp +++ b/projects/miopen/src/solver/softmax/softmax.cpp @@ -1,6 +1,9 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#include "miopen/execution_context.hpp" +#include "miopen/miopen.h" +#include "miopen/softmax/problem_description.hpp" #include #include @@ -11,12 +14,10 @@ #include #include -#define LOCAL_SIZE 256 - namespace miopen { namespace { -constexpr int nextPow2(int v) +constexpr uint64_t nextPow2(uint64_t v) { if(v == 1) { @@ -30,6 +31,7 @@ constexpr int nextPow2(int v) v |= v >> 4; v |= v >> 8; v |= v >> 16; + v |= v >> 32; v++; return v; } @@ -89,8 +91,27 @@ bool Softmax::IsApplicable( return true; } +PerformanceConfigSoftmax +Softmax::GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::softmax::ProblemDescription& problem) const +{ + PerformanceConfigSoftmax config; + config.HeuristicInit(problem); + config.local_size = PerformanceConfigSoftmax::default_local_size; + MIOPEN_LOG_I(config.ToString()); + return config; +} + +bool Softmax::IsValidPerformanceConfig(const ExecutionContext& context, + const miopen::softmax::ProblemDescription& problem, + const PerformanceConfigSoftmax& config) const +{ + return config.IsValid(context, problem); +} + ConvSolution Softmax::GetSolution([[maybe_unused]] const ExecutionContext& context, - const miopen::softmax::ProblemDescription& problem) const + const miopen::softmax::ProblemDescription& problem, + const PerformanceConfigSoftmax& config) const { auto result = ConvSolution{miopenStatusSuccess}; @@ -106,14 +127,13 @@ ConvSolution Softmax::GetSolution([[maybe_unused]] const ExecutionContext& conte auto spatial_dim = mode == MIOPEN_SOFTMAX_MODE_INSTANCE ? 1 : lengths[2] * lengths[3]; auto vector_size = mode == MIOPEN_SOFTMAX_MODE_INSTANCE ? lengths[1] * lengths[2] * lengths[3] : lengths[1]; - auto num_batch = vector_size < LOCAL_SIZE ? nextPow2(LOCAL_SIZE / vector_size) : 1; - auto workgroups = num_batch == 1 ? grid_size - : grid_size % num_batch == 0 ? grid_size / num_batch - : grid_size / num_batch + 1; - auto batch_size = LOCAL_SIZE / num_batch; + auto num_batch = + vector_size < config.local_size ? nextPow2(config.local_size / vector_size) : 1; + auto workgroups = num_batch == 1 ? grid_size : (grid_size + num_batch - 1) / num_batch; + auto batch_size = config.local_size / num_batch; auto u_batch_size = vector_size > batch_size ? nextPow2(vector_size / batch_size) : 1; - size_t xlocalsize = LOCAL_SIZE; + size_t xlocalsize = config.local_size; size_t xgridsize = workgroups * xlocalsize; size_t ylocalsize = 1; size_t ygridsize = 1; @@ -125,34 +145,34 @@ ConvSolution Softmax::GetSolution([[maybe_unused]] const ExecutionContext& conte kernel.kernel_file = "MIOpenSoftmax.cpp"; kernel.kernel_name = problem.IsForward() ? "SoftmaxFwd" : "SoftmaxBwd"; - const auto build_params = - KernelBuildParameters{{"MIOPEN_USE_FP16", static_cast(dtype == miopenHalf)}, - {"MIOPEN_USE_FP32", static_cast(dtype == miopenFloat)}, - {"MIOPEN_USE_BFP16", static_cast(dtype == miopenBFloat16)}, - {"DATA_TYPE", data_dtype == "bfloat16" ? "ushort" : data_dtype}, - {"USE_SOFTMAX_FAST", algorithm == MIOPEN_SOFTMAX_FAST}, - {"USE_SOFTMAX_ACCURATE", algorithm == MIOPEN_SOFTMAX_ACCURATE}, - {"USE_SOFTMAX_LOG", algorithm == MIOPEN_SOFTMAX_LOG}, - {"USE_SOFTMAX_MODE_INSTANCE", mode == MIOPEN_SOFTMAX_MODE_INSTANCE}, - {"USE_SOFTMAX_MODE_CHANNEL", mode == MIOPEN_SOFTMAX_MODE_CHANNEL}, - {"HEIGHT", lengths[2]}, - {"WIDTH", lengths[3]}, - {"N_STRIDE", strides[0]}, - {"C_STRIDE", strides[1]}, - {"H_STRIDE", strides[2]}, - {"W_STRIDE", strides[3]}, - {"LOCAL_SIZE", LOCAL_SIZE}, - {"WORKGROUPS", workgroups}, - {"GRID_SIZE", grid_size}, - {"SPATIAL_DIM", spatial_dim}, - {"VECTOR_SIZE", vector_size}, - {"NUM_BATCH", num_batch}, - {"BATCH_SIZE", batch_size}, - {"U_BATCH_SIZE", u_batch_size}, - {"IS_INPUT_CONTIGUOUS", problem.GetXDesc().IsContiguous()}, - {"IS_OUTPUT_CONTIGUOUS", problem.GetYDesc().IsContiguous()}, - {"IS_DINPUT_CONTIGUOUS", problem.GetdXDesc().IsContiguous()}, - {"IS_DOUTPUT_CONTIGUOUS", problem.GetdYDesc().IsContiguous()}}; + const auto build_params = KernelBuildParameters{ + {"MIOPEN_USE_FP16", static_cast(dtype == miopenHalf)}, + {"MIOPEN_USE_FP32", static_cast(dtype == miopenFloat)}, + {"MIOPEN_USE_BFP16", static_cast(dtype == miopenBFloat16)}, + {"DATA_TYPE", data_dtype == "bfloat16" ? "ushort" : data_dtype}, + {"USE_SOFTMAX_FAST", algorithm == MIOPEN_SOFTMAX_FAST}, + {"USE_SOFTMAX_ACCURATE", algorithm == MIOPEN_SOFTMAX_ACCURATE}, + {"USE_SOFTMAX_LOG", algorithm == MIOPEN_SOFTMAX_LOG}, + {"USE_SOFTMAX_MODE_INSTANCE", mode == MIOPEN_SOFTMAX_MODE_INSTANCE}, + {"USE_SOFTMAX_MODE_CHANNEL", mode == MIOPEN_SOFTMAX_MODE_CHANNEL}, + {"HEIGHT", lengths[2]}, + {"WIDTH", lengths[3]}, + {"N_STRIDE", strides[0]}, + {"C_STRIDE", strides[1]}, + {"H_STRIDE", strides[2]}, + {"W_STRIDE", strides[3]}, + {"LOCAL_SIZE", config.local_size}, + {"WORKGROUPS", workgroups}, + {"GRID_SIZE", grid_size}, + {"SPATIAL_DIM", spatial_dim}, + {"VECTOR_SIZE", vector_size}, + {"NUM_BATCH", num_batch}, + {"BATCH_SIZE", batch_size}, + {"U_BATCH_SIZE", u_batch_size}, + {"IS_INPUT_CONTIGUOUS", problem.IsForward() && problem.GetXDesc().IsContiguous()}, + {"IS_OUTPUT_CONTIGUOUS", problem.GetYDesc().IsContiguous()}, + {"IS_DINPUT_CONTIGUOUS", !problem.IsForward() && problem.GetdXDesc().IsContiguous()}, + {"IS_DOUTPUT_CONTIGUOUS", !problem.IsForward() && problem.GetdYDesc().IsContiguous()}}; kernel.comp_options = build_params.GenerateFor(kbp::HIP{}); @@ -204,6 +224,80 @@ ConvSolution Softmax::GetSolution([[maybe_unused]] const ExecutionContext& conte return result; } +void PerformanceConfigSoftmax::HeuristicInit(const miopen::softmax::ProblemDescription& problem) +{ +#if !MIOPEN_BACKEND_HIP + std::ignore = problem; +#else + switch(problem.GetYDesc().GetType()) + { + case miopenHalf: + case miopenFloat: + case miopenBFloat16: local_size = PerformanceConfigSoftmax::start_local_size; break; + case miopenDouble: + case miopenFloat8_fnuz: + case miopenBFloat8_fnuz: + case miopenInt8: + case miopenInt32: + case miopenInt64: + default: MIOPEN_THROW("Unsupported datatype"); + } +#endif + initialized = true; +} + +bool PerformanceConfigSoftmax::SetNextValue(const miopen::softmax::ProblemDescription& problem) +{ +#if !MIOPEN_BACKEND_HIP + std::ignore = problem; + return false; +#else + if(!initialized) + { + HeuristicInit(problem); + } + if(local_size <= 0) + { + MIOPEN_THROW(miopenStatusInvalidValue, "Local size zero or negative"); + } + local_size *= 2; + return local_size <= max_local_size; +#endif +} + +bool PerformanceConfigSoftmax::IsValidValue() const +{ + return local_size > 0 && local_size <= max_local_size; +} + +bool PerformanceConfigSoftmax::IsValid(const ExecutionContext&, + const miopen::softmax::ProblemDescription& problem) const +{ +#if !MIOPEN_BACKEND_HIP + std::ignore = problem; + return false; +#else + switch(problem.GetXDesc().GetType()) + { + case miopenHalf: + case miopenFloat: + case miopenBFloat16: return true; + case miopenDouble: + case miopenFloat8_fnuz: + case miopenBFloat8_fnuz: + case miopenInt8: + case miopenInt32: + case miopenInt64: + default: MIOPEN_THROW("Unsupported datatype"); + } +#endif +} + +bool PerformanceConfigSoftmax::operator==(const PerformanceConfigSoftmax& other) const +{ + return local_size == other.local_size; +} + } // namespace softmax } // namespace solver diff --git a/projects/miopen/src/target_properties.cpp b/projects/miopen/src/target_properties.cpp index 21fa3adf8e2d..7dc265be168b 100644 --- a/projects/miopen/src/target_properties.cpp +++ b/projects/miopen/src/target_properties.cpp @@ -57,12 +57,12 @@ static std::string GetDeviceNameFromMap(const std::string& in) if(!dev_str.empty()) return dev_str; - const auto name = in.substr(0, in.find(':')); // str.substr(0, npos) returns str. + std::string name = in.substr(0, in.find(':')); // str.substr(0, npos) returns str. auto match = device_name_map.find(name); if(match != device_name_map.end()) - return match->second; - return name; // NOLINT (performance-no-automatic-move) + name = match->second; + return name; } // See https://github.com/llvm/llvm-project/commit/1ed4caff1d5cd49233c1ae7b9f6483a946ed5eea. diff --git a/projects/miopen/src/tensor.cpp b/projects/miopen/src/tensor.cpp index 399528ca6362..0f522e8cae39 100644 --- a/projects/miopen/src/tensor.cpp +++ b/projects/miopen/src/tensor.cpp @@ -1,28 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + #include #include @@ -37,15 +15,17 @@ #include #include -#include - #include #include +#include #include #include #include +#include #include +#include +#include namespace miopen { @@ -911,11 +891,14 @@ std::string TensorDescriptor::ToString() const std::string result; if(this->lens.empty()) return result; + for(auto i : this->lens) { result += std::to_string(i) + ", "; } - return result.substr(0, result.length() - 2); + result = result.substr(0, result.length() - 2); + + return result; } std::ostream& operator<<(std::ostream& stream, const TensorDescriptor& t) @@ -970,19 +953,24 @@ TensorDescriptor GetFlattenedTensorDescriptor(const TensorDescriptor& desc) std::vector flat_lengths; std::vector flat_strides; - auto non1_length_strides = boost::combine(desc.GetLengths(), desc.GetStrides()) | - boost::adaptors::filtered(f_length_is_not_1_t()); + const auto& length_ = desc.GetLengths(); + const auto& strides_ = desc.GetStrides(); + auto non1_length_strides = std::views::iota(std::size_t(0), length_.size()) | + std::views::transform([&](std::size_t i) { + return std::make_tuple(length_[i], strides_[i]); + }) | + std::views::filter([](const auto& v) { return std::get<0>(v) > 1; }); auto i = non1_length_strides.begin(); - std::size_t flat_len = boost::get<0>(*i); + std::size_t flat_len = std::get<0>(*i); auto i_previous = i++; // the 0-th dimension full-length doesn't matter for(; i != non1_length_strides.end(); ++i) { - std::size_t len = boost::get<0>(*i); - std::size_t stride = boost::get<1>(*i); - std::size_t previous_stride = boost::get<1>(*i_previous); + std::size_t len = std::get<0>(*i); + std::size_t stride = std::get<1>(*i); + std::size_t previous_stride = std::get<1>(*i_previous); std::size_t full_len = previous_stride / stride; if(len == full_len) @@ -998,7 +986,7 @@ TensorDescriptor GetFlattenedTensorDescriptor(const TensorDescriptor& desc) i_previous = i; } flat_lengths.push_back(flat_len); - flat_strides.push_back(boost::get<1>(*i_previous)); + flat_strides.push_back(std::get<1>(*i_previous)); return {desc.GetType(), flat_lengths, flat_strides}; } @@ -1890,10 +1878,9 @@ void TransformTensor(const Handle& handle, } else { - auto x_y_len = boost::combine(x_len, y_len); - bool same_spatial_len = std::all_of(x_y_len.begin(), x_y_len.end(), [](auto v) { - return boost::get<0>(v) == boost::get<1>(v); - }); + bool same_spatial_len = + std::ranges::all_of(std::views::iota(std::size_t(0), x_len.size()), + [&](std::size_t i) { return x_len[i] == y_len[i]; }); if(!same_spatial_len) { diff --git a/projects/miopen/test/CMakeLists.txt b/projects/miopen/test/CMakeLists.txt index 170cbcb875fc..5908ed3dcee3 100755 --- a/projects/miopen/test/CMakeLists.txt +++ b/projects/miopen/test/CMakeLists.txt @@ -215,8 +215,8 @@ else() add_definitions(-DMIOPEN_TEST_DRIVER_MODE=1) endif() -# Exclude *-suite tests added as part of test filter standardization(#3513) from check target to avoid running them during build. -add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --verbose -C ${CMAKE_CFG_INTDIR} --exclude-regex ".*-suite") +# Exclude *_suite tests added as part of test filter standardization(#3513) from check target to avoid running them during build. +add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --verbose -C ${CMAKE_CFG_INTDIR} --exclude-regex ".*_suite") add_custom_target(tests) if(MIOPEN_TEST_GPU_DETECTION_FAILED AND NOT (MIOPEN_NO_GPU)) @@ -305,8 +305,7 @@ endif() if(MIOPEN_NO_GPU) set(SKIP_ALL_EXCEPT_TESTS - test_sqlite_perfdb - test_perfdb) + test_sqlite_perfdb) endif() #TODO WORKAROUND_ISSUE_1424 @@ -506,7 +505,6 @@ endforeach() if(MIOPEN_TEST_ENABLE_TEST_SQLITE_PERFDB) set_tests_properties(test_sqlite_perfdb PROPERTIES RUN_SERIAL On) endif() -set_tests_properties(test_perfdb PROPERTIES RUN_SERIAL On) # add_sanitize_test(perfdb.cpp) # add_sanitize_test(cache.cpp) diff --git a/projects/miopen/test/args.hpp b/projects/miopen/test/args.hpp index fa9cffb562b6..d7a785c7f347 100644 --- a/projects/miopen/test/args.hpp +++ b/projects/miopen/test/args.hpp @@ -184,7 +184,7 @@ auto any_construct_impl(miopen::rank<1>, } template -T any_construct_impl(miopen::rank<0>, std::index_sequence, const Data&) +[[noreturn]] T any_construct_impl(miopen::rank<0>, std::index_sequence, const Data&) { throw std::runtime_error("Cannot construct: " + miopen::get_type_name()); } diff --git a/projects/miopen/test/driver.hpp b/projects/miopen/test/driver.hpp index 1dc8bde662f7..7267bcd633da 100644 --- a/projects/miopen/test/driver.hpp +++ b/projects/miopen/test/driver.hpp @@ -1,28 +1,5 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #ifndef GUARD_MIOPEN_TEST_DRIVER_HPP #define GUARD_MIOPEN_TEST_DRIVER_HPP @@ -31,6 +8,7 @@ #include "get_handle.hpp" #include "network_data.hpp" #include "serialize.hpp" +#include "miopen/stringutils.hpp" #include "tensor_holder.hpp" #include "test.hpp" #include "verify.hpp" @@ -47,8 +25,6 @@ #include #include #include -#include -#include namespace env = miopen::env; @@ -320,9 +296,8 @@ struct test_driver for(auto&& arg : this->arguments) { - std::string value = arg.read_value(); - std::vector value_vector; - boost::split(value_vector, value, boost::is_any_of(" "), boost::token_compress_on); + std::string value = arg.read_value(); + std::vector value_vector = miopen::SplitSpaceSeparated(value); if(not value.empty()) { ret.emplace_back("--" + arg.name); diff --git a/projects/miopen/test/gru_common.hpp b/projects/miopen/test/gru_common.hpp index e796e02825a4..6e047854df57 100644 --- a/projects/miopen/test/gru_common.hpp +++ b/projects/miopen/test/gru_common.hpp @@ -3041,7 +3041,7 @@ struct gru_basic_driver : test_driver clCreateBuffer(ctx, CL_MEM_READ_WRITE, statesSizeInBytes, nullptr, nullptr); #elif MIOPEN_BACKEND_HIP void* dropout_state_buf; - hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); + (void)hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); #endif miopenSetDropoutDescriptor(DropoutDesc, diff --git a/projects/miopen/test/gtest/CMakeLists.txt b/projects/miopen/test/gtest/CMakeLists.txt index cdd80c0614e0..9d08e6be515d 100644 --- a/projects/miopen/test/gtest/CMakeLists.txt +++ b/projects/miopen/test/gtest/CMakeLists.txt @@ -35,6 +35,45 @@ else() list(APPEND SKIP_TESTS ck_builder_xdl.cpp ck_builder_shared.cpp) endif() +# Add files to SKIP_TESTS when ASAN packaging is enabled. These are temporary until a fix is found for +# the hang that occurs with CK kernels. +if(THEROCK_SANITIZER STREQUAL "ASAN") + # Disable tests temporarily due to a bug that cause a hang when using CK kernels while Address Sanitizer is enabled. + list(APPEND SKIP_TESTS bad_fusion_plan.cpp cba_find2_infer.cpp cba_infer.cpp ck_builder_shared.cpp ck_builder_xdl.cpp conv_activ_infer.cpp conv_ai_3d_kernel_tuning_utils.cpp conv_ck_igemm_fwd_v6r1_dlops_nchw.cpp conv_hip_igemm_xdlops.cpp conv_igemm_mlir_xdlops_bwd_wrw.cpp conv_igemm_mlir_xdlops_fwd.cpp find_2_conv.cpp find_db.cpp fused_conv_bias_res_add_activ.cpp graphapi_conv_bias_res_add_activ_fwd.cpp group_conv_deterministic_split_k.cpp group_conv2d_fwd.cpp group_conv2d_bwd.cpp group_conv2d_wrw.cpp group_conv3d_fwd.cpp group_conv3d_bwd.cpp group_conv3d_wrw.cpp find_mode_trust_verify.cpp kernel_tuning_net.cpp miopendriver_conv_immed.cpp miopendriver_conv2d_trans.cpp miopendriver_gemm.cpp miopendriver_regression_big_tensor.cpp miopendriver_regression_big_tensor.cpp miopendriver_regression_half_gfx9.cpp miopendriver_regression_half.cpp nonpack_conv3d_fwd.cpp perf_config_HipImplicitGemm3DGroupFwdXdlops.cpp smoke_solver_ConvCkIgemmFwdV6r1DlopsNchw.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicBwdXdlops.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicFwdXdlops.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicWrwXdlops.cpp unit_conv_solver_ConvCkGroupedConvFwd.cpp unit_conv_solver_ConvHipImplicitGemm3DGroupBwdXdlops.cpp unit_conv_solver_ConvHipImplicitGemm3DGroupFwdXdlops.cpp unit_conv_solver_ConvHipImplicitGemm3DGroupWrwXdlops.cpp unit_conv_solver_ConvHipImplicitGemmBwdDataV1R1Xdlops.cpp unit_conv_solver_ConvHipImplicitGemmBwdDataV4R1Xdlops.cpp unit_conv_solver_ConvHipImplicitGemmFwdXdlops.cpp unit_conv_solver_ConvHipImplicitGemmGroupBwdXdlops.cpp unit_conv_solver_ConvHipImplicitGemmForwardV4R4Xdlops.cpp unit_conv_solver_ConvHipImplicitGemmForwardV4R5Xdlops.cpp unit_conv_solver_ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm.cpp unit_conv_solver_ConvHipImplicitGemmWrwV4R4Xdlops.cpp unit_conv_solver_ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm.cpp unit_conv_solver_ConvHipImplicitGemmBwdXdlops.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC.cpp unit_conv_solver_ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC.cpp unit_conv_solver_ConvHipImplicitGemmGroupFwdXdlops.cpp unit_conv_solver_ConvHipImplicitGemmGroupWrwXdlops.cpp unit_implicitgemm_ck_util.cpp smoke_solver_ConvAsmImplicitGemmGTCDynamicXdlopsNHWC_fp32_fp16.cpp smoke_solver_ConvAsmImplicitGemmGTCDynamicXdlopsNHWC_bf16.cpp) + # Disable test temporarily due to a Memory Fault Error while Address Sanitizer is enabled. Needs to be investigated further. + list(APPEND SKIP_TESTS tensor_fwd_bias_ocl_hip.cpp) + # temporarily disable due to test failures + list(APPEND SKIP_TESTS unit_conv_solver_ConvWinoRageRxS.cpp) + # Disable tests that leak memory when Address Sanitizer is enabled. These tests need to be investigated and fixed. + list(APPEND SKIP_TESTS + conv_api.cpp + deterministic_conv_api.cpp + fusion_aux.cpp + fusion_test.cpp + gpu_mha_backward.cpp + gpu_mha_forward.cpp + graphapi_convolution.cpp + graphapi_enginecfg.cpp + graphapi_engineheur.cpp + graphapi_execution_plan.cpp + graphapi_matmul.cpp + graphapi_operationgraph_descriptor.cpp + graphapi_operation_reduction.cpp + graphapi_operation_reshape.cpp + graphapi_operation_matmul.cpp + graphapi_operation_pointwise.cpp + graphapi_operation_rng.cpp + graphapi_pointwise.cpp + graphapi_reduction.cpp + graphapi_rng.cpp + graphapi_tensor.cpp + graphapi_variant_pack.cpp + log_test.cpp + log_test_neg.cpp + mha_find20.cpp + w_supertensor.cpp) +endif() + function(add_gtest_negative_filter NEGATIVE_FILTER_TO_ADD) set (TMP_FILTER ${MIOPEN_GTEST_FILTER_NEGATIVE}) @@ -77,7 +116,7 @@ function(add_gtest TEST_NAME TEST_CPP) # Workaround : change in rocm-cmake was causing linking error so had to add ${CMAKE_DL_LIBS} # We can remove ${CMAKE_DL_LIBS} once root cause is identified. - target_link_libraries(${TEST_NAME} ${CMAKE_DL_LIBS} GTest::gtest GTest::gtest_main GTest::gmock MIOpen ${Boost_LIBRARIES} hip::host ) + target_link_libraries(${TEST_NAME} ${CMAKE_DL_LIBS} GTest::gtest GTest::gtest_main GTest::gmock MIOpen hip::host ) if(NOT MIOPEN_EMBED_DB STREQUAL "") target_link_libraries(${TEST_NAME} $) endif() @@ -101,10 +140,6 @@ function(add_gtest TEST_NAME TEST_CPP) set(MIOPEN_GTEST_PREFIX "Smoke") endif() - if (NOT MIOPEN_TEST_MLIR) - add_gtest_negative_filter("*MLIRTest*") - endif() - if (NOT MIOPEN_TEST_DBSYNC) add_gtest_negative_filter("*DBSync*") endif() @@ -152,10 +187,6 @@ function(add_gtest TEST_NAME TEST_CPP) set(MIOPEN_GTEST_PREFIX "Smoke") endif() - if (NOT MIOPEN_TEST_MLIR) - add_gtest_negative_filter("*MLIRTest*") - endif() - if (NOT MIOPEN_TEST_DBSYNC) add_gtest_negative_filter("*DBSync*") endif() diff --git a/projects/miopen/test/gtest/activ_common.hpp b/projects/miopen/test/gtest/activ_common.hpp index c381ec9bd997..11bb29783979 100644 --- a/projects/miopen/test/gtest/activ_common.hpp +++ b/projects/miopen/test/gtest/activ_common.hpp @@ -56,13 +56,13 @@ class ActivationPASTHRU { public: template - static T Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static T Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) { return x; } template - static T Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static T Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T /*x*/, T /*y*/) { return dy; } @@ -72,13 +72,13 @@ class ActivationLOGISTIC { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) { return 1 / (1 + std::exp(-x)); } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T /*x*/, T y) { return static_cast(dy) * y * (static_cast(1) - y); } @@ -88,13 +88,13 @@ class ActivationTANH { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam alpha, Tparam beta, Tparam /*gamma*/, T x) { return beta * std::tanh(alpha * x); } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam alpha, Tparam beta, Tparam /*gamma*/, T dy, T /*x*/, T y) { return dy * alpha * (beta - static_cast(y) * y / beta); } @@ -104,13 +104,13 @@ class ActivationRELU { public: template - static T Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static T Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) { return (x > static_cast(0)) ? x : static_cast(0); } template - static T Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static T Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) { return (x > static_cast(0)) ? dy : static_cast(0); } @@ -120,13 +120,13 @@ class ActivationSOFTRELU { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) { return std::log1p(std::exp(x)); } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) { const Tparam threshold = 50.0; const Tparam expval = std::exp(std::min(static_cast(x), threshold)); @@ -138,13 +138,13 @@ class ActivationABS { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) { return std::abs(x); } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) { return dy * static_cast((x > static_cast(0)) ? 1 : -1); } @@ -161,7 +161,7 @@ class ActivationPOWER } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T /*dy*/, T x, T y) { const auto v = alpha + beta * x; return v <= std::numeric_limits::epsilon() ? 0 : gamma * beta * y / v; @@ -172,13 +172,13 @@ class ActivationCLIPPEDRELU { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) { return std::clamp(static_cast(x), static_cast(0), alpha); } template - static T Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static T Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) { Tparam x_native = x; return (x_native > 0 && x_native <= alpha) ? dy : static_cast(0); @@ -189,14 +189,14 @@ class ActivationLEAKYRELU { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) { Tparam x_native = x; return (x_native > 0) ? x_native : x_native * alpha; } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) { return dy * ((x > static_cast(0)) ? 1 : alpha); } @@ -206,14 +206,14 @@ class ActivationELU { public: template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) + static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) { Tparam x_native = x; return (x_native > 0) ? x_native : alpha * std::expm1(x_native); } template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T dy, T x, T y) + static Tparam Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T y) { return dy * ((x > static_cast(0)) ? 1 : y + alpha); } @@ -356,7 +356,7 @@ void CpuActivationPacked(std::size_t num_items, Ts&&... xs) } template -void CpuActivationNonPacked(std::size_t num_items, Ts&&... xs) +[[noreturn]] void CpuActivationNonPacked(std::size_t /*num_items*/, Ts&&... /*xs*/) { throw std::runtime_error("CpuActivationNonPacked is not implemented yet"); } diff --git a/projects/miopen/test/gtest/bn_activ_infer.cpp b/projects/miopen/test/gtest/bn_activ_infer.cpp index 33b5354a2776..d149d6c119c4 100644 --- a/projects/miopen/test/gtest/bn_activ_infer.cpp +++ b/projects/miopen/test/gtest/bn_activ_infer.cpp @@ -47,9 +47,9 @@ void BatchNormInferenceGPU(const miopen::Handle& handle, const float activ_gamma, const miopen::TensorDescriptor& xDesc, ConstData_t x, - const miopen::TensorDescriptor& yDesc, + const miopen::TensorDescriptor& /*yDesc*/, Data_t y, - const miopen::TensorDescriptor& bnScaleBiasMeanVarDesc, + const miopen::TensorDescriptor& /*bnScaleBiasMeanVarDesc*/, ConstData_t bnScale, ConstData_t bnBias, ConstData_t estimatedMean, @@ -330,7 +330,7 @@ TEST_P(GPU_bn_activ_infer_spatial_FP32, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} TEST_P(GPU_bn_activ_infer_per_act_FP32, PortTest) { @@ -340,7 +340,7 @@ TEST_P(GPU_bn_activ_infer_per_act_FP32, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} TEST_P(GPU_bn_activ_infer_spatial_FP16, PortTest) { @@ -350,7 +350,7 @@ TEST_P(GPU_bn_activ_infer_spatial_FP16, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} TEST_P(GPU_bn_activ_infer_per_act_FP16, PortTest) { @@ -360,7 +360,7 @@ TEST_P(GPU_bn_activ_infer_per_act_FP16, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} TEST_P(GPU_bn_activ_infer_spatial_BFP16, PortTest) { @@ -370,7 +370,7 @@ TEST_P(GPU_bn_activ_infer_spatial_BFP16, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} TEST_P(GPU_bn_activ_infer_per_act_BFP16, PortTest) { @@ -380,7 +380,7 @@ TEST_P(GPU_bn_activ_infer_per_act_BFP16, PortTest) RunTestGPU(); // Compare the outputs Verify(); -}; +} INSTANTIATE_TEST_SUITE_P( Smoke, diff --git a/projects/miopen/test/gtest/bn_fwd_infer.cpp b/projects/miopen/test/gtest/bn_fwd_infer.cpp index fe9dfb6e96a5..ab03eb381bf7 100644 --- a/projects/miopen/test/gtest/bn_fwd_infer.cpp +++ b/projects/miopen/test/gtest/bn_fwd_infer.cpp @@ -48,7 +48,6 @@ void BatchNormInferenceGPU(const miopen::Handle& handle, ConstData_t x, const miopen::TensorDescriptor& yDesc, Data_t y, - const miopen::TensorDescriptor& bnScaleBiasMeanVarDesc, ConstData_t bnScale, ConstData_t bnBias, ConstData_t estimatedMean, @@ -138,7 +137,7 @@ void BatchNormInferenceGPU(const miopen::Handle& handle, std::string network_config = ss.str(); // add the kernel to the handle - [[maybe_unused]] auto kernelInvoke = + auto kernelInvoke = handle.AddKernel(kernel_name, network_config, kernel_file, kernel_name, vld, vgd, params); if constexpr(PERF_ENABLE) @@ -222,7 +221,6 @@ struct BatchNormFwdInferTester : public BatchNormInferTesterin_dev.get(), this->output.desc, this->out_dev.get(), - this->scale.desc, this->scale_dev.get(), this->shift_dev.get(), this->estMean_dev.get(), diff --git a/projects/miopen/test/gtest/cba_find2_infer.cpp b/projects/miopen/test/gtest/cba_find2_infer.cpp index 05d9ce2413c8..b3741df33d83 100644 --- a/projects/miopen/test/gtest/cba_find2_infer.cpp +++ b/projects/miopen/test/gtest/cba_find2_infer.cpp @@ -142,11 +142,6 @@ TEST_P(GPU_ConvBiasActivFind2Infer_FP16, ConvWinoFuryRxSf2x3Find2Fused) RunSolver>( fused_problem, invoke_params, conv_config, test_skipped); } -TEST_P(GPU_ConvBiasActivFind2Infer_FP16, ConvWinoRageRxSf2x3Find2Fused) -{ - RunSolver>( - fused_problem, invoke_params, conv_config, test_skipped); -} TEST_P(GPU_ConvBiasActivFind2Infer_FP16, ConvCKIgemmFwdBiasActivFind2Fused) { diff --git a/projects/miopen/test/gtest/cba_infer.cpp b/projects/miopen/test/gtest/cba_infer.cpp index 0414dcb7ff6f..c819a60d511f 100644 --- a/projects/miopen/test/gtest/cba_infer.cpp +++ b/projects/miopen/test/gtest/cba_infer.cpp @@ -190,10 +190,6 @@ TEST_P(GPU_ConvBiasActivInfer_FP16, ConvWinoFuryRxSf2x3Fused) { RunSolver(miopen::solver::fusion::ConvWinoFuryRxSFused<2, 3>{}); } -TEST_P(GPU_ConvBiasActivInfer_FP16, ConvWinoRageRxSf2x3Fused) -{ - RunSolver(miopen::solver::fusion::ConvWinoRageRxSFused<2, 3>{}); -} TEST_P(GPU_ConvBiasActivInfer_FP16, ConvCKIgemmFwdBiasActivFused) { diff --git a/projects/miopen/test/gtest/compare_helper.hpp b/projects/miopen/test/gtest/compare_helper.hpp index 6315dcaaf089..b9d9b27f6d61 100644 --- a/projects/miopen/test/gtest/compare_helper.hpp +++ b/projects/miopen/test/gtest/compare_helper.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -60,13 +61,13 @@ std::vector const& GetResultData(tensor const& result) } template -constexpr double GetResultDataErrorMargin(std::vector const& result, double tolerance) +constexpr double GetResultDataErrorMargin(std::vector const& /*result*/, double tolerance) { return (std::numeric_limits::epsilon() * tolerance); } template -constexpr double GetResultDataErrorMargin(tensor const& result, double tolerance) +constexpr double GetResultDataErrorMargin(tensor const& /*result*/, double tolerance) { return (std::numeric_limits::epsilon() * tolerance); } @@ -83,19 +84,20 @@ bool CompareFloatTensorTuples(LeftTuple const& left, ...); } -template -std::enable_if_t() || any_float_tensors()), bool> -Compare(std::tuple const& cpu_result, std::tuple const& gpu_result, double) +template +bool Compare(std::tuple const& cpu_result, + std::tuple const& gpu_result, + double tolerance) { - return cpu_result == gpu_result; -} - -template -std::enable_if_t<(any_float_tensors() || any_float_tensors()), bool> Compare( - std::tuple const& cpu_result, std::tuple const& gpu_result, double tolerance) -{ - return CompareFloatTensorTuples( - cpu_result, gpu_result, tolerance, std::index_sequence_for{}); + if constexpr(any_float_tensors() || any_float_tensors()) + { + return CompareFloatTensorTuples( + cpu_result, gpu_result, tolerance, std::index_sequence_for{}); + } + else + { + return cpu_result == gpu_result; + } } template diff --git a/projects/miopen/test/gtest/db_paths.cpp b/projects/miopen/test/gtest/db_paths.cpp index 705d77529eed..65ebc46fed14 100644 --- a/projects/miopen/test/gtest/db_paths.cpp +++ b/projects/miopen/test/gtest/db_paths.cpp @@ -116,8 +116,7 @@ TEST_F(CPU_DbPaths_NONE, UserDbPath_NetworkFS_NoEnvVar) { #if MIOPEN_BUILD_DEV GTEST_SKIP() << "Network filesystem detection is disabled in MIOPEN_BUILD_DEV mode"; -#endif - +#else // Ensure environment variable is NOT set for this test ScopedEnvironment unset_user_db(MIOPEN_USER_DB_PATH, ""); @@ -142,6 +141,7 @@ TEST_F(CPU_DbPaths_NONE, UserDbPath_NetworkFS_NoEnvVar) << "User DB path should contain '.config' folder"; EXPECT_TRUE(PathContains(user_db_path, "miopen")) << "User DB path should contain 'miopen' folder"; +#endif } TEST_F(CPU_DbPaths_NONE, UserDbPath_NetworkFS_EnvVarSet) @@ -220,8 +220,7 @@ TEST_F(CPU_DbPaths_NONE, CachePath_NetworkFS_NoEnvVar) { #if MIOPEN_BUILD_DEV GTEST_SKIP() << "Network filesystem detection is disabled in MIOPEN_BUILD_DEV mode"; -#endif - +#else // Ensure environment variable is NOT set for this test ScopedEnvironment unset_cache(MIOPEN_CUSTOM_CACHE_DIR, ""); @@ -248,6 +247,7 @@ TEST_F(CPU_DbPaths_NONE, CachePath_NetworkFS_NoEnvVar) const std::string expected_version = GetExpectedVersionString(); EXPECT_TRUE(PathContains(cache_path, expected_version)) << "Cache path should contain version string '" << expected_version << "'"; +#endif } TEST_F(CPU_DbPaths_NONE, CachePath_NetworkFS_EnvVarSet) diff --git a/projects/miopen/test/gtest/db_sync.cpp b/projects/miopen/test/gtest/db_sync.cpp index 48e20c592f45..433a4c00542b 100644 --- a/projects/miopen/test/gtest/db_sync.cpp +++ b/projects/miopen/test/gtest/db_sync.cpp @@ -121,6 +121,7 @@ static bool TransformFindDbItem10to20(std::string& id, std::string& values) #endif namespace miopen { + conv::Direction GetDirectionFromString(const std::string& direction) { if(direction == "F") @@ -131,6 +132,7 @@ conv::Direction GetDirectionFromString(const std::string& direction) return conv::Direction::BackwardWeights; throw std::runtime_error("Invalid Direction"); } + miopenTensorLayout_t GetLayoutFromString(const std::string& layout) { if(layout == "NCHW") @@ -143,6 +145,7 @@ miopenTensorLayout_t GetLayoutFromString(const std::string& layout) return miopenTensorNDHWC; throw std::runtime_error("Invalid Layout"); } + miopenDataType_t GetDataTypeFromString(const std::string& data_type) { if(data_type == "FP32") @@ -159,6 +162,7 @@ miopenDataType_t GetDataTypeFromString(const std::string& data_type) return miopenDouble; throw std::runtime_error("Invalid data type in find db key"); } + void ParseProblemKey(const std::string& key_, conv::ProblemDescription& prob_desc) { std::string key = key_; @@ -576,29 +580,23 @@ TEST(CPU_DBSync_NONE, KDBTargetID) #endif } -bool LogBuildMessage() -{ - MIOPEN_LOG_W("Unable to produce missing binary due to COMGR being enabled"); - return true; -} - void BuildKernel(const fs::path& program_file, const std::string& program_args, - [[maybe_unused]] const miopen::Handle& handle) + const miopen::Handle& handle) { // Build the code object entry // This will write the code object in the user kdb which Jenkins can archive // This has to be done with the offline clang compiler and not COMGR (or hipRTC) otherwise the // code object would be target ID specific #if MIOPEN_USE_COMGR - static const bool discard = LogBuildMessage(); - std::ignore = discard; - std::ignore = program_file; - std::ignore = program_args; + std::ignore = program_file; + std::ignore = program_args; + std::ignore = handle; + MIOPEN_LOG_W("Unable to produce missing binary due to COMGR being enabled"); #else try { - auto p = handle.LoadProgram(program_file, program_args, ""); + (void)handle.LoadProgram(program_file, program_args, ""); } catch(std::exception&) { diff --git a/projects/miopen/test/gtest/deepbench_lstm.cpp b/projects/miopen/test/gtest/deepbench_lstm.cpp index df9ec3c608d0..afb977c41335 100644 --- a/projects/miopen/test/gtest/deepbench_lstm.cpp +++ b/projects/miopen/test/gtest/deepbench_lstm.cpp @@ -23,89 +23,102 @@ * SOFTWARE. * *******************************************************************************/ -#include #include "lstm.hpp" -#include "get_handle.hpp" -#include -#include -#include "gtest_common.hpp" -namespace deepbench_lstm { +namespace { -void GetArgs(const std::string& param, std::vector& tokens) +struct TestCase { - std::stringstream ss(param); - std::istream_iterator begin(ss); - std::istream_iterator end; - while(begin != end) - tokens.push_back(*begin++); -} + TestCase(int batch_size_, int seq_len_, int vector_len_, int hidden_size_) + : batch_size(batch_size_), + seq_len(seq_len_), + vector_len(vector_len_), + hidden_size(hidden_size_) + { + } -auto GetTestCases(std::string precision) + int batch_size; + int seq_len; + int vector_len; + int hidden_size; + + friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) + { + return os << "batch-size:" << tc.batch_size << " seq-len:" << tc.seq_len + << " vector-len:" << tc.vector_len << " hidden-size:" << tc.hidden_size; + } +}; + +std::vector GetTestCases() { - std::string flags = "test_lstm --verbose " + precision; - std::string commonFlags = - " --num-layers 1 --in-mode 1 --bias-mode 0 -dir-mode 0 --rnn-mode 0 --flat-batch-fill"; - - // clang-format off - return std::vector{ - {flags + " --batch-size 16 --seq-len 25 --vector-len 512 --hidden-size 512" + commonFlags}, - {flags + " --batch-size 32 --seq-len 25 --vector-len 512 --hidden-size 512" + commonFlags}, - {flags + " --batch-size 64 --seq-len 25 --vector-len 512 --hidden-size 512" + commonFlags}, - {flags + " --batch-size 128 --seq-len 25 --vector-len 512 --hidden-size 512" + commonFlags}, - {flags + " --batch-size 16 --seq-len 25 --vector-len 1024 --hidden-size 1024" + commonFlags}, - {flags + " --batch-size 32 --seq-len 25 --vector-len 1024 --hidden-size 1024" + commonFlags}, - {flags + " --batch-size 64 --seq-len 25 --vector-len 1024 --hidden-size 1024" + commonFlags}, - {flags + " --batch-size 128 --seq-len 25 --vector-len 1024 --hidden-size 1024" + commonFlags}, - {flags + " --batch-size 16 --seq-len 25 --vector-len 2048 --hidden-size 2048" + commonFlags}, - {flags + " --batch-size 32 --seq-len 25 --vector-len 2048 --hidden-size 2048" + commonFlags}, - {flags + " --batch-size 64 --seq-len 25 --vector-len 2048 --hidden-size 2048" + commonFlags}, - {flags + " --batch-size 128 --seq-len 25 --vector-len 2048 --hidden-size 2048" + commonFlags}, - {flags + " --batch-size 16 --seq-len 25 --vector-len 4096 --hidden-size 4096" + commonFlags}, - {flags + " --batch-size 32 --seq-len 25 --vector-len 4096 --hidden-size 4096" + commonFlags}, - {flags + " --batch-size 64 --seq-len 25 --vector-len 4096 --hidden-size 4096" + commonFlags}, - {flags + " --batch-size 128 --seq-len 25 --vector-len 4096 --hidden-size 4096" + commonFlags}, - {flags + " --batch-size 8 --seq-len 50 --vector-len 1536 --hidden-size 1536" + commonFlags}, - {flags + " --batch-size 16 --seq-len 50 --vector-len 1536 --hidden-size 1536" + commonFlags}, - {flags + " --batch-size 32 --seq-len 50 --vector-len 1536 --hidden-size 1536" + commonFlags}, - {flags + " --batch-size 16 --seq-len 150 --vector-len 256 --hidden-size 256" + commonFlags}, - {flags + " --batch-size 32 --seq-len 150 --vector-len 256 --hidden-size 256" + commonFlags}, - {flags + " --batch-size 64 --seq-len 150 --vector-len 256 --hidden-size 256" + commonFlags} + return std::vector{ + // clang-format off + // batch-size seq-len vector-len hidden-size + TestCase( 16, 25, 512, 512 ), + TestCase( 32, 25, 512, 512 ), + TestCase( 64, 25, 512, 512 ), + TestCase(128, 25, 512, 512 ), + TestCase( 16, 25, 1024, 1024 ), + TestCase( 32, 25, 1024, 1024 ), + TestCase( 64, 25, 1024, 1024 ), + TestCase(128, 25, 1024, 1024 ), + TestCase( 16, 25, 2048, 2048 ), + TestCase( 32, 25, 2048, 2048 ), + TestCase( 64, 25, 2048, 2048 ), + TestCase(128, 25, 2048, 2048 ), + TestCase( 16, 25, 4096, 4096 ), + TestCase( 32, 25, 4096, 4096 ), + TestCase( 64, 25, 4096, 4096 ), + TestCase(128, 25, 4096, 4096 ), + TestCase( 8, 50, 1536, 1536 ), + TestCase( 16, 50, 1536, 1536 ), + TestCase( 32, 50, 1536, 1536 ), + TestCase( 16, 150, 256, 256 ), + TestCase( 32, 150, 256, 256 ), + TestCase( 64, 150, 256, 256 ) + // clang-format on }; - // clang-format on } -using TestCase = decltype(GetTestCases({}))::value_type; +} // namespace -class GPU_DeepBench_lstm_FP32 : public testing::TestWithParam> +template +struct GPU_DeepBench_LSTM_test : LSTM_test, testing::TestWithParam { - MIOPEN_DECLARE_GTEST_USES_TEST_DRIVE(); -}; +protected: + void SetUp() override + { + this->dataType = miopen_type{}; -void Run2dDriverFloat(void) -{ - std::vector params = GPU_DeepBench_lstm_FP32::GetParam(); + auto [batchSize, seqLength, inVecLen, hiddenSize] = GetParam(); - for(const auto& test_value : params) - { - std::vector tokens; - GetArgs(test_value, tokens); - std::vector ptrs; - - std::transform(tokens.begin(), tokens.end(), std::back_inserter(ptrs), [](const auto& str) { - return str.data(); - }); - testing::internal::CaptureStderr(); - test_drive(ptrs.size(), ptrs.data()); - auto capture = testing::internal::GetCapturedStderr(); - std::cout << capture; + this->numLayers = 1; + this->inputMode = 1; + this->biasMode = 0; + this->dirMode = 0; + this->flatBatchFill = 1; + this->batchSize = batchSize; + this->seqLength = seqLength; + this->inVecLen = inVecLen; + this->hiddenSize = hiddenSize; } }; -} // namespace deepbench_lstm -using namespace deepbench_lstm; +using GPU_DeepBench_LSTM_FP16 = GPU_DeepBench_LSTM_test; + +TEST_P(GPU_DeepBench_LSTM_FP16, HalfTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP16, testing::ValuesIn(GetTestCases())); + +using GPU_DeepBench_LSTM_FP32 = GPU_DeepBench_LSTM_test; + +TEST_P(GPU_DeepBench_LSTM_FP32, FloatTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP32, testing::ValuesIn(GetTestCases())); + +using GPU_DeepBench_LSTM_FP64 = GPU_DeepBench_LSTM_test; -TEST_P(GPU_DeepBench_lstm_FP32, FloatTest_deepbench_lstm) { Run2dDriverFloat(); }; +TEST_P(GPU_DeepBench_LSTM_FP64, DoubleTest) { RunTest(); } -INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_lstm_FP32, testing::Values(GetTestCases("--float"))); +INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/execute_primitive.cpp b/projects/miopen/test/gtest/execute_primitive.cpp index 30841e9d2d4c..534ccc66fef3 100644 --- a/projects/miopen/test/gtest/execute_primitive.cpp +++ b/projects/miopen/test/gtest/execute_primitive.cpp @@ -87,7 +87,7 @@ struct TestProblemDescription : miopen::ProblemDescriptionBase, #endif template - static void Visit(TSelf&& self, std::function visitor) + static void Visit(TSelf&& /*self*/, std::function /*visitor*/) { } @@ -122,8 +122,8 @@ struct TestPerfConfig : miopen::solver::PerfConfig TestPerfConfig(std::uint32_t check_value_ = 0) : check_value(check_value_) {} - void Serialize(std::ostream& stream) const override {} - auto Deserialize(const std::string&) -> bool override { return true; } + void Serialize(std::ostream&) const override {} + bool Deserialize(const std::string&) override { return true; } }; static auto MakeNoopSolution() -> miopen::solver::ConvSolution diff --git a/projects/miopen/test/gtest/find_2_conv.cpp b/projects/miopen/test/gtest/find_2_conv.cpp index 8ff2b630556a..0030fa2b4da7 100644 --- a/projects/miopen/test/gtest/find_2_conv.cpp +++ b/projects/miopen/test/gtest/find_2_conv.cpp @@ -276,6 +276,19 @@ TEST_P(GPU_Find2Conv_FP32, Find2ConvTest) std::get<2>(param), std::get<3>(param), std::get<4>(param)}; + + // Temporarily disable 4 test cases. + // When the entire test suite is executed, all test cases pass. + // However, running any of these test cases in isolation results in a failure. + // This issue is also reproducible with the original ctest when executing tests individually. + if((tc.tune == 1) && (tc.preallocate == 1) && + (tc.workspace_limit == std::numeric_limits::max()) && + ((tc.direction == miopenProblemDirectionForward) || + (tc.direction == miopenProblemDirectionBackwardWeights))) + { + GTEST_SKIP(); + } + RunFind2ConvTest(tc); } diff --git a/projects/miopen/test/gtest/fusion_test.cpp b/projects/miopen/test/gtest/fusion_test.cpp index 6a539428e0a7..f2c4505e9df6 100644 --- a/projects/miopen/test/gtest/fusion_test.cpp +++ b/projects/miopen/test/gtest/fusion_test.cpp @@ -196,6 +196,7 @@ TEST(CPU_FusionCreateOpConvForward_FP32, TestInvalidConvLayout) } MIOPEN_LIB_ENV_VAR(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1) +MIOPEN_LIB_ENV_VAR(MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3) // The test uses a specific fusion configuration that triggers the solver. // ConvCKIgemmGrpFwdBiasActivFused @@ -263,6 +264,7 @@ class GPU_CBAFind2FusionWorkspace : public ConvBiasActivInferTest miopenConvFwdAlgorithm_t algo{}; // not used in GetWorkSpaceSize { ScopedEnvironment find_mode_env1(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1, false); + ScopedEnvironment find_mode_env2(MIOPEN_DEBUG_AMD_WINOGRAD_RAGE_RXS_F2X3, false); ASSERT_TRUE( IsOnlyConvCKIgemmGrpFwdBiasActivFusedSolverApplicable(cba_base::fusePlanDesc)) << "Test configuration is invalid as other solvers are applicable. Please update " diff --git a/projects/miopen/test/gtest/getitem.hpp b/projects/miopen/test/gtest/getitem.hpp index 83119150ee02..4f45212fbba6 100644 --- a/projects/miopen/test/gtest/getitem.hpp +++ b/projects/miopen/test/gtest/getitem.hpp @@ -43,8 +43,7 @@ void cpu_getitem_backward(tensor dy, uint32_t dimCount, int32_t* dims, uint32_t sliceCount, - int32_t* slices, - uint32_t offset) + int32_t* slices) { auto dy_dims = dy.desc.GetLengths(); auto dy_numel = std::accumulate(dy_dims.begin(), dy_dims.end(), 1L, std::multiplies()); @@ -307,6 +306,7 @@ struct GetitemBwdTest : public ::testing::TestWithParam dx_dev = handle.Write(dx.data); error_dev = handle.Write(error.data); } + void RunTest() { auto&& handle = get_handle(); @@ -318,8 +318,7 @@ struct GetitemBwdTest : public ::testing::TestWithParam dims.size(), dims.data(), slices.size(), - slices_flat.data(), - offset); + slices_flat.data()); std::vector indexDescs; std::vector indexData; diff --git a/projects/miopen/test/gtest/gpu_reference_kernel.cpp b/projects/miopen/test/gtest/gpu_reference_kernel.cpp index c25266d96cf0..7a0cf3411936 100644 --- a/projects/miopen/test/gtest/gpu_reference_kernel.cpp +++ b/projects/miopen/test/gtest/gpu_reference_kernel.cpp @@ -579,9 +579,9 @@ struct gpu_reference_conv_2d : public ::testing::TestWithParam miopenDestroyTensorDescriptor(weiDesc); miopenDestroyTensorDescriptor(outDesc); - hipFree(in_dev); - hipFree(wei_dev); - hipFree(out_dev); + (void)hipFree(in_dev); + (void)hipFree(wei_dev); + (void)hipFree(out_dev); } }; @@ -848,9 +848,9 @@ struct gpu_reference_conv_3d : public ::testing::TestWithParam miopenDestroyTensorDescriptor(weiDesc); miopenDestroyTensorDescriptor(outDesc); - hipFree(in_dev); - hipFree(wei_dev); - hipFree(out_dev); + (void)hipFree(in_dev); + (void)hipFree(wei_dev); + (void)hipFree(out_dev); } }; } // namespace diff --git a/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp b/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp index a56485b3ffa9..753c4d935239 100644 --- a/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp +++ b/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp @@ -81,18 +81,18 @@ class GMockBackendOperationDescriptor : public BackendDescriptor public: GMockBackendOperationDescriptor(bool finalized) { mFinalized = finalized; } - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override + void setAttribute(miopenBackendAttributeName_t /*attributeName*/, + miopenBackendAttributeType_t /*attributeType*/, + int64_t /*elementCount*/, + void* /*arrayOfElements*/) override { } void finalize() override {} - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override + void getAttribute(miopenBackendAttributeName_t /*attributeName*/, + miopenBackendAttributeType_t /*attributeType*/, + int64_t /*requestedElementCount*/, + int64_t* /*elementCount*/, + void* /*arrayOfElements*/) override { } OpNode* getOperation() override { return &mNode; } @@ -107,8 +107,8 @@ using miopen::graphapi::GTestGraphApiExecute; TEST(CPU_GraphApiOperationGraphDescriptor_NONE, CFunctions) { - miopenHandle_t handle = nullptr; - auto status = miopenCreate(&handle); + miopenHandle_t handle{nullptr}; + auto status = miopenCreate(&handle); ASSERT_EQ(status, miopenStatusSuccess) << "Handle wasn't obtained, cannot continue"; GTestDescriptorSingleValueAttribute handleAttribute{ diff --git a/projects/miopen/test/gtest/gtest_common.hpp b/projects/miopen/test/gtest/gtest_common.hpp index 437ad1ffeb37..6fbdfba281e6 100644 --- a/projects/miopen/test/gtest/gtest_common.hpp +++ b/projects/miopen/test/gtest/gtest_common.hpp @@ -325,6 +325,7 @@ MIOPEN_LIB_ENV_VAR(MIOPEN_DEBUG_CONV_DIRECT) MIOPEN_LIB_ENV_VAR(MIOPEN_DEBUG_CONV_GEMM) MIOPEN_LIB_ENV_VAR(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM) MIOPEN_LIB_ENV_VAR(MIOPEN_LOG_LEVEL) +MIOPEN_LIB_ENV_VAR(MIOPEN_LOG_BUFFER_SIZE) MIOPEN_LIB_ENV_VAR(MIOPEN_FIND_ENFORCE) // TODO: GTests using test_drive<> disabled until gtest-aware version of test/driver.hpp is built diff --git a/projects/miopen/test/gtest/handle_hip_device.cpp b/projects/miopen/test/gtest/handle_hip_device.cpp index 1f9e1a256551..23330f8eea39 100644 --- a/projects/miopen/test/gtest/handle_hip_device.cpp +++ b/projects/miopen/test/gtest/handle_hip_device.cpp @@ -29,8 +29,8 @@ class CPU_HandleHipDevice_NONE : public ::testing::Test void TearDown() override { // Clear any HIP error triggered by asking for an invalid device ordinal - hipGetLastError(); - hipExtGetLastError(); + (void)hipGetLastError(); + (void)hipExtGetLastError(); } int device_count_ = 0; diff --git a/projects/miopen/test/gtest/hipblaslt_gemm.cpp b/projects/miopen/test/gtest/hipblaslt_gemm.cpp index 4cd94d7b7746..4fd4fcb12d32 100644 --- a/projects/miopen/test/gtest/hipblaslt_gemm.cpp +++ b/projects/miopen/test/gtest/hipblaslt_gemm.cpp @@ -272,6 +272,7 @@ template static void CheckExceptionsWithSkip(miopenDataType_t dataType) { #ifdef WORKAROUND_SWDEV_473387 + (void)dataType; GTEST_SKIP(); #else if(!IsTestSupportedForDevMask()) diff --git a/projects/miopen/test/gtest/layout_transpose.cpp b/projects/miopen/test/gtest/layout_transpose.cpp index ea235529fb07..f67c7a0387de 100644 --- a/projects/miopen/test/gtest/layout_transpose.cpp +++ b/projects/miopen/test/gtest/layout_transpose.cpp @@ -212,7 +212,7 @@ struct transpose_dims template auto gen_value = - [](auto... is) { return static_cast(prng::gen_A_to_B(RAND_INTEGER_MIN, RAND_INTEGER_MAX)); }; + [](auto...) { return static_cast(prng::gen_A_to_B(RAND_INTEGER_MIN, RAND_INTEGER_MAX)); }; } // namespace diff --git a/projects/miopen/test/gtest/log.cpp b/projects/miopen/test/gtest/log.cpp index aeb9b5fc5924..2caed4aacce0 100644 --- a/projects/miopen/test/gtest/log.cpp +++ b/projects/miopen/test/gtest/log.cpp @@ -133,7 +133,7 @@ struct Tensor #if MIOPEN_BACKEND_OPENCL clReleaseMemObject(data); #elif MIOPEN_BACKEND_HIP - hipFree(data); + (void)hipFree(data); #endif } }; @@ -347,7 +347,7 @@ void TestLogCmdBNormFusion(std::function log_level_env(MIOPEN_LOG_LEVEL, "5"); // miopen::LoggingLevel::Info + ScopedEnvironment log_buffer_on_env(MIOPEN_LOG_BUFFER_SIZE, + "128"); // enable logging // test log dump after error - miopen::ClearBufferLog(); + miopen::ClearLogBuffer(); MIOPEN_LOG_W("warn"); MIOPEN_LOG_I("info"); MIOPEN_LOG_I2("info2"); @@ -376,23 +378,19 @@ void TestLogBufferOn() case 0: ASSERT_TRUE(isSubStr(line, "warn")); break; case 1: ASSERT_TRUE(isSubStr(line, "info")); break; case 2: ASSERT_TRUE(isSubStr(line, "info2")); break; - case 3: - ASSERT_FALSE(isSubStr(line, "trace")); - ASSERT_TRUE(isSubStr(line, "error")); - break; + case 3: ASSERT_TRUE(isSubStr(line, "error")); break; + case 4: ASSERT_TRUE(isSubStr(line, "")); break; } line_i++; } } - fs::remove(filename); // test log dump after throw - miopen::ClearBufferLog(); - MIOPEN_LOG_W("warn"); - MIOPEN_LOG_I("info"); - MIOPEN_LOG_I2("info2"); - MIOPEN_LOG_T("trace"); - EXPECT_ANY_THROW({ MIOPEN_THROW("throw"); }); + MIOPEN_LOG_W("warn 2nd"); + MIOPEN_LOG_I("info 2nd"); + MIOPEN_LOG_I2("info2 2nd"); + MIOPEN_LOG_T("trace 2nd"); + MIOPEN_LOG_E("error 2nd"); EXPECT_TRUE(fs::exists(filename)); { @@ -405,12 +403,22 @@ void TestLogBufferOn() case 0: ASSERT_TRUE(isSubStr(line, "warn")); break; case 1: ASSERT_TRUE(isSubStr(line, "info")); break; case 2: ASSERT_TRUE(isSubStr(line, "info2")); break; - case 3: ASSERT_TRUE(isSubStr(line, "throw")); break; + case 3: ASSERT_TRUE(isSubStr(line, "error")); break; + case 4: ASSERT_TRUE(isSubStr(line, "")); break; + case 5: ASSERT_TRUE(isSubStr(line, "warn 2nd")); break; + case 6: ASSERT_TRUE(isSubStr(line, "info 2nd")); break; + case 7: ASSERT_TRUE(isSubStr(line, "info2 2nd")); break; + case 8: ASSERT_TRUE(isSubStr(line, "error 2nd")); break; + case 9: ASSERT_TRUE(isSubStr(line, "")); break; } line_i++; } + ASSERT_TRUE(line_i == 10); } fs::remove(filename); + + EXPECT_ANY_THROW({ MIOPEN_THROW("throw"); }); + ASSERT_FALSE(fs::exists(filename)); } void TestLogBufferOff() @@ -420,10 +428,33 @@ void TestLogBufferOff() fs::remove(filename); + ScopedEnvironment log_level_env(MIOPEN_LOG_LEVEL, + "5"); // miopen::LoggingLevel::Info + + miopen::ClearLogBuffer(); + // log messages + MIOPEN_LOG_W("warn"); + MIOPEN_LOG_I("info"); + MIOPEN_LOG_I2("info2"); + MIOPEN_LOG_T("trace"); + // test log dump after error + MIOPEN_LOG_E("error"); + ASSERT_FALSE(fs::exists(filename)); +} + +void TestLogBufferOffAtHighLevel() +{ + auto filename = + fs::temp_directory_path() / ("miopen_error_" + std::to_string(getpid()) + ".log"); + + fs::remove(filename); + ScopedEnvironment log_level_env(MIOPEN_LOG_LEVEL, "6"); // miopen::LoggingLevel::Info2 + ScopedEnvironment log_buffer_on_env(MIOPEN_LOG_BUFFER_SIZE, + "128"); // enable logging - miopen::ClearBufferLog(); + miopen::ClearLogBuffer(); // log messages MIOPEN_LOG_W("warn"); MIOPEN_LOG_I("info"); diff --git a/projects/miopen/test/gtest/log.hpp b/projects/miopen/test/gtest/log.hpp index b89af35208d2..5f47bb783ff0 100644 --- a/projects/miopen/test/gtest/log.hpp +++ b/projects/miopen/test/gtest/log.hpp @@ -83,6 +83,8 @@ void TestLogCmdBNormFusion(std::function GetTestCases() +{ + return std::vector{ + // clang-format off + // use-padding input-mode, bias-mode, dir-mode, algo-mode + TestCase(0, 0, 0, 0, 0), + TestCase(0, 0, 0, 0, 1), + TestCase(0, 0, 0, 1, 0), + TestCase(0, 0, 0, 1, 1), + TestCase(0, 0, 1, 0, 0), + TestCase(0, 0, 1, 0, 1), + TestCase(0, 0, 1, 1, 0), + TestCase(0, 0, 1, 1, 1), + TestCase(0, 1, 0, 0, 0), + TestCase(0, 1, 0, 0, 1), + TestCase(0, 1, 0, 1, 0), + TestCase(0, 1, 0, 1, 1), + TestCase(0, 1, 1, 0, 0), + TestCase(0, 1, 1, 0, 1), + TestCase(0, 1, 1, 1, 0), + TestCase(0, 1, 1, 1, 1), + TestCase(1, 0, 0, 0, 0), + TestCase(1, 0, 0, 0, 1), + TestCase(1, 0, 0, 1, 0), + TestCase(1, 0, 0, 1, 1), + TestCase(1, 0, 1, 0, 0), + TestCase(1, 0, 1, 0, 1), + TestCase(1, 0, 1, 1, 0), + TestCase(1, 0, 1, 1, 1), + TestCase(1, 1, 0, 0, 0), + TestCase(1, 1, 0, 0, 1), + TestCase(1, 1, 0, 1, 0), + TestCase(1, 1, 0, 1, 1), + TestCase(1, 1, 1, 0, 0), + TestCase(1, 1, 1, 0, 1), + TestCase(1, 1, 1, 1, 0), + TestCase(1, 1, 1, 1, 1) + // clang-format on + }; +} + +} // namespace + +template +struct GPU_LSTM_test : LSTM_test, testing::TestWithParam +{ +protected: + void SetUp() override + { + this->dataType = miopen_type{}; + + auto [usePadding, inputMode, biasMode, dirMode, algoMode] = GetParam(); + + int batchSize{17}; + int seqLength{2}; + this->batchSize = batchSize; + this->seqLength = seqLength; + this->inVecLen = batchSize; + this->batchSeq = generate_batchSeq(batchSize, seqLength)[0]; + this->hiddenSize = 67; + this->usePadding = usePadding; + this->inputMode = inputMode; + this->biasMode = biasMode; + this->dirMode = dirMode; + this->algoMode = algoMode; + } +}; + +using GPU_LSTM_FP16 = GPU_LSTM_test; + +TEST_P(GPU_LSTM_FP16, HalfTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_FP16, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_FP32 = GPU_LSTM_test; + +TEST_P(GPU_LSTM_FP32, FloatTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_FP32, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_FP64 = GPU_LSTM_test; + +TEST_P(GPU_LSTM_FP64, DoubleTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/lstm_common.hpp b/projects/miopen/test/gtest/lstm.hpp similarity index 87% rename from projects/miopen/test/lstm_common.hpp rename to projects/miopen/test/gtest/lstm.hpp index dfd3c4875183..83c3b32b9081 100644 --- a/projects/miopen/test/lstm_common.hpp +++ b/projects/miopen/test/gtest/lstm.hpp @@ -1,8 +1,7 @@ /******************************************************************************* * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,38 +22,29 @@ * SOFTWARE. * *******************************************************************************/ +#pragma once -#ifndef GUARD_MIOPEN_TEST_LSTM_COMMON_HPP -#define GUARD_MIOPEN_TEST_LSTM_COMMON_HPP - -#include "workspace.hpp" -#include "driver.hpp" +#include #include "dropout_util.hpp" -#include "get_handle.hpp" -#include "tensor_holder.hpp" -#include "test.hpp" -#include "verify.hpp" #include "rnn_util.hpp" -#include "random.hpp" #include "cpu_rnn.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "workspace.hpp" +#include "verify.hpp" + +#include +#include +#include +#include #define MIO_LSTM_TEST_DEBUG 0 #define MIO_RNN_TIME_EVERYTHING 0 -#define WORKAROUND_ISSUE_692 1 +#if(MIO_LSTM_TEST_DEBUG > 0) +#include +#endif +#if(MIO_RNN_TIME_EVERYTHING > 0) +#include +#endif //**************************************************** // FORWARD BASE @@ -202,8 +192,7 @@ struct verify_backward_data_lstm void fail(int badtensor) const { - - std::cout << "./bin/MIOpenDriver rnn -n "; + std::cout << "MIOpenDriver rnn -n "; for(int i = 0; i < seqLength; i++) { if(i < seqLength - 1) @@ -320,7 +309,7 @@ struct verify_backward_weights_lstm void fail(int) const { - std::cout << "./bin/MIOpenDriver rnn -n "; + std::cout << "MIOpenDriver rnn -n "; for(int i = 0; i < seqLength; i++) { if(i < seqLength - 1) @@ -428,11 +417,9 @@ struct verify_forward_infer_lstm : verify_forward_lstm std::vector cpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); int bi = dirMode != 0 ? 2 : 1; @@ -503,13 +490,12 @@ struct verify_forward_infer_lstm : verify_forward_lstm #if(MIO_LSTM_TEST_DEBUG == 2) for(int i = 0; i < output.size(); i++) { - printf("CPU outdata[%d]: %f\n", i, output[i]); + std::cout << "CPU outdata[" << i << "]: " << output[i] << std::endl; } #endif #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: CPU forward inference LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; @@ -523,11 +509,9 @@ struct verify_forward_infer_lstm : verify_forward_lstm std::vector gpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); size_t out_sz = 0; @@ -599,13 +583,12 @@ struct verify_forward_infer_lstm : verify_forward_lstm auto outdata = handle.Read(output_dev, output.size()); for(int i = 0; i < outdata.size(); i++) { - printf("GPU outdata[%d]: %f\n", i, outdata[i]); + std::cout << "GPU outdata[" << i << "]: " << outdata[i] << std::endl; } #endif #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: GPU forward_infer LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; @@ -618,7 +601,7 @@ struct verify_forward_infer_lstm : verify_forward_lstm void fail(int) const { - std::cout << "./bin/MIOpenDriver rnn -n "; + std::cout << "MIOpenDriver rnn -n "; for(int i = 0; i < seqLength; i++) { if(i < seqLength - 1) @@ -736,11 +719,9 @@ struct verify_forward_train_lstm : verify_forward_lstm std::tuple, std::vector, std::vector> cpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); int bi = dirMode != 0 ? 2 : 1; @@ -849,7 +830,6 @@ struct verify_forward_train_lstm : verify_forward_lstm #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: CPU forward train LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; @@ -862,7 +842,8 @@ struct verify_forward_train_lstm : verify_forward_lstm if(reserveSpace.size() != RSVcpu.size()) { - std::abort(); + // std::abort(); + return {}; } std::copy(reserveSpace.begin(), reserveSpace.end(), RSVcpu.begin()); @@ -878,11 +859,9 @@ struct verify_forward_train_lstm : verify_forward_lstm std::tuple, std::vector, std::vector> gpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); std::vector inputCPPDescs; @@ -967,7 +946,7 @@ struct verify_forward_train_lstm : verify_forward_lstm auto outdata = handle.Read(output_dev, output.size()); for(int i = 0; i < outdata.size(); i++) { - printf("GPU outdata[%d]: %f\n", i, outdata[i]); + std::cout << "GPU outdata[" << i << "]: " << outdata[i] << std::endl; } #endif rspace.ReadTo(RSVgpu); @@ -980,7 +959,6 @@ struct verify_forward_train_lstm : verify_forward_lstm #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: GPU forward_train LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; @@ -993,8 +971,7 @@ struct verify_forward_train_lstm : verify_forward_lstm void fail(int badtensor) const { - - std::cout << "./bin/MIOpenDriver rnn -n "; + std::cout << "MIOpenDriver rnn -n "; for(int i = 0; i < seqLength; i++) { if(i < seqLength - 1) @@ -1035,11 +1012,9 @@ template std::tuple, std::vector, std::vector, std::vector> verify_backward_data_lstm::cpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); int bi = dirMode != 0 ? 2 : 1; @@ -1086,7 +1061,8 @@ verify_backward_data_lstm::cpu() const if(reserveSpaceSize != RSVcpu.size()) { - std::abort(); + // std::abort(); + return {}; } std::vector reserveSpace(RSVcpu); @@ -1166,7 +1142,6 @@ verify_backward_data_lstm::cpu() const #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: CPU backward data LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; #endif @@ -1204,11 +1179,9 @@ template std::tuple, std::vector, std::vector, std::vector> verify_backward_data_lstm::gpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); std::vector inputCPPDescs; @@ -1228,7 +1201,8 @@ verify_backward_data_lstm::gpu() const miopenGetRNNWorkspaceSize(&handle, rnnDesc, seqLength, inputDescs.data(), &workspace_size); if(workspace_size % sizeof(T) != 0) { - std::abort(); + // std::abort(); + return {}; } Workspace wspace{}; // Needed to zero out the workspace (happens in std::vector's constructor) @@ -1247,7 +1221,8 @@ verify_backward_data_lstm::gpu() const if(reserveSpaceSize != (RSVgpu.size() * sizeof(T))) { - std::abort(); + // std::abort(); + return {}; } Workspace rspace{}; rspace.Write(RSVgpu); @@ -1315,7 +1290,6 @@ verify_backward_data_lstm::gpu() const #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: GPU backward data LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; #endif @@ -1332,7 +1306,6 @@ verify_backward_data_lstm::gpu() const template std::vector verify_backward_weights_lstm::cpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif @@ -1409,7 +1382,6 @@ std::vector verify_backward_weights_lstm::cpu() const #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: CPU backward_weights LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; #endif @@ -1423,11 +1395,9 @@ std::vector verify_backward_weights_lstm::cpu() const template std::vector verify_backward_weights_lstm::gpu() const { - #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_start = std::chrono::high_resolution_clock::now(); #endif - auto&& handle = get_handle(); std::vector inputCPPDescs; @@ -1479,7 +1449,6 @@ std::vector verify_backward_weights_lstm::gpu() const #if(MIO_RNN_TIME_EVERYTHING == 1) auto t_end = std::chrono::high_resolution_clock::now(); - std::cout << "Wall clock: GPU backwards_weights LSTM pass time: " << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; #endif @@ -1491,48 +1460,214 @@ std::vector verify_backward_weights_lstm::gpu() const } //~~~~~~~~~~~~ END BACKWARD WEIGHTS CPU & GPU ~~~~~~~~~~~~~~~~~~~~~~~~ -//====================== DRIVER ============================ -template -struct lstm_basic_driver : test_driver +struct Verifier { - std::vector batchSeq; - int seqLength{}; - int inVecLen{}; - int hiddenSize{}; - int numLayers{}; - int inputMode{}; - int biasMode{}; - int dirMode{}; - int algoMode{}; - int batchSize{}; - int useDropout{}; - - // Null pointer input - bool nohx = false; - bool nodhy = false; - bool nocx = false; - bool nodcy = false; - bool nohy = false; - bool nodhx = false; - bool nocy = false; - bool nodcx = false; - bool flatBatchFill = false; - bool usePadding = false; - - lstm_basic_driver() {} - - void run() + bool time{false}; + int time_iter{1}; + int warmup_iter{0}; + double tolerance{80.0}; + + template + bool compare_and_report( + const CpuRange& out_cpu, const GpuRange& out_gpu, Compare compare, Report report, Fail fail) { - const double Data_scale = 0.001; -#if(MIOPEN_BACKEND_OPENCL == 1) -#if WORKAROUND_ISSUE_692 == 1 - std::cout << "Skip test for Issue #692: " << std::endl; - exit(EXIT_SUCCESS); // NOLINT (concurrency-mt-unsafe) -#endif - if(type == miopenHalf) - exit(EXIT_SUCCESS); // NOLINT (concurrency-mt-unsafe) -#endif + std::vector error; + bool pass = compare(error, out_cpu, out_gpu); + return report(pass, error, out_cpu, out_gpu, fail); + } + + template + bool compare_and_report(const std::tuple& out_cpu, + const std::tuple& out_gpu, + Compare compare, + Report report, + Fail fail) + { + static_assert(sizeof...(CpuRanges) == sizeof...(GpuRanges), "CPU and GPU mismatch"); + return miopen::sequence([&](auto... is) { + bool continue_ = true; + miopen::each_args( + [&](auto i) { + // cppcheck-suppress knownConditionTrueFalse + if(continue_) + { + continue_ = this->compare_and_report( + std::get(out_cpu), std::get(out_gpu), compare, report, [&](int) { + return fail(i); + }); + } + }, + is...); + return continue_; + })(std::integral_constant{}); + } + auto verify_reporter() + { + return [=](bool pass, + std::vector error, + const auto& out_cpu, + const auto& out_gpu, + auto fail) { + if(not pass) + { + if(not error.empty()) + { + std::cout << "FAILED: " << error.front() << std::endl; + fail(-1); + } + + auto mxdiff = miopen::max_diff(out_cpu, out_gpu); + std::cout << "Max diff: " << mxdiff << std::endl; + + if(miopen::range_zero(out_cpu)) + std::cout << "CPU data is all zeros" << std::endl; + if(miopen::range_zero(out_gpu)) + std::cout << "GPU data is all zeros" << std::endl; + + auto idx = miopen::mismatch_idx(out_cpu, out_gpu, miopen::float_equal); + if(idx < miopen::range_distance(out_cpu)) + { + std::cout << "Mismatch at " << idx << ": " << out_cpu[idx] + << " != " << out_gpu[idx] << std::endl; + } + + auto cpu_nan_idx = find_idx(out_cpu, miopen::not_finite); + if(cpu_nan_idx >= 0) + { + std::cout << "Non finite number found in CPU data at " << cpu_nan_idx << ": " + << out_cpu[cpu_nan_idx] << std::endl; + } + + auto gpu_nan_idx = find_idx(out_gpu, miopen::not_finite); + if(gpu_nan_idx >= 0) + { + std::cout << "Non finite number found in GPU data at " << gpu_nan_idx << ": " + << out_gpu[gpu_nan_idx] << std::endl; + } + } + else if(miopen::range_zero(out_cpu) and miopen::range_zero(out_gpu) and + (miopen::range_distance(out_cpu) != 0)) + { + std::cout << "Warning: Both CPU and GPU data is all zero" << std::endl; + fail(-1); + } + return true; + }; + } + + template + auto cpu_async(V& v, Ts&&... xs) -> std::future + { + return std::async(std::launch::deferred, [&] { return v.cpu(xs...); }); + } + + template + auto verify_impl(F&& f, V&& v, Ts&&... xs) -> decltype(std::make_pair(v.cpu(xs...), + v.gpu(xs...))) + { + decltype(v.cpu(xs...)) cpu; + decltype(v.gpu(xs...)) gpu; + + try + { + auto&& h = get_handle(); + // Compute cpu + std::future cpuf; + { + cpuf = cpu_async(v, xs...); + } + // Compute gpu + if(time) + { + for(size_t i = 0; i < warmup_iter; ++i) + { + v.gpu(xs...); + } + h.EnableProfiling(); + h.ResetKernelTime(); + } + gpu = v.gpu(xs...); + if(time) + { + float total_time = h.GetKernelTime(); + for(size_t i = 1; i < time_iter; ++i) + { + h.ResetKernelTime(); + v.gpu(xs...); + total_time += h.GetKernelTime(); + } + std::cout << "Kernel time: " << (total_time / time_iter) << " ms" << std::endl; + h.EnableProfiling(false); + } + + cpu = cpuf.get(); + auto report = verify_reporter(); + compare_and_report(cpu, gpu, f, report, [&](int mode) { v.fail(mode, xs...); }); + + if(time) + v.fail(std::integral_constant{}, xs...); + } + catch(const std::exception& ex) + { + std::cout << "FAILED: " << ex.what() << std::endl; + v.fail(-1, xs...); + } + catch(...) + { + std::cout << "FAILED with unknown exception" << std::endl; + v.fail(-1, xs...); + } + + return std::make_pair(cpu, gpu); + } + + template + auto verify(V&& v, Ts&&... xs) -> decltype(std::make_pair(v.cpu(xs...), v.gpu(xs...))) + { + return verify_impl( + [&](std::vector& error, auto&& cpu, auto&& gpu) { + EXPECT_EQ(miopen::range_distance(cpu), miopen::range_distance(gpu)); + + using value_type = miopen::range_value; + double threshold = std::numeric_limits::epsilon() * tolerance; + error = {miopen::rms_range(cpu, gpu)}; + return error.front() <= threshold; + }, + v, + xs...); + } +}; + +template +struct LSTM_test : Verifier +{ + int batchSize{4}; + int seqLength{10}; + int inVecLen{32}; + int numLayers{1}; + int hiddenSize{0}; + int useDropout{0}; + int usePadding{0}; + int flatBatchFill{0}; + int inputMode{0}; + int biasMode{0}; + int dirMode{0}; + int algoMode{0}; + bool nohx{false}; + bool nodhy{false}; + bool nocx{false}; + bool nodcy{false}; + bool nohy{false}; + bool nodhx{false}; + bool nocy{false}; + bool nodcx{false}; + std::vector batchSeq; + const double dataScale{0.001}; + miopenDataType_t dataType{miopenFloat}; + + void RunTest() + { if(batchSeq.empty() || 0 == batchSeq[0]) { std::cout << "Empty batch sequence. Filling uniformly with batch size: " << batchSize @@ -1550,9 +1685,7 @@ struct lstm_basic_driver : test_driver if(batchSeq.size() != seqLength) { - std::cerr << "FAILED: Batch sequence vector length, does not match sequence length." - << std::endl; - std::abort(); + GTEST_SKIP() << "FAILED: Batch sequence vector length, does not match sequence length."; } #if(MIO_LSTM_TEST_DEBUG == 2) @@ -1563,9 +1696,6 @@ struct lstm_basic_driver : test_driver #endif auto&& handle = get_handle(); - - int batch_n = std::accumulate(batchSeq.begin(), batchSeq.end(), 0); - miopenRNNDescriptor_t rnnDesc; miopenCreateRNNDescriptor(&rnnDesc); miopenDropoutDescriptor_t DropoutDesc; @@ -1574,29 +1704,15 @@ struct lstm_basic_driver : test_driver if(useDropout != 0) { -// Workaround for issue #2335. -// OpenCL error creating buffer: 0 Invalid Buffer Size -#if MIOPEN_BACKEND_OPENCL - std::cout << "Skip test for Issue #2335: " << std::endl; - return; -#endif miopenHandle_t mio_handle; miopenCreateWithStream(&mio_handle, handle.GetStream()); - float dropout_rate = 0.5; - unsigned long long dropout_seed = 0ULL; + float dropout_rate{0.5f}; + unsigned long long dropout_seed{0ULL}; miopenDropoutGetStatesSize(mio_handle, &statesSizeInBytes); -#if MIOPEN_BACKEND_OPENCL - cl_context ctx; - clGetCommandQueueInfo( - handle.GetStream(), CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); - cl_mem dropout_state_buf = - clCreateBuffer(ctx, CL_MEM_READ_WRITE, statesSizeInBytes, nullptr, nullptr); -#elif MIOPEN_BACKEND_HIP void* dropout_state_buf; - hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); -#endif + (void)hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); miopenSetDropoutDescriptor(DropoutDesc, mio_handle, @@ -1617,7 +1733,7 @@ struct lstm_basic_driver : test_driver miopenLSTM, miopenRNNBiasMode_t(biasMode), miopenRNNAlgo_t(algoMode), - type); + dataType); } else { @@ -1629,7 +1745,7 @@ struct lstm_basic_driver : test_driver miopenLSTM, miopenRNNBiasMode_t(biasMode), miopenRNNAlgo_t(algoMode), - type); // defined in superclass testdriver + dataType); } if(usePadding) @@ -1654,7 +1770,7 @@ struct lstm_basic_driver : test_driver std::vector input(in_sz); for(std::size_t i = 0; i < in_sz; i++) { - input[i] = prng::gen_descreet_unsigned(Data_scale, 100); + input[i] = prng::gen_descreet_unsigned(dataScale, 100); } std::size_t hx_sz = ((dirMode != 0) ? 2ULL : 1ULL) * hiddenSize * batchSize * numLayers; @@ -1667,17 +1783,17 @@ struct lstm_basic_driver : test_driver std::vector inlens(2, 0); inlens.at(0) = batchSeq.at(0); inlens.at(1) = inVecReal; - auto firstInputDesc = miopen::TensorDescriptor(miopen::deref(rnnDesc).dataType, inlens); - miopenGetRNNParamsSize( - &handle, rnnDesc, &firstInputDesc, &wei_bytes, miopen::deref(rnnDesc).dataType); + auto firstInputDesc = miopen::TensorDescriptor(dataType, inlens); + miopenGetRNNParamsSize(&handle, rnnDesc, &firstInputDesc, &wei_bytes, dataType); auto wei_sz = int(wei_bytes / sizeof(T)); std::vector weights(wei_sz); for(std::size_t i = 0; i < wei_sz; i++) { - weights[i] = prng::gen_descreet_uniform_sign(Data_scale, 100); + weights[i] = prng::gen_descreet_uniform_sign(dataScale, 100); } -#if(MIO_LSTM_TEST_DEBUG > 0) + int batch_n = std::accumulate(batchSeq.begin(), batchSeq.end(), 0); +#if(MIO_LSTM_TEST_DEBUG == 2) printf("inputMode: %d, biasMode: %d, dirMode: %d\n", inputMode, biasMode, dirMode); printf("hz: %d, batch_n: %d, seqLength: %d, inputLen: %d, numLayers: %d\n", hiddenSize, @@ -1699,7 +1815,7 @@ struct lstm_basic_driver : test_driver { for(std::size_t i = 0; i < hx_sz; i++) { - hx[i] = prng::gen_descreet_unsigned(Data_scale, 100); + hx[i] = prng::gen_descreet_unsigned(dataScale, 100); } } @@ -1707,7 +1823,7 @@ struct lstm_basic_driver : test_driver { for(std::size_t i = 0; i < hx_sz; i++) { - dhyin[i] = prng::gen_descreet_unsigned(Data_scale, 100); + dhyin[i] = prng::gen_descreet_unsigned(dataScale, 100); } } @@ -1715,7 +1831,7 @@ struct lstm_basic_driver : test_driver { for(std::size_t i = 0; i < hx_sz; i++) { - cx[i] = prng::gen_descreet_unsigned(Data_scale, 100); + cx[i] = prng::gen_descreet_unsigned(dataScale, 100); } } @@ -1723,25 +1839,21 @@ struct lstm_basic_driver : test_driver { for(std::size_t i = 0; i < hx_sz; i++) { - dcyin[i] = prng::gen_descreet_unsigned(Data_scale, 100); + dcyin[i] = prng::gen_descreet_unsigned(dataScale, 100); } } std::vector inputCPPDescs; std::vector inputDescs; - createTensorDescArray( - inputCPPDescs, inputDescs, batchSeq, inVecLen, miopen::deref(rnnDesc).dataType); + createTensorDescArray(inputCPPDescs, inputDescs, batchSeq, inVecLen, dataType); size_t reserveSpaceSize; miopenGetRNNTrainingReserveSize( &handle, rnnDesc, seqLength, inputDescs.data(), &reserveSpaceSize); std::vector outputCPPDescs; std::vector outputDescs; - createTensorDescArray(outputCPPDescs, - outputDescs, - batchSeq, - hiddenSize * ((dirMode != 0) ? 2 : 1), - miopen::deref(rnnDesc).dataType); + createTensorDescArray( + outputCPPDescs, outputDescs, batchSeq, hiddenSize * ((dirMode != 0) ? 2 : 1), dataType); size_t out_sz = getSuperTensorSize(batchSeq, seqLength, @@ -1763,10 +1875,9 @@ struct lstm_basic_driver : test_driver size_t device_mem = handle.GetGlobalMemorySize(); if(total_mem >= device_mem) { - show_command(); std::cout << "Config requires " << total_mem << " Bytes to write all necessary tensors to GPU. GPU has " << device_mem - << " Bytes of memory." << std::endl; + << " bytes of memory." << std::endl; } reserveSpaceSize = (reserveSpaceSize + sizeof(T) - 1) / sizeof(T); @@ -1811,10 +1922,10 @@ struct lstm_basic_driver : test_driver std::vector dyin(out_sz); for(std::size_t i = 0; i < out_sz; i++) { - dyin[i] = prng::gen_descreet_unsigned(Data_scale, 100); + dyin[i] = prng::gen_descreet_unsigned(dataScale, 100); } -#if(MIO_LSTM_TEST_DEBUG > 0) +#if(MIO_LSTM_TEST_DEBUG == 2) printf("Running backward data LSTM.\n"); #endif auto bwdDataOutputPair = @@ -1832,9 +1943,9 @@ struct lstm_basic_driver : test_driver // RETURNS: std::make_tuple(dx, dhx, dcx, reserveSpace, workSpace); auto workSpaceBwdData = std::get<3>(bwdDataOutputPair.second); -#if(MIO_LSTM_TEST_DEBUG > 0) +#if(MIO_LSTM_TEST_DEBUG == 2) printf("Running backward weights LSTM.\n"); - printf("reserve sz: %d, workSpace sz: %d, weight sz: %d\n", + printf("reserve sz: %zu, workSpace sz: %zu, weight sz: %d\n", rsvcpu.size(), workSpaceBwdData.size(), wei_sz); @@ -1845,50 +1956,5 @@ struct lstm_basic_driver : test_driver rnnDesc, input, dyin, hx, rsvgpu, rsvcpu, workSpaceBwdData, batchSeq, hiddenSize, wei_sz, batch_n, seqLength, numLayers, biasMode, dirMode, inputMode, inVecReal, hx_sz, nohx, bool(useDropout), usePadding}); - -/// \todo Resolve the issue and remove workaround. -/// ROCm3.3, Radeon VII: Many test cases always fail with: -/// "Forward Inference LSTM:" -/// "Output tensor output failed verification." -#if 0 - if(useDropout == 0) - { - verify(verify_forward_infer_lstm{rnnDesc, - input, - hx, - cx, - weights, - batchSeq, - hiddenSize, - batch_n, - seqLength, - numLayers, - biasMode, - dirMode, - inputMode, - inVecReal, - hx_sz, - nohx, - nocx, - nohy, - nocy}); - } -#endif - /* normal hx/cx/dhy/dcy input test end */ - - // DLOWELL: Subtracting delta weights may produce NAN and infinities. Further investigation - // is needed. - // auto dweights = std::get<1>(dweights_pair); - // std::transform(weightData.begin( ), weightData.end( ), dweights.begin( ), - // weightData.begin( ),std::minus( )); - // verify(verify_forward_infer_lstm{rnnDesc, inputData, - // curHiddenState, curCellState, weightData, - // batchSeq, - // hiddenSize, batch_n, - // seqLength, numLayers, - // biasMode, dirMode, - // inputMode, inVecReal}); } }; - -#endif diff --git a/projects/miopen/test/gtest/lstm_dropout.cpp b/projects/miopen/test/gtest/lstm_dropout.cpp new file mode 100644 index 000000000000..d2a023b65b34 --- /dev/null +++ b/projects/miopen/test/gtest/lstm_dropout.cpp @@ -0,0 +1,99 @@ +/******************************************************************************* + * + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "lstm.hpp" + +namespace { + +struct TestCase +{ + TestCase(int dirMode_, int flatBatchFill_) : dirMode(dirMode_), flatBatchFill(flatBatchFill_) {} + + int dirMode; + int flatBatchFill; + + friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) + { + return os << "dir-mode:" << tc.dirMode << " flat-batch-fill:" << tc.flatBatchFill; + } +}; + +std::vector GetTestCases() +{ + return std::vector{ + // clang-format off + // dir-mode flat-batch-fill + TestCase(0, 0), + TestCase(0, 1), + TestCase(1, 0), + TestCase(1, 1) + // clang-format on + }; +} + +} // namespace + +template +struct GPU_LSTM_dropout_test : LSTM_test, testing::TestWithParam +{ +protected: + void SetUp() override + { + this->dataType = miopen_type{}; + + auto [dirMode, flatBatchFill] = GetParam(); + + int useDropout{1}; + int batchSize{17}; + int seqLength{25}; + this->useDropout = useDropout; + this->batchSize = batchSize; + this->seqLength = seqLength; + this->inVecLen = batchSize; + this->batchSeq = generate_batchSeq(batchSize, seqLength)[0]; + this->numLayers = 3; + this->hiddenSize = 67; + this->dirMode = dirMode; + this->flatBatchFill = flatBatchFill; + } +}; + +using GPU_LSTM_dropout_FP16 = GPU_LSTM_dropout_test; + +TEST_P(GPU_LSTM_dropout_FP16, HalfTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP16, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_dropout_FP32 = GPU_LSTM_dropout_test; + +TEST_P(GPU_LSTM_dropout_FP32, FloatTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP32, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_dropout_FP64 = GPU_LSTM_dropout_test; + +TEST_P(GPU_LSTM_dropout_FP64, DoubleTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/lstm_extra.cpp b/projects/miopen/test/gtest/lstm_extra.cpp index fe030256b858..7ed311664e84 100644 --- a/projects/miopen/test/gtest/lstm_extra.cpp +++ b/projects/miopen/test/gtest/lstm_extra.cpp @@ -25,105 +25,136 @@ *******************************************************************************/ #include "lstm.hpp" -#include "get_handle.hpp" -#include -#include -#include -namespace lstm_extra { -void GetArgs(const std::string& param, std::vector& tokens) -{ - std::stringstream ss(param); - std::istream_iterator begin(ss); - std::istream_iterator end; - while(begin != end) - tokens.push_back(*begin++); -} +namespace { -auto GetTestCases(std::string precision) +struct TestCase { - std::string flags = "test_lstm --verbose " + precision; - std::string commonFlags = " --batch-size 32 --seq-len 3 --batch-seq 32 32 32 --vector-len 128 " - "--hidden-size 128 --num-layers 1 --in-mode 0 --bias-mode 0"; - - // clang-format off - return std::vector{ - {flags + commonFlags + " -dir-mode 0 --no-hx"}, - {flags + commonFlags + " -dir-mode 0 --no-dhy"}, - {flags + commonFlags + " -dir-mode 0 --no-hx --no-dhy"}, - {flags + commonFlags + " -dir-mode 0 --no-cx"}, - {flags + commonFlags + " -dir-mode 0 --no-hx --no-cx"}, - {flags + commonFlags + " -dir-mode 0 --no-dcy"}, - {flags + commonFlags + " -dir-mode 0 --no-cx --no-dcy"}, - {flags + commonFlags + " -dir-mode 1 --no-hx"}, - {flags + commonFlags + " -dir-mode 1 --no-dhy"}, - {flags + commonFlags + " -dir-mode 1 --no-hx --no-dhy"}, - {flags + commonFlags + " -dir-mode 1 --no-cx"}, - {flags + commonFlags + " -dir-mode 1 --no-hx --no-cx"}, - {flags + commonFlags + " -dir-mode 1 --no-dcy"}, - {flags + commonFlags + " -dir-mode 1 --no-cx --no-dcy"}, - {flags + commonFlags + " -dir-mode 0 --no-hy"}, - {flags + commonFlags + " -dir-mode 0 --no-dhx"}, - {flags + commonFlags + " -dir-mode 0 --no-hy --no-dhx"}, - {flags + commonFlags + " -dir-mode 0 --no-cy"}, - {flags + commonFlags + " -dir-mode 0 --no-hy --no-cy"}, - {flags + commonFlags + " -dir-mode 0 --no-dcx"}, - {flags + commonFlags + " -dir-mode 0 --no-cy --no-dcx"}, - {flags + commonFlags + " -dir-mode 1 --no-hy"}, - {flags + commonFlags + " -dir-mode 1 --no-dhx"}, - {flags + commonFlags + " -dir-mode 1 --no-hy --no-dhx"}, - {flags + commonFlags + " -dir-mode 1 --no-cy"}, - {flags + commonFlags + " -dir-mode 1 --no-hy --no-cy"}, - {flags + commonFlags + " -dir-mode 1 --no-dcx"}, - {flags + commonFlags + " -dir-mode 1 --no-cy --no-dcx"}, - {flags + commonFlags + " -dir-mode 0 --no-hx --no-dhy --no-cx --no-dcy --no-hy --no-dhx --no-cy --no-dcx"}, - {flags + commonFlags + " -dir-mode 1 --no-hx --no-dhy --no-cx --no-dcy --no-hy --no-dhx --no-cy --no-dcx"} - }; - // clang-format on -} + TestCase(int dir_mode_, + int no_hx_, + int no_dhy_, + int no_cx_, + int no_dcy_, + int no_hy_, + int no_dhx_, + int no_cy_, + int no_dcx_) + : dir_mode(dir_mode_), + no_hx(no_hx_), + no_dhy(no_dhy_), + no_cx(no_cx_), + no_dcy(no_dcy_), + no_hy(no_hy_), + no_dhx(no_dhx_), + no_cy(no_cy_), + no_dcx(no_dcx_) + { + } -using TestCase = decltype(GetTestCases({}))::value_type; + int dir_mode; + int no_hx; + int no_dhy; + int no_cx; + int no_dcy; + int no_hy; + int no_dhx; + int no_cy; + int no_dcx; -class GPU_lstm_extra_FP32 : public testing::TestWithParam> -{ - MIOPEN_DECLARE_GTEST_USES_TEST_DRIVE(); + friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) + { + return os << "dir-mode:" << tc.dir_mode << " no-hx:" << tc.no_hx << " no-dhy:" << tc.no_dhy + << " no-cx:" << tc.no_cx << " no-dcy:" << tc.no_dcy << " no-hy:" << tc.no_hy + << " no-dhx:" << tc.no_dhx << " no-cy:" << tc.no_cy << " no-dcx:" << tc.no_dcx + << " no-dcx:" << tc.no_dcx; + } }; -bool IsTestSupportedForDevice() +std::vector GetTestCases() { - using namespace miopen::debug; - using e_mask = enabled; - using d_mask = disabled; - return ::IsTestSupportedForDevMask(); + return std::vector{ + // clang-format off + // dir-mode no-hx no-dhy no-cx no-dcy no-hy no-dhx no-cy no-dcx + TestCase(0, 1, 0, 0, 0, 0, 0, 0, 0), + TestCase(0, 0, 1, 0, 0, 0, 0, 0, 0), + TestCase(0, 1, 1, 0, 0, 0, 0, 0, 0), + TestCase(0, 0, 0, 1, 0, 0, 0, 0, 0), + TestCase(0, 1, 0, 1, 0, 0, 0, 0, 0), + TestCase(0, 0, 0, 0, 1, 0, 0, 0, 0), + TestCase(0, 0, 0, 1, 1, 0, 0, 0, 0), + TestCase(1, 1, 0, 0, 0, 0, 0, 0, 0), + TestCase(1, 0, 1, 0, 0, 0, 0, 0, 0), + TestCase(1, 1, 1, 0, 0, 0, 0, 0, 0), + TestCase(1, 0, 0, 1, 0, 0, 0, 0, 0), + TestCase(1, 1, 0, 1, 0, 0, 0, 0, 0), + TestCase(1, 0, 0, 0, 1, 0, 0, 0, 0), + TestCase(1, 0, 0, 1, 1, 0, 0, 0, 0), + TestCase(0, 0, 0, 0, 0, 1, 0, 0, 0), + TestCase(0, 0, 0, 0, 0, 0, 1, 0, 0), + TestCase(0, 0, 0, 0, 0, 1, 1, 0, 0), + TestCase(0, 0, 0, 0, 0, 0, 0, 1, 0), + TestCase(0, 0, 0, 0, 0, 1, 0, 1, 0), + TestCase(0, 0, 0, 0, 0, 0, 0, 0, 1), + TestCase(0, 0, 0, 0, 0, 0, 0, 1, 1), + TestCase(1, 0, 0, 0, 0, 1, 0, 0, 0), + TestCase(1, 0, 0, 0, 0, 0, 1, 0, 0), + TestCase(1, 0, 0, 0, 0, 1, 1, 0, 1), + TestCase(1, 0, 0, 0, 0, 0, 0, 1, 0), + TestCase(1, 0, 0, 0, 0, 1, 0, 1, 0), + TestCase(1, 0, 0, 0, 0, 0, 0, 0, 1), + TestCase(1, 0, 0, 0, 0, 0, 0, 1, 1), + TestCase(0, 1, 1, 1, 1, 1, 1, 1, 1), + TestCase(1, 1, 1, 1, 1, 1, 1, 1, 1) + // clang-format on + }; } -void Run2dDriver(miopenDataType_t prec) +} // namespace + +template +struct GPU_LSTM_extra_test : LSTM_test, testing::TestWithParam { - if(!IsTestSupportedForDevice()) +protected: + void SetUp() override { - GTEST_SKIP(); - } - std::vector params = GPU_lstm_extra_FP32::GetParam(); + this->dataType = miopen_type{}; - for(const auto& test_value : params) - { - std::vector tokens; - GetArgs(test_value, tokens); - std::vector ptrs; - - std::transform(tokens.begin(), tokens.end(), std::back_inserter(ptrs), [](const auto& str) { - return str.data(); - }); - testing::internal::CaptureStderr(); - test_drive(ptrs.size(), ptrs.data()); - auto capture = testing::internal::GetCapturedStderr(); - std::cout << capture; + auto [dirMode, nohx, nodhy, nocx, nodcy, nohy, nodhx, nocy, nodcx] = GetParam(); + + this->batchSize = 32; + this->seqLength = 3; + this->batchSeq = {32, 32, 32}; + this->inVecLen = 128; + this->hiddenSize = 128; + this->numLayers = 1; + this->inputMode = 0; + this->biasMode = 0; + this->dirMode = dirMode; + this->nohx = bool(nohx); + this->nodhy = bool(nodhy); + this->nocx = bool(nocx); + this->nodcy = bool(nodcy); + this->nohy = bool(nohy); + this->nodhx = bool(nodhx); + this->nocy = bool(nocy); + this->nodcx = bool(nodcx); } }; -} // namespace lstm_extra -using namespace lstm_extra; +using GPU_LSTM_extra_FP16 = GPU_LSTM_extra_test; + +TEST_P(GPU_LSTM_extra_FP16, HalfTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP16, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_extra_FP32 = GPU_LSTM_extra_test; + +TEST_P(GPU_LSTM_extra_FP32, FloatTest) { RunTest(); } + +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP32, testing::ValuesIn(GetTestCases())); + +using GPU_LSTM_extra_FP64 = GPU_LSTM_extra_test; -TEST_P(GPU_lstm_extra_FP32, FloatTest_lstm_extra) { Run2dDriver(miopenFloat); }; +TEST_P(GPU_LSTM_extra_FP64, DoubleTest) { RunTest(); } -INSTANTIATE_TEST_SUITE_P(Full, GPU_lstm_extra_FP32, testing::Values(GetTestCases("--float"))); +INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/main_hip.cpp b/projects/miopen/test/gtest/main_hip.cpp index 0f5eac853322..c8e2e0126b63 100644 --- a/projects/miopen/test/gtest/main_hip.cpp +++ b/projects/miopen/test/gtest/main_hip.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + // This test event listener ensures that HIP errors are cleaned up after every test, and will flag // tests that don't clean up their own errors class HIPErrorHandler : public testing::EmptyTestEventListener @@ -26,6 +29,25 @@ class HIPErrorHandler : public testing::EmptyTestEventListener int main(int argc, char** argv) { + // Child processes spawned by multi-process tests (e.g., perfdb) inherit + // GTEST_TOTAL_SHARDS/GTEST_SHARD_INDEX from the parent's CI shard. + // This causes children to run only a subset of their assigned work. + // Clear sharding env vars so each child runs all filtered tests. + for(int i = 1; i < argc; ++i) + { + if(std::string_view(argv[i]) == "--reset-sharding") + { +#ifdef _WIN32 + _putenv_s("GTEST_TOTAL_SHARDS", ""); + _putenv_s("GTEST_SHARD_INDEX", ""); +#else + unsetenv("GTEST_TOTAL_SHARDS"); + unsetenv("GTEST_SHARD_INDEX"); +#endif + break; + } + } + testing::InitGoogleTest(&argc, argv); testing::TestEventListeners& listeners = testing::UnitTest::GetInstance()->listeners(); diff --git a/projects/miopen/test/gtest/mha_helper.hpp b/projects/miopen/test/gtest/mha_helper.hpp index 7a00c5cc6967..f38ee353b2f7 100644 --- a/projects/miopen/test/gtest/mha_helper.hpp +++ b/projects/miopen/test/gtest/mha_helper.hpp @@ -468,8 +468,6 @@ void MultiHeadAttentionBackwardDataf32(const tensor& q_val, const tensor& O_val, // attention (O) const tensor& dO_val, const tensor& softmax, - const tensor& attn_max, - const tensor& z_sum, tensor& dQ_val, tensor& dK_val, tensor& dV_val) diff --git a/projects/miopen/test/gtest/pooling2d_asymmetric.cpp b/projects/miopen/test/gtest/pooling2d_asymmetric.cpp index 6ac02fd29459..424ea5ccb95d 100644 --- a/projects/miopen/test/gtest/pooling2d_asymmetric.cpp +++ b/projects/miopen/test/gtest/pooling2d_asymmetric.cpp @@ -100,7 +100,7 @@ void Run2dDriver(miopenDataType_t prec) } }; -bool IsTestSupportedForDevice(const miopen::Handle& handle) { return true; } +bool IsTestSupportedForDevice(const miopen::Handle& /*handle*/) { return true; } std::vector GetTestCases(const std::string& precision) { diff --git a/projects/miopen/test/gtest/pooling2d_wide.cpp b/projects/miopen/test/gtest/pooling2d_wide.cpp index 4db13691343e..7bd474409eb9 100644 --- a/projects/miopen/test/gtest/pooling2d_wide.cpp +++ b/projects/miopen/test/gtest/pooling2d_wide.cpp @@ -24,9 +24,6 @@ * *******************************************************************************/ -#include -#include -#include "get_handle.hpp" #include "gtest_common.hpp" #include "pooling2d.hpp" @@ -62,7 +59,6 @@ void GetArgs(const std::string& param, std::vector& tokens) void Run2dDriver(miopenDataType_t prec) { - std::vector params; switch(prec) { @@ -98,9 +94,9 @@ void Run2dDriver(miopenDataType_t prec) auto capture = testing::internal::GetCapturedStderr(); std::cout << capture; } -}; +} -bool IsTestSupportedForDevice(const miopen::Handle& handle) { return true; } +bool IsTestSupportedForDevice(const miopen::Handle& /*handle*/) { return true; } std::vector GetTestCases(const std::string& precision) { @@ -132,7 +128,7 @@ TEST_P(GPU_WidePooling2d_FP32, FloatTest_pooling2d_wide) { GTEST_SKIP(); } -}; +} TEST_P(GPU_WidePooling2d_FP16, HalfTest_pooling2d_wide) { @@ -145,7 +141,7 @@ TEST_P(GPU_WidePooling2d_FP16, HalfTest_pooling2d_wide) { GTEST_SKIP(); } -}; +} TEST_P(GPU_WidePooling2d_BFP16, BFloat16Test_pooling2d_wide) { @@ -158,7 +154,7 @@ TEST_P(GPU_WidePooling2d_BFP16, BFloat16Test_pooling2d_wide) { GTEST_SKIP(); } -}; +} INSTANTIATE_TEST_SUITE_P(Full, GPU_WidePooling2d_FP32, testing::Values(GetTestCases("--float"))); diff --git a/projects/miopen/test/gtest/reducecalculation.hpp b/projects/miopen/test/gtest/reducecalculation.hpp index 138d04c758e4..2f2867423d5f 100644 --- a/projects/miopen/test/gtest/reducecalculation.hpp +++ b/projects/miopen/test/gtest/reducecalculation.hpp @@ -55,7 +55,7 @@ void cpu_calculation_forward(tensor input, T calculation = static_cast(0); - miopen::ford(reduce_size)([&](size_t i) { + miopen::ford(reduce_size)([&](size_t /*i*/) { T val = input[input_idx]; if(nanPropagation && std::isnan(val)) { diff --git a/projects/miopen/test/gtest/rnn_seq_api.hpp b/projects/miopen/test/gtest/rnn_seq_api.hpp index 3279013c53b8..b96a7537c3fd 100644 --- a/projects/miopen/test/gtest/rnn_seq_api.hpp +++ b/projects/miopen/test/gtest/rnn_seq_api.hpp @@ -1766,7 +1766,7 @@ struct RNNSeqApiCommon : public ::testing::TestWithParam miopenDropoutGetStatesSize(&handle, &statesSizeInBytes); void* dropout_state_buf; - hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); + (void)hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); miopenSetDropoutDescriptor(&dropoutDesc, &handle, diff --git a/projects/miopen/test/gtest/rnn_vanilla_common.hpp b/projects/miopen/test/gtest/rnn_vanilla_common.hpp index 8a57bcdfab90..6d9c01668ca9 100644 --- a/projects/miopen/test/gtest/rnn_vanilla_common.hpp +++ b/projects/miopen/test/gtest/rnn_vanilla_common.hpp @@ -1228,7 +1228,7 @@ struct RNNVanillaCommon : ::testing::TestWithParam clCreateBuffer(ctx, CL_MEM_READ_WRITE, statesSizeInBytes, nullptr, nullptr); #elif MIOPEN_BACKEND_HIP void* dropout_state_buf; - hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); + (void)hipMalloc(static_cast(&dropout_state_buf), statesSizeInBytes); #endif miopenSetDropoutDescriptor(DropoutDesc, diff --git a/projects/miopen/test/gtest/t5layernorm.hpp b/projects/miopen/test/gtest/t5layernorm.hpp index 78bf49127495..e082830f374c 100644 --- a/projects/miopen/test/gtest/t5layernorm.hpp +++ b/projects/miopen/test/gtest/t5layernorm.hpp @@ -113,12 +113,11 @@ void cpu_t5layernorm_backward(tensor dy, } template -void cpu_t5layernorm_backward_weight( - tensor dy, tensor x, tensor rstd, tensor& ref_dw, miopenNormMode_t mode) +void cpu_t5layernorm_backward_weight(tensor dy, tensor x, tensor rstd, tensor& ref_dw) { auto dims = dy.desc.GetLengths(); - size_t outer_size = 1; size_t inner_size = dims[dims.size() - 1]; + size_t outer_size = 1; for(size_t i = 0ULL; i < dims.size() - 1; ++i) { @@ -434,7 +433,7 @@ struct T5LayerNormBwdTest : public ::testing::TestWithParam { auto&& handle = get_handle(); cpu_t5layernorm_backward(dy, x, weight, rstd, ref_dx, ln_mode); - cpu_t5layernorm_backward_weight(dy, x, rstd, ref_dw, ln_mode); + cpu_t5layernorm_backward_weight(dy, x, rstd, ref_dw); miopenStatus_t status; diff --git a/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp index fe0267dce0fb..2cfc3d8abb23 100644 --- a/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_4d_generic_ocl_hip.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (c) 2025 Advanced Micro Devices, Inc. + * Copyright (c) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,13 +23,14 @@ * SOFTWARE. * *******************************************************************************/ -#include -#include - #include "get_handle.hpp" -#include "verify.hpp" #include "perf_helper.hpp" +#include +#include + +#include + #define MAX_TENSOR_ELEM 17 #define PERF_ENABLE 0 #define POW_2 1 @@ -83,34 +84,41 @@ std::vector TensorsConfigs() if constexpr(PERF_ENABLE) { - // Determine a cache-aware cap on total tensor elements for HIP/AMD: - // 1) Query L2 size via HIP and use 2x L2 as working set - // 2) Fallback to per-architecture table if L2 is not reported - size_t maxTotalSize = 0; - - // 1) HIP L2 cache query - int dev = -1; - if(hipSuccess == hipGetDevice(&dev)) + + const auto& handle = get_handle(); + size_t maxTotalSize; + + // Generate all NCHW tensors that are limited by L3 cache size + // or 2xL2 cache size when L3 is not available + if(miopen::StartsWith(handle.GetDeviceName(), "gfx90a") || + miopen::StartsWith(handle.GetDeviceName(), "gfx908")) { - int L2_bytes = 0; - if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && - L2_bytes > 0) - { - // Use 2x L2 as a working-set heuristic - maxTotalSize = 2ul * static_cast(L2_bytes); - // Convert bytes -> elements of type T - maxTotalSize /= sizeof(T); - } + maxTotalSize = 16; // twice the 8MB L2 } - - // 2) Fallback table by architecture family - if(maxTotalSize == 0) + else if(miopen::StartsWith(handle.GetDeviceName(), "gfx803")) { - maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); + maxTotalSize = 4; // twice the 2MB L2 + } + else if(miopen::StartsWith(handle.GetDeviceName(), "gfx900") || + miopen::StartsWith(handle.GetDeviceName(), "gfx906")) + { + maxTotalSize = 8; // twice the 4MB L2 + } + else if(miopen::StartsWith(handle.GetDeviceName(), "gfx942")) + { + maxTotalSize = 256; // 256MB L3 + } + else if(miopen::StartsWith(handle.GetDeviceName(), "gfx103")) + { + maxTotalSize = 128; // 128MB L3 + } + else + { + maxTotalSize = 4; // twice the 2MB L2, default case. } - // Generate all NCHW tensors that are limited by L3 cache size - // or 2xL2 cache size when L3 is not available + maxTotalSize = maxTotalSize * 1024ull * 1024ull / sizeof(T); + if constexpr(POW_2) { for(size_t N = 1; N <= maxTotalSize; N *= 2) @@ -345,9 +353,9 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg_orig); tensC_ocl.data = handle.Read(tensC_dev, tensC_ocl.data.size()); @@ -357,6 +365,7 @@ struct Op4DTensorGenericTest ph.perfTest(handle, kernel_name, network_config_ocl, + false, tensA_dev.get(), static_cast(tensorsConfig.acstrides[0]), static_cast(tensorsConfig.acstrides[1]), @@ -380,9 +389,9 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg_orig); } } @@ -427,9 +436,9 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg_orig); tensC_hip.data = handle.Read(tensC_dev, tensC_hip.data.size()); @@ -439,6 +448,7 @@ struct Op4DTensorGenericTest ph.perfTest(handle, kernel_name, network_config_hip, + false, tensA_dev.get(), static_cast(tensorsConfig.acstrides[0]), static_cast(tensorsConfig.acstrides[1]), @@ -462,9 +472,9 @@ struct Op4DTensorGenericTest beta, bitmap, work_per_wg, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg_orig); } } @@ -478,7 +488,7 @@ struct Op4DTensorGenericTest void verifyCPU() { auto error = miopen::rms_range(tensC_hip, tensC_cpu); - EXPECT_TRUE(error == 0) << "GPU outputs do not match CPU results. Error: " << error; + EXPECT_TRUE(error == 0) << "GPU outputs do not match each other. Error: " << error; } void TearDown() override @@ -505,7 +515,7 @@ struct Op4DTensorGenericTest stats += "_alpha0_" + std::to_string(alpha0) + "_alpha1_" + std::to_string(alpha1) + "_beta_" + std::to_string(beta) + "_" + miopen::GetDataType(data_type); - ph.writeStatsToCSV("tensor_4d_generic.csv", stats); + ph.writeStatsToCSV("tensor_4d.csv", stats); } } diff --git a/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp index 8b1661a2c456..b6a47e50e49e 100644 --- a/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_fwd_bias_generic_ocl_hip.cpp @@ -105,9 +105,7 @@ std::vector TensorsConfigs() // 2) Fallback table by architecture family if(maxTotalSize == 0) - { maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); - } for(int N = 1; N < maxTotalSize; N *= 4) { @@ -290,9 +288,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, incr_wg); @@ -317,9 +315,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, incr_wg); #endif @@ -359,9 +357,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + uint64_t(0), // Aoffset + uint64_t(0), // Boffset + uint64_t(0), // Coffset num_wg, incr_wg); @@ -386,9 +384,9 @@ struct OpTensorFwdBiasGenericTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + uint64_t(0), + uint64_t(0), + uint64_t(0), num_wg, incr_wg); #endif @@ -432,7 +430,7 @@ struct OpTensorFwdBiasGenericTest std::string network_config{}; std::string params{}; std::vector vld, vgd; - const int max_num_wg{4096}; + const int max_num_wg = 4096; int work_per_wg, num_wg, incr_wg{0}; tensor tensA; diff --git a/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp b/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp index e099b0c4a267..f5b9f11708bc 100644 --- a/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp +++ b/projects/miopen/test/gtest/tensor_fwd_bias_ocl_hip.cpp @@ -86,30 +86,7 @@ std::vector TensorsConfigs() }; #if PERF_ENABLE - // Determine a cache-aware cap on total tensor elements for HIP/AMD: - // 1) Query L2 size via HIP and use 2x L2 as working set - // 2) Fallback to per-architecture table if L2 is not reported - size_t maxTotalSize = 0; - - // 1) HIP L2 cache query - int dev = -1; - if(hipSuccess == hipGetDevice(&dev)) - { - int L2_bytes = 0; - if(hipSuccess == hipDeviceGetAttribute(&L2_bytes, hipDeviceAttributeL2CacheSize, dev) && - L2_bytes > 0) - { - // Use 2x L2 as a working-set heuristic - maxTotalSize = 2ul * static_cast(L2_bytes); - // Convert bytes -> elements of type T - maxTotalSize /= sizeof(T); - } - } - // 2) Fallback table by architecture family - if(maxTotalSize == 0) - { - maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); - } + size_t maxTotalSize = getCacheSizeLimit(get_handle().GetDeviceName()); // Generate all NCHW tensors that are limited by L3 cache size // or 2xL2 cache size when L3 is not available @@ -278,9 +255,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + static_cast(0), // Aoffset + static_cast(0), // Boffset + static_cast(0), // Coffset num_wg, incr_wg); @@ -302,9 +279,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg, incr_wg); #endif @@ -336,9 +313,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - 0L, // Aoffset - 0L, // Boffset - 0L, // Coffset + static_cast(0), // Aoffset + static_cast(0), // Boffset + static_cast(0), // Coffset num_wg, incr_wg); @@ -360,9 +337,9 @@ struct OpTensorFwdBiasTest alpha0, alpha1, beta, - 0L, - 0L, - 0L, + static_cast(0), + static_cast(0), + static_cast(0), num_wg, incr_wg); #endif @@ -405,7 +382,7 @@ struct OpTensorFwdBiasTest std::string network_config{}; std::string params{}; std::vector vld, vgd; - const int max_num_wg{4096}; + const int max_num_wg = 4096; int work_per_wg, num_wg, incr_wg{0}; tensor tensA; diff --git a/projects/miopen/test/gtest/test_categories.yaml b/projects/miopen/test/gtest/test_categories.yaml index 95aaf954f0eb..f8815cb999a7 100644 --- a/projects/miopen/test/gtest/test_categories.yaml +++ b/projects/miopen/test/gtest/test_categories.yaml @@ -63,6 +63,7 @@ positive_patterns: &positive_patterns - "*/GPU_MhaBackward_*" - "*/GPU_MhaForward_*" - "*GPU_TestMhaFind20*" + - "*/GPU_MIOpenDriver*" test_categories: @@ -151,7 +152,8 @@ test_categories: - "Full/GPU_ConvGrpActivInfer3D_FP32*" - "Full/GPU_ConvGrpActivInfer3D_FP16*" - "Smoke/GPU_UnitTestConvSolverHipImplicitGemmV4R1Fwd_BFP16.ConvHipImplicitGemmV4R1Fwd/0" - - "*/GPU_MIOpenDriver*" + - "Full/GPU_MIOpenDriverConv2dTransTest*" + - "Full/GPU_MIOpenDriverRegressionBigTensorTest_FP32" - "*GPU_TestMhaFind20_FP32*" #rocm-1016 fixed in develop. Re-enable once bumped - "*GPU_TestMhaFind20_FP16*" #rocm-1016 fixed in develop. Re-enable once bumped @@ -198,7 +200,6 @@ test_categories: test_patterns: *positive_patterns exclude: - - "*/GPU_MIOpenDriver*" - "*GPU_TestMhaFind20_FP32*" #rocm-1016 fixed in develop. Re-enable once bumped - "*GPU_TestMhaFind20_FP16*" #rocm-1016 fixed in develop. Re-enable once bumped @@ -250,7 +251,6 @@ test_categories: - "*" exclude: - - "*/GPU_MIOpenDriver*" test_files: # All test files in the suite @@ -307,7 +307,11 @@ exclude_gpu: - "ex_gpu_gfx120X" exclude_gpu_gfx110X: - test_patterns: *gfx11_common_patterns + test_patterns: + - *gfx11_common_patterns + - "*/GPU_MIOpenDriver*" + - "Smoke/CPU_Handle_NONE*" + - "Full/GPU_reduce_custom_fp32*" labels: - "quick" - "standard" diff --git a/projects/miopen/test/gtest/unit_FinInterface.cpp b/projects/miopen/test/gtest/unit_FinInterface.cpp index 2e4744ce714a..c13033433339 100644 --- a/projects/miopen/test/gtest/unit_FinInterface.cpp +++ b/projects/miopen/test/gtest/unit_FinInterface.cpp @@ -44,7 +44,7 @@ namespace { struct TestParams { - friend std::ostream& operator<<(std::ostream& os, const TestParams& tp) + friend std::ostream& operator<<(std::ostream& os, const TestParams&) { os << "none"; return os; @@ -53,8 +53,6 @@ struct TestParams struct SolverInfo { - [[maybe_unused]] SolverInfo() = default; - SolverInfo(uint64_t id_, bool dynamic_, bool tunable_) : id(id_), dynamic(dynamic_), tunable(tunable_) { @@ -118,7 +116,7 @@ struct ConvSolverConfig : SolverConfig, private miopen::unit_tests::ConvTestCase { } - [[maybe_unused]] auto GetProblemDescription() const { return GetProblemDescription(direction); } + // auto GetProblemDescription() const { return GetProblemDescription(direction); } friend std::ostream& operator<<(std::ostream& os, const ConvSolverConfig& config) { @@ -146,12 +144,12 @@ struct ConvSolverConfig : SolverConfig, private miopen::unit_tests::ConvTestCase struct BatchNormSolverConfig : SolverConfig { BatchNormSolverConfig(int dummy) : SolverConfig(false) { std::ignore = dummy; } - - [[maybe_unused]] auto GetProblemDescription() const + /* + auto GetProblemDescription() const { return miopen::batchnorm::ProblemDescription{{}, {}, {}, {}, {}, {}, {}, {}}; } - + */ friend std::ostream& operator<<(std::ostream& os, const BatchNormSolverConfig& config) { os << "("; @@ -291,9 +289,6 @@ const auto& GetSolversInfo() {"ConvMlirIgemmWrW", {100, false, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, {"GemmWrw1x1_stride1", {101, true, false, "miopenConvolutionFwdAlgoGEMM"}}, {"GemmWrwUniversal", {102, true, false, "miopenConvolutionFwdAlgoGEMM"}}, - {"ConvMlirIgemmFwdXdlops", {103, false, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, - {"ConvMlirIgemmBwdXdlops", {104, false, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, - {"ConvMlirIgemmWrWXdlops", {105, false, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, {"ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC", {107, true, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, {"ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC", {108, true, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, {"ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC", {110, true, true, "miopenConvolutionFwdAlgoImplicitGEMM"}}, @@ -371,6 +366,7 @@ const auto& GetSolverConfigs() return configs; } +#if MIOPEN_ENABLE_FIN_INTERFACE template const auto& GetSolverNames() { @@ -384,6 +380,7 @@ const auto& GetSolverNames() }(); return names; } +#endif template const auto& GetTestCases() @@ -620,9 +617,9 @@ using CPU_FinInterfaceTestGetAllConvSolvers_NONE = TestGetAllSolvers; using GPU_FinInterfaceTestGetConvSolver_FP32 = TestGetSolver; -TEST_P(CPU_FinInterfaceTestGetAllConvSolvers_NONE, FinInterface) { this->RunTest(); }; -TEST_P(CPU_FinInterfaceTestGetConvSolvers_NONE, FinInterface) { this->RunTest(); }; -TEST_P(GPU_FinInterfaceTestGetConvSolver_FP32, FinInterface) { this->RunTest(); }; +TEST_P(CPU_FinInterfaceTestGetAllConvSolvers_NONE, FinInterface) { this->RunTest(); } +TEST_P(CPU_FinInterfaceTestGetConvSolvers_NONE, FinInterface) { this->RunTest(); } +TEST_P(GPU_FinInterfaceTestGetConvSolver_FP32, FinInterface) { this->RunTest(); } INSTANTIATE_TEST_SUITE_P(Full, CPU_FinInterfaceTestGetAllConvSolvers_NONE, @@ -642,9 +639,9 @@ using CPU_FinInterfaceTestGetAllBatchNormSolvers_NONE = TestGetAllSolvers; using GPU_FinInterfaceTestGetBatchNormSolver_FP32 = TestGetSolver; -TEST_P(CPU_FinInterfaceTestGetAllBatchNormSolvers_NONE, FinInterface) { this->RunTest(); }; -TEST_P(CPU_FinInterfaceTestGetBatchNormSolvers_NONE, FinInterface) { this->RunTest(); }; -TEST_P(GPU_FinInterfaceTestGetBatchNormSolver_FP32, FinInterface) { this->RunTest(); }; +TEST_P(CPU_FinInterfaceTestGetAllBatchNormSolvers_NONE, FinInterface) { this->RunTest(); } +TEST_P(CPU_FinInterfaceTestGetBatchNormSolvers_NONE, FinInterface) { this->RunTest(); } +TEST_P(GPU_FinInterfaceTestGetBatchNormSolver_FP32, FinInterface) { this->RunTest(); } INSTANTIATE_TEST_SUITE_P(Full, CPU_FinInterfaceTestGetAllBatchNormSolvers_NONE, diff --git a/projects/miopen/test/gtest/unit_conv_solver.cpp b/projects/miopen/test/gtest/unit_conv_solver.cpp index bfc799675c79..48fed58aa846 100644 --- a/projects/miopen/test/gtest/unit_conv_solver.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver.cpp @@ -339,10 +339,7 @@ miopen::solver::ConvSolution FindSolution(const miopen::solver::conv::ConvSolver } template -double GetThreshold(miopenConvAlgorithm_t algo, - miopen::conv::Direction direction, - const Tolerances& tolerances, - const bool use_tf32_compute) +double GetThreshold(const Tolerances& tolerances, const bool use_tf32_compute) { double tolerance = tolerances.Get(GetDevGpuType(), miopen_type{}); double threshold = std::numeric_limits::epsilon() * tolerance; @@ -359,8 +356,6 @@ double GetThreshold(miopenConvAlgorithm_t algo, template void VerifyData(const std::vector& data, const std::vector& ref_data, - miopenConvAlgorithm_t algo, - miopen::conv::Direction direction, const Tolerances& tolerances, bool use_tf32_compute = false) { @@ -389,7 +384,7 @@ void VerifyData(const std::vector& data, else { const auto error = miopen::rms_range(ref_data, data); - const double threshold = GetThreshold(algo, direction, tolerances, use_tf32_compute); + const double threshold = GetThreshold(tolerances, use_tf32_compute); ASSERT_LT(error, threshold) << "Error beyond tolerance"; // std::cout << "error: " << error << " threshold: " << threshold << std::endl; } @@ -401,8 +396,7 @@ void VerifyData(const std::vector& data, template void RunSolverFwd(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { //********************************** // Prepare @@ -512,21 +506,15 @@ void RunSolverFwd(const miopen::solver::conv::ConvSolverInterface& solv, output.data = handle.Read(out_dev, output.data.size()); - VerifyData(output.data, - ref_out.data, - algo, - miopen::conv::Direction::Forward, - params.tolerances, - problem.UseTF32()); + VerifyData(output.data, ref_out.data, params.tolerances, problem.UseTF32()); } template void RunSolverFwd(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { - RunSolverFwd(solv, params, conv_config, algo); + RunSolverFwd(solv, params, conv_config); } //********************************** @@ -535,8 +523,7 @@ void RunSolverFwd(const miopen::solver::conv::ConvSolverInterface& solv, template void RunSolverBwd(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { //********************************** // Prepare @@ -638,21 +625,15 @@ void RunSolverBwd(const miopen::solver::conv::ConvSolverInterface& solv, input.data = handle.Read(in_dev, input.data.size()); - VerifyData(input.data, - ref_in.data, - algo, - miopen::conv::Direction::BackwardData, - params.tolerances, - problem.UseTF32()); + VerifyData(input.data, ref_in.data, params.tolerances, problem.UseTF32()); } template void RunSolverBwd(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { - RunSolverBwd(solv, params, conv_config, algo); + RunSolverBwd(solv, params, conv_config); } //********************************** @@ -661,8 +642,7 @@ void RunSolverBwd(const miopen::solver::conv::ConvSolverInterface& solv, template void RunSolverWrw(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { //********************************** // Prepare @@ -764,41 +744,34 @@ void RunSolverWrw(const miopen::solver::conv::ConvSolverInterface& solv, weights.data = handle.Read(wei_dev, weights.data.size()); - VerifyData(weights.data, - ref_weights.data, - algo, - miopen::conv::Direction::BackwardWeights, - params.tolerances, - problem.UseTF32()); + VerifyData(weights.data, ref_weights.data, params.tolerances, problem.UseTF32()); } template void RunSolverWrw(const miopen::solver::conv::ConvSolverInterface& solv, const UnitTestConvSolverParams& params, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { - RunSolverWrw(solv, params, conv_config, algo); + RunSolverWrw(solv, params, conv_config); } template void RunSolver(const miopen::solver::conv::ConvSolverInterface& solver, const UnitTestConvSolverParams& params, miopen::conv::Direction direction, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { // clang-format off switch(direction) { case miopen::conv::Direction::Forward: - RunSolverFwd(solver, params, conv_config, algo); + RunSolverFwd(solver, params, conv_config); return; case miopen::conv::Direction::BackwardData: - RunSolverBwd(solver, params, conv_config, algo); + RunSolverBwd(solver, params, conv_config); return; case miopen::conv::Direction::BackwardWeights: - RunSolverWrw(solver, params, conv_config, algo); + RunSolverWrw(solver, params, conv_config); return; default: throw std::runtime_error("unknown direction"); @@ -809,8 +782,7 @@ void RunSolver(const miopen::solver::conv::ConvSolverInterface& solver, void RunSolver(const miopen::solver::conv::ConvSolverInterface& solver, const UnitTestConvSolverParams& params, miopen::conv::Direction direction, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { if(conv_config.GetXDataType() == conv_config.GetWDataType() && conv_config.GetWDataType() == conv_config.GetYDataType()) @@ -819,16 +791,16 @@ void RunSolver(const miopen::solver::conv::ConvSolverInterface& solver, switch(conv_config.GetXDataType()) { case miopenHalf: - RunSolver(solver, params, direction, conv_config, algo); + RunSolver(solver, params, direction, conv_config); return; case miopenFloat: - RunSolver(solver, params, direction, conv_config, algo); + RunSolver(solver, params, direction, conv_config); return; case miopenBFloat16: - RunSolver(solver, params, direction, conv_config, algo); + RunSolver(solver, params, direction, conv_config); return; case miopenInt8: - RunSolver(solver, params, direction, conv_config, algo); + RunSolver(solver, params, direction, conv_config); return; default: throw std::runtime_error("handling of this data type is not yet implemented"); @@ -839,7 +811,7 @@ void RunSolver(const miopen::solver::conv::ConvSolverInterface& solver, conv_config.GetXDataType() == miopenInt8 && conv_config.GetWDataType() == miopenInt8 && conv_config.GetYDataType() == miopenInt32) { - RunSolverFwd(solver, params, conv_config, algo); + RunSolverFwd(solver, params, conv_config); return; } @@ -863,14 +835,13 @@ void UnitTestConvSolverBase::SetUpImpl(const UnitTestConvSolverParams& params) void UnitTestConvSolverBase::RunTestImpl(const miopen::solver::conv::ConvSolverInterface& solver, const UnitTestConvSolverParams& params, miopen::conv::Direction direction, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo) + const ConvTestCase& conv_config) { ConvAttrFp16AltScopedSetter conv_attr_fp16_alt_setter; if(params.conv_attr_fp16_alt) conv_attr_fp16_alt_setter.SetValue(params.conv_attr_fp16_alt.value()); - RunSolver(solver, params, direction, conv_config, algo); + RunSolver(solver, params, direction, conv_config); } //************************************************************************************ diff --git a/projects/miopen/test/gtest/unit_conv_solver.hpp b/projects/miopen/test/gtest/unit_conv_solver.hpp index 2186c6644edb..15aea7489862 100644 --- a/projects/miopen/test/gtest/unit_conv_solver.hpp +++ b/projects/miopen/test/gtest/unit_conv_solver.hpp @@ -171,8 +171,7 @@ class UnitTestConvSolverBase void RunTestImpl(const miopen::solver::conv::ConvSolverInterface& solver, const UnitTestConvSolverParams& params, miopen::conv::Direction direction, - const ConvTestCase& conv_config, - miopenConvAlgorithm_t algo); + const ConvTestCase& conv_config); protected: void SetUpImpl(const UnitTestConvSolverParams& params); @@ -191,7 +190,7 @@ class UnitTestConvSolver miopenConvAlgorithm_t algo; ConvTestCase conv_config; std::tie(params, algo, conv_config) = GetParam(); - this->RunTestImpl(solver, params, direction, conv_config, algo); + this->RunTestImpl(solver, params, direction, conv_config /*, algo*/); } protected: diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinograd3x3U.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinograd3x3U.cpp index f3f4db690eb4..f8de43d8b9a4 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinograd3x3U.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinograd3x3U.cpp @@ -91,3 +91,65 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverBinWinograd3x3UDevApplicabilityFwd_NONE, testing::Combine(testing::Values(GetTestParams()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvBinWinograd3x3U (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 20, 20, 20}}, + {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, + datatype, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedBinWinograd3x3UFwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinograd3x3UBwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; + +using CPU_UnitTestConvSolverTransposedBinWinograd3x3UDevApplicabilityFwd_NONE = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinograd3x3UFwd_FP32, TransposedConvBinWinograd3x3U) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinograd3x3U{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinograd3x3UBwd_FP32, TransposedConvBinWinograd3x3U) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinograd3x3U{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinograd3x3UDevApplicabilityFwd_NONE, + TransposedConvBinWinograd3x3U) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinograd3x3U{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinograd3x3UFwd_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinograd3x3UBwd_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedBinWinograd3x3UDevApplicabilityFwd_NONE, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxS.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxS.cpp index 8f2dfa9c06d8..984b52928fdd 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxS.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxS.cpp @@ -172,3 +172,139 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverBinWinogradRxSDevApplicabilityFwd_FP32, testing::Combine(testing::Values(GetTestParamsFloat()), testing::Values(GetConvTestCasesFloat()[0]))); + +// ===================================================================== +// TransposedConvBinWinogradRxS (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWCHalfFwd() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenHalf, miopenTensorNHWC, {1, 40, 20, 20}}, + {miopenHalf, miopenTensorNHWC, {20, 40, 3, 3}}, + miopenHalf, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +auto GetConvTestCasesNHWCHalfBwd() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenHalf, miopenTensorNHWC, {1, 20, 20, 20}}, + {miopenHalf, miopenTensorNHWC, {40, 20, 3, 3}}, + miopenHalf, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +auto GetConvTestCasesNHWCFloat() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenFloat, miopenTensorNHWC, {1, 20, 20, 20}}, + {miopenFloat, miopenTensorNHWC, {20, 20, 3, 3}}, + miopenFloat, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSWrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP32 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP16, TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP16, TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP32, TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP32, TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSWrw_FP32, TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP16, + TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP32, + TransposedConvBinWinogradRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxS{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCHalfFwd()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCHalfBwd()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSFwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSBwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSWrw_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(GetConvTestCasesNHWCHalfFwd()[0]))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSDevApplicabilityFwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(GetConvTestCasesNHWCFloat()[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3.cpp index 51ded6398cab..e4e0d7279f3d 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3.cpp @@ -167,3 +167,104 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverBinWinogradRxSf2x3DevApplicabilityFwd_FP32, testing::Combine(testing::Values(GetTestParamsFloat()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvBinWinoRxS<2, 3> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{ + {datatype, miopenTensorNHWC, {1, 40, 20, 20}}, + {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, + datatype, + {{1, 1}, {1, 1}, {1, 1}, 2} + }, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; +// Note: DevApplicability tests are not included for TransposedConvBinWinoRxS<2,3> because +// the inner solver has group_count==1 restriction (WORKAROUND_ISSUE_1681) for Fwd/Bwd directions, +// which doesn't apply uniformly across all devices listed in supported_gpus + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP16, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP16, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP16, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP32, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP32, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP32, TransposedConvBinWinoRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<2, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Fwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Bwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3g1.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3g1.cpp index 7de584def9e5..5c598be10700 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3g1.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf2x3g1.cpp @@ -198,3 +198,172 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverBinWinogradRxSf2x3g1DevApplicabilityFwd_FP32, testing::Combine(testing::Values(GetTestParamsFloat()), testing::Values(GetConvTestCasesFloat()[0]))); + +// ===================================================================== +// TransposedConvBinWinogradRxSf2x3g1 (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWCHalfFwd() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenHalf, miopenTensorNHWC, {1, 40, 20, 20}}, + {miopenHalf, miopenTensorNHWC, {20, 40, 3, 3}}, + miopenHalf, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +auto GetConvTestCasesNHWCHalfBwd() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenHalf, miopenTensorNHWC, {1, 20, 20, 20}}, + {miopenHalf, miopenTensorNHWC, {40, 20, 3, 3}}, + miopenHalf, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +auto GetConvTestCasesNHWCHalfWrw() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenHalf, miopenTensorNHWC, {1, 20, 20, 20}}, + {miopenHalf, miopenTensorNHWC, {20, 20, 3, 3}}, + miopenHalf, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +auto GetConvTestCasesNHWCFloat() +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{miopenFloat, miopenTensorNHWC, {1, 20, 20, 20}}, + {miopenFloat, miopenTensorNHWC, {20, 20, 3, 3}}, + miopenFloat, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP32 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP16, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP16, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP16, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP32, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP32, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP32, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP16, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP32, + TransposedConvBinWinogradRxSf2x3g1) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinogradRxSf2x3g1{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCHalfFwd()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCHalfBwd()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCHalfWrw()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Fwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Bwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWCFloat()))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(GetConvTestCasesNHWCHalfFwd()[0]))); + +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSf2x3g1DevApplicabilityFwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(GetConvTestCasesNHWCFloat()[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf3x2.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf3x2.cpp index 788b62a0f042..c67798a5c264 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf3x2.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvBinWinogradRxSf3x2.cpp @@ -199,3 +199,128 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverBinWinogradRxSf3x2DevApplicabilityFwd_FP32, testing::Combine(testing::Values(GetTestParamsFloat()), testing::Values(GetConvTestCasesFloat()[0]))); + +// ===================================================================== +// TransposedConvBinWinoRxS<3, 2> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{ + {datatype, miopenTensorNHWC, {1, 40, 20, 20}}, + {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, + datatype, + {{1, 1}, {1, 1}, {1, 1}, 2} + }, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; +using CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP32 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP16, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP16, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP16, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP32, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP32, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP32, TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP16, + TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP32, + TransposedConvBinWinoRxSf3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvBinWinoRxS<3, 2>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Fwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Bwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParamsHalf()), + testing::Values(GetConvTestCasesNHWC(miopenHalf)[0]))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedBinWinogradRxSf3x2DevApplicabilityFwd_FP32, + testing::Combine(testing::Values(GetTestParamsFloat()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvCkGroupedConvFwd.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvCkGroupedConvFwd.cpp index e43b91a8432b..1347db8f9feb 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvCkGroupedConvFwd.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvCkGroupedConvFwd.cpp @@ -115,7 +115,7 @@ auto GetConvFullTestCases(miopenDataType_t datatype) }; } -auto GetTestParams(miopenDataType_t datatype) +auto GetTestParams(miopenDataType_t /*datatype*/) { // Solution is only applicable to devices that default to 64-lane wavefronts #if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvMPBidirectWinograd.hpp b/projects/miopen/test/gtest/unit_conv_solver_ConvMPBidirectWinograd.hpp index cd4e7f15282f..15ac9abdcd0f 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvMPBidirectWinograd.hpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvMPBidirectWinograd.hpp @@ -216,3 +216,118 @@ INSTANTIATE_TEST_SUITE_P(Smoke, TESTSUITE_NAME_DEVAPP, testing::Combine(testing::Values(GetTestParams()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvMPBidirectWinograd (NHWC layout) +// ===================================================================== + +#define TRANSPOSED_SHORT_SOLVER_NAME \ + CONCAT2(TransposedMPBidirectWinogradF, MAKE_SUFFIX_LOWER(WINO_DATA_H, WINO_FILTER_H)) + +#define TRANSPOSED_SOLVER_NAME \ + CONCAT2(TransposedConvMPBidirectWinogradF, MAKE_SUFFIX_LOWER(WINO_DATA_H, WINO_FILTER_H)) + +#define TRANSPOSED_TESTSUITE_NAME(hw_type, direction, datatype) \ + TESTSUITE_NAME_GENERIC_DIR( \ + hw_type, CONCAT2(UnitTestConvSolver, TRANSPOSED_SHORT_SOLVER_NAME), direction, datatype) + +#define TRANSPOSED_TESTSUITE_NAME_DEV_APP(hw_type, direction, datatype) \ + TESTSUITE_NAME_GENERIC_DIR( \ + hw_type, \ + CONCAT3(UnitTestConvSolver, TRANSPOSED_SHORT_SOLVER_NAME, DevApplicability), \ + direction, \ + datatype) + +#define TRANSPOSED_TESTSUITE_NAME_FWD_FP16 TRANSPOSED_TESTSUITE_NAME(GPU, Fwd, FP16) +#define TRANSPOSED_TESTSUITE_NAME_BWD_FP16 TRANSPOSED_TESTSUITE_NAME(GPU, Bwd, FP16) +#define TRANSPOSED_TESTSUITE_NAME_FWD_FP32 TRANSPOSED_TESTSUITE_NAME(GPU, Fwd, FP32) +#define TRANSPOSED_TESTSUITE_NAME_BWD_FP32 TRANSPOSED_TESTSUITE_NAME(GPU, Bwd, FP32) + +#define TRANSPOSED_TESTSUITE_NAME_DEVAPP TRANSPOSED_TESTSUITE_NAME_DEV_APP(CPU, Fwd, NONE) + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {8, 8, 8, 8}}, + {datatype, miopenTensorNHWC, {8, 8, 3, 3}}, + datatype, {{0, 0}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using TRANSPOSED_TESTSUITE_NAME_FWD_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using TRANSPOSED_TESTSUITE_NAME_BWD_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using TRANSPOSED_TESTSUITE_NAME_FWD_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using TRANSPOSED_TESTSUITE_NAME_BWD_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using TRANSPOSED_TESTSUITE_NAME_DEVAPP = CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(TRANSPOSED_TESTSUITE_NAME_FWD_FP16, TRANSPOSED_SOLVER_NAME) +{ + SolverEnabler solver_enabler; + this->RunTest( + miopen::solver::conv::TransposedConvMPBidirectWinograd{}); +}; + +TEST_P(TRANSPOSED_TESTSUITE_NAME_BWD_FP16, TRANSPOSED_SOLVER_NAME) +{ + SolverEnabler solver_enabler; + this->RunTest( + miopen::solver::conv::TransposedConvMPBidirectWinograd{}); +}; + +TEST_P(TRANSPOSED_TESTSUITE_NAME_FWD_FP32, TRANSPOSED_SOLVER_NAME) +{ + SolverEnabler solver_enabler; + this->RunTest( + miopen::solver::conv::TransposedConvMPBidirectWinograd{}); +}; + +TEST_P(TRANSPOSED_TESTSUITE_NAME_BWD_FP32, TRANSPOSED_SOLVER_NAME) +{ + SolverEnabler solver_enabler; + this->RunTest( + miopen::solver::conv::TransposedConvMPBidirectWinograd{}); +}; + +TEST_P(TRANSPOSED_TESTSUITE_NAME_DEVAPP, TRANSPOSED_SOLVER_NAME) +{ + SolverEnabler solver_enabler; + this->RunTest( + miopen::solver::conv::TransposedConvMPBidirectWinograd{}); +}; + +INSTANTIATE_TEST_SUITE_P(Smoke, + TRANSPOSED_TESTSUITE_NAME_FWD_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + TRANSPOSED_TESTSUITE_NAME_BWD_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + TRANSPOSED_TESTSUITE_NAME_FWD_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + TRANSPOSED_TESTSUITE_NAME_BWD_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + TRANSPOSED_TESTSUITE_NAME_DEVAPP, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoFuryRxS.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinoFuryRxS.cpp index 4ee81d077155..9f8ac5e4e3bd 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoFuryRxS.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinoFuryRxS.cpp @@ -114,3 +114,67 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinoFury2x3DevApplicabilityFwd_NONE, testing::Combine(testing::Values(GetTestParams()), testing::Values(GetConvTestCases(miopenHalf)[0]))); + +// ===================================================================== +// TransposedConvWinoFuryRxS (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{ + {datatype, miopenTensorNHWC, {1, 20, 20, 20}}, + {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, + datatype, + {{1, 1}, {1, 1}, {1, 1}} + }, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinoFuryRxSFwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedWinoFuryRxSBwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using CPU_UnitTestConvSolverTransposedWinoFuryRxSDevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinoFuryRxSFwd_FP16, TransposedConvWinoFuryRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoFuryRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinoFuryRxSBwd_FP16, TransposedConvWinoFuryRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoFuryRxS<2, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinoFuryRxSDevApplicabilityFwd_FP16, + TransposedConvWinoFuryRxS) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoFuryRxS<2, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinoFuryRxSFwd_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinoFuryRxSBwd_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedWinoFuryRxSDevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(GetConvTestCasesNHWC(miopenHalf)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp index 834ce2079dad..4d7640ccb070 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp @@ -34,10 +34,10 @@ auto GetConvTestCases(miopenDataType_t datatype) return std::vector{ // clang-format off - // rage v4.6 + // rage v4.6 and v4.9(fp32 and bf16) TestCase{{1, 16, 135, 240}, {16, 16, 3, 3}, {1, 1}, {1, 1}, {1, 1}, datatype}, TestCase{{2, 4, 64, 64}, {16, 4, 3, 3}, {1, 1}, {1, 1}, {1, 1}, datatype}, - // rage v4.7 + // rage v4.9 TestCase{{1, 16, 135, 240}, {16, 16, 5, 5}, {2, 2}, {1, 1}, {1, 1}, datatype}, TestCase{{2, 4, 64, 64}, {16, 4, 5, 5}, {2, 2}, {1, 1}, {1, 1}, datatype}, // group convs @@ -66,6 +66,16 @@ const auto& GetTestParams() { static const auto params = [] { auto p = miopen::unit_tests::UnitTestConvSolverParams(Gpu::gfx94X); + p.CheckXnackDisabled(); + return p; + }(); + return params; +} + +const auto& GetTestParamsFP16() +{ + static const auto params = [] { + auto p = miopen::unit_tests::UnitTestConvSolverParams(Gpu::gfx94X | Gpu::gfx120X); return p; }(); return params; @@ -73,10 +83,20 @@ const auto& GetTestParams() } // namespace -using GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; -using GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; -using GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; -using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_NONE = +using GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; +using GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; +using GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; +using GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16 = GPU_UnitTestConvSolverFwd_BFP16; +using GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16 = GPU_UnitTestConvSolverBwd_BFP16; +using GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16 = GPU_UnitTestConvSolverWrw_BFP16; +using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; +using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; +using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16 = CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16, ConvWinoRageRxSf2x3) @@ -94,7 +114,47 @@ TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16, ConvWinoRageRxSf2x3) this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); }; -TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_NONE, ConvWinoRageRxSf2x3) +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32, ConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16, ConvWinoRageRxSf2x3) { this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); }; @@ -102,24 +162,146 @@ TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_NONE, ConvWinoRageRx // Smoke tests INSTANTIATE_TEST_SUITE_P(Smoke, GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16, - testing::Combine(testing::Values(GetTestParams()), + testing::Combine(testing::Values(GetTestParamsFP16()), testing::Values(miopenConvolutionAlgoWinograd), testing::ValuesIn(GetConvTestCases(miopenHalf)))); INSTANTIATE_TEST_SUITE_P(Smoke, GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16, - testing::Combine(testing::Values(GetTestParams()), + testing::Combine(testing::Values(GetTestParamsFP16()), testing::Values(miopenConvolutionAlgoWinograd), testing::ValuesIn(GetConvTestCases(miopenHalf)))); INSTANTIATE_TEST_SUITE_P(Smoke, GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16, - testing::Combine(testing::Values(GetTestParams()), + testing::Combine(testing::Values(GetTestParamsFP16()), testing::Values(miopenConvolutionAlgoWinograd), testing::ValuesIn(GetConvTestCasesWrw(miopenHalf)))); -// Device applicability test INSTANTIATE_TEST_SUITE_P(Smoke, - CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_NONE, + GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32, testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCases(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCases(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesWrw(miopenFloat)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCases(miopenBFloat16)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCases(miopenBFloat16)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesWrw(miopenBFloat16)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), testing::Values(GetConvTestCases(miopenHalf)[0]))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(GetConvTestCases(miopenFloat)[0]))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(GetConvTestCases(miopenBFloat16)[0]))); + +// ===================================================================== +// TransposedConvWinoRageRxS (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{ + {datatype, miopenTensorNHWC, {1, 40, 20, 20}}, + {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, + datatype, + {{1, 1}, {1, 1}, {1, 1}, 2} + }, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; +using GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; +using GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; +using CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16 = + CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16, TransposedConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16, TransposedConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16, TransposedConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16, + TransposedConvWinoRageRxSf2x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16, + testing::Combine(testing::Values(GetTestParams()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P(Smoke, + CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(GetConvTestCasesNHWC(miopenHalf)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x2.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x2.cpp index 5159d676f586..fdc65faabc89 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x2.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x2.cpp @@ -116,3 +116,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF1x1x7x2DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 1, 7}}, + datatype, {{0, 3}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF1x1x7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF1x1x7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF1x1x7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 2>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x2DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x3.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x3.cpp index adf07b8fe1fe..0ec2e4f90898 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x3.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF1x1x7x3.cpp @@ -116,3 +116,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF1x1x7x3DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 1, 7}}, + datatype, {{0, 3}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF1x1x7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF1x1x7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF1x1x7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<1, 1, 7, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF1x1x7x3DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x2.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x2.cpp index ab705216dcd8..639da0659ebe 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x2.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x2.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF3x2DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<3, 2> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 3, 3}}, + datatype, {{1, 1}, {2, 2}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 2>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF3x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 2>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x2DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x3.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x3.cpp index dd2cc56e72d4..ea3e52e0b01b 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x3.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x3.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF3x3DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<3, 3> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 3, 3}}, + datatype, {{1, 1}, {2, 2}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF3x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF3x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF3x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x3DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x4.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x4.cpp index 7ef0ae7563aa..c0462a5a027d 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x4.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x4.cpp @@ -161,3 +161,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF3x4DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<3, 4> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 3, 3}}, + datatype, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF3x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 4>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF3x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 4>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF3x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 4>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x4DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x5.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x5.cpp index 1ce72c96ea64..343b1e1a328d 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x5.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x5.cpp @@ -161,3 +161,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF3x5DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<3, 5> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 3, 3}}, + datatype, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF3x5) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 5>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF3x5) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 5>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF3x5) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 5>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x5DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x6.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x6.cpp index fb8b928311a4..413a5a35b7e2 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x6.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF3x6.cpp @@ -161,3 +161,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF3x6DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<3, 6> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 3, 3}}, + datatype, {{1, 1}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF3x6) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 6>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF3x6) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 6>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF3x6) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<3, 6>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF3x6DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x3.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x3.cpp index a5c09a519d04..f54d34428643 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x3.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x3.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF5x3DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<5, 3> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 5, 5}}, + datatype, {{2, 2}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF5x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF5x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF5x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x3DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x4.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x4.cpp index c1f1d43518a6..7c8b0a596e07 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x4.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF5x4.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF5x4DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<5, 4> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 5, 5}}, + datatype, {{2, 2}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF5x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 4>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF5x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 4>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF5x4) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<5, 4>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF5x4DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2.cpp index 9ae3dc64ac25..8aefeb14fcab 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF7x2DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<7, 2> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 7, 7}}, + datatype, {{3, 3}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF7x2) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2x1x1.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2x1x1.cpp index fadfd7d92c7c..432a1ef167e2 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2x1x1.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x2x1x1.cpp @@ -116,3 +116,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF7x2x1x1DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 7, 1}}, + datatype, {{3, 0}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF7x2x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF7x2x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF7x2x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 2, 1, 1>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x2x1x1DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3.cpp index 3ad6c0ff903b..22e23197663b 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3.cpp @@ -113,3 +113,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF7x3DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<7, 3> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 7, 7}}, + datatype, {{3, 3}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF7x3) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3x1x1.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3x1x1.cpp index 704e05697284..30861aa1a07a 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3x1x1.cpp +++ b/projects/miopen/test/gtest/unit_conv_solver_ConvWinograd3x3MultipassWrWF7x3x1x1.cpp @@ -116,3 +116,70 @@ INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestConvSolverWinograd3x3MultipassF7x3x1x1DevApplicabilityWrw_NONE, testing::Combine(testing::Values(GetTestParamsFP32()), testing::Values(GetConvTestCases(miopenFloat)[0]))); + +// ===================================================================== +// TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1> (NHWC layout) +// ===================================================================== + +namespace { + +auto GetConvTestCasesNHWC(miopenDataType_t datatype) +{ + using TestCase = miopen::unit_tests::ConvTestCase; + + return std::vector{ + // clang-format off + TestCase{{datatype, miopenTensorNHWC, {1, 16, 24, 24}}, + {datatype, miopenTensorNHWC, {16, 16, 7, 1}}, + datatype, {{3, 0}, {1, 1}, {1, 1}}}, + // clang-format on + }; +} + +} // namespace + +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP16 = + GPU_UnitTestConvSolverWrw_FP16; +using GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP32 = + GPU_UnitTestConvSolverWrw_FP32; + +using CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1DevApplicabilityWrw_NONE = + CPU_UnitTestConvSolverDevApplicabilityWrw_NONE; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP16, + TransposedConvWinograd3x3MultipassWrWF7x3x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>{}); +}; + +TEST_P(GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP32, + TransposedConvWinograd3x3MultipassWrWF7x3x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>{}); +}; + +TEST_P(CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1DevApplicabilityWrw_NONE, + TransposedConvWinograd3x3MultipassWrWF7x3x1x1) +{ + this->RunTest(miopen::solver::conv::TransposedConvWinograd3x3MultipassWrW<7, 3, 1, 1>{}); +}; + +// Smoke tests +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP16, + testing::Combine(testing::Values(GetTestParamsFP16()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); + +INSTANTIATE_TEST_SUITE_P(Smoke, + GPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1Wrw_FP32, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(miopenConvolutionAlgoWinograd), + testing::ValuesIn(GetConvTestCasesNHWC(miopenFloat)))); + +// Device applicability test +INSTANTIATE_TEST_SUITE_P( + Smoke, + CPU_UnitTestConvSolverTransposedWinograd3x3MultipassF7x3x1x1DevApplicabilityWrw_NONE, + testing::Combine(testing::Values(GetTestParamsFP32()), + testing::Values(GetConvTestCasesNHWC(miopenFloat)[0]))); diff --git a/projects/miopen/test/gtest/unit_conv_solver_group_xdlops.hpp b/projects/miopen/test/gtest/unit_conv_solver_group_xdlops.hpp index b715914b1d07..09089f3b8de6 100644 --- a/projects/miopen/test/gtest/unit_conv_solver_group_xdlops.hpp +++ b/projects/miopen/test/gtest/unit_conv_solver_group_xdlops.hpp @@ -88,7 +88,7 @@ class UnitTestConvSolverGroupXDlops GetConvTestForGroupXdlops(layout, std::move(conv_numeric_data)); this->RunTestImpl( - solver, params, direction, conv_config, miopenConvolutionAlgoImplicitGEMM); + solver, params, direction, conv_config /*, miopenConvolutionAlgoImplicitGEMM*/); } protected: diff --git a/projects/miopen/test/gtest/unit_implicitgemm_ck_util.cpp b/projects/miopen/test/gtest/unit_implicitgemm_ck_util.cpp index 37b0474ae82f..ccf3ee17443d 100644 --- a/projects/miopen/test/gtest/unit_implicitgemm_ck_util.cpp +++ b/projects/miopen/test/gtest/unit_implicitgemm_ck_util.cpp @@ -95,7 +95,7 @@ std::vector StubbedDeviceOps::deviceOps = {}; struct StubbedCKArgs { - StubbedCKArgs(const ProblemDescription& problem) {} + StubbedCKArgs(const ProblemDescription& /*problem*/) {} template bool IsSupportedBy(const ConvPtr&) const @@ -152,7 +152,7 @@ TEST_P(CPU_UnitTestImplicitGemmCKUtil_NONE, TestParsing) #else GTEST_SKIP(); #endif -}; +} INSTANTIATE_TEST_SUITE_P(Smoke, CPU_UnitTestImplicitGemmCKUtil_NONE, diff --git a/projects/miopen/test/gtest/unit_lib_env_var.cpp b/projects/miopen/test/gtest/unit_lib_env_var.cpp index e753741d721f..b013e6f7aee2 100644 --- a/projects/miopen/test/gtest/unit_lib_env_var.cpp +++ b/projects/miopen/test/gtest/unit_lib_env_var.cpp @@ -60,7 +60,7 @@ class EnvVarRestorer struct TestParams { - friend std::ostream& operator<<(std::ostream& os, const TestParams& tp) + friend std::ostream& operator<<(std::ostream& os, const TestParams&) { os << "none"; return os; diff --git a/projects/miopen/test/lstm.cpp b/projects/miopen/test/lstm.cpp deleted file mode 100644 index b0af61102e8f..000000000000 --- a/projects/miopen/test/lstm.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "lstm.hpp" - -int main(int argc, const char* argv[]) -{ -#if(MIO_RNN_TIME_EVERYTHING > 0) - auto t_start = std::chrono::high_resolution_clock::now(); -#endif - test_drive(argc, argv); - -#if(MIO_RNN_TIME_EVERYTHING > 0) - auto t_end = std::chrono::high_resolution_clock::now(); - - std::cout << "Wall clock: RNN test pass time: " - << std::chrono::duration(t_end - t_start).count() << " seconds." << std::endl; -#endif - exit(0); // NOLINT (concurrency-mt-unsafe) -} diff --git a/projects/miopen/test/lstm.hpp b/projects/miopen/test/lstm.hpp deleted file mode 100644 index 154fdf47a5d4..000000000000 --- a/projects/miopen/test/lstm.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include "lstm_common.hpp" - -template -struct lstm_driver : lstm_basic_driver -{ - lstm_driver() : lstm_basic_driver() - { - std::vector modes(2, 0); - modes[1] = 1; - std::vector defaultBS(1); - - this->add(this->batchSize, "batch-size", this->generate_data(get_lstm_batchSize(), {17})); - this->add(this->seqLength, "seq-len", this->generate_data(get_lstm_seq_len(), {2})); - this->add(this->inVecLen, "vector-len", this->generate_data(get_lstm_vector_len())); - this->add(this->hiddenSize, "hidden-size", this->generate_data(get_lstm_hidden_size())); - this->add(this->numLayers, "num-layers", this->generate_data(get_lstm_num_layers())); - this->add(this->nohx, "no-hx", this->flag()); - this->add(this->nodhy, "no-dhy", this->flag()); - this->add(this->nocx, "no-cx", this->flag()); - this->add(this->nodcy, "no-dcy", this->flag()); - this->add(this->nohy, "no-hy", this->flag()); - this->add(this->nodhx, "no-dhx", this->flag()); - this->add(this->nocy, "no-cy", this->flag()); - this->add(this->nodcx, "no-dcx", this->flag()); - this->add(this->flatBatchFill, "flat-batch-fill", this->flag()); - this->add(this->useDropout, "use-dropout", this->generate_data({0})); - this->add(this->usePadding, "use-padding", this->generate_data({false, true})); - -#if(MIO_LSTM_TEST_DEBUG == 3) - this->biasMode = 0; - this->dirMode = 0; - this->inputMode = 0; - this->algoMode = 0; -#else - this->add(this->inputMode, "in-mode", this->generate_data(modes)); - this->add(this->biasMode, "bias-mode", this->generate_data(modes)); - this->add(this->dirMode, "dir-mode", this->generate_data(modes)); - this->add(this->algoMode, "algo-mode", this->generate_data(modes)); -#endif - this->add( - this->batchSeq, - "batch-seq", - this->lazy_generate_data( - [=] { return generate_batchSeq(this->batchSize, this->seqLength); }, defaultBS)); - } -}; diff --git a/projects/miopen/test/lstm_dropout.cpp b/projects/miopen/test/lstm_dropout.cpp deleted file mode 100644 index c0cc1da88e75..000000000000 --- a/projects/miopen/test/lstm_dropout.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "lstm_common.hpp" - -template -struct lstm_dropout_driver : lstm_basic_driver -{ - lstm_dropout_driver() : lstm_basic_driver() - { - std::vector modes(2, 0); - modes[1] = 1; - std::vector defaultBS(1); - - this->add(this->batchSize, "batch-size", this->generate_data({17})); - this->add(this->seqLength, "seq-len", this->generate_data({25})); - this->add(this->inVecLen, "vector-len", this->generate_data({17})); - this->add(this->hiddenSize, "hidden-size", this->generate_data({67})); - this->add(this->numLayers, "num-layers", this->generate_data({3})); - this->add(this->nohx, "no-hx", this->generate_data({false})); - this->add(this->nodhy, "no-dhy", this->generate_data({false})); - this->add(this->nocx, "no-cx", this->generate_data({false})); - this->add(this->nodcy, "no-dcy", this->generate_data({false})); - this->add(this->nohy, "no-hy", this->generate_data({false})); - this->add(this->nodhx, "no-dhx", this->generate_data({false})); - this->add(this->nocy, "no-cy", this->generate_data({false})); - this->add(this->nodcx, "no-dcx", this->generate_data({false})); - this->add(this->flatBatchFill, "flat-batch-fill", this->generate_data({false, true})); - this->add(this->useDropout, "use-dropout", this->generate_data({1})); - -#if(MIO_LSTM_TEST_DEBUG == 3) - biasMode = 0; - dirMode = 0; - inputMode = 0; - algoMode = 0; -#else - this->add(this->inputMode, "in-mode", this->generate_data({0})); - this->add(this->biasMode, "bias-mode", this->generate_data({1})); - this->add(this->dirMode, "dir-mode", this->generate_data(modes)); - this->add(this->algoMode, "algo-mode", this->generate_data({0})); -#endif - this->add(this->batchSeq, "batch-seq", this->generate_data(defaultBS)); - } -}; - -int main(int argc, const char* argv[]) { test_drive(argc, argv); } diff --git a/projects/miopen/test/rnn_util.hpp b/projects/miopen/test/rnn_util.hpp index 308ce4d511ba..d993d0df4c57 100644 --- a/projects/miopen/test/rnn_util.hpp +++ b/projects/miopen/test/rnn_util.hpp @@ -33,10 +33,10 @@ #include #include #include -#include "random.hpp" #include -#include "../test/gemm.hpp" +#include "gemm.hpp" +#include "random.hpp" #include diff --git a/projects/miopen/test/sqlite_perfdb.cpp b/projects/miopen/test/sqlite_perfdb.cpp index 4023031f1819..8d1b0ba4c995 100644 --- a/projects/miopen/test/sqlite_perfdb.cpp +++ b/projects/miopen/test/sqlite_perfdb.cpp @@ -1,28 +1,5 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT #include "test.hpp" #include "driver.hpp" @@ -36,14 +13,13 @@ #include #include -#include - #include #include #include #include #include #include +#include #include #include #include @@ -841,7 +817,7 @@ class DbMultiProcessTest : public DbTest std::cout << "Launching test processes..." << std::endl; { auto& file_lock = LockFile::Get(lock_file_path.c_str()); - boost::shared_lock lock(file_lock); + std::shared_lock lock(file_lock); auto id = 0; @@ -918,7 +894,7 @@ class DbMultiProcessReadTest : public DbTest std::cout << "Launching test processes..." << std::endl; { auto& file_lock = LockFile::Get(lock_file_path.c_str()); - boost::shared_lock lock(file_lock); + std::shared_lock lock(file_lock); auto id = 0; diff --git a/projects/miopen/test/tensor_holder.hpp b/projects/miopen/test/tensor_holder.hpp index 5c6c4606e6e3..674637edc138 100644 --- a/projects/miopen/test/tensor_holder.hpp +++ b/projects/miopen/test/tensor_holder.hpp @@ -151,7 +151,6 @@ struct tensor #if defined(__clang__) || defined(__GNUG__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif tensor() : desc(miopen_type{}) {} @@ -307,15 +306,15 @@ struct tensor } }; - void operator()(any = {}, - any = {}, - any = {}, - any = {}, - any = {}, - any = {}, - any = {}, - any = {}, - any = {}) const + [[noreturn]] void operator()(any = {}, + any = {}, + any = {}, + any = {}, + any = {}, + any = {}, + any = {}, + any = {}, + any = {}) const { throw std::runtime_error( "Arguments to for_each do not match tensor size or the function " + diff --git a/shared/ctest/parse_test_categories.py b/shared/ctest/parse_test_categories.py index 829a395ec039..28341edceadc 100644 --- a/shared/ctest/parse_test_categories.py +++ b/shared/ctest/parse_test_categories.py @@ -97,6 +97,10 @@ def main(): execution_settings = config.get("execution_settings", {}) timeouts = execution_settings.get("category_timeouts", {}) timeout_multiplier = execution_settings.get("timeout_multiplier", 1) + env_dict = execution_settings.get("environment", {}) or {} + env_string = ( + ";".join(f"{k}={v}" for k, v in env_dict.items()) if env_dict else None + ) exclude_gpu_config = config.get("exclude_gpu", {}) # Detect OS @@ -163,16 +167,18 @@ def main(): # Write category test to CMake file and install file. # ======================================================================= print("add_test(") - print(f" NAME {target_name}-{category_name}-suite") + print(f" NAME {target_name}_{category_name}_suite") print(f" COMMAND {target_name} --gtest_filter={pattern_string}") print(f" WORKING_DIRECTORY {working_dir}") print(")") print( - f"set_tests_properties({target_name}-{category_name}-suite PROPERTIES" + f"set_tests_properties({target_name}_{category_name}_suite PROPERTIES" ) print(f" LABELS {label_string}") print(f" TIMEOUT {timeout}") + if env_string: + print(f' ENVIRONMENT "{env_string}"') print(")") print() @@ -180,10 +186,11 @@ def main(): if install_file_handle: try: install_file_handle.write( - f'add_test({target_name}-{category_name}-suite "../{target_name}" --gtest_filter={pattern_string})\n' + f'add_test({target_name}_{category_name}_suite "../{target_name}" --gtest_filter={pattern_string})\n' ) + env_prop = f' ENVIRONMENT "{env_string}"' if env_string else "" install_file_handle.write( - f"set_tests_properties({target_name}-{category_name}-suite PROPERTIES LABELS {label_string} TIMEOUT {timeout})\n\n" + f"set_tests_properties({target_name}_{category_name}_suite PROPERTIES LABELS {label_string} TIMEOUT {timeout}{env_prop})\n\n" ) install_file_handle.flush() except OSError as e: @@ -206,7 +213,7 @@ def main(): # - Uses wildcard 'X' for pattern matching (e.g., gfx11X matches gfx1100, gfx1150, etc.) # - Generates one test per category (quick, standard, etc.) per unique ex_gpu_* label - # - Test name format: {target_name}-{category}-{gpu_arch}-suite + # - Test name format: {target_name}_{category}_{gpu_arch}_suite # - Uses gtest filter: "{category_patterns}:-{gpu_exclusion_patterns}" # ======================================================================== @@ -243,7 +250,11 @@ def main(): if gpu_arch_matches(gpu_arch, config_arch): patterns = gpu_config.get("test_patterns", []) if patterns: - all_applicable_patterns.extend(patterns) + for p in patterns: + if isinstance(p, list): + all_applicable_patterns.extend(p) + else: + all_applicable_patterns.append(p) # Collect applicable categories from this config gpu_labels = gpu_config.get("labels", []) @@ -293,16 +304,18 @@ def main(): # ======================================================================= print(f"# GPU exclusion for {gpu_arch} - {category_name} category") print("add_test(") - print(f" NAME {target_name}-{category_name}-{gpu_arch}-suite") + print(f" NAME {target_name}_{category_name}_{gpu_arch}_suite") print(f" COMMAND {target_name} --gtest_filter={pattern_string}") print(f" WORKING_DIRECTORY {working_dir}") print(")") print( - f"set_tests_properties({target_name}-{category_name}-{gpu_arch}-suite PROPERTIES" + f"set_tests_properties({target_name}_{category_name}_{gpu_arch}_suite PROPERTIES" ) print(f" LABELS {label_string}") print(f" TIMEOUT {timeout}") + if env_string: + print(f' ENVIRONMENT "{env_string}"') print(")") print() @@ -310,20 +323,21 @@ def main(): if install_file_handle: try: install_file_handle.write( - f'add_test({target_name}-{category_name}-{gpu_arch}-suite "../{target_name}" --gtest_filter={pattern_string})\n' + f'add_test({target_name}_{category_name}_{gpu_arch}_suite "../{target_name}" --gtest_filter={pattern_string})\n' ) + env_prop = f' ENVIRONMENT "{env_string}"' if env_string else "" install_file_handle.write( - f"set_tests_properties({target_name}-{category_name}-{gpu_arch}-suite PROPERTIES LABELS {label_string} TIMEOUT {timeout})\n\n" + f"set_tests_properties({target_name}_{category_name}_{gpu_arch}_suite PROPERTIES LABELS {label_string} TIMEOUT {timeout}{env_prop})\n\n" ) install_file_handle.flush() except OSError as e: print( - f"Warning: I/O error writing GPU exclude {category_name}-{gpu_arch} to install test file: {e}", + f"Warning: I/O error writing GPU exclude {category_name}_{gpu_arch} to install test file: {e}", file=sys.stderr, ) except Exception as e: print( - f"Warning: Unexpected error writing GPU exclude {category_name}-{gpu_arch} to install test file: {type(e).__name__}: {e}", + f"Warning: Unexpected error writing GPU exclude {category_name}_{gpu_arch} to install test file: {type(e).__name__}: {e}", file=sys.stderr, ) From b7f58fa59fe130ff0aff214e1a69c01a4d150dce Mon Sep 17 00:00:00 2001 From: Tulebaev Date: Thu, 26 Mar 2026 23:33:53 +0100 Subject: [PATCH 33/33] For resolving merge conflicts --- .../miopen/cmake/EnableCompilerWarnings.cmake | 124 -- .../include/miopen/conv/solver_finders.hpp | 198 --- .../include/miopen/graphapi/convolution.hpp | 675 ---------- .../src/include/miopen/graphapi/engine.hpp | 228 ---- .../src/include/miopen/graphapi/enginecfg.hpp | 142 -- .../miopen/graphapi/execution_plan.hpp | 222 --- .../src/include/miopen/graphapi/matmul.hpp | 198 --- .../src/include/miopen/graphapi/opgraph.hpp | 462 ------- .../src/include/miopen/graphapi/pointwise.hpp | 310 ----- .../src/include/miopen/graphapi/reduction.hpp | 168 --- .../src/include/miopen/graphapi/reshape.hpp | 104 -- .../src/include/miopen/graphapi/rng.hpp | 211 --- .../src/include/miopen/graphapi/tensor.hpp | 169 --- .../src/include/miopen/pooling/solvers.hpp | 527 -------- .../miopen/src/solver/conv/conv_winoRxS.cpp | 1188 ----------------- projects/miopen/test/gtest/activ_common.hpp | 477 ------- projects/miopen/test/gtest/deepbench_lstm.cpp | 124 -- .../graphapi_operationgraph_descriptor.cpp | 179 --- projects/miopen/test/gtest/lstm_dropout.cpp | 99 -- projects/miopen/test/gtest/lstm_extra.cpp | 160 --- .../miopen/test/gtest/test_categories.yaml | 350 ----- .../unit_conv_solver_ConvWinoRageRxS.cpp | 307 ----- 22 files changed, 6622 deletions(-) delete mode 100644 projects/miopen/cmake/EnableCompilerWarnings.cmake delete mode 100644 projects/miopen/src/include/miopen/conv/solver_finders.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/convolution.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/engine.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/enginecfg.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/execution_plan.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/matmul.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/opgraph.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/pointwise.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/reduction.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/reshape.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/rng.hpp delete mode 100644 projects/miopen/src/include/miopen/graphapi/tensor.hpp delete mode 100644 projects/miopen/src/include/miopen/pooling/solvers.hpp delete mode 100644 projects/miopen/src/solver/conv/conv_winoRxS.cpp delete mode 100644 projects/miopen/test/gtest/activ_common.hpp delete mode 100644 projects/miopen/test/gtest/deepbench_lstm.cpp delete mode 100644 projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp delete mode 100644 projects/miopen/test/gtest/lstm_dropout.cpp delete mode 100644 projects/miopen/test/gtest/lstm_extra.cpp delete mode 100644 projects/miopen/test/gtest/test_categories.yaml delete mode 100644 projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp diff --git a/projects/miopen/cmake/EnableCompilerWarnings.cmake b/projects/miopen/cmake/EnableCompilerWarnings.cmake deleted file mode 100644 index 3b487676b751..000000000000 --- a/projects/miopen/cmake/EnableCompilerWarnings.cmake +++ /dev/null @@ -1,124 +0,0 @@ -################################################################################ -# -# MIT License -# -# Copyright (c) 2017 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ -# - Enable warning all for gcc/clang or use /W4 for visual studio - -## Strict warning level -set(__msvc_cxx_compile_options /W4) - -set(__default_cxx_compile_options - -Wall - -Wextra - -Wcomment - -Wendif-labels - -Wformat - -Winit-self - -Wreturn-type - -Wsequence-point - -Wswitch - -Wtrigraphs - -Wundef - -Wuninitialized - -Wunreachable-code - -Wunused - -Wno-ignored-qualifiers - -Wno-sign-compare -) - -set(__clang_cxx_compile_options - -Weverything - -Wno-c++98-compat - -Wno-c++98-compat-pedantic - -Wno-conversion - -Wno-double-promotion - -Wno-exit-time-destructors - -Wno-extra-semi - -Wno-extra-semi-stmt - -Wno-float-conversion - -Wno-gnu-anonymous-struct - -Wno-gnu-zero-variadic-macro-arguments - -Wno-missing-prototypes - -Wno-nested-anon-types - -Wno-option-ignored - -Wno-padded - -Wno-return-std-move-in-c++11 - -Wno-shorten-64-to-32 - -Wno-sign-conversion - -Wno-unknown-warning-option - -Wno-unused-command-line-argument - -Wno-weak-vtables - -Wno-covered-switch-default - -Wno-unsafe-buffer-usage - -Wno-deprecated-declarations - -Wno-shadow-uncaptured-local - -Wno-global-constructors - -Wno-reserved-identifier - -Wno-zero-as-null-pointer-constant - -Wno-ignored-attributes - -Wno-deprecated - -Wno-incompatible-pointer-types - -Wno-old-style-cast - -Wno-unknown-attributes - -Wno-microsoft-cpp-macro - -Wno-microsoft-enum-value - -Wno-language-extension-token - -Wno-c++11-narrowing - -Wno-float-equal - -Wno-redundant-parens - -Wno-format-nonliteral - -Wno-comma - -Wno-suggest-destructor-override - -Wno-switch-enum - -Wno-shift-sign-overflow - -Wno-suggest-override - -Wno-inconsistent-missing-destructor-override - -Wno-cast-function-type - -Wno-nonportable-system-include-path - -Wno-incompatible-pointer-types - -Wno-documentation - -Wno-deprecated-builtins - -Wno-enum-constexpr-conversion - -Wno-missing-noreturn - -Wno-tautological-constant-out-of-range-compare - -Wno-nrvo - -Wno-c++20-extensions) -if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "19") - list(APPEND __clang_cxx_compile_options - -Wno-unique-object-duplication - -Wno-switch-default - -Wno-nontrivial-memcall) -endif() -if(WIN32) - list(APPEND __clang_cxx_compile_options - -fms-extensions - -fms-compatibility) -endif() - -add_compile_options( - "$<$,$>:${__default_cxx_compile_options};${__clang_cxx_compile_options}>" -) - -unset(__default_cxx_compile_options) -unset(__clang_cxx_compile_options) diff --git a/projects/miopen/src/include/miopen/conv/solver_finders.hpp b/projects/miopen/src/include/miopen/conv/solver_finders.hpp deleted file mode 100644 index 6d9704ef5215..000000000000 --- a/projects/miopen/src/include/miopen/conv/solver_finders.hpp +++ /dev/null @@ -1,198 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace miopen { - -namespace conv { -struct ProblemDescription; -} // namespace conv - -struct Solution; - -class DbRecord; - -// This can be used to pass some primitive-specific pre-computed data to finders. -struct PrimitiveFindParameters -{ - PrimitiveFindParameters() = default; -}; - -class ISolversFinder -{ -public: - virtual ~ISolversFinder() = default; - - [[nodiscard]] virtual AlgorithmName - GetAlgorithmName(const ProblemDescriptionBase& problem) const = 0; - - [[nodiscard]] inline std::vector - Find(const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const AnyInvokeParams& invoke_ctx, - const PrimitiveFindParameters& parameters, - const std::optional& find_options) const - { - if(!IsEnabled(ctx, problem, parameters)) - { - MIOPEN_LOG_I2("Skipping " << GetAlgorithmName(problem).ToString()); - return {}; - } - - try - { - MIOPEN_LOG_I2("Starting find for " << GetAlgorithmName(problem).ToString()); - return FindImpl(ctx, problem, invoke_ctx, parameters, find_options); - } - catch(Exception& ex) - { - MIOPEN_LOG_WE(ex.what()); - return {}; - } - } - -protected: - [[nodiscard]] virtual bool IsEnabled(const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const PrimitiveFindParameters& parameters) const = 0; - [[nodiscard]] virtual std::vector - FindImpl(const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const AnyInvokeParams& invoke_ctx, - const PrimitiveFindParameters& parameters, - const std::optional& options) const = 0; -}; - -template -class SolversFinderMixin : public ISolversFinder -{ -public: - static_assert(std::is_base_of_v); - static_assert(std::is_base_of_v); - - [[nodiscard]] AlgorithmName GetAlgorithmName(const ProblemDescriptionBase& problem) const final - { - return GetAlgorithmName(static_cast(problem)); - } - - [[nodiscard]] std::vector - FindImpl(const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const AnyInvokeParams& invoke_ctx, - const PrimitiveFindParameters& parameters, - const std::optional& options) const final - { - return FindImpl(ctx, - static_cast(problem), - invoke_ctx, - static_cast(parameters), - options); - } - - [[nodiscard]] bool IsEnabled(const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const PrimitiveFindParameters& parameters) const final - { - return IsEnabled(ctx, - static_cast(problem), - static_cast(parameters)); - } - -protected: - [[nodiscard]] virtual AlgorithmName - GetAlgorithmName(const ProblemDescription& problem) const = 0; - - [[nodiscard]] virtual std::vector - FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const FindParameters& parameters, - const std::optional& options) const = 0; - - [[nodiscard]] virtual bool IsEnabled(const ExecutionContext& ctx, - const ProblemDescription& problem, - const FindParameters& parameters) const = 0; -}; - -namespace conv { - -const std::vector>& GetConvSolverFinders(); - -} // namespace conv - -struct FindCoreResult -{ - std::vector solutions; - float find_search_best_time = std::numeric_limits::max(); - bool is_optimal; -}; - -std::vector EvaluateInvokers(const Handle& handle, - const std::vector& solutions, - const AlgorithmName& algorithm_name, - const NetworkConfig& network_config, - const AnyInvokeParams& invoke_ctx, - FindCoreResult& core_result, - bool force_attach_binary); - -FindCoreResult FindCore(const AnyInvokeParams& invoke_ctx, - const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const PrimitiveFindParameters& parameters, - const std::vector>& finders, - const std::optional& options = std::nullopt, - bool force_attach_binary = false); - -namespace conv { -MIOPEN_INTERNALS_EXPORT bool IsAlgorithmDisabled(miopenConvAlgorithm_t algo); -MIOPEN_INTERNALS_EXPORT bool IsEnoughWorkspace(std::string_view where, - const miopen::solver::Id& solver_id, - std::size_t required_size, - const miopen::AnyInvokeParams* invokeParams, - bool log_as_warning = true); - -struct ConvFindParameters : PrimitiveFindParameters -{ - bool use_winograd_only; - ConvFindParameters(bool use_winograd_only_) : use_winograd_only(use_winograd_only_) {} -}; - -} // namespace conv -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/convolution.hpp b/projects/miopen/src/include/miopen/graphapi/convolution.hpp deleted file mode 100644 index cfb0690437c7..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/convolution.hpp +++ /dev/null @@ -1,675 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include -#include - -#include - -namespace miopen { - -namespace graphapi { - -class Convolution -{ -private: - int64_t mSpatialDims = 0; - std::vector mDilations; - std::vector mFilterStrides; - std::vector mPrePaddings; - std::vector mPostPaddings; - miopenDataType_t mCompType = miopenFloat; - miopenConvolutionMode_t mMode = miopenConvolution; - -public: - Convolution() noexcept = default; - Convolution(const Convolution&) = default; - Convolution(Convolution&&) noexcept = default; - Convolution& operator=(const Convolution&) = default; - Convolution& operator=(Convolution&&) noexcept = default; - Convolution(miopenDataType_t compType, - miopenConvolutionMode_t mode, - size_t spatialDims, - const std::vector& prePaddings, - const std::vector& filterStrides, - const std::vector& dilations, - const std::vector& postPaddings) - : mSpatialDims(spatialDims), - mDilations(dilations), - mFilterStrides(filterStrides), - mPrePaddings(prePaddings), - mPostPaddings(postPaddings), - mCompType(compType), - mMode(mode) - { - } - Convolution(miopenDataType_t compType, - miopenConvolutionMode_t mode, - size_t spatialDims, - std::vector&& prePaddings, - std::vector&& filterStrides, - std::vector&& dilations, - std::vector&& postPaddings) - : mSpatialDims(spatialDims), - mDilations(std::move(dilations)), - mFilterStrides(std::move(filterStrides)), - mPrePaddings(std::move(prePaddings)), - mPostPaddings(std::move(postPaddings)), - mCompType(compType), - mMode(mode) - { - } - - miopenDataType_t getCompType() const noexcept { return mCompType; } - miopenConvolutionMode_t getMode() const noexcept { return mMode; } - int64_t getSpatialDims() const noexcept { return mSpatialDims; } - const std::vector& getDilations() const noexcept { return mDilations; } - const std::vector& getFilterStrides() const noexcept { return mFilterStrides; } - const std::vector& getPrePaddings() const noexcept { return mPrePaddings; } - const std::vector& getPostPaddings() const noexcept { return mPostPaddings; } - - friend void to_json(nlohmann::json& json, const Convolution& conv); - friend void from_json(const nlohmann::json& json, Convolution& conv); - - struct JsonFields - { - static constexpr const char* SpatialDims = "spatial_dims"; - static constexpr const char* Dilations = "dilations"; - static constexpr const char* FilterStrides = "filter_dtrides"; - static constexpr const char* PrePaddings = "pre_paddings"; - static constexpr const char* PostPaddings = "post_paddings"; - static constexpr const char* CompType = "comp_type"; - static constexpr const char* Mode = "mode"; - }; - -private: - friend class ConvolutionBuilder; -}; - -class MIOPEN_INTERNALS_EXPORT ConvolutionBuilder -{ -private: - Convolution mConvolution; - bool mCompTypeSet = false; - bool mModeSet = false; - bool mSpatialDimsSet = false; - bool mDilationsSet = false; - bool mFilterStridesSet = false; - bool mPrePaddingsSet = false; - bool mPostPaddingsSet = false; - -public: - ConvolutionBuilder& setCompType(miopenDataType_t compType) & noexcept; - ConvolutionBuilder& setMode(miopenConvolutionMode_t mode) & noexcept; - ConvolutionBuilder& setSpatialDims(int64_t spatialDims) & noexcept; - ConvolutionBuilder& setDilations(const std::vector& dilations) &; - ConvolutionBuilder& setDilations(std::vector&& dilations) & noexcept; - ConvolutionBuilder& setFilterStrides(const std::vector& filterStrides) &; - ConvolutionBuilder& setFilterStrides(std::vector&& filterStrides) & noexcept; - ConvolutionBuilder& setPrePaddings(const std::vector& prePaddings) &; - ConvolutionBuilder& setPrePaddings(std::vector&& prePaddings) & noexcept; - ConvolutionBuilder& setPostPaddings(const std::vector& postPaddings) &; - ConvolutionBuilder& setPostPaddings(std::vector&& postPaddings) & noexcept; - - ConvolutionBuilder&& setCompType(miopenDataType_t compType) && noexcept - { - return std::move(setCompType(compType)); - } - ConvolutionBuilder&& setMode(miopenConvolutionMode_t mode) && noexcept - { - return std::move(setMode(mode)); - } - ConvolutionBuilder&& setSpatialDims(int64_t spatialDims) && noexcept - { - return std::move(setSpatialDims(spatialDims)); - } - ConvolutionBuilder&& setDilations(const std::vector& dilations) && - { - return std::move(setDilations(dilations)); - } - ConvolutionBuilder&& setDilations(std::vector&& dilations) && noexcept - { - return std::move(setDilations(std::move(dilations))); - } - ConvolutionBuilder&& setFilterStrides(const std::vector& filterStrides) && - { - return std::move(setFilterStrides(filterStrides)); - } - ConvolutionBuilder&& setFilterStrides(std::vector&& filterStrides) && noexcept - { - return std::move(setFilterStrides(std::move(filterStrides))); - } - ConvolutionBuilder&& setPrePaddings(const std::vector& prePaddings) && - { - return std::move(setPrePaddings(prePaddings)); - } - ConvolutionBuilder&& setPrePaddings(std::vector&& prePaddings) && noexcept - { - return std::move(setPrePaddings(std::move(prePaddings))); - } - ConvolutionBuilder&& setPostPaddings(const std::vector& postPaddings) && - { - return std::move(setPostPaddings(postPaddings)); - } - ConvolutionBuilder&& setPostPaddings(std::vector&& postPaddings) && noexcept - { - return std::move(setPostPaddings(std::move(postPaddings))); - } - - Convolution build() const&; - Convolution build() &&; - -private: - MIOPEN_INTERNALS_NO_EXPORT bool validate() const; -}; - -class MIOPEN_INTERNALS_EXPORT BackendConvolutionDescriptor : public BackendDescriptor -{ -private: - ConvolutionBuilder mBuilder; - Convolution mConvolution; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Convolution* getConvolution() const noexcept { return &mConvolution; } - Convolution* getConvolution() noexcept { return &mConvolution; } - -private: - void setCompType(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setMode(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setSpatialDims(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setDilations(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setFilterStrides(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setPrePaddings(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setPostPaddings(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - - void getCompType(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getMode(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getSpatialDims(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getDilations(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getFilterStrides(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getPrePaddings(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getPostPaddings(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); -}; - -class OperationConvolution : public OpNode -{ -private: - Convolution* mConvolution; - Tensor* mX; - Tensor* mY; - Tensor* mW; - double mAlpha; - double mBeta; - -public: - OperationConvolution() = default; - OperationConvolution(Convolution* convolution, - Tensor* x, - Tensor* w, - Tensor* y, - double alpha, - double beta) noexcept - : mConvolution(convolution), mX(x), mY(y), mW(w), mAlpha(alpha), mBeta(beta) - { - } - - Convolution* getConvolution() const noexcept { return mConvolution; } - Tensor* getX() const noexcept { return mX; } - Tensor* getY() const noexcept { return mY; } - Tensor* getW() const noexcept { return mW; } - double getAlpha() const noexcept { return mAlpha; } - double getBeta() const noexcept { return mBeta; } -}; - -class OperationConvolutionForward : public OperationConvolution -{ -public: - OperationConvolutionForward() = default; - OperationConvolutionForward(Convolution* convolution, - Tensor* x, - Tensor* w, - Tensor* y, - double alpha, - double beta) noexcept - : OperationConvolution(convolution, x, w, y, alpha, beta) - { - } - virtual const std::string& signName() const override - { - static const std::string name = "OP_CONVOLUTION_FORWARD"; - return name; - } - virtual std::vector getInTensors() const override { return {getX(), getW()}; } - virtual std::vector getOutTensors() const override { return {getY()}; } -}; - -class OperationConvolutionBuilder -{ -protected: - Convolution* mConvolution = nullptr; - Tensor* mX = nullptr; - Tensor* mY = nullptr; - Tensor* mW = nullptr; - double mAlpha = 1.0; - double mBeta = 0.0; - bool mAlphaSet = false; - bool mBetaSet = false; - -public: - OperationConvolutionBuilder& setConvolution(Convolution* convolution) noexcept - { - mConvolution = convolution; - return *this; - } - OperationConvolutionBuilder& setX(Tensor* x) noexcept - { - mX = x; - return *this; - } - OperationConvolutionBuilder& setY(Tensor* y) noexcept - { - mY = y; - return *this; - } - OperationConvolutionBuilder& setW(Tensor* w) noexcept - { - mW = w; - return *this; - } - OperationConvolutionBuilder& setAlpha(double alpha) noexcept - { - mAlpha = alpha; - mAlphaSet = true; - return *this; - } - OperationConvolutionBuilder& setBeta(double beta) noexcept - { - mBeta = beta; - mBetaSet = true; - return *this; - } - -protected: - OperationConvolutionBuilder() = default; -}; - -class OperationConvolutionForwardBuilder : public OperationConvolutionBuilder -{ -public: - OperationConvolutionForwardBuilder& setConvolution(Convolution* convolution) noexcept - { - OperationConvolutionBuilder::setConvolution(convolution); - return *this; - } - OperationConvolutionForwardBuilder& setX(Tensor* x) noexcept - { - OperationConvolutionBuilder::setX(x); - return *this; - } - OperationConvolutionForwardBuilder& setY(Tensor* y) noexcept - { - OperationConvolutionBuilder::setY(y); - return *this; - } - OperationConvolutionForwardBuilder& setW(Tensor* w) noexcept - { - OperationConvolutionBuilder::setW(w); - return *this; - } - OperationConvolutionForwardBuilder& setAlpha(double alpha) noexcept - { - OperationConvolutionBuilder::setAlpha(alpha); - return *this; - } - OperationConvolutionForwardBuilder& setBeta(double beta) noexcept - { - OperationConvolutionBuilder::setBeta(beta); - return *this; - } - - OperationConvolutionForward build() const - { - if(mConvolution == nullptr || mX == nullptr || mY == nullptr || mW == nullptr || - !mAlphaSet || !mBetaSet) - { - MIOPEN_THROW(miopenStatusNotInitialized); - } - return {mConvolution, mX, mW, mY, mAlpha, mBeta}; - } -}; - -class BackendOperationConvolutionDescriptor : public BackendDescriptor -{ -protected: - miopenBackendDescriptor_t mConvolutionDescriptor = nullptr; - miopenBackendDescriptor_t mXDescriptor = nullptr; - miopenBackendDescriptor_t mWDescriptor = nullptr; - miopenBackendDescriptor_t mYDescriptor = nullptr; - - virtual OperationConvolutionBuilder& getBuilder() = 0; - virtual OperationConvolution& getOperationConvolution() = 0; - - void setConvolution(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void - setX(miopenBackendAttributeType_t attributeType, int64_t elementCount, void* arrayOfElements); - void - setW(miopenBackendAttributeType_t attributeType, int64_t elementCount, void* arrayOfElements); - void - setY(miopenBackendAttributeType_t attributeType, int64_t elementCount, void* arrayOfElements); - void setAlpha(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void setBeta(miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - void getConvolution(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getX(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getW(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getY(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getAlpha(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - void getBeta(miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); -}; - -class BackendOperationConvolutionForwardDescriptor : public BackendOperationConvolutionDescriptor -{ -private: - OperationConvolutionForwardBuilder mBuilder; - OperationConvolutionForward mOperation; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - virtual OpNode* getOperation() override; - -protected: - virtual OperationConvolutionBuilder& getBuilder() override; - virtual OperationConvolution& getOperationConvolution() override; -}; - -class OperationConvolutionBackwardData : public OperationConvolution -{ -public: - OperationConvolutionBackwardData() = default; - OperationConvolutionBackwardData(Convolution* convolution, - Tensor* x, - Tensor* w, - Tensor* y, - double alpha, - double beta) noexcept - : OperationConvolution(convolution, x, w, y, alpha, beta) - { - } - virtual const std::string& signName() const override - { - static const std::string name = "OP_CONVOLUTION_BACKWARD_DATA"; - return name; - } - virtual std::vector getInTensors() const override { return {getW(), getY()}; } - virtual std::vector getOutTensors() const override { return {getX()}; } -}; - -class OperationConvolutionBackwardDataBuilder : public OperationConvolutionBuilder -{ -public: - OperationConvolutionBackwardDataBuilder& setConvolution(Convolution* convolution) noexcept - { - OperationConvolutionBuilder::setConvolution(convolution); - return *this; - } - OperationConvolutionBackwardDataBuilder& setX(Tensor* x) noexcept - { - OperationConvolutionBuilder::setX(x); - return *this; - } - OperationConvolutionBackwardDataBuilder& setY(Tensor* y) noexcept - { - OperationConvolutionBuilder::setY(y); - return *this; - } - OperationConvolutionBackwardDataBuilder& setW(Tensor* w) noexcept - { - OperationConvolutionBuilder::setW(w); - return *this; - } - OperationConvolutionBackwardDataBuilder& setAlpha(double alpha) noexcept - { - OperationConvolutionBuilder::setAlpha(alpha); - return *this; - } - OperationConvolutionBackwardDataBuilder& setBeta(double beta) noexcept - { - OperationConvolutionBuilder::setBeta(beta); - return *this; - } - - OperationConvolutionBackwardData build() const - { - if(mConvolution == nullptr || mX == nullptr || mY == nullptr || mW == nullptr || - !mAlphaSet || !mBetaSet) - { - MIOPEN_THROW(miopenStatusNotInitialized); - } - return {mConvolution, mX, mW, mY, mAlpha, mBeta}; - } -}; - -class BackendOperationConvolutionBackwardDataDescriptor - : public BackendOperationConvolutionDescriptor -{ -private: - OperationConvolutionBackwardDataBuilder mBuilder; - OperationConvolutionBackwardData mOperation; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - virtual OpNode* getOperation() override; - -private: - virtual OperationConvolutionBuilder& getBuilder() override; - virtual OperationConvolution& getOperationConvolution() override; -}; - -class OperationConvolutionBackwardFilter : public OperationConvolution -{ -public: - OperationConvolutionBackwardFilter() = default; - OperationConvolutionBackwardFilter(Convolution* convolution, - Tensor* x, - Tensor* w, - Tensor* y, - double alpha, - double beta) noexcept - : OperationConvolution(convolution, x, w, y, alpha, beta) - { - } - virtual const std::string& signName() const override - { - static const std::string name = "OP_CONVOLUTION_BACKWARD_FILTER"; - return name; - } - virtual std::vector getInTensors() const override { return {getX(), getY()}; } - virtual std::vector getOutTensors() const override { return {getW()}; } -}; - -class OperationConvolutionBackwardFilterBuilder : public OperationConvolutionBuilder -{ -public: - OperationConvolutionBackwardFilterBuilder& setConvolution(Convolution* convolution) noexcept - { - OperationConvolutionBuilder::setConvolution(convolution); - return *this; - } - OperationConvolutionBackwardFilterBuilder& setX(Tensor* x) noexcept - { - OperationConvolutionBuilder::setX(x); - return *this; - } - OperationConvolutionBackwardFilterBuilder& setY(Tensor* y) noexcept - { - OperationConvolutionBuilder::setY(y); - return *this; - } - OperationConvolutionBackwardFilterBuilder& setW(Tensor* w) noexcept - { - OperationConvolutionBuilder::setW(w); - return *this; - } - OperationConvolutionBackwardFilterBuilder& setAlpha(double alpha) noexcept - { - OperationConvolutionBuilder::setAlpha(alpha); - return *this; - } - OperationConvolutionBackwardFilterBuilder& setBeta(double beta) noexcept - { - OperationConvolutionBuilder::setBeta(beta); - return *this; - } - - OperationConvolutionBackwardFilter build() const - { - if(mConvolution == nullptr || mX == nullptr || mY == nullptr || mW == nullptr || - !mAlphaSet || !mBetaSet) - { - MIOPEN_THROW(miopenStatusNotInitialized); - } - return {mConvolution, mX, mW, mY, mAlpha, mBeta}; - } -}; - -class BackendOperationConvolutionBackwardFilterDescriptor - : public BackendOperationConvolutionDescriptor -{ -private: - OperationConvolutionBackwardFilterBuilder mBuilder; - OperationConvolutionBackwardFilter mOperation; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - virtual OpNode* getOperation() override; - -private: - virtual OperationConvolutionBuilder& getBuilder() override; - virtual OperationConvolution& getOperationConvolution() override; -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/engine.hpp b/projects/miopen/src/include/miopen/graphapi/engine.hpp deleted file mode 100644 index 7cc7058452bc..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/engine.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -namespace miopen { - -namespace graphapi { - -class Engine; -class OpGraph; - -// Pattern is a family of solvers for the same graph shape -class GraphPatternMatcher -{ - -public: - virtual bool matches(const OpGraph* graph) const = 0; - virtual std::vector getEngines(OpGraph* graph) const = 0; - virtual std::string_view name() const = 0; - - virtual ~GraphPatternMatcher(); -}; - -struct TensorInfo -{ - miopenTensorArgumentId_t mEnumId = miopenTensorArgumentIdInvalid; - Tensor* mGraphTensor = nullptr; - Data_t mDevBuf = nullptr; - - TensorInfo(miopenTensorArgumentId_t enum_id, Tensor* tens_ptr) - : mEnumId(enum_id), mGraphTensor(tens_ptr) - { - assert(mEnumId != miopenTensorArgumentIdInvalid); - } - - void setDevBuf(Data_t ptr) - { - assert(ptr); - mDevBuf = ptr; - } -}; - -// int64_t is the graph tensor id -using TensorInfoMap = std::unordered_map; - -class MIOPEN_INTERNALS_EXPORT GraphPatternExecutor -{ -public: - virtual void execute(miopenHandle_t handle, const VariantPack& vpk) = 0; - virtual size_t getWorkspaceSize() const = 0; - virtual nlohmann::json getJson() = 0; - virtual ~GraphPatternExecutor(); - - struct JsonFields - { - static constexpr const char* Name = "name"; - }; -}; - -// generic executor that uses Find 2.0 Solution -class GraphExecutorFind20 : public GraphPatternExecutor -{ - Solution mSolution; - std::shared_ptr mTensorInfoMap; - -public: - GraphExecutorFind20(const Solution& sol, const std::shared_ptr& tmap) - : GraphPatternExecutor(), mSolution(sol), mTensorInfoMap(tmap) - { - MIOPEN_THROW_IF(tmap.get() == nullptr, - "TensorInfoMap should be allocated before GraphExecutorFind20 creation"); - } - - GraphExecutorFind20(Solution&& sol, const std::shared_ptr& tmap) - : GraphPatternExecutor(), mSolution(std::move(sol)), mTensorInfoMap(tmap) - { - MIOPEN_THROW_IF(tmap.get() == nullptr, - "TensorInfoMap should be allocated before GraphExecutorFind20 creation"); - } - - GraphExecutorFind20(const nlohmann::json& json); - - void execute(miopenHandle_t handle, const VariantPack& vpk) final; - - size_t getWorkspaceSize() const final; - - nlohmann::json getJson() final; - - static constexpr const char* name = "GraphExecutorFind20"; - - struct JsonFields - { - static constexpr const char* Solution = "solution"; - static constexpr const char* Id2ArgumentMap = "id_to_argument_map"; - }; -}; - -class Engine -{ -private: - std::shared_ptr mExecutor; - OpGraph* mGraph = nullptr; - int64_t mGlobalIndex = -1; - int32_t mSmCount = 0; - friend class EngineBuilder; - -public: - Engine() = default; - Engine(const Engine&) = default; - Engine(Engine&&) = default; - Engine& operator=(const Engine&) = default; - Engine& operator=(Engine&&) = default; - - GraphPatternExecutor* getExecutor() noexcept { return mExecutor.get(); } - - const std::shared_ptr& getExecutor() const noexcept { return mExecutor; } - - int64_t getGlobalIndex() const noexcept { return mGlobalIndex; } - int32_t getSmCount() const noexcept { return mSmCount; } - - const OpGraph* getOpGraph() const { return mGraph; } - OpGraph* getOpGraph() { return mGraph; } - - friend void to_json(nlohmann::json& json, const Engine& engine); - friend void from_json(const nlohmann::json& json, Engine& engine); - - struct JsonFields - { - static constexpr const char* Executor = "executor"; - static constexpr const char* GlobalIndex = "global_index"; - static constexpr const char* SmCount = "sm_count"; - }; -}; - -class MIOPEN_INTERNALS_EXPORT EngineBuilder -{ - friend class BackendEngineDescriptor; - - std::shared_ptr mExecutor = nullptr; - OpGraph* mGraph = nullptr; - int64_t mGlobalIndex = -1; - int32_t mSmCount = 0; - bool mGraphSet = false; - bool mExecSet = false; - bool mIndexSet = false; - -public: - EngineBuilder& setGraph(OpGraph* g); - - EngineBuilder& setGlobalIndex(int64_t globalIndex); - - MIOPEN_INTERNALS_NO_EXPORT EngineBuilder& setSmCount(int32_t smCount); - - EngineBuilder& setExecutor(const std::shared_ptr& e); - - Engine build(); -}; - -class MIOPEN_INTERNALS_EXPORT BackendEngineDescriptor : public BackendDescriptor -{ -private: - EngineBuilder mBuilder; - Engine mEngine; - - miopenBackendDescriptor_t mOpGraphDescriptor = nullptr; - -public: - BackendEngineDescriptor() = default; - - BackendEngineDescriptor(const Engine& engine, miopenBackendDescriptor_t opGraphDescriptor) - : mEngine(engine), mOpGraphDescriptor(opGraphDescriptor) - { - mFinalized = true; - } - - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Engine& getEngine() const noexcept { return mEngine; } - Engine& getEngine() noexcept { return mEngine; } -}; - -MIOPEN_INTERNALS_EXPORT std::vector findEngines(OpGraph*); - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp b/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp deleted file mode 100644 index ff71f0ba3ee3..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/enginecfg.hpp +++ /dev/null @@ -1,142 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -class EngineCfg -{ -private: - /* we don't use a pointer here to allow a user - * to have several configs for an Engine. Each - * config might modify its Engine in future so - * their instance of Engine shouldn't be shared - */ - Engine mEngine; - - friend class EngineCfgBuilder; - -public: - EngineCfg() = default; - EngineCfg(const EngineCfg&) = default; - EngineCfg(EngineCfg&&) = default; - EngineCfg& operator=(const EngineCfg&) = default; - EngineCfg& operator=(EngineCfg&&) = default; - - EngineCfg(const Engine& engine) : mEngine(engine) {} - EngineCfg(Engine&& engine) : mEngine(std::move(engine)) {} - - const Engine& getEngine() const noexcept { return mEngine; } - Engine& getEngine() noexcept { return mEngine; } - - friend void to_json(nlohmann::json& json, const EngineCfg& engineCfg) - { - json = engineCfg.mEngine; - } - - friend void from_json(const nlohmann::json& json, EngineCfg& engineCfg) - { - json.get_to(engineCfg.mEngine); - } -}; - -/* For now we don't support tuning and a builder is not needed, - * but in future it will be needed. - */ -class EngineCfgBuilder -{ - EngineCfg mEngineCfg; - bool mEngineSet = false; - -public: - EngineCfgBuilder& setEngine(const Engine& engine) & - { - mEngineCfg.mEngine = engine; - mEngineSet = true; - return *this; - } - EngineCfgBuilder& setEngine(Engine&& engine) & - { - mEngineCfg.mEngine = std::move(engine); - mEngineSet = true; - return *this; - } - EngineCfgBuilder&& setEngine(const Engine& engine) && { return std::move(setEngine(engine)); } - EngineCfgBuilder&& setEngine(Engine&& engine) && - { - return std::move(setEngine(std::move(engine))); - } - EngineCfg build() &; - MIOPEN_INTERNALS_EXPORT EngineCfg build() &&; -}; - -class MIOPEN_INTERNALS_EXPORT BackendEngineCfgDescriptor : public BackendDescriptor -{ -protected: - EngineCfgBuilder mBuilder; - EngineCfg mEngineCfg; - - miopenBackendDescriptor_t mEngineDescriptor = nullptr; - - BackendEngineCfgDescriptor(const EngineCfg& engineCfg, - miopenBackendDescriptor_t engineDescriptor) - : mEngineCfg(engineCfg), mEngineDescriptor(engineDescriptor) - { - mFinalized = true; - } - BackendEngineCfgDescriptor(EngineCfg&& engineCfg, miopenBackendDescriptor_t engineDescriptor) - : mEngineCfg(std::move(engineCfg)), mEngineDescriptor(engineDescriptor) - { - mFinalized = true; - } - -public: - BackendEngineCfgDescriptor() = default; - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const EngineCfg& getEngineCfg() const { return mEngineCfg; } - EngineCfg& getEngineCfg() { return mEngineCfg; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp b/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp deleted file mode 100644 index 629ac1f061e8..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/execution_plan.hpp +++ /dev/null @@ -1,222 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include - -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -class ExecutionPlan -{ -private: - /* we don't use a pointer for mEngineCfg - * because we need to support serialization - * and deserialization - */ - EngineCfg mEngineCfg; - miopenHandle_t mHandle = nullptr; - std::vector mIntermediateIds; - - friend class ExecutionPlanBuilder; - -public: - ExecutionPlan() = default; - ExecutionPlan(const ExecutionPlan&) = default; - ExecutionPlan(ExecutionPlan&&) = default; - ExecutionPlan& operator=(const ExecutionPlan&) = default; - ExecutionPlan& operator=(ExecutionPlan&&) = default; - - miopenHandle_t getHandle() const noexcept { return mHandle; } - const EngineCfg& getEngineCfg() const noexcept { return mEngineCfg; } - EngineCfg& getEngineCfg() noexcept { return mEngineCfg; } - const std::vector& getIntermediateIds() const noexcept { return mIntermediateIds; } - MIOPEN_INTERNALS_EXPORT std::string getJsonRepresentation() const; - - void execute(miopenHandle_t handle, const VariantPack& variantPack) - { - checkPtr(handle); - mEngineCfg.getEngine().getExecutor()->execute(handle, variantPack); - } - - size_t getWorkspaceSize() const - { - return mEngineCfg.getEngine().getExecutor()->getWorkspaceSize(); - } - - struct SerializationMetadata - { - uint64_t mMagicNumber; - uint64_t mVersion; - - constexpr static SerializationMetadata current() { return {0xA4C9C1597ED9E1AC, 1}; } - - struct JsonFields - { - static constexpr const char* MagicNumber = "magic_number"; - static constexpr const char* Version = "version"; - }; - - friend void to_json(nlohmann::json& json, const SerializationMetadata& metadata) - { - json = nlohmann::json{ - {JsonFields::MagicNumber, metadata.mMagicNumber}, - {JsonFields::Version, metadata.mVersion}, - }; - } - - friend void from_json(const nlohmann::json& json, SerializationMetadata& metadata) - { - json.at(JsonFields::MagicNumber).get_to(metadata.mMagicNumber); - json.at(JsonFields::Version).get_to(metadata.mVersion); - } - }; - - struct JsonFields - { - static constexpr const char* Metadata = "header"; - static constexpr const char* EngineCfg = "engine_cfg"; - static constexpr const char* IntermediateIds = "intermediate_ids"; - }; - - friend void to_json(nlohmann::json& json, const ExecutionPlan& executionPlan) - { - json = nlohmann::json{ - {JsonFields::Metadata, SerializationMetadata::current()}, - {JsonFields::EngineCfg, executionPlan.mEngineCfg}, - {JsonFields::IntermediateIds, executionPlan.mIntermediateIds}, - }; - } - - friend void from_json(const nlohmann::json& json, ExecutionPlan& executionPlan) - { - constexpr auto supportedMetadata = SerializationMetadata::current(); - - auto metadataIterator = json.find(JsonFields::Metadata); - if(metadataIterator == json.end()) - { - MIOPEN_THROW(miopenStatusInvalidValue, - "Unsupported JSON has been passed as an ExecutionPlan"); - } - - const auto metadata = metadataIterator->get(); - - if(metadata.mMagicNumber != supportedMetadata.mMagicNumber) - { - MIOPEN_THROW(miopenStatusInvalidValue, - "JSON with unsupported magic number has been passed as an ExecutionPlan"); - } - - if(metadata.mVersion != supportedMetadata.mVersion) - { - MIOPEN_THROW(miopenStatusVersionMismatch, - "JSON of an invalid version has been passed as an ExecutionPlan"); - } - - json.at(JsonFields::EngineCfg).get_to(executionPlan.mEngineCfg); - json.at(JsonFields::IntermediateIds).get_to(executionPlan.mIntermediateIds); - executionPlan.mHandle = nullptr; - } -}; - -class MIOPEN_INTERNALS_EXPORT ExecutionPlanBuilder -{ -private: - ExecutionPlan mExecutionPlan; - bool mEngineCfgSet = false; - bool mJsonRepresentationSet = false; - -public: - ExecutionPlanBuilder& setHandle(miopenHandle_t handle) &; - ExecutionPlanBuilder& setEngineCfg(const EngineCfg& engineCfg) &; - MIOPEN_INTERNALS_NO_EXPORT ExecutionPlanBuilder& setEngineCfg(EngineCfg&& engineCfg) &; - MIOPEN_INTERNALS_NO_EXPORT ExecutionPlanBuilder& - setIntermediateIds(const std::vector& ids) &; - ExecutionPlanBuilder& setIntermediateIds(std::vector&& ids) &; - ExecutionPlanBuilder& setJsonRepresentation(const std::string_view& s) &; - - ExecutionPlanBuilder&& setHandle(miopenHandle_t handle) && - { - return std::move(setHandle(handle)); - } - ExecutionPlanBuilder&& setEngineCfg(const EngineCfg& engineCfg) && - { - return std::move(setEngineCfg(engineCfg)); - } - ExecutionPlanBuilder&& setEngineCfg(EngineCfg&& engineCfg) && - { - return std::move(setEngineCfg(std::move(engineCfg))); - } - ExecutionPlanBuilder&& setIntermediateIds(const std::vector& ids) && - { - return std::move(setIntermediateIds(ids)); - } - ExecutionPlanBuilder&& setIntermediateIds(std::vector&& ids) && - { - return std::move(setIntermediateIds(std::move(ids))); - } - ExecutionPlanBuilder&& setJsonRepresentation(const std::string_view& s) && - { - return std::move(setJsonRepresentation(s)); - } - - MIOPEN_INTERNALS_NO_EXPORT ExecutionPlan build() &; - ExecutionPlan build() &&; -}; - -class BackendExecutionPlanDescriptor : public BackendDescriptor -{ -private: - ExecutionPlanBuilder mBuilder; - ExecutionPlan mExecutionPlan; - - miopenBackendDescriptor_t mEngineCfgDescriptor = nullptr; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - void execute(miopenHandle_t handle, miopenBackendDescriptor_t variantPack) override; -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/matmul.hpp b/projects/miopen/src/include/miopen/graphapi/matmul.hpp deleted file mode 100644 index ee07d2d8ae54..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/matmul.hpp +++ /dev/null @@ -1,198 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include - -#include -#include - -namespace miopen { -namespace graphapi { - -class Matmul -{ -private: - miopenDataType_t mCompType; - -public: - Matmul() = default; - Matmul(miopenDataType_t computeType) : mCompType(computeType) {} - miopenDataType_t getComputeType() { return mCompType; } - -private: - friend class MatmulBuilder; -}; - -class MatmulBuilder -{ - -private: - Matmul mMatmul; - bool mComputeTypeSet = false; - -public: - MatmulBuilder& setComputeType(miopenDataType_t computeType) - { - mMatmul.mCompType = computeType; - mComputeTypeSet = true; - return *this; - } - - MIOPEN_INTERNALS_EXPORT Matmul build() const; -}; - -class MIOPEN_INTERNALS_EXPORT BackendMatmulDescriptor : public BackendDescriptor -{ -private: - MatmulBuilder mBuilder; - Matmul mMatmul; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Matmul* getMatmul() const noexcept { return &mMatmul; } - Matmul* getMatmul() noexcept { return &mMatmul; } -}; - -class OperationMatmul : public OpNode -{ -private: - Tensor* mA; - Tensor* mB; - Tensor* mC; - int64_t mBatchCount = 1; - Tensor* mGemmMOverride; - Tensor* mGemmNOverride; - Tensor* mGemmKOverride; - Matmul* mMatmul; - -public: - OperationMatmul(Tensor* A, - Tensor* B, - Tensor* C, - int batchCount, - Tensor* MOverride, - Tensor* NOverride, - Tensor* KOverride, - Matmul* matmul) noexcept - : mA(A), - mB(B), - mC(C), - mBatchCount(batchCount), - mGemmMOverride(MOverride), - mGemmNOverride(NOverride), - mGemmKOverride(KOverride), - mMatmul(matmul) - { - } - - OperationMatmul() = default; - Tensor* getA() const { return mA; } - Tensor* getB() const { return mB; } - Tensor* getC() const { return mC; } - int64_t getBatchCount() const { return mBatchCount; } - Tensor* getMOverride() { return mGemmMOverride; } - Tensor* getNOverride() { return mGemmNOverride; } - Tensor* getKOverride() { return mGemmKOverride; } - Matmul* getMatmul() { return mMatmul; } - virtual std::vector getInTensors() const override { return {getA(), getB()}; } - virtual std::vector getOutTensors() const override { return {getC()}; } - virtual const std::string& signName() const override - { - static const std::string name = "OP_MATMUL"; - return name; - } - -private: - friend class OperationMatmulBuilder; -}; - -class MIOPEN_INTERNALS_EXPORT OperationMatmulBuilder -{ -private: - OperationMatmul mOperationMatmul; - bool mASet = false; - bool mBSet = false; - bool mCSet = false; - bool mMatmulSet = false; - -public: - OperationMatmulBuilder& setA(Tensor* A); - - OperationMatmulBuilder& setB(Tensor* B); - - OperationMatmulBuilder& setC(Tensor* C); - - OperationMatmulBuilder& setBatchCount(int64_t count); - - OperationMatmulBuilder& setGemmMOverride(Tensor* overrideTensor); - - OperationMatmulBuilder& setGemmNOverride(Tensor* overrideTensor); - - OperationMatmulBuilder& setGemmKOverride(Tensor* overrideTensor); - - OperationMatmulBuilder& setMatmulDescriptor(Matmul* mMatmul); - - OperationMatmul build(); -}; - -class BackendOperationMatmulDescriptor : public BackendDescriptor -{ -private: - OperationMatmulBuilder mBuilder; - OperationMatmul mMatmul; - miopenBackendDescriptor_t mA = nullptr; - miopenBackendDescriptor_t mB = nullptr; - miopenBackendDescriptor_t mC = nullptr; - miopenBackendDescriptor_t mGemmMOverride = nullptr; - miopenBackendDescriptor_t mGemmNOverride = nullptr; - miopenBackendDescriptor_t mGemmKOverride = nullptr; - miopenBackendDescriptor_t mMatmuDescriptor = nullptr; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - OpNode* getOperation() override; -}; - -} // namespace graphapi -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/opgraph.hpp b/projects/miopen/src/include/miopen/graphapi/opgraph.hpp deleted file mode 100644 index c75c0551c064..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/opgraph.hpp +++ /dev/null @@ -1,462 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -#include - -namespace miopen { -namespace graphapi { - -namespace internal { -template -bool contains(const C& container, const T& val) noexcept -{ - return std::find(container.cbegin(), container.cend(), val) != container.cend(); -} - -template -bool noRepetitions(const R& r) -{ - auto begin = r.cbegin(); - auto end = r.cend(); - - if(std::distance(begin, end) < threshold) - { - // time = O(n^2) mem = O(1) - bool result = true; - while(result && begin != end) - { - const auto& val = *begin++; - result = std::find(begin, end, val) == end; - } - return result; - } - else - { - // time = O(n) mem = O(n) - std::unordered_set>> seen; - for(; begin != end; ++begin) - { - const auto& val = *begin; - if(seen.find(val) != seen.end()) - { - return false; - } - seen.insert(val); - } - return true; - } -} -} // end namespace internal - -class OpGraphBuilder; -class OpGraph; - -class MIOPEN_INTERNALS_EXPORT OpNode -{ -public: - using Edge = std::pair; - virtual ~OpNode(); - - virtual const std::string& signName() const = 0; - -private: - std::vector mInEdges; - std::vector mOutEdges; - - friend class OpGraphBuilder; - friend class OpGraph; - -protected: - static Edge makeEdge(OpNode* n, Tensor* t) { return Edge{n, t}; } - static Edge makeEdge(const OpNode* n, const Tensor* t) - { - return Edge{ - const_cast(n), // NOLINT (cppcoreguidelines-pro-type-const-cast) - const_cast(t) // NOLINT (cppcoreguidelines-pro-type-const-cast) - }; - } - virtual std::vector getInTensors() const = 0; - - virtual std::vector getOutTensors() const = 0; - - const auto& getInEdges() const { return mInEdges; } - - const auto& getOutEdges() const { return mOutEdges; } - - bool hasInEdge(const OpNode* src, const Tensor* tens_ptr) const - { - assert(src); - assert(tens_ptr); - auto e = makeEdge(src, tens_ptr); - return internal::contains(mInEdges, e); - } - - bool hasOutEdge(const OpNode* dst, const Tensor* tens_ptr) const - { - assert(dst); - assert(tens_ptr); - auto e = makeEdge(dst, tens_ptr); - return internal::contains(mOutEdges, e); - } - - void addOutEdge(OpNode* dst, Tensor* tens_ptr) - { - assert(dst); - assert(tens_ptr); - if(!hasOutEdge(dst, tens_ptr)) - { - mOutEdges.emplace_back(dst, tens_ptr); - } - } - - void addInEdge(OpNode* src, Tensor* tens_ptr) - { - assert(src); - assert(tens_ptr); - if(!hasInEdge(src, tens_ptr)) - { - mInEdges.emplace_back(src, tens_ptr); - } - } - - size_t getInDegree() const { return mInEdges.size(); } - size_t getOutDegree() const { return mOutEdges.size(); } -}; - -using Edge = OpNode::Edge; - -class SourceOpNode : public OpNode -{ -protected: - std::vector mOutTensors; - friend class OpGraph; - - const std::string& signName() const final - { - static const std::string s = "INTERNAL::SRC"; - return s; - } - - std::vector getInTensors() const final { return {}; } - - std::vector getOutTensors() const final { return mOutTensors; } - - bool hasOutTensor(const Tensor* tensor) const - { - return internal::contains(mOutTensors, tensor); - } - - void addOutTensor(Tensor* tens_ptr) - { - if(!hasOutTensor(tens_ptr)) - { - mOutTensors.emplace_back(tens_ptr); - } - } -}; - -class SinkOpNode : public OpNode -{ -protected: - std::vector mInTensors; - friend class OpGraph; - - const std::string& signName() const final - { - static const std::string s = "INTERNAL::SINK"; - return s; - } - - std::vector getInTensors() const final { return mInTensors; } - - std::vector getOutTensors() const final { return {}; } - - bool hasInTensor(Tensor* tensor) const { return internal::contains(mInTensors, tensor); } - - void addInTensor(Tensor* tens_ptr) - { - if(!hasInTensor(tens_ptr)) - { - mInTensors.emplace_back(tens_ptr); - } - } -}; - -using Path = std::vector; -using VecOfPaths = std::vector; - -class Engine; - -class OpGraph -{ - // NOTE: mSrcNode and mSinkNode need to reside on the heap because the graph may move - // to a new memory location after building, while the nodes maintain address - // of SourceOpNode and SinkOpNode in their in and out edge lists - std::unique_ptr mSrcNode = std::make_unique(); - std::unique_ptr mSinkNode = std::make_unique(); - std::vector mNodes{}; - - // Descriptor related members - miopenHandle_t mHandle = nullptr; - std::vector mEngines{}; - -public: - OpGraph(const OpGraph&) = delete; - OpGraph& operator=(const OpGraph&) = delete; - - OpGraph() = default; - OpGraph(OpGraph&&) = default; - OpGraph& operator=(OpGraph&&) = default; - ~OpGraph() = default; - - SourceOpNode* getSourceNode() const noexcept { return mSrcNode.get(); } - - SinkOpNode* getSinkNode() const noexcept { return mSinkNode.get(); } - - bool hasNode(const OpNode* n) const { return internal::contains(mNodes, n); } - - bool hasEdge(const OpNode* src, const Tensor* tens_ptr, const OpNode* dst) const - { - assert(src); - assert(dst); - return src->hasOutEdge(dst, tens_ptr) && dst->hasInEdge(src, tens_ptr); - } - - size_t numNodes() const { return mNodes.size(); } - - size_t numEdges() const - { - size_t ret = 0; - for(OpNode* n : mNodes) - { - ret += n->getOutDegree(); - } - // ignore the edges that lead to mSinkNode - assert(ret >= mSinkNode->getInDegree()); - ret -= mSinkNode->getInDegree(); - - return ret; - } - - const std::vector& getNodes() const noexcept { return mNodes; } - - const std::vector& getOutEdges(const OpNode* n) const noexcept - { - assert(n); - return n->getOutEdges(); - } - - const std::vector& getInEdges(const OpNode* n) const noexcept - { - assert(n); - return n->getInEdges(); - } - - OpNode* findNodeByName(const std::string& name) const noexcept - { - for(auto* n : getNodes()) - { - if(n->signName() == name) - { - return n; - } - } - return nullptr; - } - - OpNode* findOutNeighByName(const OpNode* node, const std::string& neigh_name) const noexcept - { - assert(node); - for(auto [m, t] : getOutEdges(node)) - { - std::ignore = t; - if(m->signName() == neigh_name) - { - return m; - } - } - return nullptr; - } - - OpNode* findInNeighByName(const OpNode* node, const std::string& neigh_name) const - { - assert(node); - for(auto [m, t] : getInEdges(node)) - { - std::ignore = t; - if(m->signName() == neigh_name) - { - return m; - } - } - return nullptr; - } - - std::vector getNodeNames() const - { - std::vector names(mNodes.size()); - for(size_t i = 0; i < mNodes.size(); ++i) - { - names[i] = mNodes[i]->signName(); - } - return names; - } - - std::vector> getInOutDegrees() const - { - std::vector> ret(mNodes.size()); - for(size_t i = 0; i < mNodes.size(); ++i) - { - ret[i] = {mNodes[i]->getInDegree(), mNodes[i]->getOutDegree()}; - } - return ret; - } - - VecOfPaths getAllPaths() const; - - // NOTE: for testing only. May remove in the future - bool hasEdgeFromSource(OpNode* dst, Tensor* tens_ptr) const - { - return hasEdge(mSrcNode.get(), tens_ptr, dst); - } - - // NOTE: for testing only. May remove in the future - bool hasEdgeToSink(OpNode* src, Tensor* tens_ptr) const - { - return hasEdge(src, tens_ptr, mSinkNode.get()); - } - - miopenHandle_t getHandle() const noexcept { return mHandle; } - const std::vector& getEngines() const noexcept { return mEngines; } - - void initEngines(); /// \todo make private. Called in finalize, but also - /// from C++ tests --amberhassaan May, 2024 - -private: - friend class OpGraphBuilder; - - void initNodes(std::vector&& nodes) { mNodes = std::move(nodes); } - - void addEdge(OpNode* src, Tensor* tens_ptr, OpNode* dst) - { - assert(src); - assert(dst); - src->addOutEdge(dst, tens_ptr); - dst->addInEdge(src, tens_ptr); - } - - void addEdgeFromSrc(OpNode* dst, Tensor* tens_ptr) - { - mSrcNode->addOutTensor(tens_ptr); - addEdge(mSrcNode.get(), tens_ptr, dst); - } - - void addEdgeToSink(OpNode* src, Tensor* tens_ptr) - { - mSinkNode->addInTensor(tens_ptr); - addEdge(src, tens_ptr, mSinkNode.get()); - } -}; - -class OpGraphBuilder -{ -private: - std::vector mNodes; - miopenHandle_t mHandle = nullptr; - -public: - void setHandle(miopenHandle_t handle) { mHandle = checkPtr(handle); } - miopenHandle_t getHandle() const noexcept { return mHandle; } - - bool hasNode(OpNode* node) const { return internal::contains(mNodes, node); } - - void addNode(OpNode* node) - { - assert(!hasNode(node)); - mNodes.emplace_back(node); - } - - void setNodes(const std::vector& nodes) - { - assert(internal::noRepetitions(nodes)); - mNodes = nodes; - } - - void setNodes(std::vector&& nodes) - { - assert(internal::noRepetitions(nodes)); - mNodes = std::move(nodes); - } - - struct EdgeInfo - { - OpNode* mSrc = nullptr; - std::vector mDests{}; - }; - - // r-value method that consumes *this - MIOPEN_INTERNALS_EXPORT OpGraph build() &&; -}; - -MIOPEN_INTERNALS_EXPORT bool isIsomorphic(const OpGraph& left, const OpGraph& right); - -std::string pathToStr(const Path& path); - -class MIOPEN_INTERNALS_EXPORT BackendOperationGraphDescriptor : public BackendDescriptor -{ -private: - OpGraphBuilder mBuilder; - OpGraph mOpGraph; - std::vector mOps; // to return them in getAttribute - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const OpGraph* getOperationGraph() const noexcept { return &mOpGraph; } - OpGraph* getOperationGraph() noexcept { return &mOpGraph; } -}; - -} // end namespace graphapi -} // end namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/pointwise.hpp b/projects/miopen/src/include/miopen/graphapi/pointwise.hpp deleted file mode 100644 index 343a096a3e30..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/pointwise.hpp +++ /dev/null @@ -1,310 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -class Pointwise -{ -public: - using FpAttribute = std::variant; - -private: - FpAttribute mReluLowerClip = 0.0f; - FpAttribute mReluUpperClip = std::numeric_limits::max(); - FpAttribute mReluLowerClipSlope = 0.0f; - FpAttribute mEluAlpha = 1.0f; - FpAttribute mSoftPlusBeta = 1.0f; - FpAttribute mSwishBeta = 1.0f; - int64_t mAxis = -1; - miopenPointwiseMode_t mMode; - miopenDataType_t mMathPrecision; - miopenNanPropagation_t mNanPropagation = MIOPEN_NOT_PROPAGATE_NAN; - -public: - Pointwise() noexcept = default; - Pointwise(miopenPointwiseMode_t mode, - miopenDataType_t mathPrecision, - miopenNanPropagation_t nanPropagation = MIOPEN_NOT_PROPAGATE_NAN, - FpAttribute reluLowerClip = 0.0f, - FpAttribute reluUpperClip = std::numeric_limits::max(), - FpAttribute reluLowerClipSlope = 0.0f, - FpAttribute eluAlpha = 1.0f, - FpAttribute softPlusBeta = 1.0f, - FpAttribute swishBeta = 1.0f, - int64_t axis = -1) noexcept - : mReluLowerClip(reluLowerClip), - mReluUpperClip(reluUpperClip), - mReluLowerClipSlope(reluLowerClipSlope), - mEluAlpha(eluAlpha), - mSoftPlusBeta(softPlusBeta), - mSwishBeta(swishBeta), - mAxis(axis), - mMode(mode), - mMathPrecision(mathPrecision), - mNanPropagation(nanPropagation) - { - } - - miopenPointwiseMode_t getMode() const noexcept { return mMode; } - miopenDataType_t getMathPrecision() const noexcept { return mMathPrecision; } - miopenNanPropagation_t getNanPropagation() const noexcept { return mNanPropagation; } - FpAttribute getReluLowerClip() const noexcept { return mReluLowerClip; } - FpAttribute getReluUpperClip() const noexcept { return mReluUpperClip; } - FpAttribute getReluLowerClipSlope() const noexcept { return mReluLowerClipSlope; } - FpAttribute getEluAlpha() const noexcept { return mEluAlpha; } - FpAttribute getSoftPlusBeta() const noexcept { return mSoftPlusBeta; } - FpAttribute getSwishBeta() const noexcept { return mSwishBeta; } - int64_t getAxis() const noexcept { return mAxis; } - -private: - friend class PointwiseBuilder; -}; - -class PointwiseBuilder -{ -private: - Pointwise mPointwise; - bool mModeSet = false; - bool mMathPrecisionSet = false; - -public: - PointwiseBuilder& setMode(miopenPointwiseMode_t mode) noexcept - { - mPointwise.mMode = mode; - mModeSet = true; - return *this; - } - PointwiseBuilder& setMathPrecision(miopenDataType_t mathPrecision) noexcept - { - mPointwise.mMathPrecision = mathPrecision; - mMathPrecisionSet = true; - return *this; - } - PointwiseBuilder& setNanPropagation(miopenNanPropagation_t nanPropagation) noexcept - { - mPointwise.mNanPropagation = nanPropagation; - return *this; - } - PointwiseBuilder& setReluLowerClip(Pointwise::FpAttribute reluLowerClip) noexcept - { - mPointwise.mReluLowerClip = reluLowerClip; - return *this; - } - PointwiseBuilder& setReluUpperClip(Pointwise::FpAttribute reluUpperClip) noexcept - { - mPointwise.mReluUpperClip = reluUpperClip; - return *this; - } - PointwiseBuilder& setReluLowerClipSlope(Pointwise::FpAttribute reluLowerClipSlope) noexcept - { - mPointwise.mReluLowerClipSlope = reluLowerClipSlope; - return *this; - } - PointwiseBuilder& setEluAlpha(Pointwise::FpAttribute eluAlpha) noexcept - { - mPointwise.mEluAlpha = eluAlpha; - return *this; - } - PointwiseBuilder& setSoftPlusBeta(Pointwise::FpAttribute softPlusBeta) noexcept - { - mPointwise.mSoftPlusBeta = softPlusBeta; - return *this; - } - PointwiseBuilder& setSwishBeta(Pointwise::FpAttribute swishBeta) noexcept - { - mPointwise.mSwishBeta = swishBeta; - return *this; - } - PointwiseBuilder& setAxis(int64_t axis) noexcept - { - mPointwise.mAxis = axis; - return *this; - } - - MIOPEN_INTERNALS_EXPORT Pointwise build(); -}; - -class MIOPEN_INTERNALS_EXPORT BackendPointwiseDescriptor : public BackendDescriptor -{ -private: - PointwiseBuilder mBuilder; - Pointwise mPointwise; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Pointwise* getPointwise() const { return &mPointwise; } - Pointwise* getPointwise() { return &mPointwise; } -}; - -class MIOPEN_INTERNALS_EXPORT OperationPointwise : public OpNode -{ -public: - using Alpha = std::variant; - struct BackwardTag - { - }; - -private: - Pointwise* mPointwise = nullptr; - Tensor* mX = nullptr; - Tensor* mB = nullptr; - Tensor* mY = nullptr; - Tensor* mT = nullptr; - Tensor* mDx = nullptr; - Tensor* mDy = nullptr; - Alpha mAlpha1 = 1.0f; - Alpha mAlpha2 = 1.0f; - - friend class OperationPointwiseBuilder; - -public: - OperationPointwise() noexcept = default; - OperationPointwise(Pointwise* pointwise, Tensor* x, Tensor* y, Alpha alpha1 = 1.0f) - : mPointwise(pointwise), mX(x), mY(y), mAlpha1(alpha1) - { - } - OperationPointwise(Pointwise* pointwise, - Tensor* x, - Tensor* b, - Tensor* y, - Alpha alpha1 = 1.0f, - Alpha alpha2 = 1.0f) noexcept - : mPointwise(pointwise), mX(x), mB(b), mY(y), mAlpha1(alpha1), mAlpha2(alpha2) - { - } - OperationPointwise(Pointwise* pointwise, - Tensor* x, - Tensor* b, - Tensor* y, - Tensor* t, - Alpha alpha1 = 1.0f, - Alpha alpha2 = 1.0f) noexcept - : mPointwise(pointwise), mX(x), mB(b), mY(y), mT(t), mAlpha1(alpha1), mAlpha2(alpha2) - { - } - OperationPointwise(BackwardTag, - Pointwise* pointwise, - Tensor* y, - Tensor* dY, - Tensor* dX, - Alpha alpha1 = 1.0f, - Alpha alpha2 = 1.0f) noexcept - : mPointwise(pointwise), mY(y), mDx(dX), mDy(dY), mAlpha1(alpha1), mAlpha2(alpha2) - { - } - - Pointwise* getPointwise() const noexcept { return mPointwise; } - Tensor* getX() const noexcept { return mX; } - Tensor* getB() const noexcept { return mB; } - Tensor* getY() const noexcept { return mY; } - Tensor* getT() const noexcept { return mT; } - Tensor* getDx() const noexcept { return mDx; } - Tensor* getDy() const noexcept { return mDy; } - Alpha getAlpha1() const noexcept { return mAlpha1; } - Alpha getAlpha2() const noexcept { return mAlpha2; } - - const std::string& signName() const override; - std::vector getInTensors() const override; - std::vector getOutTensors() const override; -}; - -class MIOPEN_INTERNALS_EXPORT OperationPointwiseBuilder -{ -private: - OperationPointwise mOperationPointwise; - bool mAlpha2Set = false; - -public: - OperationPointwiseBuilder& setPointwise(Pointwise* pointwise); - OperationPointwiseBuilder& setX(Tensor* x); - OperationPointwiseBuilder& setB(Tensor* b); - OperationPointwiseBuilder& setY(Tensor* y); - OperationPointwiseBuilder& setT(Tensor* t); - OperationPointwiseBuilder& setDx(Tensor* dX); - OperationPointwiseBuilder& setDy(Tensor* dY); - OperationPointwiseBuilder& setAlpha1(OperationPointwise::Alpha alpha1); - OperationPointwiseBuilder& setAlpha2(OperationPointwise::Alpha alpha2); - - OperationPointwise build(); -}; - -class BackendOperationPointwiseDescriptor : public BackendDescriptor -{ -private: - OperationPointwiseBuilder mBuilder; - OperationPointwise mOperationPointwise; - - miopenBackendDescriptor_t mPointwiseDescriptor = nullptr; - miopenBackendDescriptor_t mXDescriptor = nullptr; - miopenBackendDescriptor_t mBDescriptor = nullptr; - miopenBackendDescriptor_t mYDescriptor = nullptr; - miopenBackendDescriptor_t mTDescriptor = nullptr; - miopenBackendDescriptor_t mDxDescriptor = nullptr; - miopenBackendDescriptor_t mDyDescriptor = nullptr; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - OpNode* getOperation() override; - - const OperationPointwise* getOperationPointwise() const { return &mOperationPointwise; } - OperationPointwise* getOperationPointwise() { return &mOperationPointwise; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/reduction.hpp b/projects/miopen/src/include/miopen/graphapi/reduction.hpp deleted file mode 100644 index 5720bc0c0de2..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/reduction.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -class Reduction -{ -private: - miopenReduceTensorOp_t mReductionOperator = MIOPEN_REDUCE_TENSOR_ADD; - miopenDataType_t mCompType = miopenFloat; - - friend class ReductionBuilder; - -public: - Reduction() noexcept = default; - Reduction(miopenReduceTensorOp_t reductionOperator, miopenDataType_t compType) noexcept - : mReductionOperator(reductionOperator), mCompType(compType) - { - } - - miopenReduceTensorOp_t getReductionOperator() const { return mReductionOperator; } - miopenDataType_t getCompType() const { return mCompType; } -}; - -class ReductionBuilder -{ -private: - Reduction mReduction; - bool mReductionOperatorSet = false; - bool mCompTypeSet = false; - -public: - ReductionBuilder& setReductionOperator(miopenReduceTensorOp_t reductionOperator) noexcept - { - mReduction.mReductionOperator = reductionOperator; - mReductionOperatorSet = true; - return *this; - } - - ReductionBuilder& setCompType(miopenDataType_t compType) noexcept - { - mReduction.mCompType = compType; - mCompTypeSet = true; - return *this; - } - - MIOPEN_INTERNALS_EXPORT Reduction build(); -}; - -class MIOPEN_INTERNALS_EXPORT BackendReductionDescriptor : public BackendDescriptor -{ -private: - ReductionBuilder mBuilder; - Reduction mReduction; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Reduction* getReduction() const { return &mReduction; } - Reduction* getReduction() { return &mReduction; } -}; - -class MIOPEN_INTERNALS_EXPORT OperationReduction : public OpNode -{ -private: - Reduction* mReduction = nullptr; - Tensor* mX = nullptr; - Tensor* mY = nullptr; - - friend class OperationReductionBuilder; - -public: - OperationReduction() noexcept = default; - OperationReduction(Reduction* reduction, Tensor* x, Tensor* y) noexcept - : mReduction(reduction), mX(x), mY(y) - { - } - - Reduction* getReduction() const noexcept { return mReduction; } - Tensor* getX() const noexcept { return mX; } - Tensor* getY() const noexcept { return mY; } - - const std::string& signName() const override; - std::vector getInTensors() const override; - std::vector getOutTensors() const override; -}; - -class MIOPEN_INTERNALS_EXPORT OperationReductionBuilder -{ -private: - OperationReduction mOperationReduction; - -public: - OperationReductionBuilder& setReduction(Reduction* reduction); - OperationReductionBuilder& setX(Tensor* x); - OperationReductionBuilder& setY(Tensor* y); - OperationReduction build(); -}; - -class BackendOperationReductionDescriptor : public BackendDescriptor -{ -private: - OperationReductionBuilder mBuilder; - OperationReduction mOperationReduction; - - miopenBackendDescriptor_t mReductionDescriptor = nullptr; - miopenBackendDescriptor_t mXDescriptor = nullptr; - miopenBackendDescriptor_t mYDescriptor = nullptr; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - OpNode* getOperation() override; - - const OperationReduction* getOperationReduction() const { return &mOperationReduction; } - OperationReduction* getOperationReduction() { return &mOperationReduction; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/reshape.hpp b/projects/miopen/src/include/miopen/graphapi/reshape.hpp deleted file mode 100644 index 3f097fe708b1..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/reshape.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -class MIOPEN_INTERNALS_EXPORT OperationReshape : public OpNode -{ -public: - enum class OpKind - { - GENERIC, - TRANSPOSE - }; - -private: - Tensor* mX = nullptr; - Tensor* mY = nullptr; - OpKind mOpKind = OpKind::GENERIC; - - friend class OperationReshapeBuilder; - -public: - OperationReshape() noexcept = default; - OperationReshape(Tensor* x, Tensor* y) : mX(x), mY(y) {} - - Tensor* getX() const noexcept { return mX; } - Tensor* getY() const noexcept { return mY; } - OpKind getOpKind() const noexcept { return mOpKind; } - - const std::string& signName() const override; - std::vector getInTensors() const override; - std::vector getOutTensors() const override; -}; - -class MIOPEN_INTERNALS_EXPORT OperationReshapeBuilder -{ -private: - OperationReshape mOperationReshape; - -public: - OperationReshapeBuilder& setX(Tensor* x); - OperationReshapeBuilder& setY(Tensor* y); - OperationReshape build(); -}; - -class BackendOperationReshapeDescriptor : public BackendDescriptor -{ -private: - OperationReshapeBuilder mBuilder; - OperationReshape mOperationReshape; - - miopenBackendDescriptor_t mXDescriptor = nullptr; - miopenBackendDescriptor_t mYDescriptor = nullptr; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - OpNode* getOperation() override; - - const OperationReshape* getOperationReshape() const { return &mOperationReshape; } - OperationReshape* getOperationReshape() { return &mOperationReshape; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/rng.hpp b/projects/miopen/src/include/miopen/graphapi/rng.hpp deleted file mode 100644 index 1420240ec99b..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/rng.hpp +++ /dev/null @@ -1,211 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include -#include - -#include -#include - -namespace miopen { - -namespace graphapi { - -class Rng -{ -private: - miopenRngDistribution_t mDistribution = MIOPEN_RNG_DISTRIBUTION_BERNOULLI; - double mNormalMean = -1.0; - double mNormalStdev = -1.0; - double mUniformMin = -1.0; - double mUniformMax = -1.0; - double mBernoulliProb = -1.0; - -public: - Rng() = default; - Rng(miopenRngDistribution_t distribution, - double normalMean, - double normalStdev, - double uniformMin, - double uniformMax, - double bernoulliProb) - : mDistribution(distribution), - mNormalMean(normalMean), - mNormalStdev(normalStdev), - mUniformMin(uniformMin), - mUniformMax(uniformMax), - mBernoulliProb(bernoulliProb) - { - } - - miopenRngDistribution_t getDistribution() const noexcept { return mDistribution; } - double getNormalMean() const noexcept { return mNormalMean; } - double getNormalStdev() const noexcept { return mNormalStdev; } - double getUniformMin() const noexcept { return mUniformMin; } - double getUniformMax() const noexcept { return mUniformMax; } - double getBernoulliProb() const noexcept { return mBernoulliProb; } - -private: - friend class RngBuilder; -}; - -class MIOPEN_INTERNALS_EXPORT RngBuilder -{ -private: - Rng mRng; - -public: - RngBuilder& setDistribution(miopenRngDistribution_t distribution) noexcept - { - mRng.mDistribution = distribution; - return *this; - } - RngBuilder& setNormalMean(double normalMean) noexcept - { - mRng.mNormalMean = normalMean; - return *this; - } - - RngBuilder& setNormalStdev(double normalStdev); - - RngBuilder& setUniformMin(double uniformMin) noexcept - { - mRng.mUniformMin = uniformMin; - return *this; - } - RngBuilder& setUniformMax(double uniformMax) noexcept - { - mRng.mUniformMax = uniformMax; - return *this; - } - - RngBuilder& setBernoulliProb(double bernoulliProb); - - Rng build() const; -}; - -class MIOPEN_INTERNALS_EXPORT BackendRngDescriptor : public BackendDescriptor -{ -private: - RngBuilder mBuilder; - Rng mRng; - -public: - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Rng* getRng() const noexcept { return &mRng; } - Rng* getRng() noexcept { return &mRng; } -}; - -class MIOPEN_INTERNALS_EXPORT OperationRng : public OpNode -{ -private: - Rng* mRng = nullptr; - Tensor* mOutput = nullptr; - std::variant mSeed = 0; // Don't change the order of variant alternatives - Tensor* mOffset = nullptr; - - friend class OperationRngBuilder; - -public: - OperationRng() noexcept = default; - OperationRng(Rng* rng, Tensor* output, int64_t seed, Tensor* offset) noexcept - : mRng(rng), mOutput(output), mSeed(seed), mOffset(offset) - { - } - OperationRng(Rng* rng, Tensor* output, Tensor* seed, Tensor* offset) noexcept - : mRng(rng), mOutput(output), mSeed(seed), mOffset(offset) - { - } - - Rng* getRng() const noexcept { return mRng; } - Tensor* getOutput() const noexcept { return mOutput; } - std::variant getSeed() const noexcept { return mSeed; } - Tensor* getOffset() const noexcept { return mOffset; } - - virtual const std::string& signName() const override; - virtual std::vector getInTensors() const override; - virtual std::vector getOutTensors() const override; -}; - -class MIOPEN_INTERNALS_EXPORT OperationRngBuilder -{ -private: - OperationRng mOperationRng; - -public: - OperationRngBuilder& setRng(Rng* rng); - OperationRngBuilder& setOutput(Tensor* output); - OperationRngBuilder& setSeed(int64_t seed) noexcept; - OperationRngBuilder& setSeed(Tensor* seed); - OperationRngBuilder& setOffset(Tensor* offset); - - OperationRng build(); -}; - -class BackendOperationRngDescriptor : public BackendDescriptor -{ -private: - OperationRngBuilder mBuilder; - OperationRng mOperationRng; - miopenBackendDescriptor_t mRngDescriptor = nullptr; - miopenBackendDescriptor_t mOutputDescriptor = nullptr; // sometimes called Y - miopenBackendDescriptor_t mSeedDescriptor = nullptr; - miopenBackendDescriptor_t mOffsetDescriptor = nullptr; - -public: - void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - void finalize() override; - void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - OpNode* getOperation() override; - - const OperationRng* getRng() const { return &mOperationRng; } - OperationRng* getRng() { return &mOperationRng; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/graphapi/tensor.hpp b/projects/miopen/src/include/miopen/graphapi/tensor.hpp deleted file mode 100644 index f079de84b73a..000000000000 --- a/projects/miopen/src/include/miopen/graphapi/tensor.hpp +++ /dev/null @@ -1,169 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include - -#include -#include - -namespace miopen { - -namespace graphapi { - -class Tensor : public TensorDescriptor -{ -private: - int64_t mId = 0; - bool mVirtual = false; - - // Deprecated - using TensorDescriptor::GetLayout_t; - -public: - Tensor() noexcept = default; - Tensor(const Tensor&) = default; - Tensor(Tensor&&) noexcept = default; - Tensor& operator=(const Tensor&) = default; - Tensor& operator=(Tensor&&) noexcept = default; - Tensor(const TensorDescriptor& other, int64_t id, bool isVirtual) - : TensorDescriptor(other), mId(id), mVirtual(isVirtual) - { - } - Tensor(TensorDescriptor&& other, int64_t id, bool isVirtual) - : TensorDescriptor(std::move(other)), mId(id), mVirtual(isVirtual) - { - } - Tensor(miopenDataType_t dataType, - const std::vector& dimensions, - const std::vector& strides, - int64_t id, - bool isVirtual) - : TensorDescriptor(dataType, dimensions, strides), mId(id), mVirtual(isVirtual) - { - } - Tensor(miopenDataType_t dataType, - std::vector&& dimensions, - std::vector&& strides, - int64_t id, - bool isVirtual) noexcept - : TensorDescriptor(dataType, std::move(dimensions), std::move(strides)), - mId(id), - mVirtual(isVirtual) - { - } - - int64_t getId() const noexcept { return mId; } - bool isVirtual() const noexcept { return mVirtual; } - - friend void to_json(nlohmann::json& json, const Tensor& tensor); - friend void from_json(const nlohmann::json& json, Tensor& tensor); - - struct JsonFields - { - static constexpr const char* Id = "id"; - static constexpr const char* IsVirtual = "is_virtual"; - }; -}; - -class MIOPEN_INTERNALS_EXPORT TensorBuilder -{ -private: - std::vector mDimensions; - std::vector mStrides; - int64_t mId = 0; - miopenDataType_t mDataType = miopenFloat; - bool mVirtual = false; - bool mUniqueIdSet = false; - bool mDataTypeSet = false; - bool mDimensionsSet = false; - bool mStridesSet = false; - -public: - TensorBuilder& setDataType(miopenDataType_t dataType) &; - TensorBuilder& setDim(const std::vector& dimensions) &; - MIOPEN_INTERNALS_NO_EXPORT TensorBuilder& setDim(std::vector&& dimensions) &; - TensorBuilder& setStride(const std::vector& strides) &; - MIOPEN_INTERNALS_NO_EXPORT TensorBuilder& setStride(std::vector&& strides) &; - TensorBuilder& setId(int64_t id) &; - TensorBuilder& setVirtual(bool isVirtual) &; - - TensorBuilder&& setDataType(miopenDataType_t dataType) && - { - return std::move(setDataType(dataType)); - } - TensorBuilder&& setDim(const std::vector& dimensions) && - { - return std::move(setDim(dimensions)); - } - TensorBuilder&& setDim(std::vector&& dimensions) && - { - return std::move(setDim(std::move(dimensions))); - } - TensorBuilder&& setStride(const std::vector& strides) && - { - return std::move(setStride(strides)); - } - TensorBuilder&& setStride(std::vector&& strides) && - { - return std::move(setStride(std::move(strides))); - } - TensorBuilder&& setId(int64_t id) && { return std::move(setId(id)); } - TensorBuilder&& setVirtual(bool isVirtual) && { return std::move(setVirtual(isVirtual)); } - - MIOPEN_INTERNALS_NO_EXPORT Tensor build() const&; - Tensor build() &&; -}; - -class MIOPEN_INTERNALS_EXPORT BackendTensorDescriptor : public BackendDescriptor -{ -private: - TensorBuilder mBuilder; - Tensor mDescriptor; - -public: - BackendTensorDescriptor() = default; - virtual ~BackendTensorDescriptor() override; - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Tensor* getTensor() const { return &mDescriptor; } - Tensor* getTensor() { return &mDescriptor; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/projects/miopen/src/include/miopen/pooling/solvers.hpp b/projects/miopen/src/include/miopen/pooling/solvers.hpp deleted file mode 100644 index 38f9019eaa8b..000000000000 --- a/projects/miopen/src/include/miopen/pooling/solvers.hpp +++ /dev/null @@ -1,527 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include - -namespace miopen { - -namespace solver { - -namespace pooling { - -enum class OperationType -{ - Forward, - Backward -}; - -template -using PoolingTunableSolver = - SolverBaseTunable; - -template -struct PerformanceConfigPooling2d : PerfConfigBase> -{ - static_assert(OpType == OperationType::Forward || OpType == OperationType::Backward, - "OperationType must be either Forward or Backward"); - - int out_pix_tile0; - int out_pix_tile1; - int local_size0; - int local_size1; - static constexpr int min_out_pix_tile0 = 1; - static constexpr int max_out_pix_tile0 = (OpType == OperationType::Forward) ? 1 : 4; - static constexpr int min_out_pix_tile1 = 1; - static constexpr int max_out_pix_tile1 = (OpType == OperationType::Forward) ? 16 : 8; - static constexpr int min_local_size0 = (OpType == OperationType::Forward) ? 8 : 4; - static constexpr int max_local_size0 = 32; - static constexpr int min_local_size1 = (OpType == OperationType::Forward) ? 8 : 4; - static constexpr int max_local_size1 = (OpType == OperationType::Forward) ? 128 : 16; - - PerformanceConfigPooling2d(int out_pix_tile0_, - int out_pix_tile1_, - int local_size0_, - int local_size1_) - : out_pix_tile0(out_pix_tile0_), - out_pix_tile1(out_pix_tile1_), - local_size0(local_size0_), - local_size1(local_size1_) - { - } - PerformanceConfigPooling2d() - : PerformanceConfigPooling2d( - min_out_pix_tile0, min_out_pix_tile1, min_local_size0, min_local_size1) - { - } - PerformanceConfigPooling2d(bool) - : PerformanceConfigPooling2d( - min_out_pix_tile0, min_out_pix_tile1, min_local_size0, min_local_size1) - { - } - - void HeuristicInit(const miopen::pooling::ProblemDescription&); - bool SetNextValue(const miopen::pooling::ProblemDescription&); - virtual bool IsValidValue(const miopen::pooling::ProblemDescription&) const - { - throw std::runtime_error( - "IsValidValue of PerformanceConfigPooling2d is called, but it is not " - "implemented."); - } - bool IsValid(const ExecutionContext&, const miopen::pooling::ProblemDescription&) const; - bool operator==(const PerformanceConfigPooling2d& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.out_pix_tile0, "out_pix_tile0"); - f(self.out_pix_tile1, "out_pix_tile1"); - f(self.local_size0, "local_size0"); - f(self.local_size1, "local_size1"); - } - -private: - void Init(const miopen::pooling::ProblemDescription&); -}; - -extern template struct PerformanceConfigPooling2d; -extern template struct PerformanceConfigPooling2d; - -struct PerformanceConfigPooling2dForward final - : public PerformanceConfigPooling2d -{ - using PerformanceConfigPooling2d::PerformanceConfigPooling2d; - - bool IsValidValue(const miopen::pooling::ProblemDescription&) const override; -}; - -struct PerformanceConfigPooling2dBackward final - : public PerformanceConfigPooling2d -{ - using PerformanceConfigPooling2d::PerformanceConfigPooling2d; - - bool IsValidValue(const miopen::pooling::ProblemDescription&) const override; -}; - -template -struct PerformanceConfigPoolingNd : PerfConfigBase> -{ - static_assert(OpType == OperationType::Forward || OpType == OperationType::Backward, - "OperationType must be either Forward or Backward"); - - int pix_w_per_work; - int pix_h_per_work; - int pix_d_per_work; - int local_size; - static constexpr int min_pix_per_work = 1; - static constexpr int max_pix_per_work = 4; - - PerformanceConfigPoolingNd(int pix_w_per_work_, - int pix_h_per_work_, - int pix_d_per_work_, - int local_size_) - : pix_w_per_work(pix_w_per_work_), - pix_h_per_work(pix_h_per_work_), - pix_d_per_work(pix_d_per_work_), - local_size(local_size_) - { - } - - PerformanceConfigPoolingNd() : PerformanceConfigPoolingNd(1, 1, 1, 1) {} - - PerformanceConfigPoolingNd(bool) : PerformanceConfigPoolingNd(1, 1, 1, 1) {} - - void HeuristicInit(const miopen::pooling::ProblemDescription&); - virtual bool SetNextValue(const miopen::pooling::ProblemDescription&) - { - throw std::runtime_error( - "SetNextValue of PerformanceConfigPoolingNd is called, but it is not " - "implemented."); - } - virtual bool IsValidValue() const - { - throw std::runtime_error( - "IsValidValue of PerformanceConfigPoolingNd is called, but it is not " - "implemented."); - } - bool IsValid(const ExecutionContext&, const miopen::pooling::ProblemDescription&) const; - bool operator==(const PerformanceConfigPoolingNd& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.pix_w_per_work, "pix_w_per_work"); - f(self.pix_h_per_work, "pix_h_per_work"); - f(self.pix_d_per_work, "pix_d_per_work"); - f(self.local_size, "local_size"); - } - -private: - void Init(const miopen::pooling::ProblemDescription&); -}; - -extern template struct PerformanceConfigPoolingNd; -extern template struct PerformanceConfigPoolingNd; - -struct PoolingForward2d final : PoolingTunableSolver -{ - using PerformanceConfigType = PerformanceConfigPooling2dForward; - - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const PerformanceConfigPooling2dForward& config) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - PerformanceConfigPooling2dForward - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&) const override; - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&, - const PerformanceConfigPooling2dForward&) const override; - PerformanceConfigPooling2dForward Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override - { - return GenericSearch(*this, context, problem, invoke_context); - } -}; - -struct PerformanceConfigPoolingNdForward final : PerformanceConfigPoolingNd -{ - using PerformanceConfigPoolingNd::PerformanceConfigPoolingNd; - - bool SetNextValue(const miopen::pooling::ProblemDescription&) override; - bool IsValidValue() const override; -}; - -struct PoolingForwardNd final : PoolingTunableSolver -{ - using PerformanceConfigType = PerformanceConfigPoolingNdForward; - - const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsApplicable(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const PerformanceConfigPoolingNdForward& config) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - PerformanceConfigPoolingNdForward - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&) const override; - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&, - const PerformanceConfigPoolingNdForward&) const override; - PerformanceConfigPoolingNdForward Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override - { - return GenericSearch(*this, context, problem, invoke_context); - } -}; - -struct PerformanceConfigPoolingForwardNaive : PerfConfigBase -{ - int local_size0; - int local_size1; - int local_size2; - - PerformanceConfigPoolingForwardNaive(int local_size0_, int local_size1_, int local_size2_) - : local_size0(local_size0_), local_size1(local_size1_), local_size2(local_size2_) - { - } - - PerformanceConfigPoolingForwardNaive() : PerformanceConfigPoolingForwardNaive(1, 1, 1) {} - - PerformanceConfigPoolingForwardNaive(bool) : PerformanceConfigPoolingForwardNaive(1, 1, 1) {} - - void HeuristicInit(const miopen::pooling::ProblemDescription&); - bool SetNextValue(const miopen::pooling::ProblemDescription&); - bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::pooling::ProblemDescription&) const; - bool operator==(const PerformanceConfigPoolingForwardNaive& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.local_size0, "local_size0"); - f(self.local_size1, "local_size1"); - f(self.local_size2, "local_size2"); - } - -private: - void Init(const miopen::pooling::ProblemDescription&); -}; - -struct PoolingForwardNaive final : PoolingTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsDynamic() const override { return true; } - - bool IsApplicable(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const PerformanceConfigPoolingForwardNaive& config) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - PerformanceConfigPoolingForwardNaive - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&) const override; - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&, - const PerformanceConfigPoolingForwardNaive&) const override; - PerformanceConfigPoolingForwardNaive - Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override - { - return GenericSearch(*this, context, problem, invoke_context); - } -}; - -template -struct PoolingFwdNCHWTransposingSolver - : TransposingSolver, - PoolingTunableSolver, - miopen::pooling::ProblemDescription, - miopen::pooling::FwdInvokeParams, - Inner> -{ - using Problem = miopen::pooling::ProblemDescription; - using InvokeParams = miopen::pooling::FwdInvokeParams; - - inline static auto GetTransposes(const Problem& /*problem*/) - { - return std::array, 2>{{ - { - &Problem::GetXDesc, - &InvokeParams::xDesc, - &InvokeParams::x, // x is input - nullptr, - "NCDHW", - true, - }, - { - &Problem::GetYDesc, - &InvokeParams::yDesc, - nullptr, - &InvokeParams::y, // y is output - "NCDHW", - false, - }, - }}; - } -}; - -struct TransposedPoolingFwd2d final : PoolingFwdNCHWTransposingSolver -{ - const std::string& SolverDbId() const { return GetSolverDbId(); } - PoolingForward2d::PerformanceConfigType - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const - { - return PoolingForward2d{}.GetDefaultPerformanceConfig(ctx, problem); - } - bool IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingForward2d::PerformanceConfigType& config) const - { - return PoolingForward2d{}.IsValidPerformanceConfig(ctx, problem, config); - } - PoolingForward2d::PerformanceConfigType - Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const - { - return PoolingForward2d{}.Search(context, problem, invoke_context); - } -}; - -struct TransposedPoolingFwdNd final : PoolingFwdNCHWTransposingSolver -{ - const std::string& SolverDbId() const { return GetSolverDbId(); } - PoolingForwardNd::PerformanceConfigType - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const - { - return PoolingForwardNd{}.GetDefaultPerformanceConfig(ctx, problem); - } - bool IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingForwardNd::PerformanceConfigType& config) const - { - return PoolingForwardNd{}.IsValidPerformanceConfig(ctx, problem, config); - } - PoolingForwardNd::PerformanceConfigType - Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const - { - return PoolingForwardNd{}.Search(context, problem, invoke_context); - } -}; - -struct PoolingBackward2d final : PoolingTunableSolver -{ - using PerformanceConfigType = PerformanceConfigPooling2dBackward; - - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const PerformanceConfigPooling2dBackward& config) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - PerformanceConfigPooling2dBackward - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&) const override; - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&, - const PerformanceConfigPooling2dBackward&) const override; - PerformanceConfigPooling2dBackward Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override - { - return GenericSearch(*this, context, problem, invoke_context); - } -}; - -struct PerformanceConfigPoolingNdBackward final - : PerformanceConfigPoolingNd -{ - using PerformanceConfigPoolingNd::PerformanceConfigPoolingNd; - - bool SetNextValue(const miopen::pooling::ProblemDescription&) override; - bool IsValidValue() const override; -}; - -struct PoolingBackwardNd final : PoolingTunableSolver -{ - using PerformanceConfigType = PerformanceConfigPoolingNdBackward; - - const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsApplicable(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const PerformanceConfigPoolingNdBackward& config) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem) const override; - PerformanceConfigPoolingNdBackward - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&) const override; - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::pooling::ProblemDescription&, - const PerformanceConfigPoolingNdBackward&) const override; - PerformanceConfigPoolingNdBackward Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const override - { - return GenericSearch(*this, context, problem, invoke_context); - } -}; - -template -struct PoolingBwdNCHWTransposingSolver - : TransposingSolver, - PoolingTunableSolver, - miopen::pooling::ProblemDescription, - miopen::pooling::BwdInvokeParams, - Inner> -{ - using Problem = miopen::pooling::ProblemDescription; - using InvokeParams = miopen::pooling::BwdInvokeParams; - - inline static auto GetTransposes(const Problem& /*problem*/) - { - return std::array, 2>{{ - { - &Problem::GetXDesc, - &InvokeParams::dxDesc, - nullptr, - &InvokeParams::dx, // dx is output - "NCDHW", - false, - }, - { - &Problem::GetYDesc, - &InvokeParams::dyDesc, - &InvokeParams::dy, // dy is input - nullptr, - "NCDHW", - true, - }, - }}; - } -}; - -struct TransposedPoolingBwd2d final : PoolingBwdNCHWTransposingSolver -{ - const std::string& SolverDbId() const { return GetSolverDbId(); } - PoolingBackward2d::PerformanceConfigType - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const - { - return PoolingBackward2d{}.GetDefaultPerformanceConfig(ctx, problem); - } - bool IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingBackward2d::PerformanceConfigType& config) const - { - return PoolingBackward2d{}.IsValidPerformanceConfig(ctx, problem, config); - } - PoolingBackward2d::PerformanceConfigType - Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const - { - return PoolingBackward2d{}.Search(context, problem, invoke_context); - } -}; - -struct TransposedPoolingBwdNd final : PoolingBwdNCHWTransposingSolver -{ - const std::string& SolverDbId() const { return GetSolverDbId(); } - PoolingBackwardNd::PerformanceConfigType - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem) const - { - return PoolingBackwardNd{}.GetDefaultPerformanceConfig(ctx, problem); - } - bool IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::pooling::ProblemDescription& problem, - const PoolingBackwardNd::PerformanceConfigType& config) const - { - return PoolingBackwardNd{}.IsValidPerformanceConfig(ctx, problem, config); - } - PoolingBackwardNd::PerformanceConfigType - Search(const ExecutionContext& context, - const miopen::pooling::ProblemDescription& problem, - const AnyInvokeParams& invoke_context) const - { - return PoolingBackwardNd{}.Search(context, problem, invoke_context); - } -}; - -} // namespace pooling - -} // namespace solver - -} // namespace miopen diff --git a/projects/miopen/src/solver/conv/conv_winoRxS.cpp b/projects/miopen/src/solver/conv/conv_winoRxS.cpp deleted file mode 100644 index f85a2c1be74c..000000000000 --- a/projects/miopen/src/solver/conv/conv_winoRxS.cpp +++ /dev/null @@ -1,1188 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#define CONV_BIN_WINO_RXS_CPP - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -// ConvBinWinoRxS<2,3> is intended to handle group convolutions, but -// it is able to work with non-group convolutions and shows quite good performance, -// comparable with ConvBinWinogradRxSf2x3g1. Due to the issue in #1533 (fixed in #1634) -// AND some inaccuracy (jitter) of benchmarking, it is often shows the "best" performance -// with non-group convolutions, and therefore unexpected records where ConvBinWinoRxS<2,3> -// is the "best" leaked into find-db. That is why disabling ConvBinWinoRxS<2,3> for non-group -// convolutions leads to performance drops. Let's enable ConvBinWinoRxS<2,3> for non-group -// in order to quickly W/A the perf issue. When find-db fix is ready, -// we will keep ConvBinWinoRxS<2,3> for group convolutions only. -#define WORKAROUND_ISSUE_1681 0 - -/// \anchor disable_winograd_with_huge_granularity_loss -/// This is to exclude Winograd from PyTorch correctness tests. -/// These tests are written in incorrect manner. -/// They are very sensitive to the values in the output tensors, -/// but do not provide inputs that guarantee minimal deviation -/// (a numerical difference between kernel output and theoretically -/// "ideal" output). This is especially important for Winograd. -/// As a result, the PyTorch correctness tests often produce false -/// failures when Winograd algorithm is used. As PyTorch correctness -/// tests use very small convolutions and Winograd algorithm is -/// ineffective with such small configs due to huge granularity loss, -/// we can disable Winograd without any performance implications. -#define WORKAROUND_ISSUE_2492_GRANULARITY_LOSS 0 -/// \anchor disable_winograd_with_small_tensor -/// This is alternative W/A for the same problem, which -/// resolves issues shown at -/// https://github.com/ROCm/MIOpen/issues/2492#issuecomment-1910928563 -/// The idea is to disable the solver for very small tensors. -/// PyTorch tests use sizes like 2x2x6x6 (NCHW), but let's add -/// some spare space for N anc C and disable starting from 4x4x6x6. -#define WORKAROUND_ISSUE_2492_TINY_TENSOR 1 - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_WORKAROUND_ISSUE_2492) - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3) -MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_PERF_VALS) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1) - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F3X2) -MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F3X2_PERF_VALS) - -#define MAX_CU_LIMIT 512 - -#define IS2X3 (Winodata == 2 && Winofilter == 3) -#define IS3X2 (Winodata == 3 && Winofilter == 2) - -/// \todo The model is well-defined in for filters sized up to 5. -/// However, it seems producing valid results without this limitation, -/// when used against simple GEMM WTI model (to select the fastest solver). -/// This needs to be re-tested/re-considered when we have WTI -/// models for other solvers, OR when GEMM WTI model is improved. -/// --atamazov 2020-11-07. -#define WTI_MODEL_ALLOW_ANY_RS 1 -#define WTI_MODEL_ALLOW_ANY_CK 1 - -static inline size_t Ceil(const size_t v, const size_t m) -{ - assert(m > 0); - return (v + m - 1) / m; -} - -static inline size_t RoundUpToMultiple(size_t val, size_t factor) -{ - return Ceil(val, factor) * factor; -} - -// Let's avoid clang-tidy warnings without explicit casts. -static inline size_t RoundUpToMultiple(size_t val, int factor) -{ - return Ceil(val, factor) * factor; -} - -static inline int GetBestNGroupParam(const int R, - const int S, - const int R_stride, - const int S_stride, - const int C, - const int K, - const int OH, - const int OW, - const int pad_H, - const int pad_W, - const int N, - const int idilation_w, - const int idilation_h, - const int n_groups, - const int G, - const int o_tile, - const int f_tile) -{ - int r_factor = f_tile * 2; - int s_factor = r_factor; - int c_factor = 2; - int k_factor = 32; - int nwh_factor = 32; - int w_factor = o_tile * idilation_w * S_stride; - int h_factor = o_tile * idilation_h * R_stride; - - if(S_stride == 1 && idilation_w == 1 && S <= f_tile) - s_factor = f_tile; - if((R_stride == 1 && idilation_h == 1) || (R % (f_tile * 2)) == 1) - r_factor = f_tile; - if(S_stride == 2 || R_stride == 2 || idilation_w == 2 || idilation_h == 2) - c_factor = 1; - - size_t g_s = RoundUpToMultiple(S, s_factor); - size_t g_r = RoundUpToMultiple(R, r_factor); - size_t g_c = RoundUpToMultiple(C, c_factor); - size_t g_k = RoundUpToMultiple(K, k_factor); - size_t g_w = OW; - size_t g_h = OH; - - if((pad_W % 2 == 0) && (idilation_w > 1 || S_stride > 1)) - g_w += 1; - if((pad_H % 2 == 1) && (idilation_h > 1 || R_stride > 1)) - g_h += 1; - - g_w = RoundUpToMultiple(g_w, w_factor); - g_h = RoundUpToMultiple(g_h, h_factor); - size_t g_n_w_h = RoundUpToMultiple(g_w * g_h * N, nwh_factor * w_factor * h_factor); - - int best_n_groups_cnt = 1; - double min_param = 0; - for(auto i = 1; i < n_groups; ++i) - { - size_t g_n_w_h_k = - RoundUpToMultiple(g_n_w_h * g_k, nwh_factor * w_factor * h_factor * k_factor * i); - size_t granulated_mac_count = g_n_w_h_k * g_c * g_s * g_r; - size_t n_groups_per_cu = Ceil(static_cast(i) * G, n_groups); - double perf_metric = static_cast(n_groups_per_cu) * granulated_mac_count / i; - if(static_cast(granulated_mac_count) / i > 1.0e+7) - perf_metric *= (1 + i * 0.003); - else - perf_metric *= (1 + i * 0.04); - if(i == 1) - min_param = perf_metric; - if(min_param > perf_metric) - { - best_n_groups_cnt = i; - min_param = perf_metric; - } - } - return best_n_groups_cnt; -} - -namespace miopen { -namespace solver { -namespace conv { - -using ProblemDescription = miopen::conv::ProblemDescription; - -namespace { - -auto PerfFieldRules() -{ - return seq::MakeRuleSet(std::make_tuple(seq::Span{}, - &PerformanceConfigConvBinWinogradRxS::n_groups)); -} - -// Winograd v21 is preferred on Vega10/Vega20 ASICs due to ~25% performance regression with Winograd -// v30. The exception is Winograd F(3,2) stride2 as this mode is unsupported in v21. Details: -// https://github.com/ROCm/MIOpen/pull/1927#issuecomment-1412741130 -template -inline bool IsWinogradV21Preferred(const std::string& asic, const ProblemDescription& problem) -{ - return (StartsWith(asic, "gfx900") || StartsWith(asic, "gfx906")) && - !(IS3X2 && problem.GetKernelStrideW() == 2); -} - -inline bool IsShaderConstraintsMetV21(const ProblemDescription& problem, - const int R, - const int S, - const int C, - const int K, - const int H, - const int W, - const int OH, - const int OW, - const int N) -{ - uint64_t o_K_stride = static_cast(OH) * OW; - uint64_t o_N_stride = o_K_stride * K; - uint64_t o_N_stride_OHOW = o_N_stride + o_K_stride; - - uint64_t d_C_stride = static_cast(H) * W; - uint64_t d_N_stride = d_C_stride * C; - uint64_t d_N_stride_HW = d_N_stride + d_C_stride; - - auto num_tiles = Ceil(OH, 2) * Ceil(OW, 2); - auto stride_one = problem.GetKernelStrideH() == 1 && problem.GetKernelStrideW() == 1 && - problem.GetDilationH() == 1 && problem.GetDilationW() == 1; - - // clang-format off - // Check implementation limits. - return N < std::pow(2, 16) - && C < std::pow(2, 16) - && H < std::pow(2, 16) - && W < std::pow(2, 16) - && K < std::pow(2, 16) - && S < std::pow(2, 16) - && R < std::pow(2, 16) - && OH < std::pow(2, 16) - && OW < std::pow(2, 16) - && problem.GetPadW() < std::pow(2, 16) - && problem.GetPadH() < std::pow(2, 16) - && C * R * S < std::pow(2, 22) - && K * R * S < std::pow(2, 28) - && ((o_N_stride_OHOW < std::pow(2, 29) && d_N_stride_HW < std::pow(2, 29)) - || (stride_one && o_N_stride < std::pow(2, 30) && d_N_stride < std::pow(2, 30) - && (N == 1 || num_tiles % 16 == 0))); - // clang-format on -} - -inline bool IsShaderConstraintsMetV30(const ProblemDescription& problem, - const int R, - const int S, - const int C, - const int K, - const int H, - const int W, - const int OH, - const int OW, - const int N) -{ - // clang-format off - // Check implementation limits. - return N < std::pow(2, 16) - && C < std::pow(2, 16) - && H < std::pow(2, 16) - && W < std::pow(2, 16) - && K < std::pow(2, 16) - && S < std::pow(2, 16) - && R < std::pow(2, 16) - && OH < std::pow(2, 16) - && OW < std::pow(2, 16) - && problem.GetPadW() < std::pow(2, 16) - && problem.GetPadH() < std::pow(2, 16) - && H * W < std::pow(2, 29) - && K * R * S < std::pow(2, 28) - && (C + 1) * H * W < std::pow(2, 30) - && (C + 1) * R * S < std::pow(2, 22) - && (K + 1) * OH * OW < std::pow(2, 30); - // clang-format on -} - -template -inline bool IsShaderConstraintsMet(const ProblemDescription& problem, - const int R, - const int S, - const int C, - const int K, - const int H, - const int W, - const int OH, - const int OW, - const int N, - const std::string& asic) -{ - // Padding for bwd data shall not be negative. - /// \todo Either remove WrW related code or re-use function from RxS - if(problem.IsDirectionBackwardData()) - { - if(!(0 <= problem.GetBackwardPadW() && problem.GetBackwardPadW() < std::pow(2, 16))) - return false; - if(!(0 <= problem.GetBackwardPadH() && problem.GetBackwardPadH() < std::pow(2, 16))) - return false; - } - - // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string - // This allows transposed solvers to work correctly when they modify tensor strides - { - static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; - if(!(problem.GetIn().IsPossibleLayout4D5D("NCHW", strict) && - problem.GetWeights().IsPossibleLayout4D5D("NCHW", strict) && - problem.GetOut().IsPossibleLayout4D5D("NCHW", strict))) - return false; - } - - return IsWinogradV21Preferred(asic, problem) - ? IsShaderConstraintsMetV21(problem, R, S, C, K, H, W, OH, OW, N) - : IsShaderConstraintsMetV30(problem, R, S, C, K, H, W, OH, OW, N); -} - -} // namespace - -PerformanceConfigConvBinWinogradRxS::PerformanceConfigConvBinWinogradRxS(int n_groups_) - : n_groups(n_groups_) -{ -} - -template -void PerformanceConfigConvBinWinogradRxS::HeuristicInit(const ExecutionContext& ctx, - const ProblemDescription& problem) -{ - const auto n_inputs_per_group = problem.GetInChannels() / problem.GetGroupCount(), - n_outputs_per_group = problem.GetOutChannels() / problem.GetGroupCount(); - if(problem.GetGroupCount() == 1) - { - n_groups = ctx.GetStream().GetMaxHardwareComputeUnits(); - return; - } - - if(problem.IsDirectionBackwardWrW()) - { - n_groups = GetBestNGroupParam(problem.GetInHeight(), - problem.GetInWidth(), - problem.GetDilationH(), - problem.GetDilationW(), - problem.GetBatchSize(), // N - n_inputs_per_group, // K - problem.GetWeightsHeight(), - problem.GetWeightsWidth(), - problem.GetPadW(), - problem.GetPadH(), - n_outputs_per_group, // C - problem.GetKernelStrideH(), - problem.GetKernelStrideW(), - ctx.GetStream().GetMaxHardwareComputeUnits(), - problem.GetGroupCount(), - Winodata, - Winofilter); - } - else - { - n_groups = GetBestNGroupParam(problem.GetWeightsHeight(), // RxS - problem.GetWeightsWidth(), - problem.GetKernelStrideH(), - problem.GetKernelStrideW(), - n_inputs_per_group, // C - n_outputs_per_group, // K - problem.GetOutHeight(), // OHxOW - problem.GetOutWidth(), - problem.GetPadW(), - problem.GetPadH(), - problem.GetBatchSize(), // N - problem.GetDilationH(), - problem.GetDilationW(), - ctx.GetStream().GetMaxHardwareComputeUnits(), - problem.GetGroupCount(), - Winodata, - Winofilter); - } -} - -bool PerformanceConfigConvBinWinogradRxS::SetNextValue(const ProblemDescription&) -{ - return !PerfFieldRules().Next(*this); -} - -bool PerformanceConfigConvBinWinogradRxS::IsValidValue() const -{ - return PerfFieldRules().IsIn(*this); -} - -bool PerformanceConfigConvBinWinogradRxS::IsValid(const ExecutionContext& ctx) const -{ - if(ctx.GetStream().GetMaxHardwareComputeUnits() < n_groups) - return false; - - if(!IsValidValue()) - return false; - return true; -} - -bool PerformanceConfigConvBinWinogradRxS::operator==( - const PerformanceConfigConvBinWinogradRxS& other) const -{ - return n_groups == other.n_groups; -} - -template -PerformanceConfigConvBinWinogradRxS -ConvBinWinoRxS::GetDefaultPerformanceConfig( - const ExecutionContext& ctx, const ProblemDescription& problem) const -{ - PerformanceConfigConvBinWinogradRxS pp; - pp.HeuristicInit(ctx, problem); - MIOPEN_LOG_I(pp.ToString()); - return pp; -} - -template -bool ConvBinWinoRxS::IsValidPerformanceConfig( - const ExecutionContext& ctx, - const ProblemDescription&, - const PerformanceConfigConvBinWinogradRxS& config) const -{ - return config.IsValidValue() && config.IsValid(ctx); -} - -template -PerformanceConfigConvBinWinogradRxS -ConvBinWinoRxS::Search(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx) const -{ - return GenericSearch(*this, ctx, problem, invoke_ctx); -} - -namespace { - -class ShaderModel : public UnifiedDescriptionConv2d -{ - static constexpr size_t NHW_tiles_factor = 32; - - size_t n_CU; - size_t n_groups; - - double direct_convolution_macs; - - size_t R_loops; - size_t S_loops; - size_t C_loops; - - size_t n_works_per_CU; - - bool dstride2; - bool ostride2; - - bool single_mode; - - bool is_fp16; - bool is_2x3; - - bool out_of_model_scope; // Shader model produces unreliable results. - double granularity_loss; - -public: - ShaderModel(const ExecutionContext& ctx, - const ProblemDescription& problem, - size_t Winodata, - size_t Winofilter) - : UnifiedDescriptionConv2d(problem), - n_CU{ctx.GetStream().GetMaxHardwareComputeUnits()}, /// \todo Take n_CU from PerfConfig. - direct_convolution_macs{static_cast(C * N * K) / 1e+6 * - static_cast(Ceil(S * out_w, input_stride_w) * - Ceil(R * out_h, input_stride_h))}, - is_fp16{problem.IsFp16()}, - is_2x3{IS2X3}, - out_of_model_scope{!(problem.GetGroupCount() == 1) // - || !(U == 1) // - || !(V == 1) // - || !(input_stride_h == 1) // - || !(input_stride_w == 1) // - || !(filter_stride_h == 1) // - || !(filter_stride_w == 1) // -#if !WTI_MODEL_ALLOW_ANY_RS - || !(R <= 5) // - || !(S <= 5) // -#endif -#if !WTI_MODEL_ALLOW_ANY_CK - || !(C >= 16) // - || !(K >= 16) -#endif - } - { - /// \todo add G to UnifiedDescriptionConv2d - size_t G = static_cast(problem.GetGroupCount()); - - const size_t Ts = Winofilter; - const size_t Tr = Winofilter; // Ts and Tr must be the same - - const size_t Tow = Winodata; - const size_t Toh = Winodata; // Tow and Toh must be the same - - const bool dstride2_w{input_stride_w == 2}; - const bool dstride2_h{input_stride_h == 2}; - dstride2 = dstride2_w && dstride2_h; - - const bool ostride2_w{U == 2}; - const bool ostride2_h{V == 2}; - ostride2 = ostride2_w && ostride2_h; - - const bool stride1_w{!(dstride2_w || ostride2_w)}; - const bool stride1_h{!(dstride2_h || ostride2_h)}; - - single_mode = - stride1_w && - (S <= Ts /*|| (L_F_FORCE_FILTER_TRAVERSE_MODE == 1 && L_F_FILTER_TRAVERSE_DUAL == 0)*/); - - const auto R_factor = ((stride1_h || (R % (2 * Tr) == 1)) ? Tr : 2 * Tr); - const auto Rg = RoundUpToMultiple(R, R_factor); - - const auto S_factor = single_mode ? Ts : 2 * Ts; - const auto Sg = RoundUpToMultiple(S, S_factor); - - const auto C_factor = is_fp16 ? 2 : 1 /*fp32*/; - const auto Cg = RoundUpToMultiple(C, C_factor); - - const auto K_factor = dstride2 ? 16u : 32u; - const auto Kg = Ceil(K, K_factor); - - const auto OWg = RoundUpToMultiple(out_w + ((dstride2_w && (pad_w % 2 == 0)) ? 1 : 0), - Tow * input_stride_w); - const auto OHg = RoundUpToMultiple(out_h + ((dstride2_h && (pad_h % 2 == 1)) ? 1 : 0), - Toh * input_stride_h); - - const auto NWH_tiles = N * (OHg / (Toh * input_stride_h)) * (OWg / (Tow * input_stride_w)); - const auto NHW_tiles_g = Ceil(NWH_tiles, NHW_tiles_factor); - - const auto n_works = Kg * NHW_tiles_g; - - S_loops = Sg / S_factor; - R_loops = Rg / Tr; - C_loops = Cg / C_factor; - - const auto NKWH_w = K_factor * NHW_tiles_factor * Toh * Tow; - const auto grid_g = static_cast(NKWH_w * Cg * Rg * Sg) / 1e6; - - auto compute_granularity_loss = - [n_works, G, grid_g, CU = n_CU, dc_macs = direct_convolution_macs](size_t groups) { - const auto works_per_CU = Ceil(n_works, groups) * Ceil(G * groups, CU); - const auto n_works_g = works_per_CU * CU; - const auto macs_g = static_cast(n_works_g) * grid_g; - return 1. - dc_macs / macs_g; - }; - - if(G == 1) - { - n_groups = n_CU; - n_works_per_CU = Ceil(n_works, n_CU); - granularity_loss = compute_granularity_loss(n_groups); - return; - } - - n_groups = 1; - granularity_loss = compute_granularity_loss(n_groups); - for(auto i = n_groups + 1; i < n_CU; ++i) - { - auto loss = compute_granularity_loss(i); - if(loss < granularity_loss) - { - n_groups = i; - granularity_loss = loss; - } - } - - n_works_per_CU = Ceil(n_works, n_groups) * Ceil(G * n_groups, n_CU); - } - - [[maybe_unused]] size_t GetNGroups() const noexcept { return n_groups; } - - double ComputeWti() const noexcept - { - if(out_of_model_scope) - return -1.0; // Shader model produces unreliable results. - - // perf tables for the following kernel binaries: - // {stride1.f32, ostride2.f32, dstride2.f32} : {stride1.f16, ostride2.f16, dstride2.f16} - // should be updated in case of any changes in kernels configurations or implementaion - - // F(2,3) - constexpr auto const_2_3 = - std::array, 2>{{{23652, 18349, 35391}, {28561, 58446, 40618}}}; - constexpr auto fe_2_3 = - std::array, 2>{{{648, 690, 664}, {872, 737, 800}}}; - constexpr auto ph_2_3 = - std::array, 2>{{{308, 326, 316}, {327, 337, 335}}}; - constexpr auto be_2_3 = - std::array, 2>{{{2039, 2409, 1873}, {1852, 2533, 1593}}}; - - // F(3,2) - constexpr auto const_3_2 = - std::array, 2>{{{37329, 38554, 39701}, {41417, 49740, 57820}}}; - constexpr auto fe_3_2 = - std::array, 2>{{{950, 616, 550}, {799, 779, 573}}}; - constexpr auto ph_3_2 = - std::array, 2>{{{295, 315, 308}, {309, 329, 325}}}; - constexpr auto be_3_2 = - std::array, 2>{{{6295, 9900, 7201}, {4941, 4927, 5111}}}; - - const size_t fp_idx = is_fp16 ? 1 : 0; - const size_t stride_idx = ostride2 ? 1 : dstride2 ? 2 : 0 /*stride1*/; - - const auto const_cost = (is_2x3 ? const_2_3 : const_3_2)[fp_idx][stride_idx]; - const auto fe_cost = (is_2x3 ? fe_2_3 : fe_3_2)[fp_idx][stride_idx]; - const auto ph_cost = (is_2x3 ? ph_2_3 : ph_3_2)[fp_idx][stride_idx]; - const auto be_cost = (is_2x3 ? be_2_3 : be_3_2)[fp_idx][stride_idx]; - - const auto fe_calls = n_works_per_CU * S_loops * R_loops; - const auto be_calls = n_works_per_CU * (dstride2 ? 2 : 1); - const auto phases = fe_calls * C_loops * (!single_mode && !dstride2 ? 2 : 1); - - const auto cycles_predicted = static_cast(const_cost + fe_cost * fe_calls + - ph_cost * phases + be_cost * be_calls) / - 1e6; - - if(cycles_predicted <= 0.1) - return -1.0; // Unreliable, too small work to do for the shader. - - /// \todo Adjust for different architectures, probably it should be an external source for - /// this metric - const size_t macs_per_cu_per_clock = static_cast(64) * (is_fp16 ? 2 : 1); - - const auto ideal_direct_cycles = - direct_convolution_macs / static_cast(n_CU * macs_per_cu_per_clock); - const auto WTI_predicted = ideal_direct_cycles / cycles_predicted; - - return WTI_predicted; - } - - [[maybe_unused]] double GetGranularityLoss() const { return granularity_loss; } -}; - -} // namespace - -template -static float GetWtiBase(const ExecutionContext& ctx, const ProblemDescription& problem) -{ - constexpr auto WTI_UNKNOWN = -2.0; - const auto rv = ShaderModel(ctx, problem, Winodata, Winofilter).ComputeWti(); - return rv < 0 ? WTI_UNKNOWN : rv; -} - -template -static bool IsApplicableBase(const ExecutionContext& ctx, const ProblemDescription& problem) -{ - if(!problem.Is2d()) - return false; - if(problem.HasNonPackedTensors()) - return false; - if(!problem.AllTensorsDimsFitIntoInt()) - return false; - if(!(problem.IsFp32() || problem.IsFp16())) - return false; - if(problem.IsTensorsCasted()) - return false; - if(!ctx.use_asm_kernels) - return false; - if(!ctx.rmv.IsV3()) - return false; - - const auto& target = ctx.GetStream().GetTargetProperties(); - if(target.isXnackEnabled()) - return false; - - const auto name = ctx.GetStream().GetDeviceName(); - if(!(StartsWith(name, "gfx9") || StartsWith(name, "gfx10") || StartsWith(name, "gfx11") || - StartsWith(name, "gfx12"))) - return false; - if(problem.IsFp16() && - !(name == "gfx906" || name == "gfx908" || name == "gfx90a" || name == "gfx942" || - StartsWith(name, "gfx95") || name == "gfx1011" || name == "gfx1012" || - StartsWith(name, "gfx103") || StartsWith(name, "gfx11") || StartsWith(name, "gfx12"))) - return false; - - if(name == "gfx90a" && problem.IsGfx90aFp16altRequired()) - return false; - - // Use IsPossibleLayout4D5D to check actual tensor strides rather than cached layout string - // This allows transposed solvers to work correctly when they modify tensor strides - static const auto strict = TensorDescriptor::LayoutValidationMode::StrictDecreasingStrides; - - // clang-format off - if (!((problem.GetKernelStrideW() == 1 || problem.GetKernelStrideW() == 2) - && problem.GetKernelStrideW() == problem.GetKernelStrideH() - && problem.GetDilationW() == 1 - && problem.GetDilationH() == 1 - && problem.GetBias() == 0 - && problem.GetIn().IsPossibleLayout4D5D("NCHW", strict))) - return false; - // clang-format on - -#if WORKAROUND_ISSUE_2492_GRANULARITY_LOSS - if(!env::disabled(MIOPEN_DEBUG_WORKAROUND_ISSUE_2492) && !miopen::debug::IsWarmupOngoing) - { - constexpr double max_perf_drop_due_to_granularity = 200; // Times. - const auto gl = ShaderModel(ctx, problem, Winodata, Winofilter).GetGranularityLoss(); - if(gl > (1.0 - 1.0 / max_perf_drop_due_to_granularity)) - { - MIOPEN_LOG_I("granularity_loss =" << gl); - return false; - } - } -#endif - -#if WORKAROUND_ISSUE_2492_TINY_TENSOR - if(!env::disabled(MIOPEN_DEBUG_WORKAROUND_ISSUE_2492) && !miopen::debug::IsWarmupOngoing) - { - // Group count is not taken into account intentionally. - if(problem.GetInHeight() <= 6 // - && problem.GetInWidth() <= 6 // - && problem.GetBatchSize() <= 4 // - && problem.GetInChannels() <= 4) - return false; - } -#endif - - const auto n_inputs_per_group = problem.GetInChannels() / problem.GetGroupCount(), - n_outputs_per_group = problem.GetOutChannels() / problem.GetGroupCount(); - - if(problem.IsDirectionBackwardWrW()) - { - if(problem.GetKernelStrideW() == 2) - return false; - return IsShaderConstraintsMet(problem, - problem.GetInHeight(), - problem.GetInWidth(), - problem.GetBatchSize(), // N - n_inputs_per_group, // K - problem.GetOutHeight(), - problem.GetOutWidth(), - problem.GetWeightsHeight(), - problem.GetWeightsWidth(), - n_outputs_per_group, // C - name); - } - else - { - return IsShaderConstraintsMet(problem, - problem.GetWeightsHeight(), // RxS - problem.GetWeightsWidth(), - n_inputs_per_group, // C - n_outputs_per_group, // K - problem.GetInHeight(), // HxW - problem.GetInWidth(), - problem.GetOutHeight(), // OHxOW - problem.GetOutWidth(), - problem.GetBatchSize(), // N - name); - } -} - -template -bool ConvBinWinoRxS::IsApplicable(const ExecutionContext& ctx, - const ProblemDescription& problem) const -{ - if(IS2X3) - { - if(env::disabled(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3)) - return false; -#if !WORKAROUND_ISSUE_1681 - if(problem.GetGroupCount() == 1 && !problem.IsDirectionBackwardWrW()) - return false; -#endif - } - if(IS3X2) - { - if(env::disabled(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F3X2)) - return false; - } - return IsApplicableBase(ctx, problem); -} - -template -static inline std::optional -GetPerfConfFromEnv(const ExecutionContext& ctx) -{ - PerformanceConfigConvBinWinogradRxS fromEnv; - std::string s; - const char* env_name; - - if(IS2X3) - { - s = env::value(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_PERF_VALS); - env_name = "MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_PERF_VALS"; - } - else if(IS3X2) - { - s = env::value(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F3X2_PERF_VALS); - env_name = "MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F3X2_PERF_VALS"; - } - - if(s.empty()) - return {}; - - if(!fromEnv.Deserialize(s) || !fromEnv.IsValid(ctx)) - { - MIOPEN_LOG_E(env_name << "Tuning config: Bad value or invalid format: `" << s << '\''); - return {}; - } - - MIOPEN_LOG_I("Overridden from env: " << fromEnv.ToString()); - - return fromEnv; -} - -template -ConvSolution ConvBinWinoRxS::GetSolution( - const ExecutionContext& ctx, - const ProblemDescription& problem, - const PerformanceConfigConvBinWinogradRxS& config) const -{ - const auto n_groups = config.n_groups; - // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) - static bool IsWarned = false; - if(!IsWarned) - { - if(ctx.GetStream().GetMaxHardwareComputeUnits() > MAX_CU_LIMIT) - { - MIOPEN_LOG_WE(SolverDbId() - << ": GPU has " << ctx.GetStream().GetMaxHardwareComputeUnits() - << "CUs, but this solver supports max " << MAX_CU_LIMIT - << "and thus may show sub-optimal performance."); - } - IsWarned = true; - } - - ConvSolution result; - - const PerformanceConfigConvBinWinogradRxS* pcfg = &config; - - const auto fromEnv = GetPerfConfFromEnv(ctx); - if(fromEnv) - { - pcfg = &(*fromEnv); - } - - const auto name = ctx.GetStream().GetDeviceName(); - const auto is_gfx9 = StartsWith(name, "gfx9"); - const auto is_gfx10 = StartsWith(name, "gfx10"); - const auto is_gfx11 = StartsWith(name, "gfx11"); - const auto is_gfx12 = StartsWith(name, "gfx12"); - const auto is_v21 = IsWinogradV21Preferred(name, problem); - size_t wg_size = is_gfx9 ? 512 : 256; - - KernelInfo kernel; - - kernel.g_wk.push_back(wg_size * pcfg->GetNGroups() * problem.GetGroupCount()); - kernel.g_wk.push_back(1); - kernel.g_wk.push_back(1); - - kernel.l_wk.push_back(wg_size); - kernel.l_wk.push_back(1); - kernel.l_wk.push_back(1); - - KernelBuildParameters options{ - {"ROCM_METADATA_VERSION", 5}, - {"FORCE_CACHE_BYPASS_ON_STORE", false}, - }; - kernel.comp_options = options.GenerateFor(kbp::GcnAsm{}); - kernel.comp_options += std::string(" -mcumode -mwavefrontsize64"); - - const std::string kernel_version = is_gfx12 ? "_v40_6_0" : (is_v21 ? "_v21_1_3" : "_v30_3_1"); - std::string kernel_name = "miopenSp3AsmConv" + kernel_version; - std::string kernel_file = "Conv_Winograd" + kernel_version; - std::string kernel_postfix; - - if(is_gfx9) - { - kernel_name += "_gfx9"; - } - else if(is_gfx10) - { - kernel_name += "_gfx10"; - } - else if(is_gfx11) - { - kernel_name += "_gfx11"; - } - else if(is_gfx12) - { - kernel_name += "_gfx12"; - } - else - { - MIOPEN_THROW(miopenStatusInternalError); - } - - if(problem.IsFp32()) - { - kernel_name += "_fp32"; - kernel_file += "_fp32"; - } - else // if(problem.IsFp16()) - { - kernel_name += is_gfx9 ? "_fp16_dot2_edc" : "_fp16_dot2"; - kernel_file += "_fp16_dot2"; - } - - kernel_postfix = IS2X3 ? "_f2x3" : "_f3x2"; - - if(problem.GetKernelStrideW() == 1) - { - kernel_postfix += "_stride1"; - } - else if(problem.GetKernelStrideW() == 2 && !problem.IsDirectionBackwardData()) - { - kernel_postfix += "_stride2"; - } - else // if(problem.GetDilationH() == 2) - { - kernel_postfix += "_dilation2"; - } - - kernel.kernel_name = kernel_name + kernel_postfix; - kernel.kernel_file = kernel_file + kernel_postfix + ".s"; - - result.construction_params.push_back(kernel); - - constexpr int F_REVERSE_R = 1 << 0; - constexpr int F_REVERSE_S = 1 << 1; - constexpr int F_FLIP_K_C = 1 << 2; - // These are not used yet. Nevertheless let's keep as a shader documentation. - // constexpr int F_FLIP_DATA_N_C = 1 << 3; // Unsupported in f3x2. - // constexpr int F_FLIP_OUT_N_K = 1 << 4; // Unsupported in f3x2. - // constexpr int F_ADDR_INDIRECT = 1 << 6; - // constexpr int F_BIAS = 1 << 7; - // constexpr int F_LEAKY_RELU = 1 << 8; - constexpr int F_NKC_STRIDES = 1 << 9; - constexpr int F_GROUP_STRIDES = 1 << 10; - // constexpr int F_FORCE_FILTER_TRAVERSE_MODE = 1 << 11; - // constexpr int F_FILTER_TRAVERSE_DUAL = 1 << 12; - // constexpr int F_TENSOR_OFFSETS = 1 << 13; - constexpr int F_USE_ACTIVATION_MODE = 1 << 14; - // constexpr int F_USE_EXTENDED_FLAGS_64 = 1 << 15; - int flags = F_NKC_STRIDES | F_GROUP_STRIDES | (is_v21 ? 0 : F_USE_ACTIVATION_MODE); - - constexpr uint8_t IDENTITY = 0; - // constexpr uint8_t LEAKY_RELU = 1; - // constexpr uint8_t SIGMOID = 2; - // constexpr uint8_t SCALED_TANH = 3; - uint8_t identity_activation = IDENTITY; - - int N, C, H, W, K, out_H, out_W, R, S, pad_H, pad_W; - MemLayout_t d_layout, o_layout, f_layout; - - const bool is_forward = problem.IsDirectionForward(); - const bool is_backWrW = problem.IsDirectionBackwardWrW(); - const int group_cnt = problem.GetGroupCount(); - - if(!is_backWrW) - { - int unused_n_groups = 0; - GetCompiledInParameters(ctx, - problem, - &N, - &C, - &H, - &W, - &K, - &unused_n_groups, - &out_H, - &out_W, - &R, - &S, - &pad_H, - &pad_W); - C /= group_cnt; - K /= group_cnt; - - d_layout = GetGroupConvLayout(GetMemLayout_t(problem.GetInLayout()), true); - o_layout = GetGroupConvLayout(GetMemLayout_t(problem.GetOutLayout()), true); - f_layout = GetGroupConvLayout( - is_forward ? MemLayout_t::NCHW : GetSwappedNCLayout(MemLayout_t::NCHW), false); - - flags |= is_forward ? 0 : F_REVERSE_R | F_REVERSE_S | F_FLIP_K_C; - } - else - { - int unused_n_groups = 0; - GetCompiledInParameters(ctx, - problem, - &C, - &K, - &R, - &S, - &N, - &unused_n_groups, - &H, - &W, - &out_H, - &out_W, - &pad_H, - &pad_W); - N /= group_cnt; - K /= group_cnt; - pad_H = problem.GetConv().GetConvPads()[0]; - pad_W = problem.GetConv().GetConvPads()[1]; - - d_layout = - GetGroupConvLayout(GetSwappedNCLayout(GetMemLayout_t(problem.GetInLayout())), true); - o_layout = - GetGroupConvLayout(GetSwappedNCLayout(GetMemLayout_t(problem.GetOutLayout())), false); - f_layout = GetGroupConvLayout(GetSwappedNCLayout(MemLayout_t::NCHW), true); - } - - // cppcheck-suppress unreadVariable - BuffInfo d_buf(d_layout, N, C, H, W, group_cnt, GetTypeSize(problem.GetInDataType())); - // cppcheck-suppress unreadVariable - BuffInfo o_buf(o_layout, N, K, out_H, out_W, group_cnt, GetTypeSize(problem.GetOutDataType())); - // cppcheck-suppress unreadVariable - BuffInfo f_buf(f_layout, K, C, R, S, group_cnt, GetTypeSize(problem.GetWeightsDataType())); - const auto d_strides = is_v21 ? d_buf.byte_stride : d_buf.stride; - const auto f_strides = is_v21 ? f_buf.byte_stride : f_buf.stride; - const auto o_strides = is_v21 ? o_buf.byte_stride : o_buf.stride; - - result.invoker_factory = [=](std::vector kernels) { - return [=](const Handle& handle, const AnyInvokeParams& primitive_params) { - int reserved = 0; - uint64_t reserved_offset = 0; - int* reserved_ptr = nullptr; - - const auto k = handle.Run(kernels[0]); - const auto data_tensors = - !is_backWrW ? primitive_params.CastTo().tensors.in - : primitive_params.CastTo().tensors.x; - const auto filter_tensors = - !is_backWrW ? primitive_params.CastTo().tensors.w - : primitive_params.CastTo().tensors.dy; - const auto out_tensors = - !is_backWrW ? primitive_params.CastTo().tensors.out - : primitive_params.CastTo().tensors.dw; - - // clang-format off - MIOPEN_LOG_I2(" N=" << N << " G=" << group_cnt << " C=" << C << " H=" << H << " W=" << W << " K=" << K - << " n_groups=" << n_groups << " flags=" << flags << " R=" << R << " S=" << S - << " pad_H=" << pad_H << " pad_W=" << pad_W << " out_H=" << out_H << " out_W=" << out_W - << " d_N_stride=" << d_strides.nk << " d_C_stride=" << d_strides.c - << " d_H_stride=" << d_strides.h << " d_W_stride=" << d_strides.w - << " f_K_stride=" << f_strides.nk << " f_C_stride=" << f_strides.c - << " f_R_stride=" << f_strides.h << " f_S_stride=" << f_strides.w - << " o_N_stride=" << o_strides.nk << " o_K_stride=" << o_strides.c - << " o_H_stride=" << o_strides.h << " o_W_stride=" << o_strides.w - << " d_G_stride=" << d_strides.g << " f_G_stride=" << f_strides.g - << " o_G_stride=" << o_strides.g); - // clang-format on - if(is_v21) - { - k(N, - C, - H, - W, - K, - n_groups, - flags, - reserved, - data_tensors, - filter_tensors, - out_tensors, - reserved_ptr, // Unused return_addr. - R, - S, - pad_H, - pad_W, - out_H, - out_W, - reserved_ptr, // Unused bias_addr. - reserved, // Unused relu_alpha. - reserved, // Unused reserved2. - reserved_offset, // Unused d_offset. - reserved_offset, // Unused f_offset. - reserved_offset, // Unused o_offset. - reserved_offset, // Unused b_offset. - d_strides.nk, - d_strides.c, - d_strides.h, - d_strides.w, - f_strides.nk, - f_strides.c, - f_strides.h, - f_strides.w, - o_strides.nk, - o_strides.c, - o_strides.h, - o_strides.w, - group_cnt, - d_strides.g, - f_strides.g, - o_strides.g); - } - else - { - k(N, - C, - H, - W, - K, - n_groups, - flags, - reserved, - data_tensors, - filter_tensors, - out_tensors, - reserved_ptr, // Unused return_addr. - R, - S, - pad_H, - pad_W, - out_H, - out_W, - reserved_ptr, // Unused bias_addr. - reserved, // Unused alpha. - reserved, // Unused beta. - reserved_offset, // Unused d_offset. - reserved_offset, // Unused f_offset. - reserved_offset, // Unused o_offset. - reserved_offset, // Unused b_offset. - d_strides.nk, - d_strides.c, - d_strides.h, - d_strides.w, - f_strides.nk, - f_strides.c, - f_strides.h, - f_strides.w, - o_strides.nk, - o_strides.c, - o_strides.h, - o_strides.w, - group_cnt, - d_strides.g, - f_strides.g, - o_strides.g, - identity_activation); - } - }; - }; - - return result; -} - -bool ConvBinWinogradRxSf2x3g1::IsApplicable(const ExecutionContext& ctx, - const ProblemDescription& problem) const -{ - if(env::disabled(MIOPEN_DEBUG_AMD_WINOGRAD_RXS_F2X3_G1)) - return false; - - return IsApplicableBase<2, 3>(ctx, problem) && problem.GetGroupCount() == 1; -} - -float ConvBinWinogradRxSf2x3g1::GetWti(const ExecutionContext& ctx, - const ProblemDescription& problem) const -{ - return GetWtiBase<2, 3>(ctx, problem); -} - -ConvSolution ConvBinWinogradRxSf2x3g1::GetSolution(const ExecutionContext& ctx, - const ProblemDescription& problem) const -{ - const auto tunable = ConvBinWinoRxS<2, 3>{}; - return tunable.GetSolution(ctx, problem, tunable.GetDefaultPerformanceConfig(ctx, problem)); -} -template struct MIOPEN_INTERNALS_EXPORT ConvBinWinoRxS<2, 3>; -template struct MIOPEN_INTERNALS_EXPORT ConvBinWinoRxS<3, 2>; - -template struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinoRxS<2, 3>; -template struct MIOPEN_INTERNALS_EXPORT TransposedConvBinWinoRxS<3, 2>; - -} // namespace conv -} // namespace solver -} // namespace miopen diff --git a/projects/miopen/test/gtest/activ_common.hpp b/projects/miopen/test/gtest/activ_common.hpp deleted file mode 100644 index 11bb29783979..000000000000 --- a/projects/miopen/test/gtest/activ_common.hpp +++ /dev/null @@ -1,477 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "../tensor_holder.hpp" - -namespace miopen { -namespace tests { -namespace activ { - -template && std::is_unsigned_v), bool> = false> -auto Ceil(T1 val, T2 div) -{ - assert(div != 0); - return (val + div - 1) / div; -} - -namespace activ_func { - -/// \note static_cast in below functions is needed to speed up execution time for CPU non-native -/// data types -class ActivationPASTHRU -{ -public: - template - static T Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return x; - } - - template - static T Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T /*x*/, T /*y*/) - { - return dy; - } -}; - -class ActivationLOGISTIC -{ -public: - template - static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return 1 / (1 + std::exp(-x)); - } - - template - static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T /*x*/, T y) - { - return static_cast(dy) * y * (static_cast(1) - y); - } -}; - -class ActivationTANH -{ -public: - template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam /*gamma*/, T x) - { - return beta * std::tanh(alpha * x); - } - - template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam /*gamma*/, T dy, T /*x*/, T y) - { - return dy * alpha * (beta - static_cast(y) * y / beta); - } -}; - -class ActivationRELU -{ -public: - template - static T Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return (x > static_cast(0)) ? x : static_cast(0); - } - - template - static T Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) - { - return (x > static_cast(0)) ? dy : static_cast(0); - } -}; - -class ActivationSOFTRELU -{ -public: - template - static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return std::log1p(std::exp(x)); - } - - template - static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) - { - const Tparam threshold = 50.0; - const Tparam expval = std::exp(std::min(static_cast(x), threshold)); - return dy * expval / (expval + 1); - } -}; - -class ActivationABS -{ -public: - template - static Tparam Forward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return std::abs(x); - } - - template - static Tparam Backward(Tparam /*alpha*/, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) - { - return dy * static_cast((x > static_cast(0)) ? 1 : -1); - } -}; - -class ActivationPOWER -{ -public: - template - static Tparam Forward(Tparam alpha, Tparam beta, Tparam gamma, T x) - { - const auto v = alpha + beta * x; - return v <= std::numeric_limits::epsilon() ? 0 : std::pow(v, gamma); - } - - template - static Tparam Backward(Tparam alpha, Tparam beta, Tparam gamma, T /*dy*/, T x, T y) - { - const auto v = alpha + beta * x; - return v <= std::numeric_limits::epsilon() ? 0 : gamma * beta * y / v; - } -}; - -class ActivationCLIPPEDRELU -{ -public: - template - static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - return std::clamp(static_cast(x), static_cast(0), alpha); - } - - template - static T Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) - { - Tparam x_native = x; - return (x_native > 0 && x_native <= alpha) ? dy : static_cast(0); - } -}; - -class ActivationLEAKYRELU -{ -public: - template - static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - Tparam x_native = x; - return (x_native > 0) ? x_native : x_native * alpha; - } - - template - static Tparam Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T /*y*/) - { - return dy * ((x > static_cast(0)) ? 1 : alpha); - } -}; - -class ActivationELU -{ -public: - template - static Tparam Forward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T x) - { - Tparam x_native = x; - return (x_native > 0) ? x_native : alpha * std::expm1(x_native); - } - - template - static Tparam Backward(Tparam alpha, Tparam /*beta*/, Tparam /*gamma*/, T dy, T x, T y) - { - return dy * ((x > static_cast(0)) ? 1 : y + alpha); - } -}; - -} // namespace activ_func - -enum class Direction -{ - Forward, - Backward -}; - -template -class ActivationParamDataType; - -template <> -class ActivationParamDataType -{ -public: - using Type = double; // "Type" must be a native CPU type! -}; - -template <> -class ActivationParamDataType -{ -public: - using Type = float; // "Type" must be a native CPU type! -}; - -template <> -class ActivationParamDataType -{ -public: - using Type = float; // "Type" must be a native CPU type! -}; - -template <> -class ActivationParamDataType -{ -public: - using Type = float; // "Type" must be a native CPU type! -}; - -unsigned CpuActivationGetNumThreads(std::size_t num_jobs) -{ - const unsigned max_num_hw_threads = std::thread::hardware_concurrency(); - return std::min(num_jobs, static_cast(max_num_hw_threads)); -} - -template -void DoCpuActivationForwardPacked(std::size_t offset, - std::size_t end, - Tparam alpha, - Tparam beta, - Tparam gamma, - const tensor& x, - tensor& y) -{ - for(std::size_t i = offset; i < end; i++) - y.data[i] = A::Forward(alpha, beta, gamma, x.data[i]); -} - -template -void DoCpuActivationBackwardPacked(std::size_t offset, - std::size_t end, - Tparam alpha, - Tparam beta, - Tparam gamma, - const tensor& y, - const tensor& dy, - const tensor& x, - tensor& dx) -{ - for(std::size_t i = offset; i < end; i++) - dx.data[i] = A::Backward(alpha, beta, gamma, dy.data[i], x.data[i], y.data[i]); -} - -template -void DoCpuActivationPacked(std::size_t offset, std::size_t end, Ts&&... xs) -{ - if constexpr(direction == Direction::Forward) - DoCpuActivationForwardPacked(offset, end, xs...); - else - DoCpuActivationBackwardPacked(offset, end, xs...); -} - -template -void CpuActivationPackedSingleThread(std::size_t num_items, Ts&&... xs) -{ - DoCpuActivationPacked(0, num_items, xs...); -} - -template -void CpuActivationPackedMultiThread(std::size_t num_items, Ts&&... xs) -{ - const std::size_t max_num_items_per_job = 16 * 1024 * 1024; - const std::size_t num_jobs = Ceil(num_items, max_num_items_per_job); - const auto num_threads = CpuActivationGetNumThreads(num_jobs); - if(num_threads == 1) - { - CpuActivationPackedSingleThread(num_items, xs...); - return; - } - - const std::size_t max_num_jobs_per_thread = Ceil(num_jobs, num_threads); - const std::size_t max_num_items_per_thread = max_num_items_per_job * max_num_jobs_per_thread; - const std::size_t remainder = num_items % max_num_items_per_thread; - const auto num_async_threads = num_threads - 1; - - auto func_async = [&](unsigned thread_num) { - const auto offset = max_num_items_per_thread * thread_num; - const auto end = offset + max_num_items_per_thread; - DoCpuActivationPacked(offset, end, xs...); - }; - - auto func_remainder = [&]() { - const auto offset = max_num_items_per_thread * num_async_threads; - const auto end = offset + remainder; - DoCpuActivationPacked(offset, end, xs...); - }; - - std::vector threads; - for(unsigned i = 0; i < num_async_threads; i++) - threads.push_back(std::async(std::launch::async, func_async, i)); - - if(remainder) - func_remainder(); - else - func_async(num_async_threads); - - for(auto& thread : threads) - thread.wait(); -} - -template -void CpuActivationPacked(std::size_t num_items, Ts&&... xs) -{ - CpuActivationPackedMultiThread(num_items, xs...); -} - -template -[[noreturn]] void CpuActivationNonPacked(std::size_t /*num_items*/, Ts&&... /*xs*/) -{ - throw std::runtime_error("CpuActivationNonPacked is not implemented yet"); -} - -template -void CpuActivation(std::size_t num_items, Ts&&... xs) -{ - if constexpr(is_packed) - CpuActivationPacked(num_items, xs...); - else - CpuActivationNonPacked(num_items, xs...); -} - -template -void CpuActivation(miopenActivationMode_t m, std::size_t num_items, Ts&&... xs) -{ - switch(m) - { - case miopenActivationPASTHRU: - CpuActivation(num_items, xs...); - break; - case miopenActivationLOGISTIC: - CpuActivation(num_items, xs...); - break; - case miopenActivationTANH: - CpuActivation(num_items, xs...); - break; - case miopenActivationRELU: - CpuActivation(num_items, xs...); - break; - case miopenActivationSOFTRELU: - CpuActivation(num_items, xs...); - break; - case miopenActivationABS: - CpuActivation(num_items, xs...); - break; - case miopenActivationPOWER: - CpuActivation(num_items, xs...); - break; - case miopenActivationCLIPPEDRELU: - CpuActivation(num_items, xs...); - break; - case miopenActivationLEAKYRELU: - CpuActivation(num_items, xs...); - break; - case miopenActivationELU: - CpuActivation(num_items, xs...); - break; - default: throw std::runtime_error("Unknown activation mode"); - } -} - -template -void CpuActivationForward(miopenActivationMode_t m, - double alpha, - double beta, - double gamma, - const tensor& x, - tensor& y) -{ - using Tparam = typename ActivationParamDataType::Type; - - if(x.desc.GetElementSize() != y.desc.GetElementSize()) - throw std::runtime_error("x.desc.GetElementSize() != y.desc.GetElementSize()"); - - Tparam p_alpha = alpha; - Tparam p_beta = beta; - Tparam p_gamma = gamma; - - if(x.desc.IsPacked() && y.desc.IsPacked()) - { - const auto num_items = x.data.size(); - CpuActivation(m, num_items, p_alpha, p_beta, p_gamma, x, y); - } - else - { - CpuActivation(m, 0, p_alpha, p_beta, p_gamma, x, y); - } -} - -template -void CpuActivationBackward(miopenActivationMode_t m, - double alpha, - double beta, - double gamma, - const tensor& y, - const tensor& dy, - const tensor& x, - tensor& dx) -{ - using Tparam = typename ActivationParamDataType::Type; - - if(x.desc.GetElementSize() != y.desc.GetElementSize()) - throw std::runtime_error("x.desc.GetElementSize() != y.desc.GetElementSize()"); - if(dx.desc.GetElementSize() != dy.desc.GetElementSize()) - throw std::runtime_error("dx.desc.GetElementSize() != dy.desc.GetElementSize()"); - if(x.desc.GetElementSize() != dx.desc.GetElementSize()) - throw std::runtime_error("x.desc.GetElementSize() != dx.desc.GetElementSize()"); - - Tparam p_alpha = alpha; - Tparam p_beta = beta; - Tparam p_gamma = gamma; - - if(y.desc.IsPacked() && dy.desc.IsPacked() && x.desc.IsPacked() && dx.desc.IsPacked()) - { - const auto num_items = dy.data.size(); - CpuActivation( - m, num_items, p_alpha, p_beta, p_gamma, y, dy, x, dx); - } - else - { - CpuActivation(m, 0, p_alpha, p_beta, p_gamma, y, dy, x, dx); - } -} - -} // namespace activ -} // namespace tests -} // namespace miopen diff --git a/projects/miopen/test/gtest/deepbench_lstm.cpp b/projects/miopen/test/gtest/deepbench_lstm.cpp deleted file mode 100644 index afb977c41335..000000000000 --- a/projects/miopen/test/gtest/deepbench_lstm.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "lstm.hpp" - -namespace { - -struct TestCase -{ - TestCase(int batch_size_, int seq_len_, int vector_len_, int hidden_size_) - : batch_size(batch_size_), - seq_len(seq_len_), - vector_len(vector_len_), - hidden_size(hidden_size_) - { - } - - int batch_size; - int seq_len; - int vector_len; - int hidden_size; - - friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) - { - return os << "batch-size:" << tc.batch_size << " seq-len:" << tc.seq_len - << " vector-len:" << tc.vector_len << " hidden-size:" << tc.hidden_size; - } -}; - -std::vector GetTestCases() -{ - return std::vector{ - // clang-format off - // batch-size seq-len vector-len hidden-size - TestCase( 16, 25, 512, 512 ), - TestCase( 32, 25, 512, 512 ), - TestCase( 64, 25, 512, 512 ), - TestCase(128, 25, 512, 512 ), - TestCase( 16, 25, 1024, 1024 ), - TestCase( 32, 25, 1024, 1024 ), - TestCase( 64, 25, 1024, 1024 ), - TestCase(128, 25, 1024, 1024 ), - TestCase( 16, 25, 2048, 2048 ), - TestCase( 32, 25, 2048, 2048 ), - TestCase( 64, 25, 2048, 2048 ), - TestCase(128, 25, 2048, 2048 ), - TestCase( 16, 25, 4096, 4096 ), - TestCase( 32, 25, 4096, 4096 ), - TestCase( 64, 25, 4096, 4096 ), - TestCase(128, 25, 4096, 4096 ), - TestCase( 8, 50, 1536, 1536 ), - TestCase( 16, 50, 1536, 1536 ), - TestCase( 32, 50, 1536, 1536 ), - TestCase( 16, 150, 256, 256 ), - TestCase( 32, 150, 256, 256 ), - TestCase( 64, 150, 256, 256 ) - // clang-format on - }; -} - -} // namespace - -template -struct GPU_DeepBench_LSTM_test : LSTM_test, testing::TestWithParam -{ -protected: - void SetUp() override - { - this->dataType = miopen_type{}; - - auto [batchSize, seqLength, inVecLen, hiddenSize] = GetParam(); - - this->numLayers = 1; - this->inputMode = 1; - this->biasMode = 0; - this->dirMode = 0; - this->flatBatchFill = 1; - this->batchSize = batchSize; - this->seqLength = seqLength; - this->inVecLen = inVecLen; - this->hiddenSize = hiddenSize; - } -}; - -using GPU_DeepBench_LSTM_FP16 = GPU_DeepBench_LSTM_test; - -TEST_P(GPU_DeepBench_LSTM_FP16, HalfTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP16, testing::ValuesIn(GetTestCases())); - -using GPU_DeepBench_LSTM_FP32 = GPU_DeepBench_LSTM_test; - -TEST_P(GPU_DeepBench_LSTM_FP32, FloatTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP32, testing::ValuesIn(GetTestCases())); - -using GPU_DeepBench_LSTM_FP64 = GPU_DeepBench_LSTM_test; - -TEST_P(GPU_DeepBench_LSTM_FP64, DoubleTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_DeepBench_LSTM_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp b/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp deleted file mode 100644 index 753c4d935239..000000000000 --- a/projects/miopen/test/gtest/graphapi_operationgraph_descriptor.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include - -#include - -#include "graphapi_gtest_common.hpp" - -namespace { - -using miopen::graphapi::BackendDescriptor; -using miopen::graphapi::OpNode; -using miopen::graphapi::Tensor; - -static int64_t id = 0; - -class GMockNode : public OpNode -{ -private: - std::shared_ptr mIn; - std::shared_ptr mOut; - -public: - GMockNode() - : mIn(std::make_shared(miopenFloat, - std::vector{8, 64, 64}, - std::vector{64 * 64, 64, 1}, - ++id, - false)), - mOut(std::make_shared(miopenFloat, - std::vector{8, 64, 64}, - std::vector{64 * 64, 64, 1}, - ++id, - false)) - { - } - - std::vector getInTensors() const override { return {mIn.get()}; } - - std::vector getOutTensors() const override { return {mOut.get()}; } - - const std::string& signName() const override - { - static const std::string name = "OP_MOCK"; - return name; - } -}; - -class GMockBackendOperationDescriptor : public BackendDescriptor -{ -private: - GMockNode mNode; - -public: - GMockBackendOperationDescriptor(bool finalized) { mFinalized = finalized; } - void setAttribute(miopenBackendAttributeName_t /*attributeName*/, - miopenBackendAttributeType_t /*attributeType*/, - int64_t /*elementCount*/, - void* /*arrayOfElements*/) override - { - } - void finalize() override {} - void getAttribute(miopenBackendAttributeName_t /*attributeName*/, - miopenBackendAttributeType_t /*attributeType*/, - int64_t /*requestedElementCount*/, - int64_t* /*elementCount*/, - void* /*arrayOfElements*/) override - { - } - OpNode* getOperation() override { return &mNode; } -}; - -using miopen::graphapi::GTestDescriptorAttribute; -using miopen::graphapi::GTestDescriptorSingleValueAttribute; -using miopen::graphapi::GTestDescriptorVectorAttribute; -using miopen::graphapi::GTestGraphApiExecute; - -} // namespace - -TEST(CPU_GraphApiOperationGraphDescriptor_NONE, CFunctions) -{ - miopenHandle_t handle{nullptr}; - auto status = miopenCreate(&handle); - ASSERT_EQ(status, miopenStatusSuccess) << "Handle wasn't obtained, cannot continue"; - - GTestDescriptorSingleValueAttribute handleAttribute{ - true, - "MIOPEN_ATTR_OPERATIONGRAPH_HANDLE", - MIOPEN_ATTR_OPERATIONGRAPH_HANDLE, - MIOPEN_TYPE_HANDLE, - MIOPEN_TYPE_CHAR, - 0, - handle}; - - GMockBackendOperationDescriptor goodNode(true); - - GTestDescriptorVectorAttribute goodOp{ - true, - "MIOPEN_ATTR_OPERATIONGRAPH_OPS", - MIOPEN_ATTR_OPERATIONGRAPH_OPS, - MIOPEN_TYPE_BACKEND_DESCRIPTOR, - MIOPEN_TYPE_CHAR, - -1, - {&goodNode}}; - ; - - GTestGraphApiExecute execute{ - {"MIOPEN_BACKEND_OPERATIONGRAPH_DESCRIPTOR", - MIOPEN_BACKEND_OPERATIONGRAPH_DESCRIPTOR, - true, - {&handleAttribute, &goodOp}}}; - - execute(); - - execute.descriptor.attributes = {&handleAttribute, &goodOp, &goodOp}; - execute(); - - GMockBackendOperationDescriptor badNode(false); - - GTestDescriptorVectorAttribute badOp{ - false, - "MIOPEN_ATTR_OPERATIONGRAPH_OPS", - MIOPEN_ATTR_OPERATIONGRAPH_OPS, - MIOPEN_TYPE_BACKEND_DESCRIPTOR, - MIOPEN_TYPE_CHAR, - -1, - {&badNode}}; - ; - - execute.descriptor.attrsValid = false; - - execute.descriptor.attributes = {&handleAttribute}; - execute(); - - execute.descriptor.attributes = {&goodOp}; - execute(); - - execute.descriptor.attributes = {&handleAttribute, &badOp}; - execute(); - - badOp = {false, - "MIOPEN_ATTR_OPERATIONGRAPH_OPS", - MIOPEN_ATTR_OPERATIONGRAPH_OPS, - MIOPEN_TYPE_BACKEND_DESCRIPTOR, - MIOPEN_TYPE_CHAR, - -1, - {&goodNode, &goodNode}}; - execute(); - - miopenDestroy(handle); -} diff --git a/projects/miopen/test/gtest/lstm_dropout.cpp b/projects/miopen/test/gtest/lstm_dropout.cpp deleted file mode 100644 index d2a023b65b34..000000000000 --- a/projects/miopen/test/gtest/lstm_dropout.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/******************************************************************************* - * - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "lstm.hpp" - -namespace { - -struct TestCase -{ - TestCase(int dirMode_, int flatBatchFill_) : dirMode(dirMode_), flatBatchFill(flatBatchFill_) {} - - int dirMode; - int flatBatchFill; - - friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) - { - return os << "dir-mode:" << tc.dirMode << " flat-batch-fill:" << tc.flatBatchFill; - } -}; - -std::vector GetTestCases() -{ - return std::vector{ - // clang-format off - // dir-mode flat-batch-fill - TestCase(0, 0), - TestCase(0, 1), - TestCase(1, 0), - TestCase(1, 1) - // clang-format on - }; -} - -} // namespace - -template -struct GPU_LSTM_dropout_test : LSTM_test, testing::TestWithParam -{ -protected: - void SetUp() override - { - this->dataType = miopen_type{}; - - auto [dirMode, flatBatchFill] = GetParam(); - - int useDropout{1}; - int batchSize{17}; - int seqLength{25}; - this->useDropout = useDropout; - this->batchSize = batchSize; - this->seqLength = seqLength; - this->inVecLen = batchSize; - this->batchSeq = generate_batchSeq(batchSize, seqLength)[0]; - this->numLayers = 3; - this->hiddenSize = 67; - this->dirMode = dirMode; - this->flatBatchFill = flatBatchFill; - } -}; - -using GPU_LSTM_dropout_FP16 = GPU_LSTM_dropout_test; - -TEST_P(GPU_LSTM_dropout_FP16, HalfTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP16, testing::ValuesIn(GetTestCases())); - -using GPU_LSTM_dropout_FP32 = GPU_LSTM_dropout_test; - -TEST_P(GPU_LSTM_dropout_FP32, FloatTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP32, testing::ValuesIn(GetTestCases())); - -using GPU_LSTM_dropout_FP64 = GPU_LSTM_dropout_test; - -TEST_P(GPU_LSTM_dropout_FP64, DoubleTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_dropout_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/lstm_extra.cpp b/projects/miopen/test/gtest/lstm_extra.cpp deleted file mode 100644 index 7ed311664e84..000000000000 --- a/projects/miopen/test/gtest/lstm_extra.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "lstm.hpp" - -namespace { - -struct TestCase -{ - TestCase(int dir_mode_, - int no_hx_, - int no_dhy_, - int no_cx_, - int no_dcy_, - int no_hy_, - int no_dhx_, - int no_cy_, - int no_dcx_) - : dir_mode(dir_mode_), - no_hx(no_hx_), - no_dhy(no_dhy_), - no_cx(no_cx_), - no_dcy(no_dcy_), - no_hy(no_hy_), - no_dhx(no_dhx_), - no_cy(no_cy_), - no_dcx(no_dcx_) - { - } - - int dir_mode; - int no_hx; - int no_dhy; - int no_cx; - int no_dcy; - int no_hy; - int no_dhx; - int no_cy; - int no_dcx; - - friend std::ostream& operator<<(std::ostream& os, const TestCase& tc) - { - return os << "dir-mode:" << tc.dir_mode << " no-hx:" << tc.no_hx << " no-dhy:" << tc.no_dhy - << " no-cx:" << tc.no_cx << " no-dcy:" << tc.no_dcy << " no-hy:" << tc.no_hy - << " no-dhx:" << tc.no_dhx << " no-cy:" << tc.no_cy << " no-dcx:" << tc.no_dcx - << " no-dcx:" << tc.no_dcx; - } -}; - -std::vector GetTestCases() -{ - return std::vector{ - // clang-format off - // dir-mode no-hx no-dhy no-cx no-dcy no-hy no-dhx no-cy no-dcx - TestCase(0, 1, 0, 0, 0, 0, 0, 0, 0), - TestCase(0, 0, 1, 0, 0, 0, 0, 0, 0), - TestCase(0, 1, 1, 0, 0, 0, 0, 0, 0), - TestCase(0, 0, 0, 1, 0, 0, 0, 0, 0), - TestCase(0, 1, 0, 1, 0, 0, 0, 0, 0), - TestCase(0, 0, 0, 0, 1, 0, 0, 0, 0), - TestCase(0, 0, 0, 1, 1, 0, 0, 0, 0), - TestCase(1, 1, 0, 0, 0, 0, 0, 0, 0), - TestCase(1, 0, 1, 0, 0, 0, 0, 0, 0), - TestCase(1, 1, 1, 0, 0, 0, 0, 0, 0), - TestCase(1, 0, 0, 1, 0, 0, 0, 0, 0), - TestCase(1, 1, 0, 1, 0, 0, 0, 0, 0), - TestCase(1, 0, 0, 0, 1, 0, 0, 0, 0), - TestCase(1, 0, 0, 1, 1, 0, 0, 0, 0), - TestCase(0, 0, 0, 0, 0, 1, 0, 0, 0), - TestCase(0, 0, 0, 0, 0, 0, 1, 0, 0), - TestCase(0, 0, 0, 0, 0, 1, 1, 0, 0), - TestCase(0, 0, 0, 0, 0, 0, 0, 1, 0), - TestCase(0, 0, 0, 0, 0, 1, 0, 1, 0), - TestCase(0, 0, 0, 0, 0, 0, 0, 0, 1), - TestCase(0, 0, 0, 0, 0, 0, 0, 1, 1), - TestCase(1, 0, 0, 0, 0, 1, 0, 0, 0), - TestCase(1, 0, 0, 0, 0, 0, 1, 0, 0), - TestCase(1, 0, 0, 0, 0, 1, 1, 0, 1), - TestCase(1, 0, 0, 0, 0, 0, 0, 1, 0), - TestCase(1, 0, 0, 0, 0, 1, 0, 1, 0), - TestCase(1, 0, 0, 0, 0, 0, 0, 0, 1), - TestCase(1, 0, 0, 0, 0, 0, 0, 1, 1), - TestCase(0, 1, 1, 1, 1, 1, 1, 1, 1), - TestCase(1, 1, 1, 1, 1, 1, 1, 1, 1) - // clang-format on - }; -} - -} // namespace - -template -struct GPU_LSTM_extra_test : LSTM_test, testing::TestWithParam -{ -protected: - void SetUp() override - { - this->dataType = miopen_type{}; - - auto [dirMode, nohx, nodhy, nocx, nodcy, nohy, nodhx, nocy, nodcx] = GetParam(); - - this->batchSize = 32; - this->seqLength = 3; - this->batchSeq = {32, 32, 32}; - this->inVecLen = 128; - this->hiddenSize = 128; - this->numLayers = 1; - this->inputMode = 0; - this->biasMode = 0; - this->dirMode = dirMode; - this->nohx = bool(nohx); - this->nodhy = bool(nodhy); - this->nocx = bool(nocx); - this->nodcy = bool(nodcy); - this->nohy = bool(nohy); - this->nodhx = bool(nodhx); - this->nocy = bool(nocy); - this->nodcx = bool(nodcx); - } -}; - -using GPU_LSTM_extra_FP16 = GPU_LSTM_extra_test; - -TEST_P(GPU_LSTM_extra_FP16, HalfTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP16, testing::ValuesIn(GetTestCases())); - -using GPU_LSTM_extra_FP32 = GPU_LSTM_extra_test; - -TEST_P(GPU_LSTM_extra_FP32, FloatTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP32, testing::ValuesIn(GetTestCases())); - -using GPU_LSTM_extra_FP64 = GPU_LSTM_extra_test; - -TEST_P(GPU_LSTM_extra_FP64, DoubleTest) { RunTest(); } - -INSTANTIATE_TEST_SUITE_P(Full, GPU_LSTM_extra_FP64, testing::ValuesIn(GetTestCases())); diff --git a/projects/miopen/test/gtest/test_categories.yaml b/projects/miopen/test/gtest/test_categories.yaml deleted file mode 100644 index f8815cb999a7..000000000000 --- a/projects/miopen/test/gtest/test_categories.yaml +++ /dev/null @@ -1,350 +0,0 @@ -# MIOpen Test Categories Configuration -# Time-based test organization for CI/CD pipeline efficiency -# This file defines test categories by execution time and criticality - -# Common comprehensive patterns - used by both standard and comprehensive -# Standard applies exclude list to filter out slower tests -# Comprehensive runs all patterns without exclusion -positive_patterns: &positive_patterns - - "*Fusion*" - - "*/GPU_BNBWD*_*" - - "*/GPU_BNOCLBWD*_*" - - "*/GPU_BNFWD*_*" - - "*/GPU_BNOCLFWD*_*" - - "*/GPU_BNInfer*_*" - - "*/GPU_BNActivInfer_*" - - "*/GPU_BNOCLInfer*_*" - - "*/GPU_bn_infer*_*" - - "CPU_*" - - "*/CPU_*" - - "*/GPU_Cat_*" - - "*/GPU_ConvBiasActiv*" - - "*/GPU_Conv*" - - "*/GPU_conv*" - - "*/GPU_UnitTestConv*" - - "*/GPU_GetitemBwd*" - - "*/GPU_GLU_*" - - "*/GPU_GroupConv*" - - "*/GPU_GroupNorm_*" - - "*/GPU_GRUExtra_*" - - "*/GPU_TestActivation*" - - "*/GPU_HipBLASLtGEMMTest*" - - "*/GPU_KernelTuningNetTestConv*" - - "*/GPU_Kthvalue_*" - - "*/GPU_LayerNormTest*" - - "*/GPU_LayoutTransposeTest_*" - - "*/GPU_Lrn*" - - "*/GPU_lstm_extra*" - - "*/GPU_MultiMarginLoss_*" - - "*/GPU_ConvNonpack*" - - "*/GPU_PerfConfig_HipImplicitGemm*" - - "*/GPU_AsymPooling2d_*" - - "*/GPU_WidePooling2d_*" - - "*/GPU_PReLU_*" - - "*/GPU_Reduce*" - - "*/GPU_reduce_custom_*" - - "*/GPU_regression_issue_*" - - "*/GPU_RNNExtra_*" - - "*/GPU_RoPE*" - - "*/GPU_SoftMarginLoss*" - - "*/GPU_T5LayerNormTest_*" - - "*/GPU_Op4dTensorGenericTest_*" - - "*/GPU_TernaryTensorOps_*" - - "*/GPU_unaryTensorOps_*" - - "*/GPU_Transformers*" - - "*/GPU_TunaNetTest_*" - - "*/GPU_UnitTestActivationDescriptor_*" - - "*/GPU_FinInterfaceTest*" - - "*/GPU_KernelTuningNetTest*" - - "*/GPU_Bwd_Mha_*" - - "*/GPU_Fwd_Mha_*" - - "*/GPU_Softmax*" - - "*/GPU_Dropout*" - - "*/GPU_MhaBackward_*" - - "*/GPU_MhaForward_*" - - "*GPU_TestMhaFind20*" - - "*/GPU_MIOpenDriver*" - - -test_categories: - quick: - description: "Fast sanity checks - Run pre-commit (target: < 5 min)" - test_patterns: - - "Smoke/GPU_BNCKFWDTrainLarge2D_FP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge2D_FP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge3D_FP16*" - - "Smoke/GPU_BNCKFWDTrainLarge2D_BFP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge3D_BFP16*" - - "Smoke/GPU_UnitTestConvSolverImplicitGemmFwdXdlops_FP16*" - - "Smoke/GPU_UnitTestConvSolverImplicitGemmFwdXdlops_BFP16*" - - exclude: - - "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*" - - "Smoke/GPU_BNOCLInferLarge2D_BFP16*" - - "*/GPU_MIOpenDriver*" - - "*GPU_TestMhaFind20_FP32*" #rocm-1016 fixed in develop. Re-enable once bumped - - "*GPU_TestMhaFind20_FP16*" #rocm-1016 fixed in develop. Re-enable once bumped - - exclude_windows: - # The following tests are added to show functionality - - "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*" - - "Smoke/GPU_BNOCLInferLarge2D_BFP16*" - - exclude_linux: - # The following tests are added to show functionality - - "Smoke/GPU_BNOCLInferLarge2D_BFP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*" - - - test_files: - - "gtest/smoke_*.cpp" - - "gtest/unit_ActivationDescriptor.cpp" - - "gtest/unit_TensorDescriptor.cpp" - - "gtest/unit_FinInterface.cpp" - - "gtest/unit_lib_env_var.cpp" - - "gtest/gtest_activation.cpp" - - "handle_test.cpp" - - "gtest/handle_hip_device.cpp" - - "gtest/tensor_api.cpp" - - "gtest/tensor_test.cpp" - - "gtest/log.cpp" - - "gtest/log_test.cpp" - - "gtest/test_errors.cpp" - - "gtest/pooling2d_asymmetric.cpp" - - labels: - - "quick" - - "pre-commit" - - "smoke" - - standard: - description: "Core functionality - Run on every PR (target: < 30 min)" - test_patterns: *positive_patterns - - exclude: - # Excluding slower tests for faster PR validation - - "*DeepBench*" - - "*MIOpenTestConv*" - - "Full/GPU_Reduce_FP64*" - - "Full/GPU_BNOCLFWDTrainSerialRun3D_BFP16*" - - "Full/GPU_Lrn_FP32*" - - "Full/GPU_Lrn_FP16*" - - "Full/GPU_BNOCLInferSerialRun3D_BFP16*" - - "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*" - - "Smoke/GPU_BNOCLInferLarge2D_BFP16*" - - "Full/GPU_BNOCLBWDSerialRun3D_BFP16*" - - "Smoke/GPU_BNOCLBWDLarge2D_BFP16*" - - "Full/GPU_UnitTestActivationDescriptor_FP32*" - - "Full/GPU_UnitTestActivationDescriptor_FP16*" - - "Smoke/GPU_BNOCLBWDLargeFusedActivation2D_BFP16*" - - "Smoke/GPU_BNOCLBWDLargeFusedActivation2D_FP16*" - - "Full/GPU_ConvGrpBiasActivInfer_BFP16*" - - "Full/GPU_ConvGrpBiasActivInfer_FP32*" - - "Full/GPU_ConvGrpBiasActivInfer_FP16*" - - "Full/GPU_ConvGrpActivInfer_BFP16*" - - "Full/GPU_ConvGrpActivInfer_FP32*" - - "Full/GPU_ConvGrpActivInfer_FP16*" - - "Full/GPU_ConvGrpBiasActivInfer3D_BFP16*" - - "Full/GPU_ConvGrpBiasActivInfer3D_FP32*" - - "Full/GPU_ConvGrpBiasActivInfer3D_FP16*" - - "Full/GPU_ConvGrpActivInfer3D_BFP16*" - - "Full/GPU_ConvGrpActivInfer3D_FP32*" - - "Full/GPU_ConvGrpActivInfer3D_FP16*" - - "Smoke/GPU_UnitTestConvSolverHipImplicitGemmV4R1Fwd_BFP16.ConvHipImplicitGemmV4R1Fwd/0" - - "Full/GPU_MIOpenDriverConv2dTransTest*" - - "Full/GPU_MIOpenDriverRegressionBigTensorTest_FP32" - - "*GPU_TestMhaFind20_FP32*" #rocm-1016 fixed in develop. Re-enable once bumped - - "*GPU_TestMhaFind20_FP16*" #rocm-1016 fixed in develop. Re-enable once bumped - - test_files: - # Quick tests - - "gtest/smoke_*.cpp" - - "gtest/unit_*.cpp" - - "gtest/gtest_activation.cpp" - - "handle_test.cpp" - - "gtest/handle_hip_device.cpp" - - # Standard tests - - "conv2d.cpp" - - "conv2d_bias.cpp" - - "gtest/miopen_conv.cpp" - - "bn_spatial_test.cpp" - - "bn_peract_test.cpp" - - "gtest/bn_fwd_train.cpp" - - "gtest/bn_bwd.cpp" - - "gtest/bn_infer.cpp" - - "gtest/conv_api.cpp" - - "gtest/conv_activ_infer.cpp" - - "pooling2d.cpp" - - "gtest/pooling2d_asymmetric.cpp" - - "gtest/pooling2d_wide.cpp" - - "tensor_reorder.cpp" - - "tensor_trans.cpp" - - "gtest/tensor_api.cpp" - - "gtest/unary_tensor_ops.cpp" - - "gtest/binary_tensor_ops.cpp" - - "gtest/soft_max.cpp" - - "gtest/dropout.cpp" - - "gtest/reduce.cpp" - - "gtest/layernorm.cpp" - - "gtest/cache.cpp" - - labels: - - "standard" - - "pr" - - - comprehensive: - description: "Extended testing - Run nightly (target: < 2 hours)" - test_patterns: *positive_patterns - - exclude: - - "*GPU_TestMhaFind20_FP32*" #rocm-1016 fixed in develop. Re-enable once bumped - - "*GPU_TestMhaFind20_FP16*" #rocm-1016 fixed in develop. Re-enable once bumped - - test_files: - # Quick tests - - "gtest/smoke_*.cpp" - - "gtest/unit_*.cpp" - - "gtest/gtest_activation.cpp" - - "handle_test.cpp" - - "gtest/handle_hip_device.cpp" - - # Standard tests - - "conv2d.cpp" - - "conv2d_bias.cpp" - - "gtest/miopen_conv.cpp" - - "bn_spatial_test.cpp" - - "bn_peract_test.cpp" - - "gtest/bn_fwd_train.cpp" - - "gtest/bn_bwd.cpp" - - "gtest/bn_infer.cpp" - - "gtest/conv_api.cpp" - - "gtest/conv_activ_infer.cpp" - - "pooling2d.cpp" - - "gtest/pooling2d_asymmetric.cpp" - - "gtest/pooling2d_wide.cpp" - - "tensor_reorder.cpp" - - "tensor_trans.cpp" - - "gtest/tensor_api.cpp" - - "gtest/unary_tensor_ops.cpp" - - "gtest/binary_tensor_ops.cpp" - - "gtest/soft_max.cpp" - - "gtest/dropout.cpp" - - "gtest/reduce.cpp" - - "gtest/layernorm.cpp" - - "gtest/cache.cpp" - - labels: - - "comprehensive" - - "nightly" - - full: - description: "Complete test suite - (target: hours+)" - test_patterns: - # Everything - all test suite prefixes - - "Smoke/*" - - "Full/*" - - "Perf/*" - - "Unit/*" - - "*" - - exclude: - - test_files: - # All test files in the suite - - "*.cpp" - - "gtest/*.cpp" - - exclude: - # Only exclude explicitly disabled tests - - "*DISABLED*" - - labels: - - "full" - - "weekly" - - "all" - - -# Common GPU exclusion patterns - used by multiple GPU architectures -gfx11_common_patterns: &gfx11_common_patterns - - "Smoke/GPU_BNFWDTrainLargeFusedActivation2D_FP32.BnV2LargeFWD_TrainCKfp32Activation/NCHW_BNSpatial_testBNAPIV1_Dim_2_test_id_32" - - "Smoke/GPU_BNFWDTrainLarge2D_FP32.BnV2LargeFWD_TrainCKfp32/NCHW_BNSpatial_testBNAPIV2_Dim_2_test_id_64" - - "*SerialRun3D*" - - "*gfx942*" - - "*GPU_UnitTestConvSolverFFTFwd_FP32*" - - "*GPU_UnitTestConvSolverFFTBwd_FP32*" - - "*GPU_TernaryTensorOps_FP64*" - - "*GPU_TernaryTensorOps_FP16*" - - "*GPU_TernaryTensorOps_FP32*" - - "*GPU_Op4dTensorGenericTest_FP32*" - - "*GPU_UnitTestActivationDescriptor_FP16*" - - "*GPU_UnitTestActivationDescriptor_FP32*" - - "*CPU_TuningPolicy_NONE*" - - "*GPU_Dropout_FP32*" - - "*GPU_Dropout_FP16*" - - "*GPU_GroupConv3D_BackwardData_FP16.GroupConv3D_BackwardData_half_Test*" - - "*GPU_GroupConv3D_BackwardData_BFP16.GroupConv3D_BackwardData_bfloat16_Test*" - - "*GPU_UnitTestConvSolverImplicitGemmGroupWrwXdlops_BFP16.ConvHipImplicitGemmGroupWrwXdlops*" - - "Smoke/GPU_MultiMarginLoss*" - - "*CPU_UnitTestConvSolverImplicitGemmGroupWrwXdlopsDevApplicability_FP16.ConvHipImplicitGemmGroupWrwXdlops*" - # Slow tests to exclude (from #3362) - - "Full/GPU_Softmax_FP32*" # 24 min - - "Full/GPU_Softmax_BFP16*" # 13 min - - "Full/GPU_Softmax_FP16*" # 11.5 min - - "Smoke/GPU_Reduce_FP32*" # 6.5 min - - "Smoke/GPU_Reduce_FP16*" # 4.5 min - -exclude_gpu: - exclude_gpu_gfx120X: - test_patterns: *gfx11_common_patterns - labels: - - "quick" - - "standard" - - "comprehensive" - - "full" - - "ex_gpu_gfx120X" - - exclude_gpu_gfx110X: - test_patterns: - - *gfx11_common_patterns - - "*/GPU_MIOpenDriver*" - - "Smoke/CPU_Handle_NONE*" - - "Full/GPU_reduce_custom_fp32*" - labels: - - "quick" - - "standard" - - "comprehensive" - - "full" - - "ex_gpu_gfx110X" - - exclude_gpu_gfx115X: - #Negative patterns for gfx1150 and gfx1151 will be covered here. - #Add exclude_gpu_gfx1150/gfx1151 separately only if they have unique patterns. - test_patterns: *gfx11_common_patterns - labels: - - "quick" - - "standard" - - "comprehensive" - - "full" - - "ex_gpu_gfx115X" - - exclude_gpu_gfx950: - test_patterns: - - "*DBSync*" - labels: - - "quick" - - "standard" - - "comprehensive" - - "full" - - "ex_gpu_gfx950" - -execution_settings: - default_timeout: 300 # 5 minutes per test - timeout_multiplier: 2 # Multiplier for all timeouts (1, 1.5, 1.75, 2, etc.) - category_timeouts: - quick: 300 # 5 minutes - standard: 1800 # 30 minutes - comprehensive: 7200 # 2 hours - full: 14400 # 4 hours diff --git a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp b/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp deleted file mode 100644 index 4d7640ccb070..000000000000 --- a/projects/miopen/test/gtest/unit_conv_solver_ConvWinoRageRxS.cpp +++ /dev/null @@ -1,307 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include "unit_conv_solver.hpp" - -namespace { - -auto GetConvTestCases(miopenDataType_t datatype) -{ - using TestCase = miopen::unit_tests::ConvTestCase; - - return std::vector{ - // clang-format off - // rage v4.6 and v4.9(fp32 and bf16) - TestCase{{1, 16, 135, 240}, {16, 16, 3, 3}, {1, 1}, {1, 1}, {1, 1}, datatype}, - TestCase{{2, 4, 64, 64}, {16, 4, 3, 3}, {1, 1}, {1, 1}, {1, 1}, datatype}, - // rage v4.9 - TestCase{{1, 16, 135, 240}, {16, 16, 5, 5}, {2, 2}, {1, 1}, {1, 1}, datatype}, - TestCase{{2, 4, 64, 64}, {16, 4, 5, 5}, {2, 2}, {1, 1}, {1, 1}, datatype}, - // group convs - TestCase{{2, 15, 28, 28}, {15, 3, 3, 3}, {1, 1}, {1, 1}, {1, 1}, 5, datatype}, - TestCase{{2, 15, 28, 28}, {15, 3, 5, 5}, {2, 2}, {1, 1}, {1, 1}, 5, datatype}, - // clang-format on - }; -} - -auto GetConvTestCasesWrw(miopenDataType_t datatype) -{ - using TestCase = miopen::unit_tests::ConvTestCase; - - return std::vector{ - // clang-format off - TestCase{{1, 16, 5, 5}, {16, 16, 3, 3}, {0, 0}, {1, 1}, {1, 1}, datatype}, - TestCase{{1, 32, 7, 7}, { 4, 32, 3, 3}, {0, 0}, {1, 1}, {1, 1}, datatype}, - // group convs - TestCase{{1, 16, 5, 5}, {16, 1, 3, 3}, {0, 0}, {1, 1}, {1, 1}, 16, datatype}, - TestCase{{2, 16, 28, 28}, {16, 1, 5, 5}, {2, 2}, {1, 1}, {1, 1}, 16, datatype}, - // clang-format on - }; -} - -const auto& GetTestParams() -{ - static const auto params = [] { - auto p = miopen::unit_tests::UnitTestConvSolverParams(Gpu::gfx94X); - p.CheckXnackDisabled(); - return p; - }(); - return params; -} - -const auto& GetTestParamsFP16() -{ - static const auto params = [] { - auto p = miopen::unit_tests::UnitTestConvSolverParams(Gpu::gfx94X | Gpu::gfx120X); - return p; - }(); - return params; -} - -} // namespace - -using GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; -using GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; -using GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; -using GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32 = GPU_UnitTestConvSolverFwd_FP32; -using GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32 = GPU_UnitTestConvSolverBwd_FP32; -using GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32 = GPU_UnitTestConvSolverWrw_FP32; -using GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16 = GPU_UnitTestConvSolverFwd_BFP16; -using GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16 = GPU_UnitTestConvSolverBwd_BFP16; -using GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16 = GPU_UnitTestConvSolverWrw_BFP16; -using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16 = - CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; -using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32 = - CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; -using CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16 = - CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16, ConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::ConvWinoRageRxS<2, 3>{}); -}; - -// Smoke tests -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Fwd_FP16, - testing::Combine(testing::Values(GetTestParamsFP16()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenHalf)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Bwd_FP16, - testing::Combine(testing::Values(GetTestParamsFP16()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenHalf)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Wrw_FP16, - testing::Combine(testing::Values(GetTestParamsFP16()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesWrw(miopenHalf)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Fwd_FP32, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenFloat)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Bwd_FP32, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenFloat)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Wrw_FP32, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesWrw(miopenFloat)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Fwd_BFP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenBFloat16)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Bwd_BFP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCases(miopenBFloat16)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverWinoRage2x3Wrw_BFP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesWrw(miopenBFloat16)))); - -// Device applicability test -INSTANTIATE_TEST_SUITE_P(Smoke, - CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP16, - testing::Combine(testing::Values(GetTestParamsFP16()), - testing::Values(GetConvTestCases(miopenHalf)[0]))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_FP32, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(GetConvTestCases(miopenFloat)[0]))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - CPU_UnitTestConvSolverWinoRage2x3DevApplicabilityFwd_BFP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(GetConvTestCases(miopenBFloat16)[0]))); - -// ===================================================================== -// TransposedConvWinoRageRxS (NHWC layout) -// ===================================================================== - -namespace { - -auto GetConvTestCasesNHWC(miopenDataType_t datatype) -{ - using TestCase = miopen::unit_tests::ConvTestCase; - - return std::vector{ - // clang-format off - TestCase{ - {datatype, miopenTensorNHWC, {1, 40, 20, 20}}, - {datatype, miopenTensorNHWC, {20, 20, 3, 3}}, - datatype, - {{1, 1}, {1, 1}, {1, 1}, 2} - }, - // clang-format on - }; -} - -} // namespace - -using GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16 = GPU_UnitTestConvSolverFwd_FP16; -using GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16 = GPU_UnitTestConvSolverBwd_FP16; -using GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16 = GPU_UnitTestConvSolverWrw_FP16; -using CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16 = - CPU_UnitTestConvSolverDevApplicabilityFwd_NONE; - -TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16, TransposedConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16, TransposedConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16, TransposedConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); -}; - -TEST_P(CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16, - TransposedConvWinoRageRxSf2x3) -{ - this->RunTest(miopen::solver::conv::TransposedConvWinoRageRxS<2, 3>{}); -}; - -// Smoke tests -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverTransposedWinoRageRxSFwd_FP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverTransposedWinoRageRxSBwd_FP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); - -INSTANTIATE_TEST_SUITE_P(Smoke, - GPU_UnitTestConvSolverTransposedWinoRageRxSWrw_FP16, - testing::Combine(testing::Values(GetTestParams()), - testing::Values(miopenConvolutionAlgoWinograd), - testing::ValuesIn(GetConvTestCasesNHWC(miopenHalf)))); - -// Device applicability test -INSTANTIATE_TEST_SUITE_P(Smoke, - CPU_UnitTestConvSolverTransposedWinoRageRxSDevApplicabilityFwd_FP16, - testing::Combine(testing::Values(GetTestParamsFP16()), - testing::Values(GetConvTestCasesNHWC(miopenHalf)[0])));