Various tweaks to enable Wave32 for Radeon cards with HIP.

Author: seanofthemillers

CMakeLists.txt

@@ -171,6 +171,7 @@ endif()
 
 if(RAJA_ENABLE_HIP)
   message(STATUS "HIP version: ${hip_VERSION}")
+  set(RAJA_DEFAULT_WAVESIZE "64" CACHE STRING "Default wave size for GPU architecture. E.g. MI200/MI300 this is 64.")
   if("${hip_VERSION}" VERSION_LESS "3.5")
     message(FATAL_ERROR "Trying to use HIP/ROCm version ${hip_VERSION}. RAJA requires HIP/ROCm version 3.5 or newer. ")
   endif()

include/RAJA/config.hpp.in

@@ -182,6 +182,8 @@ static_assert(RAJA_HAS_SOME_CXX14,
 #cmakedefine RAJA_ENABLE_NV_TOOLS_EXT
 #cmakedefine RAJA_ENABLE_ROCTX
 
+#cmakedefine RAJA_DEFAULT_WAVESIZE @RAJA_DEFAULT_WAVESIZE@
+
 /*!
  ******************************************************************************
  *

include/RAJA/pattern/kernel/InitLocalMem.hpp

@@ -77,23 +77,16 @@ struct StatementExecutor<statement::InitLocalMem<RAJA::cpu_tile_mem,camp::idx_se
     using varType = typename camp::tuple_element_t<Pos, typename camp::decay<Data>::param_tuple_t>::value_type;
 
     // Initialize memory
-#ifdef RAJA_COMPILER_MSVC
-    // MSVC doesn't like taking a pointer to stack allocated data?!?!
     varType *ptr = new varType[camp::get<Pos>(data.param_tuple).size()];
     camp::get<Pos>(data.param_tuple).set_data(ptr);
-#else
-    varType Array[camp::get<Pos>(data.param_tuple).size()];
-    camp::get<Pos>(data.param_tuple).set_data(&Array[0]);
-#endif
+
 
     // Initialize others and execute
     exec_expanded<others...>(data);
 
     // Cleanup and return
     camp::get<Pos>(data.param_tuple).set_data(nullptr);
-#ifdef RAJA_COMPILER_MSVC
     delete[] ptr;
-#endif
   }
 
 

include/RAJA/policy/hip/policy.hpp

@@ -324,12 +324,9 @@ struct DeviceConstants
 // values for HIP warp size and max block size.
 //
 #if defined(__HIP_PLATFORM_AMD__)
-#if defined(RAJA_HIP_WAVE64)
-constexpr DeviceConstants device_constants(RAJA_HIP_WAVESIZE, 1024, 64); // MI300A
-// constexpr DeviceConstants device_constants(64, 1024, 128); // MI250X
-#else
-constexpr DeviceConstants device_constants(32, 1024, 64); // Radeon cards (e.g. gfx1100)
-#endif
+constexpr DeviceConstants device_constants(RAJA_DEFAULT_WAVESIZE, 1024, 64); // MI300A
+// constexpr DeviceConstants device_constants(RAJA_DEFAULT_WAVESIZE, 1024, 128); // MI250X
+
 #elif defined(__HIP_PLATFORM_NVIDIA__)
 constexpr DeviceConstants device_constants(32, 1024, 32); // V100
 #endif