AIRUNTIME-2610 - implement hipKernelSetCacheConfig()

projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp

@@ -48,7 +48,7 @@
 #define HIP_API_TABLE_STEP_VERSION 0
 #define HIP_COMPILER_API_TABLE_STEP_VERSION 0
 #define HIP_TOOLS_API_TABLE_STEP_VERSION 1
-#define HIP_RUNTIME_API_TABLE_STEP_VERSION 37
+#define HIP_RUNTIME_API_TABLE_STEP_VERSION 38
 
 // HIP API interface
 // HIP compiler dispatch functions
@@ -1193,6 +1193,9 @@ typedef hipError_t (*t_hipDeviceFlushGPUDirectRDMAWrites)(
 typedef hipError_t (*t_hipLibraryGetModule)(hipModule_t* pMod, hipLibrary_t library);
 typedef hipError_t (*t_hipKernelSetAttributeForDevice)(hipKernel_t kernel, hipFuncAttribute attr,
                                                         int value, int device);
+typedef hipError_t (*t_hipKernelSetCacheConfig)(hipKernel_t kernel, hipFuncCache_t config,
+                                                hipDevice_t dev);
+
 // HIP Compiler dispatch table
 struct HipCompilerDispatchTable {
   // HIP_COMPILER_API_TABLE_STEP_VERSION == 0
@@ -1858,8 +1861,11 @@ struct HipDispatchTable {
   // HIP_RUNTIME_API_TABLE_STEP_VERSION == 37
   t_hipKernelSetAttributeForDevice hipKernelSetAttributeForDevice_fn;
 
-  // DO NOT EDIT ABOVE!
   // HIP_RUNTIME_API_TABLE_STEP_VERSION == 38
+  t_hipKernelSetCacheConfig hipKernelSetCacheConfig_fn;
+
+  // DO NOT EDIT ABOVE!
+  // HIP_RUNTIME_API_TABLE_STEP_VERSION == 39
 
   // ******************************************************************************************* //
   //

projects/clr/hipamd/include/hip/amd_detail/hip_prof_str.h

@@ -510,7 +510,8 @@ enum hip_api_id_t {
   HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites = 488,
   HIP_API_ID_hipLibraryGetModule = 489,
   HIP_API_ID_hipKernelSetAttributeForDevice = 490,
-  HIP_API_ID_LAST = 490,
+  HIP_API_ID_hipKernelSetCacheConfig = 491,
+  HIP_API_ID_LAST = 491,
 
 
   HIP_API_ID_hipBindTexture = HIP_API_ID_NONE,
@@ -798,6 +799,7 @@ static inline const char* hip_api_name(const uint32_t id) {
     case HIP_API_ID_hipKernelGetParamInfo: return "hipKernelGetParamInfo";
     case HIP_API_ID_hipKernelSetAttribute: return "hipKernelSetAttribute";
     case HIP_API_ID_hipKernelSetAttributeForDevice: return "hipKernelSetAttributeForDevice";
+    case HIP_API_ID_hipKernelSetCacheConfig: return "hipKernelSetCacheConfig";
     case HIP_API_ID_hipLaunchByPtr: return "hipLaunchByPtr";
     case HIP_API_ID_hipLaunchCooperativeKernel: return "hipLaunchCooperativeKernel";
     case HIP_API_ID_hipLaunchCooperativeKernelMultiDevice: return "hipLaunchCooperativeKernelMultiDevice";
@@ -1282,6 +1284,7 @@ static inline uint32_t hipApiIdByName(const char* name) {
   if (strcmp("hipKernelGetParamInfo", name) == 0) return HIP_API_ID_hipKernelGetParamInfo;
   if (strcmp("hipKernelSetAttribute", name) == 0) return HIP_API_ID_hipKernelSetAttribute;
   if (strcmp("hipKernelSetAttributeForDevice", name) == 0) return HIP_API_ID_hipKernelSetAttributeForDevice;
+  if (strcmp("hipKernelSetCacheConfig", name) == 0) return HIP_API_ID_hipKernelSetCacheConfig;
   if (strcmp("hipLaunchByPtr", name) == 0) return HIP_API_ID_hipLaunchByPtr;
   if (strcmp("hipLaunchCooperativeKernel", name) == 0) return HIP_API_ID_hipLaunchCooperativeKernel;
   if (strcmp("hipLaunchCooperativeKernelMultiDevice", name) == 0) return HIP_API_ID_hipLaunchCooperativeKernelMultiDevice;
@@ -2976,6 +2979,11 @@ typedef struct hip_api_data_s {
       int value;
       int device;
     } hipKernelSetAttributeForDevice;
+    struct {
+      hipKernel_t kernel;
+      hipFuncCache_t config;
+      hipDevice_t device;
+    } hipKernelSetCacheConfig;
     struct {
       const void* hostFunction;
     } hipLaunchByPtr;
@@ -5916,6 +5924,12 @@ typedef struct hip_api_data_s {
   cb_data.args.hipKernelSetAttributeForDevice.value = (int)value; \
   cb_data.args.hipKernelSetAttributeForDevice.device = (int)device; \
 };
+// hipKernelSetCacheConfig[('hipKernel_t', 'kernel'), ('hipFuncCache_t', 'config'), ('hipDevice_t', 'device')]
+#define INIT_hipKernelSetCacheConfig_CB_ARGS_DATA(cb_data) { \
+  cb_data.args.hipKernelSetCacheConfig.kernel = (hipKernel_t)kernel; \
+  cb_data.args.hipKernelSetCacheConfig.config = (hipFuncCache_t)config; \
+  cb_data.args.hipKernelSetCacheConfig.device = (hipDevice_t)device; \
+};
 // hipLaunchByPtr[('const void*', 'hostFunction')]
 #define INIT_hipLaunchByPtr_CB_ARGS_DATA(cb_data) { \
   cb_data.args.hipLaunchByPtr.hostFunction = (const void*)hostFunction; \
@@ -8475,6 +8489,9 @@ static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) {
 // hipKernelSetAttributeForDevice[('hipKernel_t', 'kernel'), ('hipFuncAttribute', 'attr'), ('int', 'value'), ('int', 'device')]
     case HIP_API_ID_hipKernelSetAttributeForDevice:
       break;
+// hipKernelSetCacheConfig[('hipKernel_t', 'kernel'), ('hipFuncCache_t', 'config'), ('hipDevice_t', 'device')]
+    case HIP_API_ID_hipKernelSetCacheConfig:
+      break;
 // hipLaunchByPtr[('const void*', 'hostFunction')]
     case HIP_API_ID_hipLaunchByPtr:
       break;
@@ -11357,6 +11374,13 @@ static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* da
       oss << ", device="; roctracer::hip_support::detail::operator<<(oss, data->args.hipKernelSetAttributeForDevice.device);
       oss << ")";
     break;
+    case HIP_API_ID_hipKernelSetCacheConfig:
+      oss << "hipKernelSetCacheConfig(";
+      oss << "kernel="; roctracer::hip_support::detail::operator<<(oss, data->args.hipKernelSetCacheConfig.kernel);
+      oss << ", config="; roctracer::hip_support::detail::operator<<(oss, data->args.hipKernelSetCacheConfig.config);
+      oss << ", device="; roctracer::hip_support::detail::operator<<(oss, data->args.hipKernelSetCacheConfig.device);
+      oss << ")";
+    break;
     case HIP_API_ID_hipLaunchByPtr:
       oss << "hipLaunchByPtr(";
       oss << "hostFunction="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLaunchByPtr.hostFunction);

projects/clr/hipamd/src/amdhip.def.in

@@ -545,6 +545,7 @@ hipMipmappedArrayGetMemoryRequirements
 hipKernelGetAttribute
 hipKernelSetAttribute
 hipKernelSetAttributeForDevice
+hipKernelSetCacheConfig
 hipKernelGetFunction
 hipOccupancyMaxPotentialClusterSize
 hipOccupancyMaxActiveClusters

projects/clr/hipamd/src/hip_api_trace.cpp

@@ -947,6 +947,7 @@ hipError_t hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget t
                                              hipFlushGPUDirectRDMAWritesScope scope);
 hipError_t hipKernelSetAttributeForDevice(hipKernel_t kernel, hipFuncAttribute attr, int value,
                                           int device);
+hipError_t hipKernelSetCacheConfig(hipKernel_t kernel, hipFuncCache_t config, hipDevice_t dev);
 }  // namespace hip
 
 namespace hip {
@@ -1517,6 +1518,7 @@ void UpdateDispatchTable(HipDispatchTable* ptrDispatchTable) {
   ptrDispatchTable->hipKernelGetAttribute_fn = hip::hipKernelGetAttribute;
   ptrDispatchTable->hipKernelSetAttribute_fn = hip::hipKernelSetAttribute;
   ptrDispatchTable->hipKernelSetAttributeForDevice_fn = hip::hipKernelSetAttributeForDevice;
+  ptrDispatchTable->hipKernelSetCacheConfig_fn = hip::hipKernelSetCacheConfig;
   ptrDispatchTable->hipKernelGetFunction_fn = hip::hipKernelGetFunction;
   ptrDispatchTable->hipGreenCtxCreate_fn = hip::hipGreenCtxCreate;
   ptrDispatchTable->hipExecutionCtxDestroy_fn = hip::hipExecutionCtxDestroy;
@@ -2287,15 +2289,18 @@ HIP_ENFORCE_ABI(HipDispatchTable, hipDeviceFlushGPUDirectRDMAWrites_fn, 545);
 HIP_ENFORCE_ABI(HipDispatchTable, hipLibraryGetModule_fn, 546);
 // HIP_RUNTIME_API_TABLE_STEP_VERSION == 37
 HIP_ENFORCE_ABI(HipDispatchTable, hipKernelSetAttributeForDevice_fn, 547);
+// HIP_RUNTIME_API_TABLE_STEP_VERSION == 38
+HIP_ENFORCE_ABI(HipDispatchTable, hipKernelSetCacheConfig_fn, 548);
+
 // if HIP_ENFORCE_ABI entries are added for each new function pointer in the table, the number below
 // will be +1 of the number in the last HIP_ENFORCE_ABI line. E.g.:
 //
 //  HIP_ENFORCE_ABI(<table>, <functor>, 8)
 //
 //  HIP_ENFORCE_ABI_VERSIONING(<table>, 9) <- 8 + 1 = 9
-HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 548)
+HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 549)
 
-static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 37,
+static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 38,
               "If you get this error, add new HIP_ENFORCE_ABI(...) code for the new function "
               "pointers and then update this check so it is true");
 #endif

projects/clr/hipamd/src/hip_hcc.map.in

@@ -707,6 +707,7 @@ local:
 hip_7.18 {
 global:
     hipDeviceFlushGPUDirectRDMAWrites;
+    hipKernelSetCacheConfig;
 local:
     *;
 } hip_7.17;

projects/clr/hipamd/src/hip_library.cpp

@@ -621,6 +621,8 @@ hipError_t hipKernelSetAttributeForDevice(hipKernel_t kernel, hipFuncAttribute a
       work_group_info->kernelPreferredShmemCarveout_ = value;
       if (!work_group_info->hasFuncPreferredShmemCarveout_) {
         work_group_info->groupMemCarveout_ = value;
+        // override whatever setting we had in coarseMemCarveout_
+        work_group_info->coarseMemCarveout_ = hipFuncCachePreferNone;
       }
       break;
     case hipFuncAttributeRequiredClusterWidth:
@@ -656,6 +658,69 @@ hipError_t hipKernelSetAttributeForDevice(hipKernel_t kernel, hipFuncAttribute a
   HIP_RETURN(hipSuccess);
 }
 
+hipError_t hipKernelSetCacheConfig(hipKernel_t kernel, hipFuncCache_t config, hipDevice_t device)
+{
+  HIP_INIT_API(hipKernelSetCacheConfig, kernel, config, device);
+  hipError_t status = hipSuccess;
+
+  if (kernel == nullptr) {
+    HIP_RETURN(hipErrorInvalidResourceHandle);
+  }
+
+  if (device < 0 || static_cast<size_t>(device) >= hip::g_devices.size()) {
+    HIP_RETURN(hipErrorInvalidDevice);
+  }
+
+  // Load code object for the target device if not already loaded
+  hipKernel_t targetKernel = kernel;
+  hipLibrary_t library = nullptr;
+
+  if (hip::PlatformState::Instance().GetFunctionLibrary(kernel, &library)) {
+    auto* container = reinterpret_cast<hip::LibraryContainer*>(library);
+    const char* kernelName = nullptr;
+    if (container->GetKernelName(&kernelName, kernel) != hipSuccess || kernelName == nullptr) {
+      HIP_RETURN(hipErrorInvalidResourceHandle);
+    }
+    hipError_t status = container->ResolveKernelForDevice(&targetKernel, kernelName, device);
+    if (status != hipSuccess) {
+      HIP_RETURN(status);
+    }
+  } else {
+    HIP_RETURN(hipErrorInvalidResourceHandle);
+  }
+
+  amd::Kernel* amdKernel = hip::asKernel(targetKernel);
+  if (amdKernel == nullptr) {
+    HIP_RETURN(hipErrorInvalidResourceHandle);
+  }
+
+  const amd::Device& amdDevice = *hip::g_devices[device]->devices()[0];
+  amd::device::Kernel* deviceKernel =
+      const_cast<amd::device::Kernel*>(amdKernel->getDeviceKernel(amdDevice));
+  if (deviceKernel == nullptr) {
+    HIP_RETURN(hipErrorInvalidDeviceFunction);
+  }
+
+  device::Kernel::WorkGroupInfo* wrkGrpInfo = deviceKernel->workGroupInfo();
+
+  if (wrkGrpInfo == nullptr) {
+    HIP_RETURN(hipErrorMissingConfiguration);
+  }
+
+  switch (config) {
+  case hipFuncCachePreferNone:
+  case hipFuncCachePreferShared:
+  case hipFuncCachePreferL1:
+  case hipFuncCachePreferEqual:
+    wrkGrpInfo->coarseMemCarveout_ = static_cast<uint32_t>(config);
+    break;
+  default:
+    status = hipErrorInvalidValue;
+  }
+
+  HIP_RETURN(status);
+}
+
 hipError_t hipKernelGetFunction(hipFunction_t* pFunc, hipKernel_t kernel) {
   HIP_INIT_API(hipKernelGetFunction, pFunc, kernel);
 

projects/clr/hipamd/src/hip_module.cpp

@@ -198,6 +198,8 @@ hipError_t hipFuncGetAttribute(int* value, hipFunction_attribute attrib, hipFunc
       *value = static_cast<int>(wrkGrpInfo->maxDynamicSharedSizeBytes_);
       break;
     case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT:
+      // note: this intentionally ignores coarseMemCarveout_; i.e. it is not returning an
+      // "effective" carveout; instead behaves just like a getter
       *value = wrkGrpInfo->groupMemCarveout_;
       break;
     default:

projects/clr/hipamd/src/hip_table_interface.cpp

@@ -3377,10 +3377,14 @@ hipError_t hipModuleEnumerateFunctions(hipFunction_t* functions, unsigned int nu
   return hip::GetHipDispatchTable()->hipModuleEnumerateFunctions_fn(functions, numFunctions, mod);
   CATCH;
 }
-
 hipError_t hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget target,
                                              hipFlushGPUDirectRDMAWritesScope scope) {
   TRY;
   return hip::GetHipDispatchTable()->hipDeviceFlushGPUDirectRDMAWrites_fn(target, scope);
   CATCH;
 }
+hipError_t hipKernelSetCacheConfig(hipKernel_t kernel, hipFuncCache_t config, hipDevice_t dev) {
+  TRY;
+  return hip::GetHipDispatchTable()->hipKernelSetCacheConfig_fn(kernel, config, dev);
+  CATCH;
+}

projects/clr/rocclr/device/devkernel.cpp

@@ -638,6 +638,7 @@ Kernel::Kernel(const amd::Device& dev, const std::string& name, const Program& p
   workGroupInfo_.hasFuncMaxDynamicSharedSize_ = false;
   workGroupInfo_.hasFuncPreferredShmemCarveout_ = false;
   workGroupInfo_.groupMemCarveout_ = -1;
+  workGroupInfo_.coarseMemCarveout_ = 0; // preference = None
 }
 
 // ================================================================================================

projects/clr/rocclr/device/devkernel.hpp

@@ -224,6 +224,7 @@ class Kernel {
     bool hasFuncMaxDynamicSharedSize_;    //!< context/function-level override was set
     bool hasFuncPreferredShmemCarveout_;  //!< context/function-level override was set
     int groupMemCarveout_;                //!< effective LDS carveout; -1 selects device default
+    uint32_t coarseMemCarveout_;      //!< LDS coarse carveout (enum value as opposed to percentage)
   };
 
   //! Default constructor

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -5164,11 +5164,36 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
     }
 
     if (dev().settings().groupMemCarveout_) {
+      uint8_t percent;
+
       const int requestedPercent = devKernel->workGroupInfo()->groupMemCarveout_;
       // Signed storage preserves -1 (device default) and explicit 0; cast only resolved 0..100.
-      const uint8_t percent = requestedPercent >= 0
-                                  ? static_cast<uint8_t>(requestedPercent)
-                                  : dev().GetGroupMemCarveout();
+      percent = requestedPercent >= 0 ?
+                    static_cast<uint8_t>(requestedPercent) :
+                    dev().GetGroupMemCarveout();
+
+      if (!devKernel->workGroupInfo()->hasFuncPreferredShmemCarveout_) {
+        // hipFuncSetCacheConfig() has priority (regardless of being called before of after this
+        // function), so only use the coarse carveout if the former was not called
+        switch (devKernel->workGroupInfo()->coarseMemCarveout_) {
+        case 0:
+          // hipFuncCachePreferNone; the coarse setting is "off"
+          break;
+        case 1: // hipFuncCachePreferShared
+          percent = 100;
+          break;
+        case 2: // hipFuncCachePreferL1
+          percent = 0;
+          break;
+        case 3: // hipFuncCachePreferEqual
+          percent = 50;
+          break;
+        default:
+          assert(false && "Unexpected hipFuncCache_t value");
+          percent = 0;
+        }
+      }
+
       auto& dispatchPacketExt = dispatchPacketUnion.extKernelDispatch;
       // Encodings [1, 127] represent a range from 0% (no group memory) to 100% (maximum
       // group memory)

projects/hip-tests/CONTRACT_COVERAGE.md

@@ -13,10 +13,10 @@ contract tier intentionally pins only small, portable semantic guarantees.
 ## Snapshot
 
 <!-- contract-coverage-snapshot
-contract_tests: 627
+contract_tests: 629
 contract_domains: 118
-declared_apis: 503
-covered_apis: 490
+declared_apis: 504
+covered_apis: 491
 uncovered_allowlisted: 13
 coverage_pct: 97.4
 -->
@@ -160,7 +160,7 @@ binaries instead of exposing portable state to assert.
 | `kernel` | 4 |
 | `kernel_launch` | 7 |
 | `kernel_name_ref` | 3 |
-| `kernel_object_attributes` | 3 |
+| `kernel_object_attributes` | 5 |
 | `library` | 14 |
 | `library_file` | 3 |
 | `logging` | 3 |

projects/hip-tests/catch/TEST_PLAN.md

@@ -13,8 +13,8 @@ Each row is one `HIP_TEST_CASE`. The API and invariant come from the `// @assert
 
 | Tier | Cases | Tagged | Missing `@asserts` |
 |---|---:|---:|---:|
-| `contract` | 627 | 627 | 0 |
-| **total** | **627** | **627** | **0** |
+| `contract` | 629 | 629 | 0 |
+| **total** | **629** | **629** | **0** |
 
 ## Tier: `contract`
 
@@ -704,7 +704,7 @@ Each row is one `HIP_TEST_CASE`. The API and invariant come from the `// @assert
 | `Contract_KernelNameRef_HipKernelNameRef_ByFunction_NamesResolvedKernel` | hipKernelNameRef | resolves a hipFunction_t (from hipGetFuncBySymbol) to a non-empty name mentioning the kernel identifier |
 | `Contract_KernelNameRef_HipKernelNameRef_NvidiaUnsupported_IsSkipped` | hipKernelNameRef | NVIDIA backend does not expose this API family; the contract is skipped until backend parity exists |
 
-### `kernel_object_attributes` (5 cases)
+### `kernel_object_attributes` (7 cases)
 
 | Case | API | Asserts |
 |---|---|---|
@@ -713,6 +713,8 @@ Each row is one `HIP_TEST_CASE`. The API and invariant come from the `// @assert
 | `Contract_KernelObjectAttributes_HipKernelSetAttributeForDevice_NonCurrentDevice_PreservesCurrentAndReadsBack` | hipKernelSetAttributeForDevice | preserves current device and target value |
 | `Contract_KernelObjectAttributes_HipKernelSetAttributeForDevice_NvidiaPre128Unsupported_IsSkipped` | hipKernelSetAttributeForDevice | unavailable before CUDA 12.8 |
 | `Contract_KernelObjectAttributes_HipKernelSetAttribute_SetMaxDynamicSharedMemory_IsAcceptedOrUnsupported` | hipKernelSetAttribute | setting max dynamic shared memory to zero is either accepted or reported as unsupported, never another error |
+| `Contract_KernelObjectAttributes_HipKernelSetCacheConfig_NvidiaPre120Unsupported_IsSkipped` | hipKernelSetCacheConfig | unavailable before CUDA 12.0 |
+| `Contract_KernelObjectAttributes_HipKernelSetCacheConfig_Values_AreSupported` | hipKernelSetCacheConfig | all hipFuncCache_t values are supported and kernel |
 
 ### `library` (21 cases)
 

projects/hip-tests/catch/config/configs/contract.yaml

@@ -1869,6 +1869,12 @@ contract:
   Contract_KernelObjectAttributes_HipKernelSetAttributeForDevice_NvidiaPre128Unsupported_IsSkipped:
     <<: *level_0
     tags: [kernel_object_attributes, P0]
+  Contract_KernelObjectAttributes_HipKernelSetCacheConfig_Values_AreSupported:
+    <<: *level_0
+    tags: [kernel_object_attributes, P0]
+  Contract_KernelObjectAttributes_HipKernelSetCacheConfig_NvidiaPre120Unsupported_IsSkipped:
+    <<: *level_0
+    tags: [kernel_object_attributes, P0]
   Contract_KernelObjectAttributes_HipKernelGetParamInfo_Default_ReturnsFirstParamLayout:
     <<: *level_0
     tags: [kernel_object_attributes, P0]

projects/hip-tests/catch/config/configs/unit/library.yaml

@@ -18,6 +18,12 @@ library:
   Unit_hipKernelGetAttribute_Positive_Basic:
     <<: *level_default
     tags: [module]
+  Unit_hipKernelSetCacheConfig_Positive:
+    <<: *level_2
+    tags: [module]
+  Unit_hipKernelSetCacheConfig_Negative:
+    <<: *level_2
+    tags: [module]
   Unit_hipKernelGetAttribute_Negative_Parameters:
     <<: *level_default
     tags: [module]