feat(rocr): Implement ROCr APIs for GL2 residency control

projects/rocprofiler-sdk/VERSION

@@ -1 +1 @@
-1.4.1
+1.4.2

projects/rocprofiler-sdk/source/include/rocprofiler-sdk/cxx/enum_string.hpp

@@ -440,6 +440,9 @@ ROCPROFILER_ENUM_LABEL(ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_interop_map_buffer
 #    if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x13
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_svm_discard_and_prefetch_batch_async);
 #    endif
+#    if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x14
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_agent_set_attribute);
+#    endif
 #endif
 
 #if HSA_AMD_EXT_API_TABLE_MAJOR_VERSION == 0x01
@@ -485,6 +488,8 @@ static_assert(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST == 89);
 static_assert(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST == 91);
 #    elif HSA_AMD_EXT_API_TABLE_STEP_VERSION == 0x13
 static_assert(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST == 92);
+#    elif HSA_AMD_EXT_API_TABLE_STEP_VERSION == 0x14
+static_assert(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST == 93);
 #    else
 #        if !defined(ROCPROFILER_UNSAFE_NO_VERSION_CHECK) &&                                       \
             (defined(ROCPROFILER_CI) && ROCPROFILER_CI > 0)

projects/rocprofiler-sdk/source/include/rocprofiler-sdk/hsa/amd_ext_api_id.h

@@ -163,6 +163,9 @@ typedef enum rocprofiler_hsa_amd_ext_api_id_t  // NOLINT(performance-enum-size)
 #    if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x13
     ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_svm_discard_and_prefetch_batch_async,
 #    endif
+#    if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x14
+    ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_agent_set_attribute,
+#    endif
 #endif
 
     ROCPROFILER_HSA_AMD_EXT_API_ID_LAST,

projects/rocprofiler-sdk/source/include/rocprofiler-sdk/hsa/api_args.h

@@ -1597,6 +1597,14 @@ typedef union rocprofiler_hsa_api_args_t
         hsa_signal_t        completion_signal;
     } hsa_amd_svm_discard_and_prefetch_batch_async;
 #    endif
+#    if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x14
+    struct
+    {
+        hsa_agent_t               agent;
+        hsa_amd_agent_attribute_t attribute;
+        void*                     value;
+    } hsa_amd_agent_set_attribute;
+#    endif
 #endif
 } rocprofiler_hsa_api_args_t;
 

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/abi.cpp

@@ -81,6 +81,8 @@ ROCP_SDK_ENFORCE_ABI_VERSIONING(::AmdExtTable, 90);
 ROCP_SDK_ENFORCE_ABI_VERSIONING(::AmdExtTable, 92);
 #elif HSA_AMD_EXT_API_TABLE_STEP_VERSION == 0x13
 ROCP_SDK_ENFORCE_ABI_VERSIONING(::AmdExtTable, 93);
+#elif HSA_AMD_EXT_API_TABLE_STEP_VERSION == 0x14
+ROCP_SDK_ENFORCE_ABI_VERSIONING(::AmdExtTable, 94);
 #else
 INTERNAL_CI_ROCP_SDK_ENFORCE_ABI_VERSIONING(::AmdExtTable, 0);
 #endif
@@ -359,6 +361,9 @@ ROCP_SDK_ENFORCE_ABI(::AmdExtTable, hsa_amd_interop_map_buffer_with_size_fn, 91)
 #if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x13
 ROCP_SDK_ENFORCE_ABI(::AmdExtTable, hsa_amd_svm_discard_and_prefetch_batch_async_fn, 92);
 #endif
+#if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x14
+ROCP_SDK_ENFORCE_ABI(::AmdExtTable, hsa_amd_agent_set_attribute_fn, 93);
+#endif
 
 ROCP_SDK_ENFORCE_ABI(::ImageExtTable, hsa_ext_image_get_capability_fn, 1);
 ROCP_SDK_ENFORCE_ABI(::ImageExtTable, hsa_ext_image_data_get_info_fn, 2);

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/hsa.def.cpp

@@ -668,6 +668,15 @@ HSA_API_INFO_DEFINITION_V(
     dep_signals,
     completion_signal)
 #        endif
+#        if HSA_AMD_EXT_API_TABLE_STEP_VERSION >= 0x14
+HSA_API_INFO_DEFINITION_V(ROCPROFILER_HSA_TABLE_ID_AmdExt,
+                          ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_agent_set_attribute,
+                          hsa_amd_agent_set_attribute,
+                          hsa_amd_agent_set_attribute_fn,
+                          agent,
+                          attribute,
+                          value)
+#        endif
 #    endif
 
 #elif defined(ROCPROFILER_LIB_ROCPROFILER_HSA_ASYNC_COPY_CPP_IMPL) &&                              \

projects/rocr-runtime/libhsakmt/include/hsakmt/drm/amdgpu_drm.h

@@ -648,11 +648,15 @@ union drm_amdgpu_sem {
 /* vm ioctl */
 #define AMDGPU_VM_OP_RESERVE_VMID	1
 #define AMDGPU_VM_OP_UNRESERVE_VMID	2
+#define AMDGPU_VM_OP_GL2_PERSISTING_L2_CACHE 3
 
 struct drm_amdgpu_vm_in {
 	/** AMDGPU_VM_OP_* */
 	__u32	op;
-	__u32	flags;
+	union {
+		 __u32 flags;
+		 __u32 size;
+	};
 };
 
 struct drm_amdgpu_vm_out {
@@ -1634,6 +1638,7 @@ struct drm_amdgpu_info_device {
 	/* Userq IP mask (1 << AMDGPU_HW_IP_*) */
 	__u32 userq_ip_mask;
 	__u32 pad;
+	__u32 persisting_gl2_cache_size_max;
 };
 
 struct drm_amdgpu_info_hw_ip {

projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt.h

@@ -1498,6 +1498,22 @@ hsaKmtGetAmdGPUDeviceFd(
   int *fd //OUT
 );
 
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtSetPersistingCacheSizeCtx(
+    HsaKFDContext *ctx,
+    HSAuint32 Node,
+    HSAuint64 CacheSize
+    );
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtSetPersistingCacheSize(
+    HSAuint32 Node,
+    HSAuint64 CacheSize
+    );
+
 #ifdef __cplusplus
 }   //extern "C"
 #endif

projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmttypes.h

@@ -455,6 +455,7 @@ typedef struct _HaCacheProperties
     HSAuint32    CacheLatency;     // Cache latency in ns
     HsaCacheType CacheType;
     HSAuint32    SiblingMap[HSA_CPU_SIBLINGS];
+    HSAuint32    PersistingCacheSizeMax; //Maximum persisting cache size in L2 Cache topology.
 } HsaCacheProperties;
 
 

projects/rocr-runtime/libhsakmt/src/fmm.c

@@ -3795,6 +3795,17 @@ HSAKMT_STATUS hsakmt_fmm_advance_vm_timeline(HsaKFDContext *ctx,
 	return HSAKMT_STATUS_SUCCESS;
 }
 
+int hsakmt_fmm_get_drm_render_fd(HsaKFDContext *ctx, HSAuint32 node_id)
+{
+	struct hsa_kfd_fmm_context *fmm_ctx = ctx->fmm_context;
+	int32_t index = gpu_mem_find_by_node_id(fmm_ctx, node_id);
+
+	if (index < 0)
+		return -1;
+
+	return fmm_ctx->gpu_mem[index].drm_render_fd;
+}
+
 HSAKMT_STATUS hsakmt_fmm_get_aperture_base_and_limit(HsaKFDContext *ctx,
 			aperture_type_e aperture_type, HSAuint32 gpu_id,
 			HSAuint64 *aperture_base, HSAuint64 *aperture_limit)

projects/rocr-runtime/libhsakmt/src/fmm.h

@@ -155,6 +155,8 @@ HSAKMT_STATUS hsakmt_fmm_advance_vm_timeline(HsaKFDContext *ctx,
 		HSAuint32 node_id, int *drm_render_fd,
 		uint32_t *vm_timeline_syncobj, uint64_t *vm_timeline_point);
 
+int hsakmt_fmm_get_drm_render_fd(HsaKFDContext *ctx, HSAuint32 node_id);
+
 int hsakmt_open_drm_render_device(HsaKFDContext *ctx, int minor);
 void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align,
 			    uint64_t guard_size, void *aper_base, void *aper_limit, int fd);

projects/rocr-runtime/libhsakmt/src/memory.c

@@ -284,6 +284,46 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryCtx(HsaKFDContext *ctx,
 				   NULL, 0, flags);
 }
 
+
+
+// Configure the persisting GL2 (L2) cache size for a GPU node.
+//
+// The kernel implements this through the amdgpu render-node ioctl
+// DRM_IOCTL_AMDGPU_VM (amdgpu_vm_ioctl -> AMDGPU_VM_OP_GL2_PERSISTING_L2_CACHE),
+// not through a KFD ioctl. So the request must be issued on the per-node DRM
+// render fd, not on the KFD device fd.
+HSAKMT_STATUS HSAKMTAPI hsaKmtSetPersistingCacheSizeCtx(HsaKFDContext *ctx,
+												HSAuint32 Node,
+												HSAuint64 CacheSize) {
+	union drm_amdgpu_vm args = {0};
+	int drm_fd;
+	int ret;
+
+	CHECK_KFD_OPEN();
+
+	pr_debug("[%s] node %d size %lu\n", __func__, Node, CacheSize);
+
+	/* Get the amdgpu render-node fd for this KFD node */
+	drm_fd = hsakmt_fmm_get_drm_render_fd(ctx, Node);
+	if (drm_fd < 0) {
+		pr_err("[%s] invalid node ID: %d\n", __func__, Node);
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	}
+
+	args.in.op = AMDGPU_VM_OP_GL2_PERSISTING_L2_CACHE;
+	if (CacheSize > UINT32_MAX)
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	args.in.size = (uint32_t)CacheSize;
+
+	ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_VM, &args);
+	if (ret) {
+		pr_err("[%s] DRM_IOCTL_AMDGPU_VM GL2 persisting failed: %d\n", __func__, ret);
+		return HSAKMT_STATUS_ERROR;
+	}
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodesCtx(HsaKFDContext *ctx,
 						    void *MemoryAddress,
 						    HSAuint64 MemorySizeInBytes,
@@ -935,6 +975,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId,
 	return hsaKmtGetAMDGPUDeviceHandleCtx(&hsakmt_primary_kfd_ctx, NodeId, DeviceHandle);
 }
 
+HSAKMT_STATUS HSAKMTAPI hsaKmtSetPersistingCacheSize(HSAuint32 Node,
+												HSAuint64 CacheSize)
+{
+	return hsaKmtSetPersistingCacheSizeCtx(&hsakmt_primary_kfd_ctx, Node, CacheSize);
+}
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtHandleExport(const HsaHandleExportDesc* desc,
 					   HsaMemoryExportResult* res,
 					   HsaHandleExportFlags* flags)

projects/rocr-runtime/libhsakmt/src/topology.c

@@ -1679,6 +1679,8 @@ static HSAKMT_STATUS topology_sysfs_get_cache_props(
 			props->CacheLatency = (uint32_t)prop_val;
 		else if (strcmp(prop_name, "type") == 0)
 			props->CacheType.Value = (uint32_t)prop_val;
+        else if(strcmp(prop_name, "persisting_cache_size_max") == 0)
+            props->PersistingCacheSizeMax = (uint32_t)prop_val;
 		else if (strcmp(prop_name, "sibling_map") == 0)
 			break;
 	}

projects/rocr-runtime/rocrtst/config/platform_config.yaml

@@ -103,6 +103,14 @@ groups:
     - rocrtstFunc.Memory_Atomic_Dec_Test
     - rocrtstFunc.Memory_Atomic_Xchg_Test
 
+  GL2Cache:
+    - rocrtstFunc.GL2_PersistingCache_QueryMax
+    - rocrtstFunc.GL2_PersistingCache_QueryRequest
+    - rocrtstFunc.GL2_PersistingCache_Set
+    - rocrtstFunc.GL2_PersistingCache_SetAndReadBack
+    - rocrtstFunc.GL2_PersistingCache_NegativeInvalidSize
+    - rocrtstFunc.GL2_PersistingCache_NegativeCPUAgent
+
   Performance:
     - rocrtstPerf.Memory_Async_Copy_On_Engine
     - rocrtstPerf.ENQUEUE_LATENCY
@@ -173,6 +181,12 @@ platforms:
       - rocrtstFunc.GpuCoreDump_InvalidPath
       - rocrtstFunc.GpuCoreDump_ContentIntegrity
       - rocrtstFunc.GpuCoreDump_PipePattern
+      - rocrtstFunc.GL2_PersistingCache_QueryMax
+      - rocrtstFunc.GL2_PersistingCache_QueryRequest
+      - rocrtstFunc.GL2_PersistingCache_Set
+      - rocrtstFunc.GL2_PersistingCache_SetAndReadBack
+      - rocrtstFunc.GL2_PersistingCache_NegativeInvalidSize
+      - rocrtstFunc.GL2_PersistingCache_NegativeCPUAgent
       - rocrtstPerf.Memory_Async_Copy_On_Engine
       - rocrtstNeg.Memory_Negative_Tests
       - rocrtstNeg.Queue_Validation_InvalidDimension