feat(rocprofiler-sdk): Add SPM aqlprofile support for gfx12/gfx1250

projects/rocprofiler-sdk/source/lib/aqlprofile/CMakeLists.txt

@@ -23,6 +23,19 @@ add_library(rocprofiler-sdk-aqlprofile STATIC)
 add_library(rocprofiler-sdk::rocprofiler-sdk-aqlprofile ALIAS rocprofiler-sdk-aqlprofile)
 target_include_directories(rocprofiler-sdk-aqlprofile
                            INTERFACE $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/source>)
+
+set(AQLPROFILE_DEBUG_TRACE
+    "0"
+    CACHE STRING "AQLProfile DEBUG_TRACE level (0, 1, or 2)")
+set_property(CACHE AQLPROFILE_DEBUG_TRACE PROPERTY STRINGS 0 1 2)
+
+if(AQLPROFILE_DEBUG_TRACE STREQUAL "1" OR AQLPROFILE_DEBUG_TRACE STREQUAL "2")
+    target_compile_definitions(rocprofiler-sdk-aqlprofile
+                               PRIVATE DEBUG_TRACE=${AQLPROFILE_DEBUG_TRACE})
+elseif(NOT AQLPROFILE_DEBUG_TRACE STREQUAL "0")
+    message(FATAL_ERROR "AQLPROFILE_DEBUG_TRACE must be one of: 0, 1, 2")
+endif()
+
 target_link_libraries(
     rocprofiler-sdk-aqlprofile
     PRIVATE $<BUILD_INTERFACE:rocprofiler-sdk::rocprofiler-sdk-headers>

projects/rocprofiler-sdk/source/lib/aqlprofile/aql_profile_v2.h

@@ -164,8 +164,7 @@ typedef enum
 {
     AQLPROFILE_SPM_DEPTH_NONE,
     AQLPROFILE_SPM_DEPTH_16_BITS,
-    AQLPROFILE_SPM_DEPTH_32_BITS,
-    AQLPROFILE_SPM_DEPTH_64_BITS
+    AQLPROFILE_SPM_DEPTH_32_BITS
 } aqlprofile_spm_depth_t;
 
 /**
@@ -745,7 +744,8 @@ typedef enum
 {
     AQLPROFILE_SPM_PARAMETER_TYPE_BUFFER_SIZE = 0,
     AQLPROFILE_SPM_PARAMETER_TYPE_SAMPLE_INTERVAL,  ///< Sample interval in clock cycles.
-                                                    ///< Must be a multiple of 32.
+                                                    ///< Exact range/alignment is GPU-family
+                                                    ///< dependent.
     AQLPROFILE_SPM_PARAMETER_TYPE_TIMEOUT,
     AQLPROFILE_SPM_PARAMETER_TYPE_SAMPLE_MODE,
     AQLPROFILE_SPM_PARAMETER_TYPE_LAST,
@@ -873,6 +873,16 @@ aqlprofile_spm_start(aqlprofile_handle_t            handle,
 hsa_status_t
 aqlprofile_spm_stop(aqlprofile_handle_t handle);
 
+/**
+ * @brief Callback invoked for each decoded SPM sample.
+ *
+ * The `shader_engine` argument uses the following contract:
+ * - `-1` for global SPM samples
+ * - otherwise a packed topology value compatible with
+ *   ::aqlprofile_spm_decode_shader_engine(). For the base non-global sample,
+ *   that packed value is numerically equal to the shader-engine index because
+ *   `sa_index` and `wgp_index` are both zero.
+ */
 typedef void (*aqlprofile_spm_decode_callback_v1_t)(uint64_t timestamp,
                                                     uint64_t value,
                                                     uint64_t index,
@@ -911,6 +921,26 @@ aqlprofile_spm_decode_query(aqlprofile_spm_buffer_desc_t  desc,
                             aqlprofile_spm_decode_query_t query,
                             uint64_t*                     param_out);
 
+/**
+ * @brief Decode the topology information packed into the `shader_engine`
+ *        value returned by ::aqlprofile_spm_decode_callback_v1_t.
+ *
+ * For global SPM events, `shader_engine` is `-1` and this helper returns
+ * `se_index = sa_index = wgp_index = -1`.
+ *
+ * @param[in] shader_engine Packed `shader_engine` value from the decode callback
+ * @param[out] se_index     Decoded shader-engine index, or `-1` for global events
+ * @param[out] sa_index     Decoded shader-array index, or `-1` if not encoded
+ * @param[out] wgp_index    Decoded workgroup-processor index, or `-1` if not encoded
+ * @retval HSA_STATUS_SUCCESS                if decode successful
+ * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT if any output pointer is null
+ */
+hsa_status_t
+aqlprofile_spm_decode_shader_engine(int  shader_engine,
+                                    int* se_index,
+                                    int* sa_index,
+                                    int* wgp_index);
+
 bool
 aqlprofile_spm_is_event_supported(aqlprofile_agent_handle_t agent, aqlprofile_pmc_event_t event);
 

projects/rocprofiler-sdk/source/lib/aqlprofile/core/aql_profile.cpp

@@ -301,7 +301,6 @@ hsa_ven_amd_aqlprofile_start(hsa_ven_amd_aqlprofile_profile_t* profile,
             pm4_builder::TraceConfig trace_config{};
             const uint64_t           se_number_total = pm4_factory->GetShaderEnginesNumber();
 
-            trace_config.spm_sq_32bit_mode = true;
             trace_config.spm_has_core1 = (pm4_factory->GetGpuId() == aql_profile::MI100_GPU_ID) ||
                                          (pm4_factory->GetGpuId() == aql_profile::MI200_GPU_ID);
             trace_config.spm_sample_delay_max = pm4_factory->GetSpmSampleDelayMax();

projects/rocprofiler-sdk/source/lib/aqlprofile/core/gfx1250_factory.cpp

@@ -46,6 +46,18 @@ class Mi450Factory : public Pm4Factory
     bool IsGFX12() const override { return true; }
     bool IsGFX1250() const override { return true; }
 
+    uint32_t EncodeSpmBlockIndex(uint32_t inst_index,
+                                 uint32_t sa_index,
+                                 uint32_t wgp_index) const override
+    {
+        return gfx12_cntx_prim::encode_spm_block_index(inst_index, sa_index, wgp_index);
+    }
+    uint32_t DecodeSpmInstanceIndex(const GpuBlockInfo* block_info,
+                                    uint32_t            block_index) const override
+    {
+        return gfx12_cntx_prim::decode_spm_instance_index(block_info, block_index);
+    }
+
     virtual int GetAccumLowID() const override { return 1; };
     virtual int GetAccumHiID() const override { return 1; };
     virtual int GetSQGAccumID() const override { return 13; };

projects/rocprofiler-sdk/source/lib/aqlprofile/core/gfx12_factory.cpp

@@ -42,7 +42,18 @@ class Gfx12Factory : public Pm4Factory
     {
         Init(agent_info);
     }
-    bool IsGFX12() const override { return true; }
+    bool     IsGFX12() const override { return true; }
+    uint32_t EncodeSpmBlockIndex(uint32_t inst_index,
+                                 uint32_t sa_index,
+                                 uint32_t wgp_index) const override
+    {
+        return gfx12_cntx_prim::encode_spm_block_index(inst_index, sa_index, wgp_index);
+    }
+    uint32_t DecodeSpmInstanceIndex(const GpuBlockInfo* block_info,
+                                    uint32_t            block_index) const override
+    {
+        return gfx12_cntx_prim::decode_spm_instance_index(block_info, block_index);
+    }
 
 protected:
     void ConstructBuilders(const AgentInfo* agent_info);

projects/rocprofiler-sdk/source/lib/aqlprofile/core/memorymanager.hpp

@@ -390,5 +390,23 @@ class SPMMemoryManager : public MemoryManager
         outputbuf_size    = size;
     }
 
+    // DEBUG_TRACE-only buffer for copying out RLC_SPM_SAMPLE_CNT.
+    // This is used during SPM integration-test debugging to compare the hardware
+    // sample-pulse count against the number of received segments. For a successful
+    // run, those two numbers should match.
+    void CreateSampleCountBuf()
+    {
+        aqlprofile_buffer_desc_flags_t flags{};
+        flags.host_access   = true;
+        flags.device_access = true;
+        flags.memory_hint   = AQLPROFILE_MEMORY_HINT_DEVICE_NONCOHERENT;
+        sample_count_buf_   = AllocMemory(sizeof(uint32_t), flags);
+    }
+
+    void* GetSampleCountBuf() const { return sample_count_buf_.get(); }
+
     pm4_builder::TraceConfig config{};
+
+private:
+    std::unique_ptr<void, MemoryDeleter> sample_count_buf_ = nullptr;
 };

projects/rocprofiler-sdk/source/lib/aqlprofile/core/pm4_factory.h

@@ -171,6 +171,19 @@ class Pm4Factory
     virtual bool IsGFX11() const { return false; }
     virtual bool IsGFX12() const { return false; }
     virtual bool IsGFX1250() const { return false; }
+
+    virtual uint32_t EncodeSpmBlockIndex(uint32_t inst_index,
+                                         uint32_t sa_index,
+                                         uint32_t wgp_index) const
+    {
+        return inst_index;
+    }
+    virtual uint32_t DecodeSpmInstanceIndex(const GpuBlockInfo* block_info,
+                                            uint32_t            block_index) const
+    {
+        return block_index;
+    }
+
     // Return number of XCC on the GPU
     uint32_t GetXccNumber() const { return agent_info_->xcc_num; }
     // Return number of XCC per AID
@@ -184,7 +197,8 @@ class Pm4Factory
         const GpuBlockInfo* info = block_map_.Get(event->block_name);
         if(info == nullptr) throw std::runtime_error("Bad Block");
         // Checking that the block index is in proper range
-        if(event->block_index >= info->instance_count) throw std::runtime_error("Bad Index");
+        if(DecodeSpmInstanceIndex(info, event->block_index) >= info->instance_count)
+            throw std::runtime_error("Bad Index");
             // Checking that the counter event index is in proper range
 #if 0
     if (event->counter_id > info->event_id_max)
@@ -199,7 +213,7 @@ class Pm4Factory
         const GpuBlockInfo* info = block_map_.Get(event->block_name);
         if(info == nullptr) throw event_exception(std::string("Bad block, "), *event);
         // Checking that the block index is in proper range
-        if(event->block_index >= info->instance_count)
+        if(DecodeSpmInstanceIndex(info, event->block_index) >= info->instance_count)
             throw event_exception(std::string("Bad block index, "), *event);
             // Checking that the counter event index is in proper range
 #if 0

projects/rocprofiler-sdk/source/lib/aqlprofile/core/spm_common.hpp

@@ -15,6 +15,12 @@ operator<(const aqlprofile_handle_t& a, const aqlprofile_handle_t& b)
 
 #define SPM_DESC_SIZE 0x1000
 
+static constexpr uint16_t SPM_COUNTER_MAP_GLOBAL_FLAG = 0x8000;
+static constexpr uint16_t SPM_COUNTER_MAP_SA_FLAG     = 0x4000;
+static constexpr uint16_t SPM_COUNTER_MAP_WGP_FLAG    = 0x2000;
+static constexpr uint16_t SPM_COUNTER_MAP_32BIT_FLAG  = 0x1000;
+static constexpr uint16_t SPM_COUNTER_MAP_INDEX_MASK  = 0x0FFF;
+
 // Once KFD change is merged, we should use the definition from linux/include/uapi/linux/kfd_ioctl.h
 struct kfd_ioctl_spm_buffer_header
 {
@@ -31,8 +37,12 @@ typedef struct SpmBufferDesc_
     uint32_t se_num_line{0};
     uint32_t num_se{0};
     uint32_t num_sa{0};
+    uint32_t num_wgp{0};
     uint32_t num_xcc{0};
     size_t   num_events{0};
 
     uint16_t* get_counter_map() { return (uint16_t*) (this + 1); }
 } SpmBufferDesc;
+
+static constexpr size_t SPM_COUNTER_MAP_CAPACITY =
+    (SPM_DESC_SIZE - sizeof(SpmBufferDesc)) / sizeof(uint16_t);

projects/rocprofiler-sdk/source/lib/aqlprofile/core/spm_decode.cpp

@@ -25,6 +25,37 @@
 #    define PUBLIC_API __attribute__((visibility("default")))
 #endif
 
+static inline int
+encode_spm_shader_engine(uint32_t se_index, uint32_t sa_index = 0, uint32_t wgp_index = 0)
+{
+    return int((wgp_index << 24) | (sa_index << 16) | se_index);
+}
+
+static inline void
+decode_spm_shader_engine(int shader_engine, int* se_index, int* sa_index, int* wgp_index)
+{
+    if(shader_engine < 0)
+    {
+        *se_index  = -1;
+        *sa_index  = -1;
+        *wgp_index = -1;
+        return;
+    }
+
+    const uint32_t packed = static_cast<uint32_t>(shader_engine);
+    *se_index             = static_cast<int>(packed & 0xFFFF);
+    *sa_index             = static_cast<int>((packed >> 16) & 0xFF);
+    *wgp_index            = static_cast<int>((packed >> 24) & 0xFF);
+}
+
+PUBLIC_API hsa_status_t
+aqlprofile_spm_decode_shader_engine(int shader_engine, int* se_index, int* sa_index, int* wgp_index)
+{
+    if(!se_index || !sa_index || !wgp_index) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+    decode_spm_shader_engine(shader_engine, se_index, sa_index, wgp_index);
+    return HSA_STATUS_SUCCESS;
+}
+
 PUBLIC_API hsa_status_t
 aqlprofile_spm_decode_query(aqlprofile_spm_buffer_desc_t  desc_bin,
                             aqlprofile_spm_decode_query_t query,
@@ -65,35 +96,86 @@ aqlprofile_spm_decode_stream_v1(aqlprofile_spm_buffer_desc_t        desc_bin,
     size_t          datasize = _size / sizeof(uint16_t);
     uint16_t* const data_end = datain + datasize;
 
+    auto decode_bufvalue = [](uint16_t lo, uint16_t hi, bool is_32bit) {
+        if(is_32bit)
+        {
+            if((lo == 0xFFFF) && (hi == 0xFFFF)) return uint64_t(-1);
+            return uint64_t(lo) | (uint64_t(hi) << 16);
+        }
+
+        if(lo == 0xFFFF) return uint64_t(-1);
+        return uint64_t(lo);
+    };
+
     while(datain < data_end)
     {
         if(datain + seg_elem > data_end) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
 
         uint64_t timestamp = *(uint64_t*) datain;
+        size_t   i_exp     = desc->num_events;
 
         for(int i = 0; i < desc->num_events; i++)
         {
-            uint64_t counter_value = 0;
-
             uint16_t index     = desc->get_counter_map()[i];
-            bool     is_global = (index & 0x8000) ? true : false;
-            index &= 0x7FFF;
+            bool     is_global = (index & SPM_COUNTER_MAP_GLOBAL_FLAG) ? true : false;
+            bool     is_sa     = (index & SPM_COUNTER_MAP_SA_FLAG) ? true : false;
+            bool     is_32bit  = (index & SPM_COUNTER_MAP_32BIT_FLAG) ? true : false;
+            bool     is_wgp    = (index & SPM_COUNTER_MAP_WGP_FLAG) ? true : false;
+            index &= SPM_COUNTER_MAP_INDEX_MASK;
+            uint32_t sa_count  = is_sa ? desc->num_sa : 1;
+            uint32_t wgp_count = is_wgp ? desc->num_wgp : 1;
+#if defined(DEBUG_TRACE)
+            if(is_wgp && wgp_count == 0)
+            {
+                std::fprintf(stderr,
+                             "Invalid SPM descriptor: WGP-flagged event has num_wgp == 0\n");
+            }
+#endif
+            size_t expanded_count = size_t(sa_count) * size_t(wgp_count) - 1;
 
             if(is_global)
             {
-                auto bufvalue = datain[index];
+                auto bufvalue =
+                    decode_bufvalue(datain[index], is_32bit ? datain[index + 16] : 0, is_32bit);
                 decode_cb(timestamp, bufvalue, i, -1, userdata);
             }
             else
             {
-                uint16_t se_base = desc->global_num_line * 16;
-                uint16_t se_step = desc->se_num_line * 16;
+                uint16_t se_base         = desc->global_num_line * 16;
+                uint16_t se_step         = desc->se_num_line * 16;
+                size_t   event_exp_start = i_exp;
                 for(int j = 0; j < desc->num_se; j++)
                 {
-                    auto bufvalue = datain[index + se_base + se_step * j];
-                    decode_cb(timestamp, bufvalue, i, j, userdata);
+                    auto bufvalue =
+                        decode_bufvalue(datain[index + se_base + se_step * j],
+                                        is_32bit ? datain[index + 16 + se_base + se_step * j] : 0,
+                                        is_32bit);
+                    decode_cb(timestamp, bufvalue, i, encode_spm_shader_engine(j), userdata);
+
+                    size_t event_i_exp = event_exp_start;
+                    for(uint32_t sa = 0; sa < sa_count; ++sa)
+                        for(uint32_t wgp = 0; wgp < wgp_count; ++wgp)
+                        {
+                            // The base SE sample already covers the original (sa=0, wgp=0) case,
+                            // so only the additional SA/WGP-expanded projections are emitted here.
+                            if((sa == 0) && (wgp == 0)) continue;
+
+                            uint16_t expanded_index =
+                                desc->get_counter_map()[event_i_exp++] & SPM_COUNTER_MAP_INDEX_MASK;
+                            auto bufvalue = decode_bufvalue(
+                                datain[expanded_index + se_base + se_step * j],
+                                is_32bit ? datain[expanded_index + 16 + se_base + se_step * j] : 0,
+                                is_32bit);
+                            decode_cb(timestamp,
+                                      bufvalue,
+                                      i,
+                                      encode_spm_shader_engine(j, sa, wgp),
+                                      userdata);
+                        }
                 }
             }
+
+            i_exp += expanded_count;
         }
 
         datain += seg_elem;