From cbddf6fb3dc7a08c2d1cd6ce64b22c81a3b5a3c8 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Mon, 20 Jul 2026 21:46:02 +0000 Subject: [PATCH 01/45] refactor(rocprofiler-sdk): migrate counter collection off the per-queue callback registry Isolates the counter-collection slice of the callback-removal effort (reference PRs #8730 / #8586) into a small, single-service change, per review guidance to land callback removal one service at a time. Independent of kernel replay. Counter collection no longer registers a queue_callbacks_t with the HSA queue controller. Instead the HSA write interceptor and async signal handler call explicit free functions: - counters::write_hook / signal_completion_hook / is_any_active (new counters/queue_hooks.{hpp,cpp}) iterating active dispatch_counter_collection contexts and calling the existing queue_cb / completed_cb - hsa/queue_hooks/client_ids.hpp: stable inst_pkt producer tags The per-queue callback registry is intentionally retained for thread-trace, pc-sampling and spm; those migrate in follow-up PRs. The callback thread (callback_thread_start/stop) is preserved. The WriteInterceptor gate and should_batch predicate detect counters via is_any_active() (it no longer contributes to queue.get_notifiers()). Removes the now-unused queue_id from counter_callback_info and updates counter tests off iterate_callbacks. Co-authored-by: Cursor --- .../rocprofiler-sdk/counters/CMakeLists.txt | 6 +- .../lib/rocprofiler-sdk/counters/core.cpp | 62 +----------- .../lib/rocprofiler-sdk/counters/core.hpp | 3 - .../rocprofiler-sdk/counters/queue_hooks.cpp | 99 +++++++++++++++++++ .../rocprofiler-sdk/counters/queue_hooks.hpp | 68 +++++++++++++ .../counters/tests/CMakeLists.txt | 2 +- .../rocprofiler-sdk/counters/tests/core.cpp | 22 +---- .../counters/tests/queue_hooks_test.cpp | 37 +++++++ .../lib/rocprofiler-sdk/hsa/CMakeLists.txt | 1 + .../source/lib/rocprofiler-sdk/hsa/queue.cpp | 30 +++++- .../hsa/queue_hooks/CMakeLists.txt | 9 ++ .../hsa/queue_hooks/client_ids.hpp | 43 ++++++++ 12 files changed, 300 insertions(+), 82 deletions(-) create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/CMakeLists.txt index 0f3b95f9dd55..7d3e09a55c0b 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/CMakeLists.txt @@ -9,7 +9,8 @@ set(ROCPROFILER_LIB_COUNTERS_SOURCES controller.cpp device_counting.cpp ioctl.cpp - firmware_restrictions.cpp) + firmware_restrictions.cpp + queue_hooks.cpp) set(ROCPROFILER_LIB_COUNTERS_HEADERS metrics.hpp dimensions.hpp @@ -22,7 +23,8 @@ set(ROCPROFILER_LIB_COUNTERS_HEADERS device_counting.hpp sample_consumer.hpp ioctl.hpp - firmware_restrictions.hpp) + firmware_restrictions.hpp + queue_hooks.hpp) target_sources(rocprofiler-sdk-object-library PRIVATE ${ROCPROFILER_LIB_COUNTERS_SOURCES} ${ROCPROFILER_LIB_COUNTERS_HEADERS}) add_subdirectory(parser) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index f53406a9104c..20753a8a5f6a 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -161,48 +161,10 @@ start_context(const context::context* ctx) if(!already_enabled) { + // Counter collection no longer registers a per-queue callback with the queue + // controller; the HSA write interceptor calls counters::write_hook / + // signal_completion_hook directly (see hsa/queue.cpp). Keep the callback thread. callback_thread_start(); - - for(auto& cb : ctx->dispatch_counter_collection->callbacks) - { - using external_corr_id_map_t = tracing::external_correlation_id_map_t; - - // Insert our callbacks into HSA Interceptor. This - // turns on counter instrumentation. - if(cb->queue_id != rocprofiler::hsa::ClientID{-1}) continue; - cb->queue_id = controller->add_callback( - std::nullopt, - hsa::queue_callbacks_t{.batch_packets = []() { return false; }, - .write_interceptor = - [=](const hsa::Queue& q, - const hsa::rocprofiler_packet& kern_pkt, - rocprofiler_kernel_id_t kernel_id, - rocprofiler_dispatch_id_t dispatch_id, - rocprofiler_user_data_t* user_data, - const external_corr_id_map_t& extern_corr_ids, - const context::correlation_id* correlation_id) { - return queue_cb(ctx, - cb, - q, - kern_pkt, - kernel_id, - dispatch_id, - user_data, - extern_corr_ids, - correlation_id); - }, - // Completion CB - .signal_completion = - [=](const hsa::Queue& /*queue*/, - hsa::rocprofiler_packet /*kern_pkt*/, - std::shared_ptr& session, - hsa::packet_data_t& packet, - inst_pkt_t& aql, - kernel_dispatch::profiling_time dispatch_time) { - completed_cb( - ctx, cb, session, packet, aql, dispatch_time); - }}); - } } } @@ -221,22 +183,8 @@ stop_context(const context::context* ctx) if(controller) { controller->disable_serialization(); - - // In attach mode, remove counter callbacks when stopping the context so - // new dispatches during detach drain can pass through without incrementing - // _active_kernels. In normal profiling, keep callbacks registered to avoid - // dropping counter data for in-flight dispatches. - if(registration::is_attached()) - { - for(auto& cb : ctx->dispatch_counter_collection->callbacks) - { - if(cb->queue_id != rocprofiler::hsa::ClientID{-1}) - { - controller->remove_callback(cb->queue_id); - cb->queue_id = rocprofiler::hsa::ClientID{-1}; - } - } - } + // No per-queue callback to remove; counters::write_hook no-ops once + // dispatch_counter_collection is disabled above. } callback_thread_stop(); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp index 588098cedad5..77ffbca63c93 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp @@ -52,9 +52,6 @@ struct counter_callback_info rocprofiler_context_id_t context{.handle = 0}; // Link to the internal context this is associated with const context::context* internal_context{nullptr}; - // HSA Queue ClientID. This is an ID we get when we insert a callback into the - // HSA queue interceptor. This ID can be used to disable the callback. - rocprofiler::hsa::ClientID queue_id{-1}; // Buffer to use for storing counter data. Used if callback is not set. std::optional buffer; diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp new file mode 100644 index 000000000000..8d4883d5cc52 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -0,0 +1,99 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" +#include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp" + +namespace rocprofiler +{ +namespace counters +{ +namespace +{ +auto +active_counter_contexts_filter() +{ + return [](const context::context* ctx) -> bool { + return ctx && ctx->dispatch_counter_collection != nullptr; + }; +} +} // namespace + +void +write_hook(const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + rocprofiler_kernel_id_t kernel_id, + rocprofiler_dispatch_id_t dispatch_id, + rocprofiler_user_data_t* user_data, + const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, + const context::correlation_id* correlation_id, + hsa::inst_pkt_t& inst_pkt, + bool& is_serialized) +{ + auto active = context::get_active_contexts(active_counter_contexts_filter()); + for(const auto* ctx : active) + { + for(auto& cb : ctx->dispatch_counter_collection->callbacks) + { + auto [packet, bSerial] = queue_cb(ctx, + cb, + queue, + kernel_packet, + kernel_id, + dispatch_id, + user_data, + ext_corr_ids, + correlation_id); + if(packet) + inst_pkt.emplace_back(std::move(packet), hsa::queue_hooks::COUNTERS_CLIENT_ID); + is_serialized |= bSerial; + } + } +} + +void +signal_completion_hook(const hsa::Queue& /*queue*/, + const hsa::rocprofiler_packet& /*kernel_packet*/, + std::shared_ptr& session, + hsa::packet_data_t& packet, + hsa::inst_pkt_t& inst_pkt, + kernel_dispatch::profiling_time dispatch_time) +{ + auto active = context::get_active_contexts(active_counter_contexts_filter()); + for(const auto* ctx : active) + { + for(auto& cb : ctx->dispatch_counter_collection->callbacks) + { + completed_cb(ctx, cb, session, packet, inst_pkt, dispatch_time); + } + } +} + +bool +is_any_active() +{ + return !context::get_active_contexts(active_counter_contexts_filter()).empty(); +} +} // namespace counters +} // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp new file mode 100644 index 000000000000..f775c8124dc2 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp @@ -0,0 +1,68 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/hsa/queue.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_info_session.hpp" +#include "lib/rocprofiler-sdk/hsa/rocprofiler_packet.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" + +#include + +namespace rocprofiler +{ +namespace counters +{ +// Explicit replacement for the dispatch-counter per-queue callback that used to +// be registered with the HSA queue controller. Iterates active +// dispatch_counter_collection contexts and calls each callback's queue_cb; +// appends produced packets to inst_pkt, OR-folding each callback's serialize flag +// into is_serialized. +void +write_hook(const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + rocprofiler_kernel_id_t kernel_id, + rocprofiler_dispatch_id_t dispatch_id, + rocprofiler_user_data_t* user_data, + const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, + const context::correlation_id* correlation_id, + hsa::inst_pkt_t& inst_pkt, + bool& is_serialized); + +// Explicit replacement for the dispatch-counter completion callback. Iterates +// active dispatch_counter_collection contexts and calls each callback's +// completed_cb. +void +signal_completion_hook(const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + std::shared_ptr& session, + hsa::packet_data_t& packet, + hsa::inst_pkt_t& inst_pkt, + kernel_dispatch::profiling_time dispatch_time); + +// True if any context currently has dispatch counter collection active. +bool +is_any_active(); +} // namespace counters +} // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt index ca1d4f1b6fd5..6ec7bd403190 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt @@ -61,7 +61,7 @@ target_link_libraries( set(ROCPROFILER_LIB_COUNTER_TEST_SOURCES metrics_test.cpp evaluate_ast_test.cpp dimension.cpp core.cpp code_object_loader.cpp - device_counting.cpp firmware_restrictions_test.cpp) + device_counting.cpp firmware_restrictions_test.cpp queue_hooks_test.cpp) set(ROCPROFILER_LIB_COUNTER_TEST_HEADERS code_object_loader.hpp device_counting.hpp) add_executable(counter-tests) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/core.cpp index ac3c77f663ee..26e77c265324 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/core.cpp @@ -560,19 +560,12 @@ TEST(core, start_stop_buffered_ctx) ASSERT_TRUE(ctx.dispatch_counter_collection->callbacks.at(0)->buffer); EXPECT_EQ(*ctx.dispatch_counter_collection->callbacks.at(0)->buffer, opt_buff_id); + // Counter collection no longer registers a per-queue callback; activeness is observable via + // the context enabled flag (and counters::is_any_active()). bool found = false; ctx.dispatch_counter_collection->enabled.rlock([&](const auto& data) { found = data; }); EXPECT_TRUE(found); - found = false; - hsa::get_queue_controller()->iterate_callbacks([&](auto cid, const auto&) { - if(cid == ctx.dispatch_counter_collection->callbacks.at(0)->queue_id) - { - found = true; - } - }); - EXPECT_TRUE(found); - /** * Check if context can be disabled correctly */ @@ -627,19 +620,12 @@ TEST(core, start_stop_callback_ctx) (void*) 0x54321); EXPECT_EQ(ctx.dispatch_counter_collection->callbacks.at(0)->context, get_client_ctx()); + // Counter collection no longer registers a per-queue callback; activeness is observable via + // the context enabled flag (and counters::is_any_active()). bool found = false; ctx.dispatch_counter_collection->enabled.rlock([&](const auto& data) { found = data; }); EXPECT_TRUE(found); - found = false; - hsa::get_queue_controller()->iterate_callbacks([&](auto cid, const auto&) { - if(cid == ctx.dispatch_counter_collection->callbacks.at(0)->queue_id) - { - found = true; - } - }); - EXPECT_TRUE(found); - /** * Check if context can be disabled correctly */ diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp new file mode 100644 index 000000000000..864a16624527 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -0,0 +1,37 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" + +#include + +namespace +{ +// is_any_active replaces the queue.get_notifiers() signal that the old per-queue +// callback registration used to provide to the HSA write interceptor gate. With +// no dispatch-counter-collection context active, it must report inactive. This +// requires no GPU / HSA runtime. +TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) +{ + EXPECT_FALSE(rocprofiler::counters::is_any_active()); +} +} // namespace diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/CMakeLists.txt index ab27514b1822..c085f2c7f7cd 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/CMakeLists.txt @@ -39,3 +39,4 @@ target_sources(rocprofiler-sdk-object-library PRIVATE ${ROCPROFILER_LIB_HSA_SOUR ${ROCPROFILER_LIB_HSA_HEADERS}) add_subdirectory(details) +add_subdirectory(queue_hooks) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp index 63561e628687..428b1bf95af6 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp @@ -28,6 +28,7 @@ #include "lib/common/utility.hpp" #include "lib/rocprofiler-sdk/code_object/code_object.hpp" #include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" #include "lib/rocprofiler-sdk/hip/graph.hpp" #include "lib/rocprofiler-sdk/hsa/details/fmt.hpp" #include "lib/rocprofiler-sdk/hsa/hsa.hpp" @@ -168,6 +169,15 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data) } }); + // Counter collection completion is migrated off the callback registry (see + // WriteInterceptor); invoke it explicitly here. + counters::signal_completion_hook(queue_info_session.queue, + packet.kernel_packet, + _session, + packet, + packet.instrumentation_packets, + dispatch_time); + if(packet.is_serialized) { CHECK_NOTNULL(hsa::get_queue_controller()) @@ -305,8 +315,11 @@ WriteInterceptor(const void* packets, auto* gls = ::rocprofiler::hip::graph::current_launch_state(); const bool graph_launch_active = (gls != nullptr); + // Counter collection no longer registers a queue-controller callback, so it does not count + // toward get_notifiers(); detect it explicitly so a counters-only run still enters the + // interceptor. const bool no_real_consumers = - (queue.get_notifiers() == 0 && + (queue.get_notifiers() == 0 && !counters::is_any_active() && context::get_active_contexts(full_packet_instrumentation_context_filter).empty()); if(pkt_count == 0 || (no_real_consumers && !graph_launch_active)) @@ -617,6 +630,18 @@ WriteInterceptor(const void* packets, } }); + // Counter collection is migrated off the per-queue callback registry: call its hook + // explicitly (the other services still flow through signal_callback above). + counters::write_hook(queue, + kernel_packet, + kernel_id, + dispatch_id, + &_packet_data.user_data, + _packet_data.tracing_data.external_correlation_ids, + corr_id, + _packet_data.instrumentation_packets, + _packet_data.is_serialized); + bool inserted_before = false; if(_packet_data.is_serialized) { @@ -751,6 +776,9 @@ WriteInterceptor(const void* packets, } }); + // Counter collection requires per-packet mode; it no longer participates in the registry. + if(counters::is_any_active()) should_batch_packets = false; + if(should_batch_packets) { ROCP_TRACE_IF(pkt_count > 1) << fmt::format( diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/CMakeLists.txt new file mode 100644 index 000000000000..5f5e8f8481f2 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/CMakeLists.txt @@ -0,0 +1,9 @@ +# +# Shared, header-only client-id tags used by the per-service queue hooks. +# +set(ROCPROFILER_LIB_HSA_QUEUE_HOOKS_SOURCES) +set(ROCPROFILER_LIB_HSA_QUEUE_HOOKS_HEADERS client_ids.hpp) + +target_sources( + rocprofiler-sdk-object-library PRIVATE ${ROCPROFILER_LIB_HSA_QUEUE_HOOKS_SOURCES} + ${ROCPROFILER_LIB_HSA_QUEUE_HOOKS_HEADERS}) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp new file mode 100644 index 000000000000..9e4879b262bc --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp @@ -0,0 +1,43 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include + +namespace rocprofiler +{ +namespace hsa +{ +namespace queue_hooks +{ +// Tags identifying the producer subsystem of each inst_pkt_t entry. +// Only distinctness matters; values are fixed for test-order stability. +// Services are migrated off the per-queue callback registry one at a time; the +// unused ids are reserved so the tag values stay stable across those PRs. +constexpr int64_t COUNTERS_CLIENT_ID = 1; +constexpr int64_t THREAD_TRACE_CLIENT_ID = 2; +constexpr int64_t PC_SAMPLING_CLIENT_ID = 3; +constexpr int64_t SPM_CLIENT_ID = 4; +} // namespace queue_hooks +} // namespace hsa +} // namespace rocprofiler From e0b0782591f6c1dceb2a7d2d07aeabbe13c4dee1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Jul 2026 04:31:35 +0000 Subject: [PATCH 02/45] fix(rocprofiler-sdk): route counter completions after stop_context Address review feedback on #8891: completion routing must use registered contexts (packet_return_map provenance), stop_context must drain in-flight GPU work before deactivation, and queue-interposed services must stop while the context remains active for serialization coordination. Adds queue_hooks regression test for stop-then-complete ordering required by kernel replay (#7960/#8622). Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/context/context.cpp | 26 ++- .../lib/rocprofiler-sdk/counters/core.cpp | 6 +- .../rocprofiler-sdk/counters/queue_hooks.cpp | 13 +- .../rocprofiler-sdk/counters/queue_hooks.hpp | 5 +- .../counters/tests/queue_hooks_test.cpp | 208 +++++++++++++++++- 5 files changed, 235 insertions(+), 23 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 0bfdeabc7387..75b2d4efb185 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -368,6 +368,20 @@ stop_context(rocprofiler_context_id_t idx) const context* _expected = itr.load(std::memory_order_acquire); if(_expected && _expected->context_idx == idx.handle) { + // Stop queue-interposed services while the context is still in the active list so + // write_hook can coordinate serialized->unserialized transitions during drain. + if(_expected->dispatch_counter_collection) + { + rocprofiler::counters::stop_context(const_cast(_expected)); + } + + if(_expected->dispatch_spm) + rocprofiler::spm::stop_context(const_cast(_expected)); + + if(_expected->device_thread_trace) _expected->device_thread_trace->stop_context(); + if(_expected->dispatch_thread_trace) + _expected->dispatch_thread_trace->stop_context(); + bool success = itr.compare_exchange_strong(_expected, nullptr); if(success) @@ -375,18 +389,6 @@ stop_context(rocprofiler_context_id_t idx) auto nactive = get_num_active_contexts().load(std::memory_order_acquire); if(nactive > 0) get_num_active_contexts().fetch_sub(1, std::memory_order_release); - if(_expected->dispatch_counter_collection) - { - rocprofiler::counters::stop_context(const_cast(_expected)); - } - - if(_expected->dispatch_spm) - rocprofiler::spm::stop_context(const_cast(_expected)); - - if(_expected->device_thread_trace) _expected->device_thread_trace->stop_context(); - if(_expected->dispatch_thread_trace) - _expected->dispatch_thread_trace->stop_context(); - rocprofiler::hsa::queue_interposition:: notify_queue_interposition_consumer_context_stopped(_expected); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 20753a8a5f6a..aaca98fbae00 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -182,9 +182,11 @@ stop_context(const context::context* ctx) if(controller) { + // Drain in-flight dispatches while the context remains in the active list (see + // context::stop_context ordering). Completions are routed via registered contexts in + // counters::signal_completion_hook. + hsa::queue_controller_sync(); controller->disable_serialization(); - // No per-queue callback to remove; counters::write_hook no-ops once - // dispatch_counter_collection is disabled above. } callback_thread_stop(); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index 8d4883d5cc52..16d60447e0a3 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -32,7 +32,7 @@ namespace counters namespace { auto -active_counter_contexts_filter() +counter_contexts_filter() { return [](const context::context* ctx) -> bool { return ctx && ctx->dispatch_counter_collection != nullptr; @@ -51,7 +51,7 @@ write_hook(const hsa::Queue& queue, hsa::inst_pkt_t& inst_pkt, bool& is_serialized) { - auto active = context::get_active_contexts(active_counter_contexts_filter()); + auto active = context::get_active_contexts(counter_contexts_filter()); for(const auto* ctx : active) { for(auto& cb : ctx->dispatch_counter_collection->callbacks) @@ -80,8 +80,11 @@ signal_completion_hook(const hsa::Queue& /*queue*/, hsa::inst_pkt_t& inst_pkt, kernel_dispatch::profiling_time dispatch_time) { - auto active = context::get_active_contexts(active_counter_contexts_filter()); - for(const auto* ctx : active) + // Route by packet provenance, not current activeness: completed_cb self-filters via + // packet_return_map, so in-flight dispatches still complete after stop_context removes the + // context from the active list (required for kernel replay and ordinary stop/drain). + auto contexts = context::get_registered_contexts(counter_contexts_filter()); + for(const auto* ctx : contexts) { for(auto& cb : ctx->dispatch_counter_collection->callbacks) { @@ -93,7 +96,7 @@ signal_completion_hook(const hsa::Queue& /*queue*/, bool is_any_active() { - return !context::get_active_contexts(active_counter_contexts_filter()).empty(); + return !context::get_active_contexts(counter_contexts_filter()).empty(); } } // namespace counters } // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp index f775c8124dc2..fdebeff1f93e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp @@ -51,8 +51,9 @@ write_hook(const hsa::Queue& queue, bool& is_serialized); // Explicit replacement for the dispatch-counter completion callback. Iterates -// active dispatch_counter_collection contexts and calls each callback's -// completed_cb. +// registered dispatch_counter_collection contexts (not only active ones) and calls +// each callback's completed_cb; completed_cb self-filters via packet_return_map so +// in-flight dispatches still complete after stop_context. void signal_completion_hook(const hsa::Queue& queue, const hsa::rocprofiler_packet& kernel_packet, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 864a16624527..e9925cd8e98e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -17,15 +17,105 @@ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +#include "lib/rocprofiler-sdk/counters/core.hpp" +#include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" #include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" +#include "lib/rocprofiler-sdk/counters/tests/hsa_tables.hpp" +#include "lib/rocprofiler-sdk/agent.hpp" +#include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/hsa/queue.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" +#include "lib/rocprofiler-sdk/registration.hpp" + +#include +#include #include +#include + +#include + +using namespace rocprofiler::counters::test_constants; +using namespace rocprofiler; + +#define ROCPROFILER_CALL(result, msg) \ + { \ + rocprofiler_status_t CHECKSTATUS = result; \ + ASSERT_EQ(CHECKSTATUS, ROCPROFILER_STATUS_SUCCESS) \ + << msg << ": " << rocprofiler_get_status_string(CHECKSTATUS); \ + } + +namespace rocprofiler +{ +namespace hsa +{ +class QueueHooksFakeQueue : public Queue +{ +public: + QueueHooksFakeQueue(const AgentCache& a, rocprofiler_queue_id_t id) + : Queue(a, get_api_table()) + , _agent(a) + , _id(id) + {} + const AgentCache& get_agent() const final { return _agent; }; + rocprofiler_queue_id_t get_id() const final { return _id; }; + + ~QueueHooksFakeQueue() override = default; + +private: + const AgentCache& _agent; + rocprofiler_queue_id_t _id = {}; +}; +} // namespace hsa +} // namespace rocprofiler namespace { +rocprofiler_context_id_t& +get_client_ctx() +{ + static rocprofiler_context_id_t ctx{0}; + return ctx; +} + +void +test_init() +{ + HsaApiTable table; + table.amd_ext_ = &get_ext_table(); + table.core_ = &get_api_table(); + agent::construct_agent_cache(&table); + ASSERT_TRUE(hsa::get_queue_controller() != nullptr); + hsa::get_queue_controller()->init(get_api_table(), get_ext_table()); +} + +struct expected_dispatch +{ + rocprofiler_counter_config_id_t id = {.handle = 0}; + rocprofiler_queue_id_t queue_id = {.handle = 0}; + rocprofiler_agent_id_t agent_id = {.handle = 0}; + uint64_t kernel_id = 0; + uint64_t dispatch_id = 0; +}; + +void +user_dispatch_cb(rocprofiler_dispatch_counting_service_data_t dispatch_data, + rocprofiler_counter_config_id_t* config, + rocprofiler_user_data_t*, + void* callback_data_args) +{ + auto& expected = *static_cast(callback_data_args); + ASSERT_NE(config, nullptr); + config->handle = expected.id.handle; + (void) dispatch_data; +} + // is_any_active replaces the queue.get_notifiers() signal that the old per-queue // callback registration used to provide to the HSA write interceptor gate. With // no dispatch-counter-collection context active, it must report inactive. This @@ -34,4 +124,118 @@ TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) { EXPECT_FALSE(rocprofiler::counters::is_any_active()); } + +// Regression for callback-registry removal: a dispatch enqueued while the context +// is active must still complete (and drain packet_return_map) when stop_context +// runs before the GPU completion callback. Kernel replay depends on this because +// each replay pass reuses the same counter-collection completion path. +TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_path) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + registration::init_logging(); + registration::set_init_status(-1); + context::push_client(1); + + ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed"); + ROCPROFILER_CALL( + rocprofiler_configure_callback_dispatch_counting_service(get_client_ctx(), + user_dispatch_cb, + nullptr, + nullptr, + nullptr), + "Could not setup counting service"); + ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context"); + + auto* ctx_p = context::get_mutable_registered_context(get_client_ctx()); + ASSERT_TRUE(ctx_p); + ASSERT_TRUE(ctx_p->dispatch_counter_collection); + ASSERT_EQ(ctx_p->dispatch_counter_collection->callbacks.size(), 1); + auto cb_info = ctx_p->dispatch_counter_collection->callbacks.front(); + + ASSERT_TRUE(hsa::get_queue_controller() != nullptr); + auto agents = hsa::get_queue_controller()->get_supported_agents(); + ASSERT_GT(agents.size(), 0); + + const auto& [_, agent] = *agents.begin(); + ASSERT_TRUE(agent.get_rocp_agent()); + + rocprofiler_counter_id_t counter_id = {.handle = 0}; + { + auto mets = rocprofiler::counters::loadMetrics(); + const auto* gfx_metrics = + rocprofiler::common::get_val(mets->arch_to_metric, std::string(agent.name())); + ASSERT_TRUE(gfx_metrics); + ASSERT_FALSE(gfx_metrics->empty()); + for(const auto& metric : *gfx_metrics) + { + if(metric.expression().empty()) + { + counter_id.handle = metric.id(); + break; + } + } + } + ASSERT_NE(counter_id.handle, 0U); + + expected_dispatch expected = {}; + ROCPROFILER_CALL(rocprofiler_create_counter_config( + agent.get_rocp_agent()->id, &counter_id, 1, &expected.id), + "Unable to create profile"); + cb_info->callback_args = &expected; + expected.queue_id = {.handle = 1}; + expected.agent_id = agent.get_rocp_agent()->id; + expected.kernel_id = 42; + expected.dispatch_id = 7; + + hsa::QueueHooksFakeQueue fq(agent, expected.queue_id); + hsa::rocprofiler_packet pkt{}; + context::correlation_id corr_id{.internal = 99}; + + auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; + auto ret_pkt = rocprofiler::counters::queue_cb(ctx_p, + cb_info, + fq, + pkt, + expected.kernel_id, + expected.dispatch_id, + &user_data, + {}, + &corr_id); + ASSERT_TRUE(ret_pkt.packet); + + size_t map_size_before_stop = 0; + cb_info->packet_return_map.rlock( + [&](const auto& data) { map_size_before_stop = data.size(); }); + ASSERT_EQ(map_size_before_stop, 1U) << "queue_cb should register instrumentation in the map"; + + // Simulate stop while the dispatch is still in flight on the GPU. + ROCPROFILER_CALL(rocprofiler_stop_context(get_client_ctx()), "stop context"); + EXPECT_FALSE(rocprofiler::counters::is_any_active()); + + auto sess = std::make_shared(hsa::queue_info_session_t{.queue = fq}); + auto& packet_data = sess->packet_data.emplace_back(); + sess->correlation_id = &corr_id; + + rocprofiler::counters::inst_pkt_t inst_pkt; + inst_pkt.emplace_back( + std::make_pair(std::move(ret_pkt.packet), hsa::queue_hooks::COUNTERS_CLIENT_ID)); + + rocprofiler::counters::signal_completion_hook(fq, + pkt, + sess, + packet_data, + inst_pkt, + rocprofiler::kernel_dispatch::profiling_time{}); + + size_t map_size_after_completion = 1; + cb_info->packet_return_map.rlock( + [&](const auto& data) { map_size_after_completion = data.size(); }); + EXPECT_EQ(map_size_after_completion, 0U) + << "packet_return_map must drain via signal_completion_hook after stop_context"; + + registration::set_init_status(1); + registration::finalize(); +} } // namespace From 8b6a4d0954fa1497f2aec7741805721c7afc0435 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Jul 2026 22:56:32 +0000 Subject: [PATCH 03/45] fix(rocprofiler-sdk): do not drain the GPU in counters::stop_context Reverts the queue_controller_sync() added earlier on this branch. The justification given for it was wrong: it claimed callback_thread_stop() could strand records belonging to dispatches still in flight. It cannot. - consumer_thread_t::exit() sets valid=false, notifies, then waits on the exited flag, and consumer_loop only sets exited once read_ptr == write_ptr, so everything already queued is consumed before the join. counters/tests/ consumer_test.cpp restart already covers this drain-on-exit path. - consumer_thread_t::add() falls back to calling consume_fn inline when valid is false, so a completion arriving after the thread is gone is still processed on the async signal handler thread. Nothing else in counters::stop_context needs an idle GPU. In-flight completions are handled by signal_completion_hook routing over registered rather than active contexts, and the serializer transition is reconciled GPU-side by the hsa_barrier that profiler_serializer::disable() pushes. This aligns the counters path with the same change on the SPM branch, prompted by review feedback from @SrirackshaNag noting that SPM was the only service with a sync in its stop path and that the hook migration removes its original reasons. The context::stop_context ordering is kept but re-justified: with no drain, the point is that disable_serialization() runs before dispatches stop being instrumented, so no dispatch is submitted without serializer packets while the serializer is still enabled. Co-authored-by: Vivek Kale --- .../source/lib/rocprofiler-sdk/context/context.cpp | 7 +++++-- .../source/lib/rocprofiler-sdk/counters/core.cpp | 12 ++++++++---- .../lib/rocprofiler-sdk/counters/queue_hooks.cpp | 3 ++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 75b2d4efb185..e112423d49e3 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -368,8 +368,11 @@ stop_context(rocprofiler_context_id_t idx) const context* _expected = itr.load(std::memory_order_acquire); if(_expected && _expected->context_idx == idx.handle) { - // Stop queue-interposed services while the context is still in the active list so - // write_hook can coordinate serialized->unserialized transitions during drain. + // Stop queue-interposed services before clearing the active slot so + // disable_serialization() always runs before dispatches stop being instrumented. + // Clearing the slot first opens the opposite window, in which write_hook sees no + // active context and a dispatch is submitted without serializer packets while the + // serializer is still enabled. if(_expected->dispatch_counter_collection) { rocprofiler::counters::stop_context(const_cast(_expected)); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index aaca98fbae00..82d1f52fbca1 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -182,13 +182,17 @@ stop_context(const context::context* ctx) if(controller) { - // Drain in-flight dispatches while the context remains in the active list (see - // context::stop_context ordering). Completions are routed via registered contexts in - // counters::signal_completion_hook. - hsa::queue_controller_sync(); + // No GPU drain here. In-flight dispatches are handled by signal_completion_hook routing + // over registered rather than active contexts; the serializer transition is reconciled + // GPU-side by the hsa_barrier that profiler_serializer::disable() pushes. controller->disable_serialization(); + // No per-queue callback to remove; counters::write_hook no-ops once + // dispatch_counter_collection is disabled above. } + // Safe with completions still outstanding: consumer_thread_t::exit() waits until the queue is + // empty before joining, and consumer_thread_t::add() consumes inline once the thread is gone, + // so a completion arriving after this point is still processed on the async handler thread. callback_thread_stop(); } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index 16d60447e0a3..a0e379c59549 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -82,7 +82,8 @@ signal_completion_hook(const hsa::Queue& /*queue*/, { // Route by packet provenance, not current activeness: completed_cb self-filters via // packet_return_map, so in-flight dispatches still complete after stop_context removes the - // context from the active list (required for kernel replay and ordinary stop/drain). + // context from the active list. This is the mechanism that makes stop_context safe without a + // GPU drain, and it is also what kernel replay needs, since each pass completes separately. auto contexts = context::get_registered_contexts(counter_contexts_filter()); for(const auto* ctx : contexts) { From e5d8ebea950e2bba683585f0504079812a175c6e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 21:39:08 +0000 Subject: [PATCH 04/45] refactor(rocprofiler-sdk): address review feedback on #8891 Applies the changes requested in review of the base PR, so that merging this stacked PR clears them on users/vkale/remove-callbacks-counters. Renames, per @vlaindic's comment that 'write' does not convey that this is a write interceptor hook, and per the name agreed in the reply on that thread: counters::write_hook -> counters::kernel_dispatch_phase_enter_hook counters::signal_completion_hook -> counters::kernel_dispatch_phase_exit_hook The exit hook is renamed for symmetry even though only the enter hook was flagged; leaving one half renamed reads worse than either alternative, and the paired names make the enqueue-versus-completion asymmetry this branch introduces easier to follow. is_any_active keeps its name, being neither phase. SPM mirrors these names on its own branch and is deliberately left alone until the review of #8887 settles, so the two branches are briefly inconsistent by choice rather than by oversight. client_ids.hpp: the four constants become an unscoped enum with int64_t as its fixed underlying type. That answers the question asked in review -- the int64_t was not arbitrary, it matches hsa::ClientID, which is how the tag is stored in inst_pkt_t. Unscoped rather than enum class so the implicit conversion to ClientID survives and no emplace site needs a cast. The regression test now constructs the pair piecewise instead of through std::make_pair, which would deduce the tag's enum type and lean on a pair-to-pair conversion; piecewise converts the tag directly and matches the production hook. Copyright headers on the four files the base PR adds move from 2025 to 2026. Verified without a build, which is not available here: clang-format 11.1.0 (the version the repo's .clang-format pins) leaves no unrelated churn, no references to the old names remain, and a standalone translation unit confirms the enum converts to ClientID in both the piecewise and make_pair forms. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/context/context.cpp | 9 ++- .../lib/rocprofiler-sdk/counters/core.cpp | 12 ++-- .../rocprofiler-sdk/counters/queue_hooks.cpp | 33 ++++++----- .../rocprofiler-sdk/counters/queue_hooks.hpp | 33 ++++++----- .../counters/tests/queue_hooks_test.cpp | 59 ++++++++----------- .../source/lib/rocprofiler-sdk/hsa/queue.cpp | 31 +++++----- .../hsa/queue_hooks/client_ids.hpp | 18 ++++-- 7 files changed, 99 insertions(+), 96 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index e112423d49e3..22ca1622933e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -370,9 +370,9 @@ stop_context(rocprofiler_context_id_t idx) { // Stop queue-interposed services before clearing the active slot so // disable_serialization() always runs before dispatches stop being instrumented. - // Clearing the slot first opens the opposite window, in which write_hook sees no - // active context and a dispatch is submitted without serializer packets while the - // serializer is still enabled. + // Clearing the slot first opens the opposite window, in which + // kernel_dispatch_phase_enter_hook sees no active context and a dispatch is submitted + // without serializer packets while the serializer is still enabled. if(_expected->dispatch_counter_collection) { rocprofiler::counters::stop_context(const_cast(_expected)); @@ -382,8 +382,7 @@ stop_context(rocprofiler_context_id_t idx) rocprofiler::spm::stop_context(const_cast(_expected)); if(_expected->device_thread_trace) _expected->device_thread_trace->stop_context(); - if(_expected->dispatch_thread_trace) - _expected->dispatch_thread_trace->stop_context(); + if(_expected->dispatch_thread_trace) _expected->dispatch_thread_trace->stop_context(); bool success = itr.compare_exchange_strong(_expected, nullptr); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 82d1f52fbca1..facfc50803cc 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -162,8 +162,8 @@ start_context(const context::context* ctx) if(!already_enabled) { // Counter collection no longer registers a per-queue callback with the queue - // controller; the HSA write interceptor calls counters::write_hook / - // signal_completion_hook directly (see hsa/queue.cpp). Keep the callback thread. + // controller; the HSA write interceptor calls counters::kernel_dispatch_phase_enter_hook / + // kernel_dispatch_phase_exit_hook directly (see hsa/queue.cpp). Keep the callback thread. callback_thread_start(); } } @@ -182,11 +182,11 @@ stop_context(const context::context* ctx) if(controller) { - // No GPU drain here. In-flight dispatches are handled by signal_completion_hook routing - // over registered rather than active contexts; the serializer transition is reconciled - // GPU-side by the hsa_barrier that profiler_serializer::disable() pushes. + // No GPU drain here. In-flight dispatches are handled by kernel_dispatch_phase_exit_hook + // routing over registered rather than active contexts; the serializer transition is + // reconciled GPU-side by the hsa_barrier that profiler_serializer::disable() pushes. controller->disable_serialization(); - // No per-queue callback to remove; counters::write_hook no-ops once + // No per-queue callback to remove; counters::kernel_dispatch_phase_enter_hook no-ops once // dispatch_counter_collection is disabled above. } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index a0e379c59549..fa12d98f2ef4 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -41,15 +41,16 @@ counter_contexts_filter() } // namespace void -write_hook(const hsa::Queue& queue, - const hsa::rocprofiler_packet& kernel_packet, - rocprofiler_kernel_id_t kernel_id, - rocprofiler_dispatch_id_t dispatch_id, - rocprofiler_user_data_t* user_data, - const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, - const context::correlation_id* correlation_id, - hsa::inst_pkt_t& inst_pkt, - bool& is_serialized) +kernel_dispatch_phase_enter_hook( + const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + rocprofiler_kernel_id_t kernel_id, + rocprofiler_dispatch_id_t dispatch_id, + rocprofiler_user_data_t* user_data, + const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, + const context::correlation_id* correlation_id, + hsa::inst_pkt_t& inst_pkt, + bool& is_serialized) { auto active = context::get_active_contexts(counter_contexts_filter()); for(const auto* ctx : active) @@ -73,12 +74,12 @@ write_hook(const hsa::Queue& queue, } void -signal_completion_hook(const hsa::Queue& /*queue*/, - const hsa::rocprofiler_packet& /*kernel_packet*/, - std::shared_ptr& session, - hsa::packet_data_t& packet, - hsa::inst_pkt_t& inst_pkt, - kernel_dispatch::profiling_time dispatch_time) +kernel_dispatch_phase_exit_hook(const hsa::Queue& /*queue*/, + const hsa::rocprofiler_packet& /*kernel_packet*/, + std::shared_ptr& session, + hsa::packet_data_t& packet, + hsa::inst_pkt_t& inst_pkt, + kernel_dispatch::profiling_time dispatch_time) { // Route by packet provenance, not current activeness: completed_cb self-filters via // packet_return_map, so in-flight dispatches still complete after stop_context removes the diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp index fdebeff1f93e..6fc716d59258 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,27 +40,28 @@ namespace counters // appends produced packets to inst_pkt, OR-folding each callback's serialize flag // into is_serialized. void -write_hook(const hsa::Queue& queue, - const hsa::rocprofiler_packet& kernel_packet, - rocprofiler_kernel_id_t kernel_id, - rocprofiler_dispatch_id_t dispatch_id, - rocprofiler_user_data_t* user_data, - const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, - const context::correlation_id* correlation_id, - hsa::inst_pkt_t& inst_pkt, - bool& is_serialized); +kernel_dispatch_phase_enter_hook( + const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + rocprofiler_kernel_id_t kernel_id, + rocprofiler_dispatch_id_t dispatch_id, + rocprofiler_user_data_t* user_data, + const hsa::queue_info_session_t::external_corr_id_map_t& ext_corr_ids, + const context::correlation_id* correlation_id, + hsa::inst_pkt_t& inst_pkt, + bool& is_serialized); // Explicit replacement for the dispatch-counter completion callback. Iterates // registered dispatch_counter_collection contexts (not only active ones) and calls // each callback's completed_cb; completed_cb self-filters via packet_return_map so // in-flight dispatches still complete after stop_context. void -signal_completion_hook(const hsa::Queue& queue, - const hsa::rocprofiler_packet& kernel_packet, - std::shared_ptr& session, - hsa::packet_data_t& packet, - hsa::inst_pkt_t& inst_pkt, - kernel_dispatch::profiling_time dispatch_time); +kernel_dispatch_phase_exit_hook(const hsa::Queue& queue, + const hsa::rocprofiler_packet& kernel_packet, + std::shared_ptr& session, + hsa::packet_data_t& packet, + hsa::inst_pkt_t& inst_pkt, + kernel_dispatch::profiling_time dispatch_time); // True if any context currently has dispatch counter collection active. bool diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index e9925cd8e98e..0d57dc9e74fa 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,12 +20,12 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "lib/rocprofiler-sdk/counters/core.hpp" -#include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" #include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" -#include "lib/rocprofiler-sdk/counters/tests/hsa_tables.hpp" #include "lib/rocprofiler-sdk/agent.hpp" #include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/counters/core.hpp" +#include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" +#include "lib/rocprofiler-sdk/counters/tests/hsa_tables.hpp" #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" #include "lib/rocprofiler-sdk/hsa/queue.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" @@ -139,13 +139,9 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat context::push_client(1); ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed"); - ROCPROFILER_CALL( - rocprofiler_configure_callback_dispatch_counting_service(get_client_ctx(), - user_dispatch_cb, - nullptr, - nullptr, - nullptr), - "Could not setup counting service"); + ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_counting_service( + get_client_ctx(), user_dispatch_cb, nullptr, nullptr, nullptr), + "Could not setup counting service"); ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context"); auto* ctx_p = context::get_mutable_registered_context(get_client_ctx()); @@ -163,7 +159,7 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat rocprofiler_counter_id_t counter_id = {.handle = 0}; { - auto mets = rocprofiler::counters::loadMetrics(); + auto mets = rocprofiler::counters::loadMetrics(); const auto* gfx_metrics = rocprofiler::common::get_val(mets->arch_to_metric, std::string(agent.name())); ASSERT_TRUE(gfx_metrics); @@ -180,18 +176,18 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat ASSERT_NE(counter_id.handle, 0U); expected_dispatch expected = {}; - ROCPROFILER_CALL(rocprofiler_create_counter_config( - agent.get_rocp_agent()->id, &counter_id, 1, &expected.id), - "Unable to create profile"); + ROCPROFILER_CALL( + rocprofiler_create_counter_config(agent.get_rocp_agent()->id, &counter_id, 1, &expected.id), + "Unable to create profile"); cb_info->callback_args = &expected; expected.queue_id = {.handle = 1}; - expected.agent_id = agent.get_rocp_agent()->id; - expected.kernel_id = 42; - expected.dispatch_id = 7; + expected.agent_id = agent.get_rocp_agent()->id; + expected.kernel_id = 42; + expected.dispatch_id = 7; hsa::QueueHooksFakeQueue fq(agent, expected.queue_id); - hsa::rocprofiler_packet pkt{}; - context::correlation_id corr_id{.internal = 99}; + hsa::rocprofiler_packet pkt{}; + context::correlation_id corr_id{.internal = 99}; auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; auto ret_pkt = rocprofiler::counters::queue_cb(ctx_p, @@ -206,34 +202,31 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat ASSERT_TRUE(ret_pkt.packet); size_t map_size_before_stop = 0; - cb_info->packet_return_map.rlock( - [&](const auto& data) { map_size_before_stop = data.size(); }); + cb_info->packet_return_map.rlock([&](const auto& data) { map_size_before_stop = data.size(); }); ASSERT_EQ(map_size_before_stop, 1U) << "queue_cb should register instrumentation in the map"; // Simulate stop while the dispatch is still in flight on the GPU. ROCPROFILER_CALL(rocprofiler_stop_context(get_client_ctx()), "stop context"); EXPECT_FALSE(rocprofiler::counters::is_any_active()); - auto sess = std::make_shared(hsa::queue_info_session_t{.queue = fq}); - auto& packet_data = sess->packet_data.emplace_back(); + auto sess = std::make_shared(hsa::queue_info_session_t{.queue = fq}); + auto& packet_data = sess->packet_data.emplace_back(); sess->correlation_id = &corr_id; rocprofiler::counters::inst_pkt_t inst_pkt; - inst_pkt.emplace_back( - std::make_pair(std::move(ret_pkt.packet), hsa::queue_hooks::COUNTERS_CLIENT_ID)); + // Construct the pair in place rather than via make_pair: make_pair would deduce the tag's + // enum type and rely on a pair-to-pair conversion, where piecewise construction converts the + // tag to ClientID directly. Also matches how the production hook emplaces. + inst_pkt.emplace_back(std::move(ret_pkt.packet), hsa::queue_hooks::COUNTERS_CLIENT_ID); - rocprofiler::counters::signal_completion_hook(fq, - pkt, - sess, - packet_data, - inst_pkt, - rocprofiler::kernel_dispatch::profiling_time{}); + rocprofiler::counters::kernel_dispatch_phase_exit_hook( + fq, pkt, sess, packet_data, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); size_t map_size_after_completion = 1; cb_info->packet_return_map.rlock( [&](const auto& data) { map_size_after_completion = data.size(); }); EXPECT_EQ(map_size_after_completion, 0U) - << "packet_return_map must drain via signal_completion_hook after stop_context"; + << "packet_return_map must drain via kernel_dispatch_phase_exit_hook after stop_context"; registration::set_init_status(1); registration::finalize(); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp index 428b1bf95af6..c9d06e0f450d 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp @@ -171,12 +171,12 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data) // Counter collection completion is migrated off the callback registry (see // WriteInterceptor); invoke it explicitly here. - counters::signal_completion_hook(queue_info_session.queue, - packet.kernel_packet, - _session, - packet, - packet.instrumentation_packets, - dispatch_time); + counters::kernel_dispatch_phase_exit_hook(queue_info_session.queue, + packet.kernel_packet, + _session, + packet, + packet.instrumentation_packets, + dispatch_time); if(packet.is_serialized) { @@ -632,15 +632,16 @@ WriteInterceptor(const void* packets, // Counter collection is migrated off the per-queue callback registry: call its hook // explicitly (the other services still flow through signal_callback above). - counters::write_hook(queue, - kernel_packet, - kernel_id, - dispatch_id, - &_packet_data.user_data, - _packet_data.tracing_data.external_correlation_ids, - corr_id, - _packet_data.instrumentation_packets, - _packet_data.is_serialized); + counters::kernel_dispatch_phase_enter_hook( + queue, + kernel_packet, + kernel_id, + dispatch_id, + &_packet_data.user_data, + _packet_data.tracing_data.external_correlation_ids, + corr_id, + _packet_data.instrumentation_packets, + _packet_data.is_serialized); bool inserted_before = false; if(_packet_data.is_serialized) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp index 9e4879b262bc..c3adc19e7863 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -34,10 +34,18 @@ namespace queue_hooks // Only distinctness matters; values are fixed for test-order stability. // Services are migrated off the per-queue callback registry one at a time; the // unused ids are reserved so the tag values stay stable across those PRs. -constexpr int64_t COUNTERS_CLIENT_ID = 1; -constexpr int64_t THREAD_TRACE_CLIENT_ID = 2; -constexpr int64_t PC_SAMPLING_CLIENT_ID = 3; -constexpr int64_t SPM_CLIENT_ID = 4; +// +// The underlying type is fixed to int64_t to match hsa::ClientID, which is how the +// tag is stored in inst_pkt_t. The enum is deliberately unscoped rather than an +// enum class: unscoped keeps the implicit conversion to ClientID, so the emplace +// sites need no cast, while still grouping the values under one type. +enum client_id : int64_t +{ + COUNTERS_CLIENT_ID = 1, + THREAD_TRACE_CLIENT_ID = 2, + PC_SAMPLING_CLIENT_ID = 3, + SPM_CLIENT_ID = 4, +}; } // namespace queue_hooks } // namespace hsa } // namespace rocprofiler From 64c01a8f7142cdd1ad54d337e62ac09e61e01baa Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 22:41:11 +0000 Subject: [PATCH 05/45] test(rocprofiler-sdk): assert in-flight dispatches are delivered, not just unmapped Strengthens the stop-while-in-flight regression test to what the review on #8891 asked for: enqueue kernels, stop the context mid-flight, and assert every kernel produces a record. The previous version had two weaknesses. It passed nullptr for the record callback, so it could not observe delivery at all -- and delivery is the failure the review described. And it drove a single dispatch, which cannot distinguish 'routed the one entry present' from 'routed by provenance'. Now three dispatches are registered while the context is active, the context is stopped while all three are in flight, and completions are fired one at a time. Two independent invariants are checked, because they fail separately: - routing: the map is asserted to drop by exactly one per completion, not merely to reach zero at the end. An aggregate check would pass even if one completion erased another dispatch's entry. - delivery: every dispatch reaches the tool's record callback. Erasure happens in completed_cb before the hand-off to the consumer, so a map-only test would pass even if nothing were ever delivered. The delivery assertion also covers the lifetime condition raised in the review: callback_thread_stop() has already run by then, so these records arrive through the inline path in consumer_thread_t::add() rather than the consumer thread. No kernel executes, so counter values are whatever the profile's output buffer holds; the assertion is about delivery, not values. Not compiled here -- no ROCm toolchain in this environment. Co-authored-by: Vivek Kale --- .../counters/tests/queue_hooks_test.cpp | 157 ++++++++++++++---- 1 file changed, 128 insertions(+), 29 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 0d57dc9e74fa..1e45a353e299 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -27,6 +27,7 @@ #include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" #include "lib/rocprofiler-sdk/counters/tests/hsa_tables.hpp" #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/hsa/aql_packet.hpp" #include "lib/rocprofiler-sdk/hsa/queue.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" #include "lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp" @@ -39,7 +40,10 @@ #include #include +#include #include +#include +#include using namespace rocprofiler::counters::test_constants; using namespace rocprofiler; @@ -116,6 +120,51 @@ user_dispatch_cb(rocprofiler_dispatch_counting_service_data_t dispatch_data, (void) dispatch_data; } +// Records delivered to the tool, keyed by dispatch id. Guarded because before stop_context the +// consumer thread delivers these, and after it consumer_thread_t::add() consumes inline on the +// caller -- so the writer can be either thread depending on when the completion lands. +struct delivered_records +{ + std::mutex mtx{}; + std::vector dispatch_ids{}; + + void add(size_t dispatch_id) + { + auto lk = std::unique_lock{mtx}; + dispatch_ids.emplace_back(dispatch_id); + } + + std::vector snapshot() + { + auto lk = std::unique_lock{mtx}; + return dispatch_ids; + } + + // Reset so a repeated run of the test does not see the previous run's deliveries. + void clear() + { + auto lk = std::unique_lock{mtx}; + dispatch_ids.clear(); + } +}; + +delivered_records& +get_delivered() +{ + static delivered_records _v{}; + return _v; +} + +void +user_record_cb(rocprofiler_dispatch_counting_service_data_t dispatch_data, + rocprofiler_counter_record_t*, + size_t, + rocprofiler_user_data_t, + void*) +{ + get_delivered().add(dispatch_data.dispatch_info.dispatch_id); +} + // is_any_active replaces the queue.get_notifiers() signal that the old per-queue // callback registration used to provide to the HSA write interceptor gate. With // no dispatch-counter-collection context active, it must report inactive. This @@ -125,12 +174,27 @@ TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) EXPECT_FALSE(rocprofiler::counters::is_any_active()); } -// Regression for callback-registry removal: a dispatch enqueued while the context -// is active must still complete (and drain packet_return_map) when stop_context -// runs before the GPU completion callback. Kernel replay depends on this because -// each replay pass reuses the same counter-collection completion path. +// Regression for callback-registry removal, per the review on #8891: dispatches enqueued while the +// context is active must still complete when stop_context runs before their GPU completions land. +// +// Two invariants are checked, because they fail independently: +// +// 1. Routing. Each dispatch's packet_return_map entry is erased by its own completion, so the map +// is asserted to drop by exactly one per completion rather than only to reach zero at the end. +// An aggregate check would pass even if one completion erased another dispatch's entry. +// 2. Delivery. Every dispatch reaches the tool's record callback. Erasure happens in completed_cb +// before the hand-off to the consumer, so a test that only checks the map would pass even if +// nothing were ever delivered -- which is the failure the review actually described. +// +// Several dispatches are in flight at once, since a single one cannot distinguish "routed the one +// entry" from "routed by provenance". +// +// No kernel executes here, so the counter values are whatever the profile's output buffer holds; +// the assertion is about delivery, not values. TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_path) { + constexpr size_t num_dispatches = 3; + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); test_init(); @@ -139,8 +203,10 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat context::push_client(1); ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed"); + // A record callback is supplied so delivery is observable; the base test passed nullptr, which + // is why it could not detect a dropped completion. ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_counting_service( - get_client_ctx(), user_dispatch_cb, nullptr, nullptr, nullptr), + get_client_ctx(), user_dispatch_cb, nullptr, user_record_cb, nullptr), "Could not setup counting service"); ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context"); @@ -189,45 +255,78 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat hsa::rocprofiler_packet pkt{}; context::correlation_id corr_id{.internal = 99}; - auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; - auto ret_pkt = rocprofiler::counters::queue_cb(ctx_p, - cb_info, - fq, - pkt, - expected.kernel_id, - expected.dispatch_id, - &user_data, - {}, - &corr_id); - ASSERT_TRUE(ret_pkt.packet); + get_delivered().clear(); + + // Enqueue several dispatches while the context is active. Each queue_cb call must register its + // own entry; if the implementation ever pooled one packet across concurrent dispatches, the + // size assertion below would catch it rather than the test silently degenerating to one. + std::vector> in_flight; + std::vector dispatch_ids; + for(size_t i = 0; i < num_dispatches; ++i) + { + const size_t dispatch_id = expected.dispatch_id + i; + auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; + auto ret_pkt = rocprofiler::counters::queue_cb( + ctx_p, cb_info, fq, pkt, expected.kernel_id, dispatch_id, &user_data, {}, &corr_id); + ASSERT_TRUE(ret_pkt.packet) << "queue_cb produced no instrumentation for dispatch " << i; + in_flight.emplace_back(std::move(ret_pkt.packet)); + dispatch_ids.emplace_back(dispatch_id); + } size_t map_size_before_stop = 0; cb_info->packet_return_map.rlock([&](const auto& data) { map_size_before_stop = data.size(); }); - ASSERT_EQ(map_size_before_stop, 1U) << "queue_cb should register instrumentation in the map"; + ASSERT_EQ(map_size_before_stop, num_dispatches) + << "queue_cb should register one packet_return_map entry per in-flight dispatch"; - // Simulate stop while the dispatch is still in flight on the GPU. + // Stop while every dispatch is still in flight on the GPU. This is the window the review + // described: the context leaves the active list, so a completion hook that routed by + // activeness would find nothing to deliver to. ROCPROFILER_CALL(rocprofiler_stop_context(get_client_ctx()), "stop context"); EXPECT_FALSE(rocprofiler::counters::is_any_active()); - auto sess = std::make_shared(hsa::queue_info_session_t{.queue = fq}); - auto& packet_data = sess->packet_data.emplace_back(); - sess->correlation_id = &corr_id; + // Complete them one at a time, checking the map drops by exactly one each time. + for(size_t i = 0; i < num_dispatches; ++i) + { + auto sess = + std::make_shared(hsa::queue_info_session_t{.queue = fq}); + auto& packet_data = sess->packet_data.emplace_back(); + sess->correlation_id = &corr_id; + packet_data.callback_record.dispatch_info.dispatch_id = dispatch_ids[i]; + + rocprofiler::counters::inst_pkt_t inst_pkt; + // Construct the pair in place rather than via make_pair: make_pair would deduce the tag's + // enum type and rely on a pair-to-pair conversion, where piecewise construction converts + // the tag to ClientID directly. Also matches how the production hook emplaces. + inst_pkt.emplace_back(std::move(in_flight[i]), hsa::queue_hooks::COUNTERS_CLIENT_ID); - rocprofiler::counters::inst_pkt_t inst_pkt; - // Construct the pair in place rather than via make_pair: make_pair would deduce the tag's - // enum type and rely on a pair-to-pair conversion, where piecewise construction converts the - // tag to ClientID directly. Also matches how the production hook emplaces. - inst_pkt.emplace_back(std::move(ret_pkt.packet), hsa::queue_hooks::COUNTERS_CLIENT_ID); + rocprofiler::counters::kernel_dispatch_phase_exit_hook( + fq, pkt, sess, packet_data, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); - rocprofiler::counters::kernel_dispatch_phase_exit_hook( - fq, pkt, sess, packet_data, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); + size_t remaining = num_dispatches; + cb_info->packet_return_map.rlock([&](const auto& data) { remaining = data.size(); }); + EXPECT_EQ(remaining, num_dispatches - (i + 1)) + << "completion " << i + << " should erase exactly its own packet_return_map entry after stop_context"; + } - size_t map_size_after_completion = 1; + size_t map_size_after_completion = num_dispatches; cb_info->packet_return_map.rlock( [&](const auto& data) { map_size_after_completion = data.size(); }); EXPECT_EQ(map_size_after_completion, 0U) << "packet_return_map must drain via kernel_dispatch_phase_exit_hook after stop_context"; + // Delivery. callback_thread_stop() has already run as part of stop_context, so these arrive + // through the inline path in consumer_thread_t::add() rather than the consumer thread -- which + // is precisely the lifetime property the review asked to be kept intact. + auto delivered = get_delivered().snapshot(); + EXPECT_EQ(delivered.size(), num_dispatches) + << "every in-flight dispatch must reach the tool's record callback after stop_context"; + for(auto dispatch_id : dispatch_ids) + { + EXPECT_NE(std::find(delivered.begin(), delivered.end(), dispatch_id), delivered.end()) + << "no record delivered for dispatch " << dispatch_id; + } + registration::set_init_status(1); registration::finalize(); } From 7d9823affb15796ec80d3a5d637642dc202fc05b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 22:48:55 +0000 Subject: [PATCH 06/45] docs(rocprofiler-sdk): record the client-id overlap with the registry's ClientID Captures the non-blocking observation from the review of #8790 at the declaration rather than only in a PR discussion, since that is where someone about to route on these values will look. The tags overlap the per-queue registry's auto-incrementing ClientID, which also starts at 1 and is still in use by the services not yet migrated. This is inert only because no consumer dispatches on the tag: counters' completed_cb and thread trace's post_kernel_call both identify their own packets by pointer lookup or dynamic_cast. The comment states what has to happen before that changes. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp index c3adc19e7863..815ae5879ee3 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp @@ -39,6 +39,14 @@ namespace queue_hooks // tag is stored in inst_pkt_t. The enum is deliberately unscoped rather than an // enum class: unscoped keeps the implicit conversion to ClientID, so the emplace // sites need no cast, while still grouping the values under one type. +// +// These values numerically overlap the per-queue registry's auto-incrementing +// ClientID, which also starts at 1 and is still in use by the services that have +// not been migrated yet. That is inert only because no consumer routes on the tag: +// counters' completed_cb and thread trace's post_kernel_call both identify their +// own packets by pointer lookup or dynamic_cast. Before adding a consumer that +// dispatches on these values, either finish migrating the remaining services off +// the registry or move this enum to a range the registry cannot produce. enum client_id : int64_t { COUNTERS_CLIENT_ID = 1, From 8129859343af5cacdae8fce2bf163d4ca2578bcd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 22:52:48 +0000 Subject: [PATCH 07/45] fix(rocprofiler-sdk): drain in-flight dispatches in counters::stop_context Restores the queue_controller_sync() this branch had earlier removed, per review priority: the review of #8891 accepted provenance-based completion routing on the condition that the callback thread and the counter_callback_info objects stay alive until in-flight dispatches drain, and the drain is what makes that hold literally rather than by argument about consumer_thread_t's fallback behaviour. An intermediate revision removed it on the reasoning that consumer_thread_t::exit() drains its queue and add() self-consumes inline afterwards. Both facts are true and they remain a useful backstop, but they are not the guarantee that was asked for -- they argue a late completion is still processed, rather than ensuring there is no late completion to process. The comment now says which is which. Ordering matters and is now stated at the call site: context::stop_context runs this function while the context is still in the active list, so for the duration of the drain the enter hook still reaches queue_cb, whose disabled path returns serialize=true and keeps the serialized->unserialized transition coordinated until disable_serialization() runs. That is what removes the need for a separate draining flag -- and it only works because the drain happens before the active slot is cleared. callback_thread_stop() now runs after the drain rather than alongside it. Not compiled here -- no ROCm toolchain in this environment. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/core.cpp | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index facfc50803cc..4f28291a77e0 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -182,17 +182,26 @@ stop_context(const context::context* ctx) if(controller) { - // No GPU drain here. In-flight dispatches are handled by kernel_dispatch_phase_exit_hook - // routing over registered rather than active contexts; the serializer transition is - // reconciled GPU-side by the hsa_barrier that profiler_serializer::disable() pushes. + // Drain in-flight dispatches before anything else is torn down. The review of #8891 + // accepted provenance-based completion routing on the condition that the callback thread + // and the counter_callback_info objects stay alive until in-flight dispatches drain, so the + // drain is what makes that condition hold literally rather than by argument. + // + // context::stop_context calls this function while the context is still in the active list, + // which is what keeps the service visible for the duration of the drain: the enter hook + // still reaches queue_cb, and queue_cb's disabled path returns serialize=true so the + // serialized->unserialized transition stays coordinated until disable_serialization() runs + // below. That ordering is why no separate "draining" flag is needed -- but it only works + // because the drain happens here, before the slot is cleared. + hsa::queue_controller_sync(); controller->disable_serialization(); // No per-queue callback to remove; counters::kernel_dispatch_phase_enter_hook no-ops once // dispatch_counter_collection is disabled above. } - // Safe with completions still outstanding: consumer_thread_t::exit() waits until the queue is - // empty before joining, and consumer_thread_t::add() consumes inline once the thread is gone, - // so a completion arriving after this point is still processed on the async handler thread. + // After the drain. consumer_thread_t::exit() also waits until its queue is empty before + // joining, and add() consumes inline once the thread is gone, so a late completion is still + // processed -- but that is a backstop, not the guarantee the review asked for. callback_thread_stop(); } From 794008d0fe5030146c0697416b578eb10b9519d2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 03:05:00 +0000 Subject: [PATCH 08/45] fix(rocprofiler-sdk): correct the exit-hook comment about the stop-path drain The comment claimed provenance routing makes stop_context safe without a GPU drain, which contradicts counters::stop_context, where the drain was kept. State what each mechanism actually guarantees: provenance routing delivers a completion that arrives, the drain bounds when completions arrive. Co-authored-by: Vivek Kale --- .../source/lib/rocprofiler-sdk/counters/queue_hooks.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index fa12d98f2ef4..e0501af3f88f 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -83,8 +83,9 @@ kernel_dispatch_phase_exit_hook(const hsa::Queue& /*queue*/, { // Route by packet provenance, not current activeness: completed_cb self-filters via // packet_return_map, so in-flight dispatches still complete after stop_context removes the - // context from the active list. This is the mechanism that makes stop_context safe without a - // GPU drain, and it is also what kernel replay needs, since each pass completes separately. + // context from the active list. This is what guarantees a completion that arrives is delivered; + // the drain in stop_context is what bounds when completions arrive. Kernel replay needs the + // same property, since each pass completes separately. auto contexts = context::get_registered_contexts(counter_contexts_filter()); for(const auto* ctx : contexts) { From b618a75046b2fcaa27320a251d2ba862c1ee500b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 03:05:01 +0000 Subject: [PATCH 09/45] docs(rocprofiler-sdk): add queue callback removal architecture doc Records how queue-interposed services are wired into dispatch interception after the per-queue callback registry is removed: why the enter hook routes over active contexts while the exit hook routes over registered ones, why that asymmetry is the difference between delivering and dropping the completion of an in-flight dispatch, and why the stop path drains the GPU. Folded in from a separate docs branch so it is reviewed alongside the code it describes. Co-authored-by: Vivek Kale --- .../queue_callback_removal_architecture.md | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 projects/rocprofiler-sdk/source/docs/conceptual/queue_hooks/queue_callback_removal_architecture.md diff --git a/projects/rocprofiler-sdk/source/docs/conceptual/queue_hooks/queue_callback_removal_architecture.md b/projects/rocprofiler-sdk/source/docs/conceptual/queue_hooks/queue_callback_removal_architecture.md new file mode 100644 index 000000000000..7edc81710e04 --- /dev/null +++ b/projects/rocprofiler-sdk/source/docs/conceptual/queue_hooks/queue_callback_removal_architecture.md @@ -0,0 +1,208 @@ +# Queue Callback Registry Removal — Software Architecture + +How queue-interposed services are wired into dispatch interception after the per-queue callback +registry is removed, why completion routing and enqueue routing deliberately use different context +sets, and why `stop_context` drains the GPU. + +Paths are relative to `projects/rocprofiler-sdk/source/`. Symbols are named rather than cited by +line number, since line numbers rot faster than the code they point at. + +The migration is one PR per service. Counter collection (#8891) introduces the shared +`hsa/queue_hooks/` infrastructure that the rest build on; SPM (#8887), thread trace (#8790) and PC +sampling (#8895) follow the same shape. This document describes the mechanism as a whole, and flags +which service each part currently lives on. + +## 1. Diagram + +```mermaid +flowchart TB + + subgraph BEFORE["BEFORE — per-queue callback registry"] + direction TB + B1["service start_context"] + B2["QueueController::add_callback
auto-incrementing ClientID"] + B3["Queue::_callbacks map
ClientID -> queue_callbacks_t
_notifiers counter"] + B4["WriteInterceptor
queue.signal_callback(...)
iterate map, call write_interceptor"] + B5["AsyncSignalHandler
queue.signal_callback(...)
iterate map, call completed_cb"] + B6["captured context + callback refs
routing by provenance:
a registered callback fires even
after its context is stopped"] + B1 --> B2 --> B3 + B3 --> B4 + B3 --> B5 + B3 --- B6 + end + + subgraph AFTER["AFTER — explicit hooks"] + direction TB + C1["service start_context
no registration"] + C2["WriteInterceptor
counters::kernel_dispatch_phase_enter_hook
called inline, unconditionally"] + C3["counters::is_any_active
folded into no_real_consumers and
forces should_batch_packets = false"] + C4["AsyncSignalHandler
counters::kernel_dispatch_phase_exit_hook
called inline"] + C5["hsa/queue_hooks/client_ids.hpp
stable producer tags
COUNTERS / SPM / THREAD_TRACE"] + C1 --> C2 + C2 --> C3 + C1 --> C4 + C2 --- C5 + C4 --- C5 + end + + subgraph ROUTE["ROUTING — the two context sets are not interchangeable"] + direction TB + R1["enter hook: get_active_contexts
new instrumentation must stop
as soon as the context stops"] + R2["exit hook: get_registered_contexts
work already on the GPU must
still be able to complete"] + R3["using active contexts in the exit hook
drops the completion of any dispatch
in flight at stop_context:
record never delivered,
packet_return_map entry leaked"] + R2 -.->|"if this were
get_active_contexts"| R3 + end + + subgraph STOP["STOP PATH — ordering is load-bearing"] + direction TB + S1["context::stop_context stops
queue-interposed services before
clearing the active slot"] + S2["counters::stop_context
1. clear the enabled flag
2. queue_controller_sync (drain)
3. disable_serialization
4. callback_thread_stop"] + S3["the service stays visible for the
whole drain, so queue_cb's disabled
path keeps returning serialize=true
and no separate draining flag is needed"] + S1 --> S2 --> S3 + end + + BEFORE ==> AFTER + AFTER ==> ROUTE + ROUTE ==> STOP + + classDef old fill:#eceff1,stroke:#546e7a,stroke-width:2px,color:#000 + classDef new fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px,color:#000 + classDef bad fill:#ffebee,stroke:#c62828,stroke-width:2px,color:#000 + classDef fix fill:#e3f2fd,stroke:#1565c0,stroke-width:2px,color:#000 + class B1,B2,B3,B4,B5,B6 old + class C1,C2,C3,C4,C5 new + class R3 bad + class R1,R2,S1,S2,S3 fix +``` + +## 2. What the migration changes + +Before, a service registered a `queue_callbacks_t` pair with the queue controller when its context +started. The write interceptor and the async signal handler both walked `Queue::_callbacks` and +invoked whatever was registered, and `Queue::get_notifiers()` counted the registrations so the +interceptor could early-out when nothing was subscribed. + +After, the service registers nothing. The write interceptor and the async signal handler call the +service's hooks directly, and each hook decides for itself whether it has work to do by querying the +context list. + +For counter collection: + +| Element | Location | +|---|---| +| `counters::kernel_dispatch_phase_enter_hook` | `counters/queue_hooks.hpp`, `counters/queue_hooks.cpp` | +| `counters::kernel_dispatch_phase_exit_hook` | `counters/queue_hooks.hpp`, `counters/queue_hooks.cpp` | +| `counters::is_any_active` | `counters/queue_hooks.hpp`, `counters/queue_hooks.cpp` | +| Context filter shared by all three | `counters/queue_hooks.cpp`, `counter_contexts_filter()` | +| Stable producer tags | `hsa/queue_hooks/client_ids.hpp` | +| Exit hook call site | `hsa/queue.cpp`, in the async signal handler | +| `no_real_consumers` gains `!counters::is_any_active()` | `hsa/queue.cpp` | +| Enter hook call site | `hsa/queue.cpp`, in the write interceptor | +| Batching disabled while counters are active | `hsa/queue.cpp`, `should_batch_packets` | +| Service stop path | `counters/core.cpp`, `stop_context()` | + +The hook names describe the dispatch phase they run in: the enter hook runs when a dispatch is being +submitted, the exit hook when its completion signal fires. `is_any_active` is neither phase, so it +keeps a plain name. Each remaining service mirrors this trio in its own `queue_hooks.{hpp,cpp}`. + +`client_ids.hpp` replaces the registry's auto-incrementing `ClientID` with fixed producer tags, so +the id attached to an instrumentation packet no longer depends on the order in which services +happened to register. The tags numerically overlap the registry's `ClientID`, which also starts at 1 +and remains in use by services that have not migrated yet. That is inert only because no consumer +dispatches on the tag — each service identifies its own packets by pointer lookup or `dynamic_cast` +— and `client_ids.hpp` records what has to change before anything routes on these values. + +## 3. Enqueue and completion use different context sets + +This is the central design point, and getting it wrong is a silent data-loss bug rather than a crash. + +The registry routed completions by **provenance**: the callback pair captured at enqueue time was +invoked when the packet completed, regardless of whether the owning context was still active. The +hooks have to reproduce that property without the registry, and they do it by choosing different +context sets for the two phases: + +- The **enter hook** iterates `get_active_contexts`. New instrumentation must stop being added as + soon as the context stops. +- The **exit hook** iterates `get_registered_contexts`. A dispatch already executing on the GPU must + still be able to deliver its record, even though its context is no longer active. + +Routing the exit hook over active contexts instead loses any dispatch that is in flight when its +context is stopped: `completed_cb` never runs, so the record is never delivered to the tool, and the +`packet_return_map` entry is never erased, leaking the AQL packet and its profile allocation. Because +pause/resume is implemented as `stop_context`/`start_context`, this shows up as missing counter +records around every pause. + +Iterating registered contexts is safe because `completed_cb` self-filters: it resolves ownership by +looking the packet up in its own context's `packet_return_map` and returns early for packets it does +not own. Each context therefore still processes only its own packets, and the guarantee is restored +without reintroducing a registry. + +## 4. The stop path + +Two orderings matter, both stated at their call sites in the code. + +**`context::stop_context` stops queue-interposed services before clearing the active slot.** Counter +collection, SPM, device thread trace and dispatch thread trace are all stopped ahead of the +`compare_exchange_strong` that nulls the context's active slot. Clearing the slot first opens a +window in which the enter hook sees no active context, so a dispatch is submitted without serializer +packets while the serializer is still enabled. + +**`counters::stop_context` drains the GPU before disabling serialization.** The order is: clear the +service's `enabled` flag, `hsa::queue_controller_sync()`, `disable_serialization()`, then +`callback_thread_stop()`. + +### 4.1 Why the drain is kept + +Provenance routing and the drain answer different questions, so one does not replace the other. +Provenance routing guarantees that a completion which *arrives* is delivered. The drain bounds +*when completions arrive at all*, which is what the teardown downstream of it depends on: it is what +makes "the callback thread and the `counter_callback_info` objects outlive every in-flight dispatch" +true literally, rather than true by an argument about what happens if they do not. + +The supporting facts are worth recording, because they are the reason a missing drain is hard to +notice rather than a reason to omit it: + +| Property | Why it holds | +|---|---| +| `counter_callback_info` lifetime | Structural. The context holds them as `std::vector>` on the service, and neither `counters::stop_context` nor `context::stop_context` clears that vector. Since the exit hook iterates *registered* contexts, the context and its callbacks are still reachable. | +| Callback thread does not strand queued work | `consumer_thread_t::exit()` clears `valid` then waits on `exited`, and `consumer_loop` only sets `exited` once `read_ptr == write_ptr`, so the queue drains before the join. Afterwards `consumer_thread_t::add()` takes the self-consume path and runs inline on the caller. Covered by `counters/tests/consumer_test.cpp`, `restart`. | +| Serializer transition with in-flight serialized dispatches | GPU-ordered independently. `profiler_serializer::disable()` records the previous state and pushes an `hsa_barrier` across the queues, and `kernel_completion_signal` reconciles in-flight dispatches against it. | +| No separate "draining" flag is needed | `context::stop_context` calls the service stop path while the context is still in the active list, so throughout the drain the enter hook still reaches `queue_cb`, whose disabled path returns `serialize=true` and keeps the serialized-to-unserialized transition coordinated. This only works because the drain happens before the slot is cleared. | + +The drain is a bound, not a hard barrier: `Queue::sync` uses a five-second hint and only warns on +timeout, and `_active_kernels` counts intercepted dispatches only. It is the ordering guarantee for +teardown, and provenance routing in the exit hook is what makes individual completions correct. + +One hazard is not covered by either mechanism: the drain incidentally protects the sequence "stop the +context, then destroy the counter config". If a tool does that, the guard belongs at the destroy +path as a drain or a refcount, since that is where the lifetime actually ends. + +## 5. Relationship to kernel replay (#8622) + +Callback removal is a soft prerequisite for kernel replay, not a blocker, and the two can land in +either order. + +It helps because with the registry there is no explicit "if this context is active" block in the +write interceptor — the decision is hidden inside the registered callbacks, so disabling a service +for a single replay pass means mutating the callback structure. Once the hooks are inline, the +interceptor has an explicit conditional on context activity, which is a much easier place to express +that a context is active globally but inactive for this pass. + +The routing rule in section 3 matters more under replay than without it. Replay runs N passes per +dispatch and reuses `process_packet_batch` for each pass, so every pass creates its own +instrumentation packet and its own `packet_return_map` entry. A dropped completion is one lost record +without replay; under replay it is N leaked packets per dispatch plus silently missing counter +groups. The replay loop also waits on its `pass_done` barrier with no timeout, so a stall on the +completion path becomes an application hang rather than data loss. + +## 6. Known gaps + +1. Thread trace, PC sampling and device counter collection still use the registry, so + `Queue::_callbacks`, `Queue::get_notifiers()` and `QueueController::add_callback` cannot be + deleted yet. `client_ids.hpp` already reserves tags for thread trace and PC sampling. +2. Several in-flight PRs edit the same `no_real_consumers` expression in `hsa/queue.cpp`. + Consolidating the predicate into one `needs_interception(queue)` helper would remove the + recurring conflict. +3. `counters::is_any_active()` forces `should_batch_packets = false` for every dispatch while any + counter context is active. The performance effect of that, combined with kernel replay's own + gate, has not been measured. From 9e97d3256d9682520ce8095e15899f67d35f785a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 5 Aug 2026 01:20:00 +0000 Subject: [PATCH 10/45] fix(rocprofiler-sdk): build the queue-hooks test's correlation_id via its constructor context::correlation_id declares constructors, deletes its copy and move, and holds private ref counters, so it is not an aggregate and a designated initializer does not compile: error: initialization of non-aggregate type 'context::correlation_id' with a designated initializer list Default-construct and assign the one member the test needs, which leaves every other member at the same default the designated initializer would have. This broke the rocprofiler-sdk build, which is why all four Core test jobs and Code Coverage failed alongside the TheRock package builds. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 1e45a353e299..574824f1d1b5 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -253,7 +253,10 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat hsa::QueueHooksFakeQueue fq(agent, expected.queue_id); hsa::rocprofiler_packet pkt{}; - context::correlation_id corr_id{.internal = 99}; + // correlation_id declares constructors and holds private ref counters, so it is not an + // aggregate and cannot take a designated initializer. + context::correlation_id corr_id{}; + corr_id.internal = 99; get_delivered().clear(); From 2063ba6f705e18f9f56206dde06112ff65783aef Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Thu, 6 Aug 2026 01:59:29 +0000 Subject: [PATCH 11/45] fix(rocprofiler-sdk): finalize client contexts before tearing down queue interception counters::stop_context() drains in-flight dispatches via queue_controller_sync() and coordinates serializer teardown, both of which require a live queue controller. finalize() ran queue_controller_fini() first and invoked the client finalizers afterwards, so stopping a counter-collection context during teardown operated on destroyed queue state and segfaulted. Also skip empty optional slots when walking the registered-context array; the array is sparse and dereferencing a disengaged optional is undefined. Fixes the hip-streams-per-thread SEGFAULT seen in CI. Co-authored-by: Cursor --- .../source/lib/rocprofiler-sdk/context/context.cpp | 13 +++++++++---- .../source/lib/rocprofiler-sdk/registration.cpp | 14 ++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 22ca1622933e..4d7b7685ce7f 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -119,11 +119,10 @@ get_registered_contexts(context_array_t& data, context_filter_t filter) data.reserve(num_ctx); for(auto& itr : *get_registered_contexts_impl()) { + if(!itr.has_value()) continue; + const auto* ctx = &itr.value(); - if(ctx) - { - if(!filter || (filter && filter(ctx))) data.emplace_back(ctx); - } + if(!filter || (filter && filter(ctx))) data.emplace_back(ctx); } return data; } @@ -422,6 +421,8 @@ get_client_contexts(rocprofiler_client_id_t id) for(auto& itr : *get_registered_contexts_impl()) { + if(!itr.has_value()) continue; + if(itr->client_idx == id.handle) { _data.emplace_back(rocprofiler_context_id_t{.handle = itr->context_idx}); @@ -438,6 +439,8 @@ stop_client_contexts(rocprofiler_client_id_t client_id) auto ret = ROCPROFILER_STATUS_SUCCESS; for(auto& itr : *get_registered_contexts_impl()) { + if(!itr.has_value()) continue; + if(itr->client_idx == client_id.handle) { auto status = stop_context(rocprofiler_context_id_t{itr->context_idx}); @@ -472,6 +475,8 @@ deregister_client_contexts(rocprofiler_client_id_t client_id) { for(auto& itr : *get_registered_contexts_impl()) { + if(!itr.has_value()) continue; + if(itr->client_idx == client_id.handle && buffer::get_buffers()) { for(auto& bitr : *buffer::get_buffers()) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp index b56e52b0d8ba..de3a16534ec7 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp @@ -1012,6 +1012,16 @@ finalize() std::call_once(_once, []() { auto num_clients = get_num_clients(); set_fini_status(-1); + + // Stop contexts and run tool finalization while queue interception is still + // intact. counters::stop_context relies on a live queue controller to drain + // in-flight dispatches and coordinate serializer teardown; running that after + // queue_controller_fini() is undefined. + if(get_init_status() > 0) + { + invoke_client_finalizers(); + } + hsa::async_copy_fini(); counters::device_counting_service_finalize(); hsa::queue_controller_fini(); @@ -1026,10 +1036,6 @@ finalize() #endif code_object::finalize(); context::correlation_id_finalize(); - if(get_init_status() > 0) - { - invoke_client_finalizers(); - } if(num_clients > 0) internal_threading::finalize(); set_fini_status(1); }); From 5752c5f00782588d926b7d357f7a57db84d0d85b Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Thu, 6 Aug 2026 06:47:10 +0000 Subject: [PATCH 12/45] fix(rocprofiler-sdk): restore the original finalize() teardown order Moving invoke_client_finalizers() ahead of the subsystem teardowns deadlocked OMPT finalization: rocprofv3-test-ompt-tracing and rocprofv3-test-ompt-all timed out on every CI OS, and openmp-tools.test_retired_correlation_ids failed with them. The reorder also did not fix the hip-streams-per-thread SEGFAULT it was meant to address, so revert it and keep the original ordering. The sparse-optional guards in context.cpp are unaffected and stay in place. Co-authored-by: Cursor --- .../source/lib/rocprofiler-sdk/registration.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp index de3a16534ec7..b56e52b0d8ba 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp @@ -1012,16 +1012,6 @@ finalize() std::call_once(_once, []() { auto num_clients = get_num_clients(); set_fini_status(-1); - - // Stop contexts and run tool finalization while queue interception is still - // intact. counters::stop_context relies on a live queue controller to drain - // in-flight dispatches and coordinate serializer teardown; running that after - // queue_controller_fini() is undefined. - if(get_init_status() > 0) - { - invoke_client_finalizers(); - } - hsa::async_copy_fini(); counters::device_counting_service_finalize(); hsa::queue_controller_fini(); @@ -1036,6 +1026,10 @@ finalize() #endif code_object::finalize(); context::correlation_id_finalize(); + if(get_init_status() > 0) + { + invoke_client_finalizers(); + } if(num_clients > 0) internal_threading::finalize(); set_fini_status(1); }); From 9681ed78fe8d7886de74bf49528f73cc1f6e6aea Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 00:16:48 +0000 Subject: [PATCH 13/45] ci: re-run to confirm mi325 test failures are flake buffered-api-tracing and the two thread-trace-api-triple-buffer tests failed on ubuntu-22.04 only, while rhel-8.8, rhel-9.5 and sles-15.6 passed and TheRock's own rocprofiler-sdk shard passed. No content change; this re-runs CI on an identical tree to confirm. Co-authored-by: Vivek Kale From 5d894e3a0911e9ecd27670e765e09bab75dbfd7e Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Fri, 7 Aug 2026 18:14:44 +0000 Subject: [PATCH 14/45] feat(rocprofiler-sdk): scope dispatch counter collection to agents Dispatch counter collection applied to the whole machine: a context had no notion of which GPUs it collected on, so serialization was enabled on every agent for as long as any counter-collection context was active, and only one such context could be active at all. A tool could decline to profile a dispatch by returning a null config, but that only suppressed instrumentation -- the dispatch was still serialized, which is the part that costs performance and the part the tool had no lever over. The public header has carried a note since the service was introduced saying the agent restriction was unresolved and left to the tool. Store the agent set on the context's counter-collection service and factor it into the three queries that need it: serialization scope at start/stop, the dispatch enter hook, and counter-collection context conflict. An empty set still means every agent, so existing tools are unaffected. Contexts with disjoint agent sets may now be active simultaneously, which is what makes "context A on GPU-0, context B on GPU-1" expressible. Serialization is now reference counted per agent. Counter collection, thread trace and SPM each enable it independently, and without a refcount whichever service stopped first silently unserialized the ones still running. That was a pre-existing bug; scoping serialization per agent makes it reachable more easily, so it is fixed here rather than left behind. Adds rocprofiler_dispatch_counting_service_set_agents() to declare the set. It is rejected while the context is active, since the dispatch path reads the set without a lock and start_context has already used it to scope serialization. Co-authored-by: Cursor --- .../dispatch_counting_service.h | 39 ++ .../lib/rocprofiler-sdk/context/context.cpp | 9 +- .../lib/rocprofiler-sdk/context/context.hpp | 28 + .../lib/rocprofiler-sdk/counters/core.cpp | 44 +- .../lib/rocprofiler-sdk/counters/core.hpp | 5 + .../rocprofiler-sdk/counters/queue_hooks.cpp | 10 + .../counters/tests/CMakeLists.txt | 3 +- .../counters/tests/per_agent_scoping_test.cpp | 481 ++++++++++++++++++ .../dispatch_counting_service.cpp | 19 +- .../rocprofiler-sdk/hsa/queue_controller.cpp | 132 ++++- .../rocprofiler-sdk/hsa/queue_controller.hpp | 54 +- 11 files changed, 785 insertions(+), 39 deletions(-) create mode 100644 projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp diff --git a/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h b/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h index 2b1e524e8c5a..9ec6c81cd17a 100644 --- a/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h +++ b/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h @@ -159,6 +159,45 @@ rocprofiler_configure_callback_dispatch_counting_service( rocprofiler_dispatch_counting_record_cb_t record_callback, void* record_callback_args) ROCPROFILER_API; +/** + * @brief (experimental) Restrict a dispatch counting service to a set of GPU agents. + * + * By default a dispatch counting service applies to every GPU agent: the dispatch + * callback is invoked for kernels on all agents, and kernel dispatches are serialized + * on all agents for as long as the context is active. A tool can already decline to + * profile a dispatch by leaving the ::rocprofiler_counter_config_id_t output of + * ::rocprofiler_dispatch_counting_service_cb_t set to zero, but that only suppresses + * counter collection -- the dispatch is still serialized. This function narrows the + * service so that agents outside @p agents are neither instrumented nor serialized. + * + * Restricting the agent set also relaxes the counter-collection context conflict: + * two contexts may each configure a dispatch counting service and be active at the + * same time provided their agent sets are disjoint. Contexts that leave the agent set + * unrestricted claim every agent and therefore still conflict with each other. + * + * Must be called after a dispatch counting service has been configured on + * @p context_id and before the context is started. Calling it more than once replaces + * the previous set. Passing @p num_agents of zero restores the default of all agents. + * + * @param [in] context_id context id with a configured dispatch counting service + * @param [in] agents array of GPU agent ids to restrict collection to + * @param [in] num_agents number of entries in @p agents + * @retval ::ROCPROFILER_STATUS_SUCCESS agent set was applied + * @retval ::ROCPROFILER_STATUS_ERROR_CONTEXT_INVALID @p context_id is not a valid context + * @retval ::ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND no dispatch counting service has been + * configured on @p context_id + * @retval ::ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT @p agents is null with a non-zero + * @p num_agents + * @retval ::ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND an entry in @p agents is not a GPU agent + * @retval ::ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED the context is currently active + * @return ::rocprofiler_status_t + */ +ROCPROFILER_SDK_EXPERIMENTAL +rocprofiler_status_t +rocprofiler_dispatch_counting_service_set_agents(rocprofiler_context_id_t context_id, + const rocprofiler_agent_id_t* agents, + size_t num_agents) ROCPROFILER_API; + /** @} */ ROCPROFILER_EXTERN_C_FINI diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 4d7b7685ce7f..e5dba095f238 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -288,9 +288,14 @@ start_context(rocprofiler_context_id_t context_id) { return ROCPROFILER_STATUS_SUCCESS; } - else if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection) + else if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection && + cfg->dispatch_counter_collection->intersects(*itr->dispatch_counter_collection)) { - // conflicting context + // Conflicting context. Two counter-collection contexts can run concurrently as + // long as they target disjoint sets of GPU agents -- the hardware counters they + // program are per-agent, so contexts that never touch the same agent cannot + // contend. A context with no agent restriction claims every agent and therefore + // still conflicts with any other counter-collection context. return ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT; } } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index 73f075f9b148..19ede673024b 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -42,6 +42,7 @@ #include #include #include +#include namespace rocprofiler { @@ -82,10 +83,37 @@ struct dispatch_counter_collection_service // Contains callback information along with other data needed to collect/process // counters. std::vector> callbacks{}; + // GPU agents (by rocprofiler_agent_id_t::handle) this context collects on. An empty set + // means every GPU agent, which is what a plain configure_{buffer,callback}_dispatch call + // produces; rocprofiler_dispatch_counting_service_set_agents narrows it. The set is read + // on the dispatch path and when scoping serialization, so it is fixed at configure time + // and never mutated once the context is started. + std::unordered_set agents{}; // A flag to state whether or not the counter set is currently enabled. This is primarily // to protect against multithreaded calls to enable a context (and enabling already enabled // counters). common::Synchronized enabled{false}; + + // An empty agent set means "all GPU agents", so an unrestricted context keeps behaving + // exactly as it did before per-agent scoping existed. + bool collects_on(rocprofiler_agent_id_t agent_id) const + { + return agents.empty() || agents.count(agent_id.handle) > 0; + } + + // Two counter-collection contexts can coexist as long as they do not both want the same + // GPU. An unrestricted context claims every agent, so it intersects with everything. + bool intersects(const dispatch_counter_collection_service& rhs) const + { + if(agents.empty() || rhs.agents.empty()) return true; + const auto& small = (agents.size() < rhs.agents.size()) ? agents : rhs.agents; + const auto& large = (agents.size() < rhs.agents.size()) ? rhs.agents : agents; + for(auto handle : small) + { + if(large.count(handle) > 0) return true; + } + return false; + } }; struct spm_dispatch_counter_collection_service diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 4f28291a77e0..396a6c41c036 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -25,6 +25,7 @@ #include "lib/common/container/small_vector.hpp" #include "lib/common/synchronized.hpp" #include "lib/common/utility.hpp" +#include "lib/rocprofiler-sdk/agent.hpp" #include "lib/rocprofiler-sdk/aql/packet_construct.hpp" #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" @@ -101,8 +102,7 @@ counter_callback_info::setup_counter_config(std::shared_ptr& pro config.asts.back().set_dimensions(config.agent->id); } catch(std::runtime_error& e) { - ROCP_ERROR << metric.name() << " has improper dimensions" - << " " << e.what(); + ROCP_ERROR << metric.name() << " has improper dimensions" << " " << e.what(); return ROCPROFILER_STATUS_ERROR_AST_NOT_FOUND; } } @@ -152,7 +152,9 @@ start_context(const context::context* ctx) auto* controller = hsa::get_queue_controller(); bool already_enabled = true; - CHECK_NOTNULL(controller)->enable_serialization(); + // Scope serialization to the agents this context collects on. An empty set still means + // every agent, so an unrestricted context serializes the whole machine as before. + CHECK_NOTNULL(controller)->enable_serialization(ctx->dispatch_counter_collection->agents); ctx->dispatch_counter_collection->enabled.wlock([&](auto& enabled) { if(enabled) return; already_enabled = false; @@ -194,7 +196,7 @@ stop_context(const context::context* ctx) // below. That ordering is why no separate "draining" flag is needed -- but it only works // because the drain happens here, before the slot is cleared. hsa::queue_controller_sync(); - controller->disable_serialization(); + controller->disable_serialization(ctx->dispatch_counter_collection->agents); // No per-queue callback to remove; counters::kernel_dispatch_phase_enter_hook no-ops once // dispatch_counter_collection is disabled above. } @@ -205,6 +207,40 @@ stop_context(const context::context* ctx) callback_thread_stop(); } +rocprofiler_status_t +set_dispatch_agents(rocprofiler_context_id_t context_id, + const rocprofiler_agent_id_t* agents, + size_t num_agents) +{ + if(num_agents > 0 && agents == nullptr) return ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT; + + auto* ctx_p = context::get_mutable_registered_context(context_id); + if(!ctx_p) return ROCPROFILER_STATUS_ERROR_CONTEXT_INVALID; + if(!ctx_p->dispatch_counter_collection) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; + + // The agent set is read without a lock on the dispatch path and is what scopes + // serialization at start, so it may only change while the context is stopped. + for(const auto* itr : context::get_active_contexts()) + { + if(itr && itr->context_idx == ctx_p->context_idx) + return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED; + } + + auto selected = std::unordered_set{}; + selected.reserve(num_agents); + for(size_t i = 0; i < num_agents; ++i) + { + const auto* agent = rocprofiler::agent::get_agent(agents[i]); + if(!agent || agent->type != ROCPROFILER_AGENT_TYPE_GPU) + return ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND; + selected.emplace(agents[i].handle); + } + + ctx_p->dispatch_counter_collection->agents = std::move(selected); + + return ROCPROFILER_STATUS_SUCCESS; +} + rocprofiler_status_t configure_agent_collection(rocprofiler_context_id_t context_id, rocprofiler_buffer_id_t buffer_id, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp index 77ffbca63c93..07a6a3a7b22f 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.hpp @@ -88,6 +88,11 @@ configure_callback_dispatch(rocprofiler_context_id_t context_i rocprofiler_dispatch_counting_record_cb_t record_callback, void* record_callback_args); +rocprofiler_status_t +set_dispatch_agents(rocprofiler_context_id_t context_id, + const rocprofiler_agent_id_t* agents, + size_t num_agents); + rocprofiler_status_t configure_agent_collection(rocprofiler_context_id_t context_id, rocprofiler_buffer_id_t buffer_id, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index e0501af3f88f..c3572409a75a 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -23,6 +23,8 @@ #include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" +#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/hsa/queue.hpp" #include "lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp" namespace rocprofiler @@ -52,9 +54,17 @@ kernel_dispatch_phase_enter_hook( hsa::inst_pkt_t& inst_pkt, bool& is_serialized) { + const auto agent_id = CHECK_NOTNULL(queue.get_agent().get_rocp_agent())->id; + auto active = context::get_active_contexts(counter_contexts_filter()); for(const auto* ctx : active) { + // Skip contexts that do not collect on this dispatch's agent. This is what keeps a + // GPU-1-only context from serializing GPU-0: queue_cb is what returns serialize=true, + // including on its disabled path, so filtering has to happen before the call rather + // than inside it. + if(!ctx->dispatch_counter_collection->collects_on(agent_id)) continue; + for(auto& cb : ctx->dispatch_counter_collection->callbacks) { auto [packet, bSerial] = queue_cb(ctx, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt index 6ec7bd403190..b1bcc0b410e6 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt @@ -61,7 +61,8 @@ target_link_libraries( set(ROCPROFILER_LIB_COUNTER_TEST_SOURCES metrics_test.cpp evaluate_ast_test.cpp dimension.cpp core.cpp code_object_loader.cpp - device_counting.cpp firmware_restrictions_test.cpp queue_hooks_test.cpp) + device_counting.cpp firmware_restrictions_test.cpp queue_hooks_test.cpp + per_agent_scoping_test.cpp) set(ROCPROFILER_LIB_COUNTER_TEST_HEADERS code_object_loader.hpp device_counting.hpp) add_executable(counter-tests) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp new file mode 100644 index 000000000000..a696f0403310 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp @@ -0,0 +1,481 @@ +// MIT License +// +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "lib/rocprofiler-sdk/agent.hpp" +#include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/counters/core.hpp" +#include "lib/rocprofiler-sdk/counters/queue_hooks.hpp" +#include "lib/rocprofiler-sdk/counters/tests/hsa_tables.hpp" +#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/hsa/queue.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/registration.hpp" + +#include +#include + +#include +#include + +#include + +using namespace rocprofiler::counters::test_constants; +using namespace rocprofiler; + +namespace rocprofiler +{ +namespace hsa +{ +class PerAgentFakeQueue : public Queue +{ +public: + PerAgentFakeQueue(const AgentCache& a, rocprofiler_queue_id_t id) + : Queue(a, get_api_table()) + , _agent(a) + , _id(id) + {} + const AgentCache& get_agent() const final { return _agent; }; + rocprofiler_queue_id_t get_id() const final { return _id; }; + + ~PerAgentFakeQueue() override = default; + +private: + const AgentCache& _agent; + rocprofiler_queue_id_t _id = {}; +}; +} // namespace hsa +} // namespace rocprofiler + +namespace +{ +// Two GPU agents are required to say anything about scoping: with one agent, "scoped to agent +// A" and "unscoped" are indistinguishable. +struct agent_pair +{ + const hsa::AgentCache* first = nullptr; + const hsa::AgentCache* second = nullptr; +}; + +void +test_init() +{ + static bool _initialized = false; + if(_initialized) return; + _initialized = true; + + HsaApiTable table; + table.amd_ext_ = &get_ext_table(); + table.core_ = &get_api_table(); + agent::construct_agent_cache(&table); + ASSERT_TRUE(hsa::get_queue_controller() != nullptr); + hsa::get_queue_controller()->init(get_api_table(), get_ext_table()); + registration::init_logging(); + registration::set_init_status(-1); + context::push_client(1); +} + +agent_pair +get_two_agents() +{ + auto result = agent_pair{}; + auto& agents = hsa::get_queue_controller()->get_supported_agents(); + for(const auto& [_, agent] : agents) + { + if(!agent.get_rocp_agent()) continue; + if(!result.first) + result.first = &agent; + else if(!result.second) + result.second = &agent; + else + break; + } + return result; +} + +void +noop_dispatch_cb(rocprofiler_dispatch_counting_service_data_t, + rocprofiler_counter_config_id_t*, + rocprofiler_user_data_t*, + void*) +{} + +void +noop_record_cb(rocprofiler_dispatch_counting_service_data_t, + rocprofiler_counter_record_t*, + size_t, + rocprofiler_user_data_t, + void*) +{} + +// Creates a context with a dispatch counting service configured on it. +rocprofiler_context_id_t +make_counter_context() +{ + auto ctx = rocprofiler_context_id_t{0}; + EXPECT_EQ(rocprofiler_create_context(&ctx), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_configure_callback_dispatch_counting_service( + ctx, noop_dispatch_cb, nullptr, noop_record_cb, nullptr), + ROCPROFILER_STATUS_SUCCESS); + return ctx; +} + +// --------------------------------------------------------------------------------------------- +// rocprofiler_dispatch_counting_service_set_agents argument handling +// --------------------------------------------------------------------------------------------- + +TEST(counters_per_agent, set_agents_requires_a_configured_service) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto ctx = rocprofiler_context_id_t{0}; + ASSERT_EQ(rocprofiler_create_context(&ctx), ROCPROFILER_STATUS_SUCCESS); + + auto agents = get_two_agents(); + if(!agents.first) GTEST_SKIP() << "no GPU agent available"; + auto agent_id = agents.first->get_rocp_agent()->id; + + // The agent set lives on the counter-collection service, so there is nothing to restrict + // until the service exists. Silently succeeding here would let a tool believe it had + // scoped collection when it had not. + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_id, 1), + ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND); +} + +TEST(counters_per_agent, set_agents_rejects_bad_arguments) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.first) GTEST_SKIP() << "no GPU agent available"; + + auto ctx = make_counter_context(); + auto agent_id = agents.first->get_rocp_agent()->id; + + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, nullptr, 1), + ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT); + + // A null array with no entries is the documented way to clear the restriction. + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, nullptr, 0), + ROCPROFILER_STATUS_SUCCESS); + + auto bogus = rocprofiler_agent_id_t{.handle = 0xDEADBEEF}; + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &bogus, 1), + ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND); + + // The rejected call must not have partially applied: a failed set leaves the previous + // (unrestricted) set in place rather than a half-built one. + auto* ctx_p = context::get_mutable_registered_context(ctx); + ASSERT_TRUE(ctx_p); + ASSERT_TRUE(ctx_p->dispatch_counter_collection); + EXPECT_TRUE(ctx_p->dispatch_counter_collection->agents.empty()); + + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_id, 1), + ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(ctx_p->dispatch_counter_collection->agents.size(), 1U); + EXPECT_TRUE(ctx_p->dispatch_counter_collection->collects_on(agent_id)); +} + +TEST(counters_per_agent, set_agents_is_locked_while_the_context_is_active) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.first) GTEST_SKIP() << "no GPU agent available"; + + auto ctx = make_counter_context(); + auto agent_id = agents.first->get_rocp_agent()->id; + + ASSERT_EQ(rocprofiler_start_context(ctx), ROCPROFILER_STATUS_SUCCESS); + + // The dispatch path reads the agent set without a lock, and start_context has already used + // it to scope serialization, so changing it mid-flight would silently desynchronize the two. + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_id, 1), + ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED); + + ASSERT_EQ(rocprofiler_stop_context(ctx), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_id, 1), + ROCPROFILER_STATUS_SUCCESS); +} + +// --------------------------------------------------------------------------------------------- +// Context conflict: the scenario from the #9586 review +// --------------------------------------------------------------------------------------------- + +// Before per-agent scoping this was impossible -- any second counter-collection context was +// rejected outright, so "context A on GPU-0, context B on GPU-1" could not even be expressed. +TEST(counters_per_agent, disjoint_agent_contexts_can_be_active_together) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.second) GTEST_SKIP() << "fewer than two GPU agents available"; + + auto agent_a = agents.first->get_rocp_agent()->id; + auto agent_b = agents.second->get_rocp_agent()->id; + + auto ctx_a = make_counter_context(); + auto ctx_b = make_counter_context(); + + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx_a, &agent_a, 1), + ROCPROFILER_STATUS_SUCCESS); + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx_b, &agent_b, 1), + ROCPROFILER_STATUS_SUCCESS); + + EXPECT_EQ(rocprofiler_start_context(ctx_a), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_start_context(ctx_b), ROCPROFILER_STATUS_SUCCESS); + + EXPECT_EQ(rocprofiler_stop_context(ctx_b), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_stop_context(ctx_a), ROCPROFILER_STATUS_SUCCESS); +} + +TEST(counters_per_agent, overlapping_agent_contexts_still_conflict) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.first) GTEST_SKIP() << "no GPU agent available"; + auto agent_a = agents.first->get_rocp_agent()->id; + + auto ctx_a = make_counter_context(); + auto ctx_b = make_counter_context(); + + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx_a, &agent_a, 1), + ROCPROFILER_STATUS_SUCCESS); + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx_b, &agent_a, 1), + ROCPROFILER_STATUS_SUCCESS); + + ASSERT_EQ(rocprofiler_start_context(ctx_a), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_start_context(ctx_b), ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT); + + ASSERT_EQ(rocprofiler_stop_context(ctx_a), ROCPROFILER_STATUS_SUCCESS); +} + +// An unrestricted context claims every agent, so it must keep conflicting with everything -- +// otherwise existing single-context tools would start silently sharing agents. +TEST(counters_per_agent, unrestricted_context_conflicts_with_a_scoped_one) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.second) GTEST_SKIP() << "fewer than two GPU agents available"; + auto agent_b = agents.second->get_rocp_agent()->id; + + auto ctx_all = make_counter_context(); + auto ctx_scoped = make_counter_context(); + + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx_scoped, &agent_b, 1), + ROCPROFILER_STATUS_SUCCESS); + + ASSERT_EQ(rocprofiler_start_context(ctx_all), ROCPROFILER_STATUS_SUCCESS); + EXPECT_EQ(rocprofiler_start_context(ctx_scoped), ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT); + + ASSERT_EQ(rocprofiler_stop_context(ctx_all), ROCPROFILER_STATUS_SUCCESS); +} + +// --------------------------------------------------------------------------------------------- +// Serialization scope and reference counting +// --------------------------------------------------------------------------------------------- + +// The actual complaint in the review: a context that only wants GPU-1 must not serialize GPU-0. +TEST(counters_per_agent, serialization_is_scoped_to_the_contexts_agents) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.second) GTEST_SKIP() << "fewer than two GPU agents available"; + + auto agent_a = agents.first->get_rocp_agent()->id; + auto agent_b = agents.second->get_rocp_agent()->id; + auto* controller = hsa::get_queue_controller(); + + // Touch both serializers so the agents are known to the controller; otherwise "not + // serialized" would be trivially true because the entry does not exist yet. + auto queue_a = hsa::PerAgentFakeQueue{*agents.first, {.handle = 101}}; + auto queue_b = hsa::PerAgentFakeQueue{*agents.second, {.handle = 102}}; + controller->serializer(&queue_a); + controller->serializer(&queue_b); + + auto ctx = make_counter_context(); + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_b, 1), + ROCPROFILER_STATUS_SUCCESS); + + auto* ctx_p = context::get_mutable_registered_context(ctx); + ASSERT_TRUE(ctx_p && ctx_p->dispatch_counter_collection); + ASSERT_EQ(ctx_p->dispatch_counter_collection->agents.size(), 1U) << "agent set not applied"; + ASSERT_TRUE(ctx_p->dispatch_counter_collection->collects_on(agent_b)); + ASSERT_FALSE(ctx_p->dispatch_counter_collection->collects_on(agent_a)); + + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); + EXPECT_FALSE(controller->is_serialization_enabled(agent_b)); + + ASSERT_EQ(rocprofiler_start_context(ctx), ROCPROFILER_STATUS_SUCCESS); + + EXPECT_TRUE(controller->is_serialization_enabled(agent_b)) + << "the context's own agent must be serialized"; + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)) + << "an agent outside the context's agent set must not be serialized"; + + ASSERT_EQ(rocprofiler_stop_context(ctx), ROCPROFILER_STATUS_SUCCESS); + EXPECT_FALSE(controller->is_serialization_enabled(agent_b)); +} + +// Counter collection, thread trace and SPM each enable serialization independently. Without +// reference counting the first one to stop unserializes the others, which is a pre-existing +// bug that per-agent scoping would otherwise make easier to hit. +TEST(counters_per_agent, serialization_is_reference_counted_across_subsystems) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.first) GTEST_SKIP() << "no GPU agent available"; + + auto agent_a = agents.first->get_rocp_agent()->id; + auto* controller = hsa::get_queue_controller(); + + auto queue_a = hsa::PerAgentFakeQueue{*agents.first, {.handle = 201}}; + controller->serializer(&queue_a); + + ASSERT_FALSE(controller->is_serialization_enabled(agent_a)); + + controller->enable_serialization(); + controller->enable_serialization(); + EXPECT_TRUE(controller->is_serialization_enabled(agent_a)); + + controller->disable_serialization(); + EXPECT_TRUE(controller->is_serialization_enabled(agent_a)) + << "one subsystem stopping must not unserialize an agent another still needs"; + + controller->disable_serialization(); + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); + + // Over-releasing must not drive the count negative, or the next enable would be swallowed. + controller->disable_serialization(); + controller->enable_serialization(); + EXPECT_TRUE(controller->is_serialization_enabled(agent_a)); + controller->disable_serialization(); + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); +} + +// A scoped request and an unscoped request overlap on the scoped agent; releasing one must not +// release the other. +TEST(counters_per_agent, scoped_and_unscoped_serialization_compose) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.second) GTEST_SKIP() << "fewer than two GPU agents available"; + + auto agent_a = agents.first->get_rocp_agent()->id; + auto agent_b = agents.second->get_rocp_agent()->id; + auto* controller = hsa::get_queue_controller(); + + auto queue_a = hsa::PerAgentFakeQueue{*agents.first, {.handle = 301}}; + auto queue_b = hsa::PerAgentFakeQueue{*agents.second, {.handle = 302}}; + controller->serializer(&queue_a); + controller->serializer(&queue_b); + + controller->enable_serialization({agent_b.handle}); + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); + EXPECT_TRUE(controller->is_serialization_enabled(agent_b)); + + controller->enable_serialization(); + EXPECT_TRUE(controller->is_serialization_enabled(agent_a)); + EXPECT_TRUE(controller->is_serialization_enabled(agent_b)); + + controller->disable_serialization(); + EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); + EXPECT_TRUE(controller->is_serialization_enabled(agent_b)) + << "the scoped request still holds agent B"; + + controller->disable_serialization({agent_b.handle}); + EXPECT_FALSE(controller->is_serialization_enabled(agent_b)); +} + +// --------------------------------------------------------------------------------------------- +// Dispatch path +// --------------------------------------------------------------------------------------------- + +// queue_cb returns serialize=true even when it declines to instrument, so a tool cannot avoid +// serializing a GPU by returning a null config -- the filtering has to happen before the call. +// This is the end-to-end form of the review's example. +TEST(counters_per_agent, enter_hook_ignores_dispatches_on_other_agents) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + auto agents = get_two_agents(); + if(!agents.second) GTEST_SKIP() << "fewer than two GPU agents available"; + + auto agent_b = agents.second->get_rocp_agent()->id; + + auto ctx = make_counter_context(); + ASSERT_EQ(rocprofiler_dispatch_counting_service_set_agents(ctx, &agent_b, 1), + ROCPROFILER_STATUS_SUCCESS); + ASSERT_EQ(rocprofiler_start_context(ctx), ROCPROFILER_STATUS_SUCCESS); + + auto packet = hsa::rocprofiler_packet{}; + auto corr_id = context::correlation_id{}; + corr_id.internal = 4242; + + // A dispatch on the agent the context does not collect on. + { + auto queue_a = hsa::PerAgentFakeQueue{*agents.first, {.handle = 401}}; + auto inst_pkt = hsa::inst_pkt_t{}; + bool is_serialized = false; + auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; + + counters::kernel_dispatch_phase_enter_hook( + queue_a, packet, 42, 1, &user_data, {}, &corr_id, inst_pkt, is_serialized); + + EXPECT_FALSE(is_serialized) + << "a dispatch on an agent outside the context's agent set must not be serialized"; + EXPECT_TRUE(inst_pkt.empty()) + << "a dispatch on an agent outside the context's agent set must not be instrumented"; + } + + // The same dispatch on the agent the context does collect on still goes through, so the + // test cannot pass just because the hook stopped doing anything at all. + { + auto queue_b = hsa::PerAgentFakeQueue{*agents.second, {.handle = 402}}; + auto inst_pkt = hsa::inst_pkt_t{}; + bool is_serialized = false; + auto user_data = rocprofiler_user_data_t{.value = corr_id.internal}; + + counters::kernel_dispatch_phase_enter_hook( + queue_b, packet, 42, 2, &user_data, {}, &corr_id, inst_pkt, is_serialized); + + EXPECT_TRUE(is_serialized) + << "a dispatch on the context's own agent must still be serialized"; + } + + ASSERT_EQ(rocprofiler_stop_context(ctx), ROCPROFILER_STATUS_SUCCESS); +} +} // namespace diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp index 6631fa7b3c92..da64f72a709f 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp @@ -25,7 +25,8 @@ #include -extern "C" { +extern "C" +{ /** * @brief Configure buffered dispatch profile Counting Service. * Collects the counters in dispatch packets and stores them @@ -75,4 +76,20 @@ rocprofiler_configure_callback_dispatch_counting_service( record_callback, record_callback_args); } + +/** + * @brief Restrict a dispatch counting service to a set of GPU agents. + * + * @param [in] context_id context id with a configured dispatch counting service + * @param [in] agents array of GPU agent ids to restrict collection to + * @param [in] num_agents number of entries in agents + * @return ::rocprofiler_status_t + */ +rocprofiler_status_t +rocprofiler_dispatch_counting_service_set_agents(rocprofiler_context_id_t context_id, + const rocprofiler_agent_id_t* agents, + size_t num_agents) +{ + return rocprofiler::counters::set_dispatch_agents(context_id, agents, num_agents); +} } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index 38070b6d19da..edc377a97d2e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -517,10 +517,19 @@ common::Synchronized& QueueController::serializer(const Queue* queue) { CHECK(queue); + const auto agent_id = queue->get_agent().get_rocp_agent()->id; + + // Read the refcount before taking the serializer lock: update_serialization() acquires + // them in that order, and reversing it here would deadlock. A concurrent enable/disable + // for this agent can therefore race with the creation below, exactly as the previous + // _serialized_enabled load could, and is resolved the same way -- the next transition + // through update_serialization() reaches the entry once it is in the map. + const bool should_serialize = is_serialization_enabled(agent_id); + common::Synchronized* ret = nullptr; _profiler_serializer.ulock( [&](const auto& m) { - if(auto ptr = m.find(queue->get_agent().get_rocp_agent()->id); ptr != m.end()) + if(auto ptr = m.find(agent_id); ptr != m.end()) { ret = ptr->second.get(); return true; @@ -528,10 +537,10 @@ QueueController::serializer(const Queue* queue) return false; }, [&](auto& m) { - ret = m.emplace(queue->get_agent().get_rocp_agent()->id, + ret = m.emplace(agent_id, std::make_shared>()) .first->second.get(); - if(_serialized_enabled.load() == true) + if(should_serialize) { ret->wlock([&](auto& serializer) { serializer.enable({}); }); } @@ -555,47 +564,114 @@ per_dev_map(const QueueController::queue_map_t& _queues_v) }; // namespace void -QueueController::disable_serialization() +QueueController::update_serialization(const agent_handle_set_t& agents, bool enable) { _queues.rlock([&](const queue_map_t& _queues_v) { - _serialized_enabled.store(false); auto pd_map = per_dev_map(_queues_v); - _profiler_serializer.wlock([&](auto& m) { - for(auto& [k, v] : m) + + // Agents whose effective refcount crossed zero in this call; only those get their + // serializer toggled. Collected under the refcount lock, applied under the serializer + // lock. + auto transitioned = std::vector{}; + bool any_enabled = false; + + _serialization_refcount.wlock([&](auto& state) { + // An agent can be covered by `all` and by an explicit entry at the same time, so + // the transition has to be decided by comparing effective counts across the + // update rather than by watching a single counter hit zero. + auto was_enabled = std::unordered_map{}; + _profiler_serializer.rlock([&](const auto& serializers) { + for(const auto& [agent_id, _] : serializers) + was_enabled.emplace(agent_id.handle, state.enabled(agent_id.handle)); + }); + + if(agents.empty()) + { + if(enable) + ++state.all; + else if(state.all > 0) + --state.all; + } + else { - if(auto it = pd_map.find(k); it != pd_map.end()) + for(auto agent_handle : agents) { - v->wlock([&](auto& serializer) { serializer.disable(it->second); }); + auto& count = state.per_agent[agent_handle]; + if(enable) + ++count; + else if(count > 0) + --count; } - else + } + + _profiler_serializer.rlock([&](const auto& serializers) { + for(const auto& [agent_id, _] : serializers) { - v->wlock([&](auto& serializer) { serializer.disable({}); }); + auto itr = was_enabled.find(agent_id.handle); + if(itr == was_enabled.end()) continue; + if(itr->second != state.enabled(agent_id.handle)) + transitioned.emplace_back(agent_id); } + }); + + any_enabled = state.any(); + }); + + _serialized_enabled.store(any_enabled); + + if(transitioned.empty()) return; + + _profiler_serializer.wlock([&](auto& m) { + for(const auto& agent_id : transitioned) + { + auto itr = m.find(agent_id); + if(itr == m.end() || !itr->second) continue; + + auto queues = hsa_barrier::queue_map_ptr_t{}; + if(auto it = pd_map.find(agent_id); it != pd_map.end()) queues = it->second; + + itr->second->wlock([&](auto& serializer) { + if(enable) + serializer.enable(queues); + else + serializer.disable(queues); + }); } }); }); } +void +QueueController::disable_serialization() +{ + update_serialization({}, false); +} + void QueueController::enable_serialization() { - _queues.rlock([&](const queue_map_t& _queues_v) { - _serialized_enabled.store(true); - auto pd_map = per_dev_map(_queues_v); - _profiler_serializer.wlock([&](auto& m) { - for(auto& [k, v] : m) - { - if(auto it = pd_map.find(k); it != pd_map.end()) - { - v->wlock([&](auto& serializer) { serializer.enable(it->second); }); - } - else - { - v->wlock([&](auto& serializer) { serializer.enable({}); }); - } - } - }); - }); + update_serialization({}, true); +} + +void +QueueController::disable_serialization(const agent_handle_set_t& agents) +{ + update_serialization(agents, false); +} + +void +QueueController::enable_serialization(const agent_handle_set_t& agents) +{ + update_serialization(agents, true); +} + +bool +QueueController::is_serialization_enabled(rocprofiler_agent_id_t agent_id) const +{ + bool enabled = false; + _serialization_refcount.rlock( + [&](const auto& state) { enabled = state.enabled(agent_id.handle); }); + return enabled; } void diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp index ed755d001616..26288e36d119 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include namespace rocprofiler @@ -92,13 +93,27 @@ class QueueController common::Synchronized& serializer(const Queue*); + // Agent handles (rocprofiler_agent_id_t::handle). An empty set means every GPU agent. + using agent_handle_set_t = std::unordered_set; + /** - * Disable serialization for QueueController, has no effect if counter collection - * is not in use (which defaults to no serialization mechanism). Should only be used for - * testing. + * Enable/disable serialization for QueueController, has no effect if counter collection + * is not in use (which defaults to no serialization mechanism). + * + * Serialization is reference counted per agent. Counter collection, thread trace and SPM + * each enable it independently, so an agent stays serialized until every subsystem that + * asked for it has released it -- otherwise whichever service stops first would silently + * unserialize the ones still running. The no-argument overloads apply to every GPU agent, + * which is the behavior every caller had before agents could be scoped. */ void enable_serialization(); void disable_serialization(); + void enable_serialization(const agent_handle_set_t& agents); + void disable_serialization(const agent_handle_set_t& agents); + + // Whether the given agent currently has serialization enabled. Exposed for tests and for + // callers that need to reason about scope rather than trigger a transition. + bool is_serialization_enabled(rocprofiler_agent_id_t agent_id) const; // Prints current state of signals for queues, used for debugging. Only prints // serialization related signals if not compiled in debug mode. @@ -124,6 +139,39 @@ class QueueController std::unordered_map>>> _profiler_serializer; + // How many subsystems currently want serialization, per agent. `all` counts the callers + // that asked for every agent; it is kept separate from `per_agent` because an agent's + // serializer is created lazily on first use and an unscoped request has to cover agents + // that do not exist yet. + struct serialization_refcount + { + int64_t all = 0; + std::unordered_map per_agent = {}; + + int64_t count(uint64_t agent_handle) const + { + auto itr = per_agent.find(agent_handle); + return all + ((itr == per_agent.end()) ? 0 : itr->second); + } + + bool enabled(uint64_t agent_handle) const { return count(agent_handle) > 0; } + + bool any() const + { + if(all > 0) return true; + for(const auto& [handle, count] : per_agent) + { + if(count > 0) return true; + } + return false; + } + }; + + common::Synchronized _serialization_refcount; + + // Applies a +1/-1 to the refcount of each agent in `agents` (empty == all GPU agents) and + // toggles only the serializers whose effective count crossed zero. + void update_serialization(const agent_handle_set_t& agents, bool enable); }; QueueController* From 8698fae384c9dc8149fe46058abe122d04233119 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Fri, 7 Aug 2026 20:23:08 +0000 Subject: [PATCH 15/45] style(rocprofiler-sdk): reformat with the versions CI pins The previous commit was formatted with clang-format 18 and left the source and cmake format checks failing. CI pins clang-format 11.1.0 and cmake-format 0.6.13, and v18 disagreed with v11 on two spots -- it joined a pre-existing ROCP_ERROR continuation in counters/core.cpp onto one line and moved the extern "C" brace in dispatch_counting_service.cpp -- neither of which this change had any reason to touch. Co-authored-by: Cursor --- .../source/lib/rocprofiler-sdk/counters/core.cpp | 3 ++- .../lib/rocprofiler-sdk/counters/tests/CMakeLists.txt | 10 ++++++++-- .../lib/rocprofiler-sdk/dispatch_counting_service.cpp | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 396a6c41c036..5815a8218d01 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -102,7 +102,8 @@ counter_callback_info::setup_counter_config(std::shared_ptr& pro config.asts.back().set_dimensions(config.agent->id); } catch(std::runtime_error& e) { - ROCP_ERROR << metric.name() << " has improper dimensions" << " " << e.what(); + ROCP_ERROR << metric.name() << " has improper dimensions" + << " " << e.what(); return ROCPROFILER_STATUS_ERROR_AST_NOT_FOUND; } } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt index b1bcc0b410e6..2e1157ac8b6b 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt @@ -60,8 +60,14 @@ target_link_libraries( rocprofiler-sdk::rocprofiler-sdk-hsa-runtime) set(ROCPROFILER_LIB_COUNTER_TEST_SOURCES - metrics_test.cpp evaluate_ast_test.cpp dimension.cpp core.cpp code_object_loader.cpp - device_counting.cpp firmware_restrictions_test.cpp queue_hooks_test.cpp + metrics_test.cpp + evaluate_ast_test.cpp + dimension.cpp + core.cpp + code_object_loader.cpp + device_counting.cpp + firmware_restrictions_test.cpp + queue_hooks_test.cpp per_agent_scoping_test.cpp) set(ROCPROFILER_LIB_COUNTER_TEST_HEADERS code_object_loader.hpp device_counting.hpp) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp index da64f72a709f..4f74ef5efa58 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/dispatch_counting_service.cpp @@ -25,8 +25,7 @@ #include -extern "C" -{ +extern "C" { /** * @brief Configure buffered dispatch profile Counting Service. * Collects the counters in dispatch packets and stores them From 0b2c074922da7b7c26b3514d3e65b508b9bdab62 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Fri, 7 Aug 2026 20:44:05 +0000 Subject: [PATCH 16/45] test(rocprofiler-sdk): cover per-agent scoping on real multi-GPU hardware The unit tests assert the internal state, which cannot show the property a user actually cares about: that a GPU outside the context's agent set keeps running its streams concurrently. This adds an integration test that measures a multi-stream workload on GPU-0 while a counter collection context is scoped to GPU-1, and fails if GPU-0 gets serialized or instrumented. Machines differ too much for an absolute time to mean anything, so every timing assertion is a ratio against a baseline measured in the same process, and the unrestricted context serves as a positive control -- when it does not visibly serialize the same workload the machine cannot demonstrate stream concurrency and the timing assertions are skipped instead of guessed at. The record and callback assertions are exact and always run. The test reports SKIP and exits 0 below two GPUs. Verified against a deliberately reverted build: without the agent filtering the scoped GPU-0 workload goes from 7.7ms to 32.0ms and three checks fail. Co-authored-by: Cursor --- projects/rocprofiler-sdk/tests/CMakeLists.txt | 1 + .../CMakeLists.txt | 52 +++ .../per_agent_scoping_app.cpp | 422 ++++++++++++++++++ 3 files changed, 475 insertions(+) create mode 100644 projects/rocprofiler-sdk/tests/counter-collection-per-agent/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp diff --git a/projects/rocprofiler-sdk/tests/CMakeLists.txt b/projects/rocprofiler-sdk/tests/CMakeLists.txt index 8e6d800dd180..03e40c3d534a 100644 --- a/projects/rocprofiler-sdk/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/tests/CMakeLists.txt @@ -84,6 +84,7 @@ add_subdirectory(thread-trace) add_subdirectory(pc_sampling) add_subdirectory(hip-graph-tracing) add_subdirectory(counter-collection) +add_subdirectory(counter-collection-per-agent) add_subdirectory(openmp-tools) add_subdirectory(rocdecode) add_subdirectory(rocjpeg) diff --git a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/CMakeLists.txt b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/CMakeLists.txt new file mode 100644 index 000000000000..84130df233ac --- /dev/null +++ b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/CMakeLists.txt @@ -0,0 +1,52 @@ +# +# +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +if(NOT CMAKE_HIP_COMPILER) + find_program( + amdclangpp_EXECUTABLE + NAMES amdclang++ + HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATH_SUFFIXES bin llvm/bin NO_CACHE) + mark_as_advanced(amdclangpp_EXECUTABLE) + + if(amdclangpp_EXECUTABLE) + set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}") + endif() +endif() + +project( + rocprofiler-sdk-tests-counter-collection-per-agent + LANGUAGES CXX HIP + VERSION 0.0.0) + +foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO) + if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "") + set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}") + endif() +endforeach() + +set(CMAKE_HIP_STANDARD 17) +set(CMAKE_HIP_EXTENSIONS OFF) +set(CMAKE_HIP_STANDARD_REQUIRED ON) + +find_package(rocprofiler-sdk REQUIRED) +find_package(Threads REQUIRED) + +# The tool and the workload live in one executable so the test can start and stop contexts +# around the measured region instead of profiling the whole process. +set_source_files_properties(per_agent_scoping_app.cpp PROPERTIES LANGUAGE HIP) +add_executable(counter-collection-per-agent-app) +target_sources(counter-collection-per-agent-app PRIVATE per_agent_scoping_app.cpp) +target_link_libraries(counter-collection-per-agent-app + PRIVATE rocprofiler-sdk::rocprofiler-sdk Threads::Threads) + +# Reports SKIP and exits 0 on a machine with fewer than two GPUs. +rocprofiler_add_integration_execute_test( + counter-collection-per-agent + TARGET counter-collection-per-agent-app + TIMEOUT 120 + LABELS "integration-tests" + SKIP_REGULAR_EXPRESSION "SKIP: per-agent scoping requires at least 2 GPUs") diff --git a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp new file mode 100644 index 000000000000..f99f3b4bc53b --- /dev/null +++ b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp @@ -0,0 +1,422 @@ +// MIT License +// +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// End-to-end check that a dispatch counter collection context restricted to one GPU leaves +// the other GPUs alone -- neither instrumented nor serialized. +// +// The unit tests in source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp cover +// the internal state directly. This test covers the property a user actually cares about: a +// concurrent multi-stream workload on a GPU outside the context's agent set must keep running +// concurrently. Serialization forces one dispatch at a time per agent, so the wall time of the +// workload is a direct proxy for whether the agent is serialized. +// +// Absolute times are meaningless across machines, so every timing assertion is a ratio against +// a baseline measured in the same process. The unrestricted context doubles as a positive +// control: if it does not visibly serialize the same workload, this machine cannot demonstrate +// stream concurrency at all and the timing assertions are skipped rather than guessed at. The +// record and callback assertions are exact and always run. + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ROCPROFILER_CALL(result, msg) \ + { \ + rocprofiler_status_t CHECKSTATUS = result; \ + if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS) \ + { \ + fprintf(stderr, \ + "[%s:%d] %s failed: %s\n", \ + __FILE__, \ + __LINE__, \ + msg, \ + rocprofiler_get_status_string(CHECKSTATUS)); \ + exit(1); \ + } \ + } + +#define HIP_CALL(cmd) \ + { \ + hipError_t e = cmd; \ + if(e != hipSuccess) \ + { \ + fprintf(stderr, "[%s:%d] HIP error: %s\n", __FILE__, __LINE__, hipGetErrorString(e)); \ + exit(1); \ + } \ + } + +namespace +{ +// Contexts are created during tool_init and started/stopped from main(). +rocprofiler_context_id_t ctx_scoped_gpu0 = {.handle = 0}; +rocprofiler_context_id_t ctx_scoped_gpu1 = {.handle = 0}; +rocprofiler_context_id_t ctx_all = {.handle = 0}; +rocprofiler_context_id_t ctx_all_second = {.handle = 0}; + +std::vector gpu_agents = {}; + +std::mutex observed_mutex = {}; +std::map dispatch_per_agent = {}; +std::map records_per_agent = {}; + +void +reset_observations() +{ + auto lk = std::unique_lock{observed_mutex}; + dispatch_per_agent.clear(); + records_per_agent.clear(); +} + +size_t +dispatches_on(rocprofiler_agent_id_t agent) +{ + auto lk = std::unique_lock{observed_mutex}; + auto itr = dispatch_per_agent.find(agent.handle); + return (itr == dispatch_per_agent.end()) ? 0 : itr->second; +} + +size_t +records_on(rocprofiler_agent_id_t agent) +{ + auto lk = std::unique_lock{observed_mutex}; + auto itr = records_per_agent.find(agent.handle); + return (itr == records_per_agent.end()) ? 0 : itr->second; +} + +void +record_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data, + rocprofiler_counter_record_t*, + size_t record_count, + rocprofiler_user_data_t, + void*) +{ + if(record_count == 0) return; + auto lk = std::unique_lock{observed_mutex}; + records_per_agent[dispatch_data.dispatch_info.agent_id.handle]++; +} + +void +dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data, + rocprofiler_counter_config_id_t* config, + rocprofiler_user_data_t*, + void*) +{ + const auto agent_id = dispatch_data.dispatch_info.agent_id; + + { + auto lk = std::unique_lock{observed_mutex}; + dispatch_per_agent[agent_id.handle]++; + } + + static std::shared_mutex cache_mutex = {}; + static std::unordered_map cache = {}; + + auto search = [&]() { + if(auto pos = cache.find(agent_id.handle); pos != cache.end()) + { + *config = pos->second; + return true; + } + return false; + }; + + { + auto rlock = std::shared_lock{cache_mutex}; + if(search()) return; + } + + auto wlock = std::unique_lock{cache_mutex}; + if(search()) return; + + auto supported = std::vector{}; + ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters( + agent_id, + [](rocprofiler_agent_id_t, + rocprofiler_counter_id_t* counters, + size_t num_counters, + void* user_data) { + auto* vec = + static_cast*>(user_data); + for(size_t i = 0; i < num_counters; i++) + vec->push_back(counters[i]); + return ROCPROFILER_STATUS_SUCCESS; + }, + static_cast(&supported)), + "iterate supported counters"); + + auto collect = std::vector{}; + for(auto& counter : supported) + { + rocprofiler_counter_info_v0_t info; + ROCPROFILER_CALL(rocprofiler_query_counter_info( + counter, ROCPROFILER_COUNTER_INFO_VERSION_0, (void*) &info), + "query counter info"); + if(strcmp(info.name, "SQ_WAVES") == 0) collect.push_back(counter); + } + + if(collect.empty()) return; + + rocprofiler_counter_config_id_t profile = {.handle = 0}; + ROCPROFILER_CALL( + rocprofiler_create_counter_config(agent_id, collect.data(), collect.size(), &profile), + "create counter config"); + + cache.emplace(agent_id.handle, profile); + *config = profile; +} + +rocprofiler_status_t +collect_gpu_agents(rocprofiler_agent_version_t, const void** agents, size_t num_agents, void*) +{ + for(size_t i = 0; i < num_agents; ++i) + { + const auto* agent = static_cast(agents[i]); + if(agent->type == ROCPROFILER_AGENT_TYPE_GPU) gpu_agents.push_back(agent->id); + } + return ROCPROFILER_STATUS_SUCCESS; +} + +void +configure_cc(rocprofiler_context_id_t* ctx, const rocprofiler_agent_id_t* agent) +{ + ROCPROFILER_CALL(rocprofiler_create_context(ctx), "create context"); + ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_counting_service( + *ctx, dispatch_callback, nullptr, record_callback, nullptr), + "configure dispatch counting service"); + if(agent) + ROCPROFILER_CALL(rocprofiler_dispatch_counting_service_set_agents(*ctx, agent, 1), + "restrict counting service to agent"); +} + +int +tool_init(rocprofiler_client_finalize_t, void*) +{ + ROCPROFILER_CALL(rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0, + collect_gpu_agents, + sizeof(rocprofiler_agent_v0_t), + nullptr), + "query available agents"); + + // Nothing to configure when the machine cannot express the scenario; main() reports the + // skip so the reason is visible in the test output. + if(gpu_agents.size() < 2) return 0; + + configure_cc(&ctx_scoped_gpu0, &gpu_agents[0]); + configure_cc(&ctx_scoped_gpu1, &gpu_agents[1]); + configure_cc(&ctx_all, nullptr); + configure_cc(&ctx_all_second, nullptr); + + return 0; +} + +void +tool_fini(void*) +{} +} // namespace + +extern "C" rocprofiler_tool_configure_result_t* +rocprofiler_configure(uint32_t, const char*, uint32_t, rocprofiler_client_id_t* id) +{ + id->name = "PerAgentScopingTest"; + static auto cfg = rocprofiler_tool_configure_result_t{ + sizeof(rocprofiler_tool_configure_result_t), &tool_init, &tool_fini, nullptr}; + return &cfg; +} + +// ------------------------------------------------------------------------------------------- +// Workload +// ------------------------------------------------------------------------------------------- + +__global__ void +spin_kernel(long long cycles, float* out) +{ + long long start = clock64(); + float acc = 0.0F; + while(clock64() - start < cycles) + acc += 1.0F; + // Never true for a 64-thread block; keeps the loop from being optimized away without + // adding a real store to the timed path. + if(threadIdx.x == 1024) *out = acc; +} + +namespace +{ +constexpr int kNumStreams = 8; +constexpr int kKernelsPer = 4; +constexpr long kSpinCycles = 2000000; + +// Launches kKernelsPer kernels on each of kNumStreams streams and waits for all of them. When +// the agent is not serialized the streams overlap and the wall time is roughly kKernelsPer +// kernels long; when it is serialized every dispatch runs alone and the wall time is roughly +// kNumStreams * kKernelsPer kernels long. +double +time_concurrent_workload(int device) +{ + HIP_CALL(hipSetDevice(device)); + + hipStream_t streams[kNumStreams]; + for(auto& s : streams) + HIP_CALL(hipStreamCreate(&s)); + + float* out = nullptr; + HIP_CALL(hipMalloc(&out, sizeof(float))); + + // Warm up so queue creation and code object loading are not inside the measurement. + for(auto& s : streams) + hipLaunchKernelGGL(spin_kernel, dim3(1), dim3(64), 0, s, kSpinCycles / 10, out); + HIP_CALL(hipDeviceSynchronize()); + + auto t0 = std::chrono::steady_clock::now(); + for(int k = 0; k < kKernelsPer; ++k) + for(auto& s : streams) + hipLaunchKernelGGL(spin_kernel, dim3(1), dim3(64), 0, s, kSpinCycles, out); + HIP_CALL(hipDeviceSynchronize()); + auto t1 = std::chrono::steady_clock::now(); + + for(auto& s : streams) + HIP_CALL(hipStreamDestroy(s)); + HIP_CALL(hipFree(out)); + + return std::chrono::duration(t1 - t0).count(); +} + +int failures = 0; + +void +check(bool cond, const std::string& what) +{ + printf("[%s] %s\n", cond ? "PASS" : "FAIL", what.c_str()); + if(!cond) ++failures; +} +} // namespace + +int +main() +{ + int device_count = 0; + HIP_CALL(hipGetDeviceCount(&device_count)); + + if(device_count < 2 || gpu_agents.size() < 2) + { + printf("SKIP: per-agent scoping requires at least 2 GPUs (found %d)\n", device_count); + return 0; + } + + const auto agent_0 = gpu_agents[0]; + const auto agent_1 = gpu_agents[1]; + + // Phase 1: no counter collection anywhere. Reference for an unserialized GPU-0. + const double baseline_ms = time_concurrent_workload(0); + + // Phase 2: unrestricted context. This is what every tool got before per-agent scoping, and + // it doubles as the positive control for the timing measurement. + reset_observations(); + ROCPROFILER_CALL(rocprofiler_start_context(ctx_all), "start unrestricted context"); + const double unrestricted_ms = time_concurrent_workload(0); + ROCPROFILER_CALL(rocprofiler_stop_context(ctx_all), "stop unrestricted context"); + const size_t unrestricted_dispatches_gpu0 = dispatches_on(agent_0); + + // Phase 3: context restricted to GPU-1. GPU-1 must still be collected on; GPU-0 must be + // untouched. + reset_observations(); + ROCPROFILER_CALL(rocprofiler_start_context(ctx_scoped_gpu1), "start scoped context"); + time_concurrent_workload(1); + const size_t scoped_records_gpu1 = records_on(agent_1); + const double scoped_ms = time_concurrent_workload(0); + ROCPROFILER_CALL(rocprofiler_stop_context(ctx_scoped_gpu1), "stop scoped context"); + const size_t scoped_dispatches_gpu0 = dispatches_on(agent_0); + const size_t scoped_records_gpu0 = records_on(agent_0); + + printf("baseline=%.2fms unrestricted=%.2fms scoped=%.2fms (%d streams x %d kernels)\n", + baseline_ms, + unrestricted_ms, + scoped_ms, + kNumStreams, + kKernelsPer); + + // Exact assertions. These hold regardless of how fast the machine is. + check(unrestricted_dispatches_gpu0 > 0, + "unrestricted context sees dispatches on GPU-0 (control)"); + check(scoped_records_gpu1 > 0, "scoped context still collects counters on its own agent"); + check(scoped_dispatches_gpu0 == 0, + "scoped context never receives a dispatch callback for GPU-0"); + check(scoped_records_gpu0 == 0, "scoped context produces no records for GPU-0"); + + // Timing assertions, but only when the positive control shows this machine can actually + // demonstrate the difference. Otherwise the workload never overlapped to begin with and a + // ratio would be measuring noise. + const double control_ratio = unrestricted_ms / baseline_ms; + if(control_ratio < 2.0) + { + printf("SKIP timing checks: unrestricted context only slowed GPU-0 by %.2fx, so this " + "machine does not show stream concurrency clearly enough to measure\n", + control_ratio); + } + else + { + const double scoped_ratio = scoped_ms / baseline_ms; + printf("serialization slowdown: unrestricted=%.2fx scoped=%.2fx\n", + control_ratio, + scoped_ratio); + check(scoped_ratio < 2.0, + "a context scoped to GPU-1 leaves GPU-0 running at baseline speed"); + } + + // Conflict semantics: disjoint agent sets may run together, overlapping ones may not. + { + auto status_a = rocprofiler_start_context(ctx_scoped_gpu0); + auto status_b = rocprofiler_start_context(ctx_scoped_gpu1); + check(status_a == ROCPROFILER_STATUS_SUCCESS && status_b == ROCPROFILER_STATUS_SUCCESS, + "two counter collection contexts on disjoint agents can both be active"); + ROCPROFILER_CALL(rocprofiler_stop_context(ctx_scoped_gpu1), "stop gpu1 context"); + ROCPROFILER_CALL(rocprofiler_stop_context(ctx_scoped_gpu0), "stop gpu0 context"); + } + + { + ROCPROFILER_CALL(rocprofiler_start_context(ctx_all), "start unrestricted context"); + auto status = rocprofiler_start_context(ctx_all_second); + check(status == ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT, + "two unrestricted counter collection contexts still conflict"); + ROCPROFILER_CALL(rocprofiler_stop_context(ctx_all), "stop unrestricted context"); + } + + if(failures > 0) + { + printf("%d check(s) failed\n", failures); + return 1; + } + + printf("all per-agent scoping checks passed\n"); + return 0; +} From 2ae929b344de5ebd41104b698743b9bf5dcb630a Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 11 Aug 2026 00:56:45 +0000 Subject: [PATCH 17/45] Apply suggestions from code review Suggested by @jrmadsen while reviewing #9868, applied as written: hsa/queue_controller.hpp using agent_handle_set_t = std::unordered_set context/context.hpp std::unordered_set agents{} context/context.hpp return agents.empty() || agents.count(agent_id) > 0 Also pulls in and , which he noted give full support for using the opaque handles in map/set/uomap/etc. Those includes are what let the sets key on rocprofiler_agent_id_t directly instead of unwrapping to the uint64_t handle. The cascading retyping this implies for the serialization refcounts and their call sites follows in the next commit, so that this one holds only the suggested text. Co-authored-by: Vivek Kale --- .../source/lib/rocprofiler-sdk/context/context.hpp | 6 ++++-- .../source/lib/rocprofiler-sdk/hsa/queue_controller.hpp | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index 19ede673024b..a3241b1b0e88 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -35,6 +35,8 @@ #include #include +#include +#include #include #include @@ -88,7 +90,7 @@ struct dispatch_counter_collection_service // produces; rocprofiler_dispatch_counting_service_set_agents narrows it. The set is read // on the dispatch path and when scoping serialization, so it is fixed at configure time // and never mutated once the context is started. - std::unordered_set agents{}; + std::unordered_set agents{}; // A flag to state whether or not the counter set is currently enabled. This is primarily // to protect against multithreaded calls to enable a context (and enabling already enabled // counters). @@ -98,7 +100,7 @@ struct dispatch_counter_collection_service // exactly as it did before per-agent scoping existed. bool collects_on(rocprofiler_agent_id_t agent_id) const { - return agents.empty() || agents.count(agent_id.handle) > 0; + return agents.empty() || agents.count(agent_id) > 0; } // Two counter-collection contexts can coexist as long as they do not both want the same diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp index 26288e36d119..2cb88db2c12b 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -94,7 +95,7 @@ class QueueController common::Synchronized& serializer(const Queue*); // Agent handles (rocprofiler_agent_id_t::handle). An empty set means every GPU agent. - using agent_handle_set_t = std::unordered_set; + using agent_handle_set_t = std::unordered_set; /** * Enable/disable serialization for QueueController, has no effect if counter collection From da8c2cdc2f56bf1ee59fa92ec1a7662c4ce91626 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Tue, 11 Aug 2026 00:57:04 +0000 Subject: [PATCH 18/45] refactor(rocprofiler-sdk): retype agent scoping cascades, move intersects out of line Follows the suggestions applied in the previous commit. Retyping the two agent sets to rocprofiler_agent_id_t leaves the surrounding code still speaking in raw uint64_t handles, so this carries the change through: - serialization_refcount::per_agent keys on rocprofiler_agent_id_t, and count(), enabled() and any() take/iterate the typed id - queue_controller.cpp and counters/core.cpp stop unwrapping to ::handle at the call sites - comments that described the sets as holding rocprofiler_agent_id_t::handle are reworded, since they no longer do Also addresses jrmadsen's fourth review thread, which asked for the body to live in the .cpp: dispatch_counter_collection_service::intersects is now declared in context.hpp and defined in context.cpp. He described that one rather than writing it, so it sits here rather than in his commit. Co-authored-by: Cursor --- .../lib/rocprofiler-sdk/context/context.cpp | 14 ++++++++++++ .../lib/rocprofiler-sdk/context/context.hpp | 22 +++++-------------- .../lib/rocprofiler-sdk/counters/core.cpp | 4 ++-- .../rocprofiler-sdk/hsa/queue_controller.cpp | 14 ++++++------ .../rocprofiler-sdk/hsa/queue_controller.hpp | 14 ++++++------ 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index e5dba095f238..0971616ad8a9 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -106,6 +106,20 @@ get_active_contexts_impl() } } // namespace +bool +dispatch_counter_collection_service::intersects( + const dispatch_counter_collection_service& rhs) const +{ + if(agents.empty() || rhs.agents.empty()) return true; + const auto& small = (agents.size() < rhs.agents.size()) ? agents : rhs.agents; + const auto& large = (agents.size() < rhs.agents.size()) ? rhs.agents : agents; + for(const auto& agent_id : small) + { + if(large.count(agent_id) > 0) return true; + } + return false; +} + context_array_t& get_registered_contexts(context_array_t& data, context_filter_t filter) { diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index a3241b1b0e88..fba9b3bda3ca 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -85,11 +85,11 @@ struct dispatch_counter_collection_service // Contains callback information along with other data needed to collect/process // counters. std::vector> callbacks{}; - // GPU agents (by rocprofiler_agent_id_t::handle) this context collects on. An empty set - // means every GPU agent, which is what a plain configure_{buffer,callback}_dispatch call - // produces; rocprofiler_dispatch_counting_service_set_agents narrows it. The set is read - // on the dispatch path and when scoping serialization, so it is fixed at configure time - // and never mutated once the context is started. + // GPU agents this context collects on. An empty set means every GPU agent, which is what a + // plain configure_{buffer,callback}_dispatch call produces; + // rocprofiler_dispatch_counting_service_set_agents narrows it. The set is read on the + // dispatch path and when scoping serialization, so it is fixed at configure time and never + // mutated once the context is started. std::unordered_set agents{}; // A flag to state whether or not the counter set is currently enabled. This is primarily // to protect against multithreaded calls to enable a context (and enabling already enabled @@ -105,17 +105,7 @@ struct dispatch_counter_collection_service // Two counter-collection contexts can coexist as long as they do not both want the same // GPU. An unrestricted context claims every agent, so it intersects with everything. - bool intersects(const dispatch_counter_collection_service& rhs) const - { - if(agents.empty() || rhs.agents.empty()) return true; - const auto& small = (agents.size() < rhs.agents.size()) ? agents : rhs.agents; - const auto& large = (agents.size() < rhs.agents.size()) ? rhs.agents : agents; - for(auto handle : small) - { - if(large.count(handle) > 0) return true; - } - return false; - } + bool intersects(const dispatch_counter_collection_service& rhs) const; }; struct spm_dispatch_counter_collection_service diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 5815a8218d01..48084ebfa602 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -227,14 +227,14 @@ set_dispatch_agents(rocprofiler_context_id_t context_id, return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED; } - auto selected = std::unordered_set{}; + auto selected = std::unordered_set{}; selected.reserve(num_agents); for(size_t i = 0; i < num_agents; ++i) { const auto* agent = rocprofiler::agent::get_agent(agents[i]); if(!agent || agent->type != ROCPROFILER_AGENT_TYPE_GPU) return ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND; - selected.emplace(agents[i].handle); + selected.emplace(agents[i]); } ctx_p->dispatch_counter_collection->agents = std::move(selected); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index edc377a97d2e..19ddd0497a08 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -579,10 +579,10 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena // An agent can be covered by `all` and by an explicit entry at the same time, so // the transition has to be decided by comparing effective counts across the // update rather than by watching a single counter hit zero. - auto was_enabled = std::unordered_map{}; + auto was_enabled = std::unordered_map{}; _profiler_serializer.rlock([&](const auto& serializers) { for(const auto& [agent_id, _] : serializers) - was_enabled.emplace(agent_id.handle, state.enabled(agent_id.handle)); + was_enabled.emplace(agent_id, state.enabled(agent_id)); }); if(agents.empty()) @@ -594,9 +594,9 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena } else { - for(auto agent_handle : agents) + for(const auto& agent_id : agents) { - auto& count = state.per_agent[agent_handle]; + auto& count = state.per_agent[agent_id]; if(enable) ++count; else if(count > 0) @@ -607,9 +607,9 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena _profiler_serializer.rlock([&](const auto& serializers) { for(const auto& [agent_id, _] : serializers) { - auto itr = was_enabled.find(agent_id.handle); + auto itr = was_enabled.find(agent_id); if(itr == was_enabled.end()) continue; - if(itr->second != state.enabled(agent_id.handle)) + if(itr->second != state.enabled(agent_id)) transitioned.emplace_back(agent_id); } }); @@ -670,7 +670,7 @@ QueueController::is_serialization_enabled(rocprofiler_agent_id_t agent_id) const { bool enabled = false; _serialization_refcount.rlock( - [&](const auto& state) { enabled = state.enabled(agent_id.handle); }); + [&](const auto& state) { enabled = state.enabled(agent_id); }); return enabled; } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp index 2cb88db2c12b..0e74b7ee9cd7 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp @@ -94,7 +94,7 @@ class QueueController common::Synchronized& serializer(const Queue*); - // Agent handles (rocprofiler_agent_id_t::handle). An empty set means every GPU agent. + // An empty set means every GPU agent. using agent_handle_set_t = std::unordered_set; /** @@ -146,21 +146,21 @@ class QueueController // that do not exist yet. struct serialization_refcount { - int64_t all = 0; - std::unordered_map per_agent = {}; + int64_t all = 0; + std::unordered_map per_agent = {}; - int64_t count(uint64_t agent_handle) const + int64_t count(rocprofiler_agent_id_t agent_id) const { - auto itr = per_agent.find(agent_handle); + auto itr = per_agent.find(agent_id); return all + ((itr == per_agent.end()) ? 0 : itr->second); } - bool enabled(uint64_t agent_handle) const { return count(agent_handle) > 0; } + bool enabled(rocprofiler_agent_id_t agent_id) const { return count(agent_id) > 0; } bool any() const { if(all > 0) return true; - for(const auto& [handle, count] : per_agent) + for(const auto& [agent_id, count] : per_agent) { if(count > 0) return true; } From fbe364eb42892133c994acc435b395d05e767c1f Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Mon, 10 Aug 2026 20:41:54 +0000 Subject: [PATCH 19/45] fix(rocprofiler-sdk): repair #9868 CI compile and clang-format failures Use rocprofiler_agent_id_t values in per_agent_scoping_test serialization calls instead of raw handles, and apply clang-format 11 to queue_controller. Co-authored-by: Cursor --- .../counters/tests/per_agent_scoping_test.cpp | 4 ++-- .../source/lib/rocprofiler-sdk/hsa/queue_controller.cpp | 6 ++---- .../source/lib/rocprofiler-sdk/hsa/queue_controller.hpp | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp index a696f0403310..f05a6155424c 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/per_agent_scoping_test.cpp @@ -402,7 +402,7 @@ TEST(counters_per_agent, scoped_and_unscoped_serialization_compose) controller->serializer(&queue_a); controller->serializer(&queue_b); - controller->enable_serialization({agent_b.handle}); + controller->enable_serialization({agent_b}); EXPECT_FALSE(controller->is_serialization_enabled(agent_a)); EXPECT_TRUE(controller->is_serialization_enabled(agent_b)); @@ -415,7 +415,7 @@ TEST(counters_per_agent, scoped_and_unscoped_serialization_compose) EXPECT_TRUE(controller->is_serialization_enabled(agent_b)) << "the scoped request still holds agent B"; - controller->disable_serialization({agent_b.handle}); + controller->disable_serialization({agent_b}); EXPECT_FALSE(controller->is_serialization_enabled(agent_b)); } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index 19ddd0497a08..33ad0691a0fe 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -609,8 +609,7 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena { auto itr = was_enabled.find(agent_id); if(itr == was_enabled.end()) continue; - if(itr->second != state.enabled(agent_id)) - transitioned.emplace_back(agent_id); + if(itr->second != state.enabled(agent_id)) transitioned.emplace_back(agent_id); } }); @@ -669,8 +668,7 @@ bool QueueController::is_serialization_enabled(rocprofiler_agent_id_t agent_id) const { bool enabled = false; - _serialization_refcount.rlock( - [&](const auto& state) { enabled = state.enabled(agent_id); }); + _serialization_refcount.rlock([&](const auto& state) { enabled = state.enabled(agent_id); }); return enabled; } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp index 0e74b7ee9cd7..531b0951dced 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.hpp @@ -146,7 +146,7 @@ class QueueController // that do not exist yet. struct serialization_refcount { - int64_t all = 0; + int64_t all = 0; std::unordered_map per_agent = {}; int64_t count(rocprofiler_agent_id_t agent_id) const From 4bfedef79f17a21f829bff09c24ec4c8d4634e89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:51:47 +0000 Subject: [PATCH 20/45] fix: hold refcount lock through serializer transitions to eliminate race in update_serialization Co-authored-by: amd-vkale <291155109+amd-vkale@users.noreply.github.com> --- .../rocprofiler-sdk/hsa/queue_controller.cpp | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index 33ad0691a0fe..72e61d38ae65 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -570,10 +570,9 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena auto pd_map = per_dev_map(_queues_v); // Agents whose effective refcount crossed zero in this call; only those get their - // serializer toggled. Collected under the refcount lock, applied under the serializer - // lock. - auto transitioned = std::vector{}; - bool any_enabled = false; + // serializer toggled. Both refcount mutation and serializer transitions happen + // together inside the refcount lock to prevent concurrent calls from interleaving. + bool any_enabled = false; _serialization_refcount.wlock([&](auto& state) { // An agent can be covered by `all` and by an explicit entry at the same time, so @@ -604,12 +603,25 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena } } - _profiler_serializer.rlock([&](const auto& serializers) { + _profiler_serializer.wlock([&](auto& serializers) { for(const auto& [agent_id, _] : serializers) { auto itr = was_enabled.find(agent_id); if(itr == was_enabled.end()) continue; - if(itr->second != state.enabled(agent_id)) transitioned.emplace_back(agent_id); + if(itr->second == state.enabled(agent_id)) continue; + + auto queues = hsa_barrier::queue_map_ptr_t{}; + if(auto it = pd_map.find(agent_id); it != pd_map.end()) + queues = it->second; + + auto ser_itr = serializers.find(agent_id); + if(ser_itr == serializers.end() || !ser_itr->second) continue; + ser_itr->second->wlock([&](auto& serializer) { + if(enable) + serializer.enable(queues); + else + serializer.disable(queues); + }); } }); @@ -617,26 +629,6 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena }); _serialized_enabled.store(any_enabled); - - if(transitioned.empty()) return; - - _profiler_serializer.wlock([&](auto& m) { - for(const auto& agent_id : transitioned) - { - auto itr = m.find(agent_id); - if(itr == m.end() || !itr->second) continue; - - auto queues = hsa_barrier::queue_map_ptr_t{}; - if(auto it = pd_map.find(agent_id); it != pd_map.end()) queues = it->second; - - itr->second->wlock([&](auto& serializer) { - if(enable) - serializer.enable(queues); - else - serializer.disable(queues); - }); - } - }); }); } From 106307de49469f4f77fc69536286ecb16cc56f3e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 01:02:24 +0000 Subject: [PATCH 21/45] fix: address all reviewer comments (header docs, test races, callback refcount, serializer lock, context mutex) Co-authored-by: amd-vkale <291155109+amd-vkale@users.noreply.github.com> --- .../dispatch_counting_service.h | 14 ---- .../lib/rocprofiler-sdk/context/context.hpp | 6 ++ .../lib/rocprofiler-sdk/counters/core.cpp | 7 +- .../counters/sample_processing.cpp | 10 ++- .../rocprofiler-sdk/hsa/queue_controller.cpp | 52 +++++++------- .../per_agent_scoping_app.cpp | 68 +++++++++++++++---- 6 files changed, 103 insertions(+), 54 deletions(-) diff --git a/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h b/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h index 9ec6c81cd17a..87bdafef8df5 100644 --- a/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h +++ b/projects/rocprofiler-sdk/source/include/rocprofiler-sdk/dispatch_counting_service.h @@ -110,20 +110,6 @@ typedef void (*rocprofiler_dispatch_counting_record_cb_t)( * one dispatch (denoted by correlation id). Will trigger the * callback based on the parameters setup in buffer_id_t. * - * NOTE: Interface is up for comment as to whether restrictions - * on agent should be made here (limiting the CB based on agent) - * or if the restriction should be performed by the tool in - * ::rocprofiler_dispatch_counting_service_cb_t (i.e. - * tool code checking the agent param to see if they want to profile - * it). - * - * Interface is up for comment as to whether restrictions - * on agent should be made here (limiting the CB based on agent) - * or if the restriction should be performed by the tool in - * ::rocprofiler_dispatch_counting_service_cb_t (i.e. - * tool code checking the agent param to see if they want to profile - * it). - * * @param [in] context_id context id * @param [in] buffer_id id of the buffer to use for the counting service * @param [in] callback callback to perform when dispatch is enqueued diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index fba9b3bda3ca..8d52e2f8108d 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,11 @@ deactivate_client_contexts(rocprofiler_client_id_t id); void deregister_client_contexts(rocprofiler_client_id_t id); +// The mutex that serialises start_context / stop_context and any mutation that must be +// atomic with activation state (e.g. setting the agent restriction on a context). +std::mutex& +get_contexts_mutex(); + inline bool default_context_filter(const context* val) { diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 48084ebfa602..848ba3eb1be0 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -220,7 +220,12 @@ set_dispatch_agents(rocprofiler_context_id_t context_id, if(!ctx_p->dispatch_counter_collection) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; // The agent set is read without a lock on the dispatch path and is what scopes - // serialization at start, so it may only change while the context is stopped. + // serialization at start, so it may only change while the context is stopped. Hold the + // contexts mutex so this check and the assignment are atomic with start_context: a + // concurrent start could otherwise activate the context after the scan (using the old + // set for serialization) and then race with the unordered-set assignment below while + // dispatch hooks read it. + auto _lk = std::unique_lock{context::get_contexts_mutex()}; for(const auto* itr : context::get_active_contexts()) { if(itr && itr->context_idx == ctx_p->context_idx) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/sample_processing.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/sample_processing.cpp index a6ca2609f070..3dc277df221d 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/sample_processing.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/sample_processing.cpp @@ -167,18 +167,26 @@ callback_thread_get() static auto*& _v = common::static_object::construct(&process_completed_cb); return *CHECK_NOTNULL(_v); } + +// Reference count for the callback consumer thread. Incremented by callback_thread_start() +// for each context that starts and decremented by callback_thread_stop() when each stops; +// the thread is only joined when the count returns to zero. Two dispatch-counting contexts +// with disjoint agent sets may be active at the same time, so without the refcount stopping +// either one would kill the consumer while the other still needs it. +std::atomic callback_thread_refcount{0}; } // namespace void callback_thread_start() { + ++callback_thread_refcount; callback_thread_get().start(); } void callback_thread_stop() { - callback_thread_get().exit(); + if(--callback_thread_refcount == 0) callback_thread_get().exit(); } /** diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index 72e61d38ae65..b0d33f419739 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -519,33 +519,35 @@ QueueController::serializer(const Queue* queue) CHECK(queue); const auto agent_id = queue->get_agent().get_rocp_agent()->id; - // Read the refcount before taking the serializer lock: update_serialization() acquires - // them in that order, and reversing it here would deadlock. A concurrent enable/disable - // for this agent can therefore race with the creation below, exactly as the previous - // _serialized_enabled load could, and is resolved the same way -- the next transition - // through update_serialization() reaches the entry once it is in the map. - const bool should_serialize = is_serialization_enabled(agent_id); - common::Synchronized* ret = nullptr; - _profiler_serializer.ulock( - [&](const auto& m) { - if(auto ptr = m.find(agent_id); ptr != m.end()) - { - ret = ptr->second.get(); + // Hold the refcount read lock across the lookup/insertion so that a concurrent + // update_serialization() cannot run between reading the refcount and inserting the + // entry: if it did, it would find no serializer to transition and the new entry would + // be created in the wrong state. update_serialization() holds the refcount write lock + // then the serializer write lock, so taking them in the same order here is deadlock-free. + _serialization_refcount.rlock([&](const auto& state) { + const bool should_serialize = state.enabled(agent_id); + _profiler_serializer.ulock( + [&](const auto& m) { + if(auto ptr = m.find(agent_id); ptr != m.end()) + { + ret = ptr->second.get(); + return true; + } + return false; + }, + [&](auto& m) { + ret = + m.emplace(agent_id, + std::make_shared>()) + .first->second.get(); + if(should_serialize) + { + ret->wlock([&](auto& serializer) { serializer.enable({}); }); + } return true; - } - return false; - }, - [&](auto& m) { - ret = m.emplace(agent_id, - std::make_shared>()) - .first->second.get(); - if(should_serialize) - { - ret->wlock([&](auto& serializer) { serializer.enable({}); }); - } - return true; - }); + }); + }); return *ret; } diff --git a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp index f99f3b4bc53b..f9b5cd723d51 100644 --- a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp +++ b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp @@ -84,8 +84,23 @@ rocprofiler_context_id_t ctx_scoped_gpu1 = {.handle = 0}; rocprofiler_context_id_t ctx_all = {.handle = 0}; rocprofiler_context_id_t ctx_all_second = {.handle = 0}; +// gpu_agents[i] is the rocprofiler agent for HIP device i. Populated in tool_init by +// matching each HIP device's PCI domain/BDF to the rocprofiler agent with the same +// coordinates so that the contexts and assertions both refer to the same physical device +// regardless of the order rocprofiler enumerates agents or how visible-device variables +// reorder HIP ordinals. std::vector gpu_agents = {}; +// Intermediate storage used to build the HIP-to-rocprofiler mapping: keeps the +// location_id (BDF) and PCI domain alongside the agent id until the mapping is done. +struct agent_info_t +{ + rocprofiler_agent_id_t id; + uint32_t location_id; // (bus<<8)|(device<<3)|function + uint32_t domain; +}; +std::vector all_gpu_agent_info = {}; + std::mutex observed_mutex = {}; std::map dispatch_per_agent = {}; std::map records_per_agent = {}; @@ -196,17 +211,6 @@ dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data, *config = profile; } -rocprofiler_status_t -collect_gpu_agents(rocprofiler_agent_version_t, const void** agents, size_t num_agents, void*) -{ - for(size_t i = 0; i < num_agents; ++i) - { - const auto* agent = static_cast(agents[i]); - if(agent->type == ROCPROFILER_AGENT_TYPE_GPU) gpu_agents.push_back(agent->id); - } - return ROCPROFILER_STATUS_SUCCESS; -} - void configure_cc(rocprofiler_context_id_t* ctx, const rocprofiler_agent_id_t* agent) { @@ -219,6 +223,18 @@ configure_cc(rocprofiler_context_id_t* ctx, const rocprofiler_agent_id_t* agent) "restrict counting service to agent"); } +rocprofiler_status_t +collect_gpu_agents(rocprofiler_agent_version_t, const void** agents, size_t num_agents, void*) +{ + for(size_t i = 0; i < num_agents; ++i) + { + const auto* agent = static_cast(agents[i]); + if(agent->type == ROCPROFILER_AGENT_TYPE_GPU) + all_gpu_agent_info.push_back({agent->id, agent->location_id, agent->domain}); + } + return ROCPROFILER_STATUS_SUCCESS; +} + int tool_init(rocprofiler_client_finalize_t, void*) { @@ -230,6 +246,32 @@ tool_init(rocprofiler_client_finalize_t, void*) // Nothing to configure when the machine cannot express the scenario; main() reports the // skip so the reason is visible in the test output. + if(all_gpu_agent_info.size() < 2) return 0; + + // Map each HIP device ordinal to its rocprofiler agent using PCI domain/BDF so that + // gpu_agents[i] is the agent for HIP device i, regardless of the order rocprofiler + // enumerates agents or how ROCR_VISIBLE_DEVICES / HIP_VISIBLE_DEVICES reorders ordinals. + int device_count = 0; + if(hipGetDeviceCount(&device_count) != hipSuccess || device_count < 2) return 0; + + for(int dev = 0; dev < device_count; ++dev) + { + hipDeviceProp_t prop{}; + if(hipGetDeviceProperties(&prop, dev) != hipSuccess) return 0; + const auto bus = static_cast(prop.pciBusID); + const auto did = static_cast(prop.pciDeviceID); + const auto dom = static_cast(prop.pciDomainID); + for(const auto& info : all_gpu_agent_info) + { + if(info.domain == dom && (info.location_id >> 8) == bus && + ((info.location_id >> 3) & 0x1Fu) == did) + { + gpu_agents.push_back(info.id); + break; + } + } + } + if(gpu_agents.size() < 2) return 0; configure_cc(&ctx_scoped_gpu0, &gpu_agents[0]); @@ -352,9 +394,9 @@ main() reset_observations(); ROCPROFILER_CALL(rocprofiler_start_context(ctx_scoped_gpu1), "start scoped context"); time_concurrent_workload(1); - const size_t scoped_records_gpu1 = records_on(agent_1); - const double scoped_ms = time_concurrent_workload(0); + const double scoped_ms = time_concurrent_workload(0); ROCPROFILER_CALL(rocprofiler_stop_context(ctx_scoped_gpu1), "stop scoped context"); + const size_t scoped_records_gpu1 = records_on(agent_1); const size_t scoped_dispatches_gpu0 = dispatches_on(agent_0); const size_t scoped_records_gpu0 = records_on(agent_0); From 70bef64ca06502317ec4896d2535fd06e78a3788 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Tue, 18 Aug 2026 05:49:45 +0000 Subject: [PATCH 22/45] fix(rocprofiler-sdk): define get_contexts_mutex once, matching the header The Aug 18 review follow-up declared get_contexts_mutex in context.hpp while leaving the original anonymous-namespace definition in context.cpp, which made every unique_lock on that mutex an ambiguous overload. Keep a single std::mutex& definition in the context namespace. Co-authored-by: Cursor --- .../source/lib/rocprofiler-sdk/context/context.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 0971616ad8a9..f9ec5de4054f 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -57,13 +57,6 @@ using active_context_vec_t = common::container::stable_vector::max(); -auto& -get_contexts_mutex() -{ - static auto _v = std::mutex{}; - return _v; -} - uint64_t get_contexts_offset() { @@ -106,6 +99,13 @@ get_active_contexts_impl() } } // namespace +std::mutex& +get_contexts_mutex() +{ + static auto _v = std::mutex{}; + return _v; +} + bool dispatch_counter_collection_service::intersects( const dispatch_counter_collection_service& rhs) const From 454b141429f8e4ea83f06fd55126ccd29d2b856e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 19 Aug 2026 20:28:14 +0000 Subject: [PATCH 23/45] perf(rocprofiler-sdk): counters exit-hook fast path and gate caching Co-authored-by: Vivek Kale --- .../rocprofiler-sdk/counters/queue_hooks.cpp | 11 +++++ .../counters/tests/queue_hooks_test.cpp | 48 +++++++++++++++++++ .../source/lib/rocprofiler-sdk/hsa/queue.cpp | 5 +- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index c3572409a75a..a40fea7f31bf 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -91,6 +91,17 @@ kernel_dispatch_phase_exit_hook(const hsa::Queue& /*queue*/, hsa::inst_pkt_t& inst_pkt, kernel_dispatch::profiling_time dispatch_time) { + bool has_counter_packets = false; + for(const auto& tagged_pkt : inst_pkt) + { + if(tagged_pkt.second == hsa::queue_hooks::COUNTERS_CLIENT_ID) + { + has_counter_packets = true; + break; + } + } + if(!has_counter_packets) return; + // Route by packet provenance, not current activeness: completed_cb self-filters via // packet_return_map, so in-flight dispatches still complete after stop_context removes the // context from the active list. This is what guarantees a completion that arrives is delivered; diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 574824f1d1b5..5ec05e5e7b4a 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -174,6 +174,54 @@ TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) EXPECT_FALSE(rocprofiler::counters::is_any_active()); } +TEST(counters_queue_hooks, exit_hook_skips_when_inst_pkt_has_no_counter_client_id) +{ + hsa::inst_pkt_t inst_pkt; + inst_pkt.emplace_back( + std::make_pair(std::make_unique(), + rocprofiler::hsa::queue_hooks::THREAD_TRACE_CLIENT_ID)); + + auto sess = std::make_shared(); + auto packet = rocprofiler::hsa::packet_data_t{}; + auto fq_pkt = rocprofiler::hsa::rocprofiler_packet{}; + + // Must return without touching registered counter contexts (no init required). + rocprofiler::counters::kernel_dispatch_phase_exit_hook( + *reinterpret_cast(nullptr), + fq_pkt, + sess, + packet, + inst_pkt, + rocprofiler::kernel_dispatch::profiling_time{}); + SUCCEED(); +} + +TEST(counters_queue_hooks, is_any_active_true_while_context_started) +{ + ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS); + test_init(); + + registration::init_logging(); + registration::set_init_status(-1); + context::push_client(1); + + ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed"); + ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_counting_service( + get_client_ctx(), user_dispatch_cb, nullptr, nullptr, nullptr), + "Could not setup counting service"); + ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context"); + + EXPECT_TRUE(rocprofiler::counters::is_any_active()); + + ROCPROFILER_CALL(rocprofiler_stop_context(get_client_ctx()), "stop context"); + EXPECT_FALSE(rocprofiler::counters::is_any_active()); + + registration::set_init_status(1); + registration::finalize(); + context::pop_client(1); + set_client_ctx(get_client_ctx()); +} + // Regression for callback-registry removal, per the review on #8891: dispatches enqueued while the // context is active must still complete when stop_context runs before their GPU completions land. // diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp index c9d06e0f450d..6e4b5f2d8ad3 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp @@ -315,11 +315,12 @@ WriteInterceptor(const void* packets, auto* gls = ::rocprofiler::hip::graph::current_launch_state(); const bool graph_launch_active = (gls != nullptr); + const bool counters_active = counters::is_any_active(); // Counter collection no longer registers a queue-controller callback, so it does not count // toward get_notifiers(); detect it explicitly so a counters-only run still enters the // interceptor. const bool no_real_consumers = - (queue.get_notifiers() == 0 && !counters::is_any_active() && + (queue.get_notifiers() == 0 && !counters_active && context::get_active_contexts(full_packet_instrumentation_context_filter).empty()); if(pkt_count == 0 || (no_real_consumers && !graph_launch_active)) @@ -778,7 +779,7 @@ WriteInterceptor(const void* packets, }); // Counter collection requires per-packet mode; it no longer participates in the registry. - if(counters::is_any_active()) should_batch_packets = false; + if(counters_active) should_batch_packets = false; if(should_batch_packets) { From 50a1d31d5ed5be4e38dcc693872b9bc791375c8e Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Tue, 25 Aug 2026 02:35:49 +0000 Subject: [PATCH 24/45] test(rocprofiler-sdk): add queue-hooks dispatch perf regression tests Adds LD_PRELOAD dispatch-counter workload and cost-model ctests to guard direct queue-hook vs callback-registry overhead without kernel replay passes. Co-authored-by: Cursor --- projects/rocprofiler-sdk/tests/CMakeLists.txt | 1 + .../tests/queue-hooks-perf/CMakeLists.txt | 84 ++++++++++++ .../tests/queue-hooks-perf/main.cpp | 83 ++++++++++++ .../tests/queue-hooks-perf/perf_cost_model.py | 35 +++++ .../queue-hooks-perf/run_and_validate.py | 123 ++++++++++++++++++ .../queue-hooks-perf/run_launch_sweep.py | 64 +++++++++ 6 files changed, 390 insertions(+) create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py diff --git a/projects/rocprofiler-sdk/tests/CMakeLists.txt b/projects/rocprofiler-sdk/tests/CMakeLists.txt index 03e40c3d534a..f15667ed1539 100644 --- a/projects/rocprofiler-sdk/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/tests/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(pc_sampling) add_subdirectory(hip-graph-tracing) add_subdirectory(counter-collection) add_subdirectory(counter-collection-per-agent) +add_subdirectory(queue-hooks-perf) add_subdirectory(openmp-tools) add_subdirectory(rocdecode) add_subdirectory(rocjpeg) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt b/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt new file mode 100644 index 000000000000..fa0c96dcc3c3 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt @@ -0,0 +1,84 @@ +# +# Queue interposition perf tests — dispatch counter path (callback registry vs direct hooks). +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +if(NOT CMAKE_HIP_COMPILER) + find_program( + amdclangpp_EXECUTABLE + NAMES amdclang++ + HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATH_SUFFIXES bin llvm/bin NO_CACHE) + mark_as_advanced(amdclangpp_EXECUTABLE) + + if(amdclangpp_EXECUTABLE) + set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}") + endif() +endif() + +project(rocprofiler-sdk-tests-queue-hooks-perf LANGUAGES CXX HIP) + +foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO) + if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "") + set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}") + endif() +endforeach() + +find_package(rocprofiler-sdk REQUIRED) +find_package(Python3 REQUIRED) + +set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP) +add_executable(queue-hooks-perf-testapp) +target_sources(queue-hooks-perf-testapp PRIVATE main.cpp) +target_link_libraries(queue-hooks-perf-testapp PRIVATE Threads::Threads) + +add_test( + NAME test-queue-hooks-perf-scaling + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_and_validate.py --testapp + $ --preload + $ --ballast-mb 8 --launches-low 8 + --launches-high 32 --max-scaling-ratio 6.0) + +set_tests_properties( + test-queue-hooks-perf-scaling + PROPERTIES TIMEOUT + 180 + LABELS + "integration-tests;queue-hooks;queue-hooks-perf" + RESOURCE_LOCK + GPU + DEPENDS + "queue-hooks-perf-testapp;rocprofiler-sdk-json-tool" + ENVIRONMENT + "QH_PERF_RESULTS_JSON=${CMAKE_CURRENT_BINARY_DIR}/qh_perf_scaling.json" + PASS_REGULAR_EXPRESSION + "\\[qh-perf-run\\] PASS" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}") + +add_test( + NAME test-queue-hooks-perf-launch-sweep + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_launch_sweep.py --testapp + $ --preload + $ --ballast-mb 8 --launches 4 8 16 32 + --slack 2.5) + +set_tests_properties( + test-queue-hooks-perf-launch-sweep + PROPERTIES TIMEOUT + 240 + LABELS + "integration-tests;queue-hooks;queue-hooks-perf" + RESOURCE_LOCK + GPU + DEPENDS + "queue-hooks-perf-testapp;rocprofiler-sdk-json-tool" + ENVIRONMENT + "QH_PERF_SWEEP_JSON=${CMAKE_CURRENT_BINARY_DIR}/qh_perf_launch_sweep.json" + PASS_REGULAR_EXPRESSION + "\\[qh-perf-sweep\\] PASS overall" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}") diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp b/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp new file mode 100644 index 000000000000..bb5041da0a62 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp @@ -0,0 +1,83 @@ +// MIT License +// +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + +#include + +#include +#include +#include +#include + +#define HIP_CHECK(call) \ + do \ + { \ + hipError_t _err = (call); \ + if(_err != hipSuccess) \ + { \ + fprintf( \ + stderr, "HIP error '%s' at %s:%d\n", hipGetErrorString(_err), __FILE__, __LINE__); \ + return EXIT_FAILURE; \ + } \ + } while(0) + +__global__ void +touch(float* buf, int n, int* counter) +{ + const int stride = blockDim.x * gridDim.x; + for(int i = blockDim.x * blockIdx.x + threadIdx.x; i < n; i += stride) + buf[i] = buf[i] * 1.0001f + 0.001f; + if(blockIdx.x == 0 && threadIdx.x == 0) atomicAdd(counter, 1); +} + +int +main(int argc, char** argv) +{ + int ballast_mb = 8; + int launches = 32; + if(argc > 1) ballast_mb = std::atoi(argv[1]); + if(argc > 2) launches = std::atoi(argv[2]); + if(ballast_mb < 1) ballast_mb = 1; + if(launches < 1) launches = 1; + + const size_t ballast_bytes = static_cast(ballast_mb) * 1024U * 1024U; + const int ballast_elems = static_cast(ballast_bytes / sizeof(float)); + + float* ballast = nullptr; + int* counter = nullptr; + HIP_CHECK(hipMalloc(&ballast, ballast_bytes)); + HIP_CHECK(hipMalloc(&counter, sizeof(int))); + HIP_CHECK(hipMemset(ballast, 0, ballast_bytes)); + HIP_CHECK(hipMemset(counter, 0, sizeof(int))); + + using clock = std::chrono::steady_clock; + const auto start = clock::now(); + for(int i = 0; i < launches; ++i) + { + touch<<<256, 64>>>(ballast, ballast_elems, counter); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + } + const auto end = clock::now(); + const double wall_ms = std::chrono::duration(end - start).count(); + + int counter_h = 0; + HIP_CHECK(hipMemcpy(&counter_h, counter, sizeof(int), hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(ballast)); + HIP_CHECK(hipFree(counter)); + + std::printf("[qh-perf] ballast_mb=%d launches=%d wall_ms=%.3f counter=%d\n", + ballast_mb, + launches, + wall_ms, + counter_h); + + if(counter_h != launches) + { + std::fprintf(stderr, "[qh-perf] FAIL counter=%d expected %d\n", counter_h, launches); + return EXIT_FAILURE; + } + + std::printf("[qh-perf] PASS\n"); + return EXIT_SUCCESS; +} diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py new file mode 100644 index 000000000000..fde2436c6756 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py @@ -0,0 +1,35 @@ +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Cost model for dispatch-counter queue interposition perf tests. +# Architecture-agnostic ceilings for CI regression detection. + +import os + +# Per-dispatch counter instrumentation allowance (ms). +PER_DISPATCH_MS = float(os.environ.get("ROCPROFILER_QH_PER_DISPATCH_MS", "3.0")) + +# Fixed attach / context setup (ms). +FIXED_OVERHEAD_MS = float(os.environ.get("ROCPROFILER_QH_FIXED_OVERHEAD_MS", "80.0")) + +# Slack over linear launch scaling. +OVERHEAD_MARGIN = float(os.environ.get("ROCPROFILER_QH_OVERHEAD_MARGIN", "8.0")) + + +def model_max_ms( + launches: int, + per_dispatch_ms: float | None = None, + fixed_ms: float | None = None, + margin: float | None = None, +) -> float: + per = PER_DISPATCH_MS if per_dispatch_ms is None else per_dispatch_ms + fixed = FIXED_OVERHEAD_MS if fixed_ms is None else fixed_ms + m = OVERHEAD_MARGIN if margin is None else margin + return fixed + launches * per * m + + +def max_launch_scaling_ratio(l_low: int, l_high: int, slack: float = 2.0) -> float: + if l_low <= 0: + return slack * float(l_high) + return (float(l_high) / float(l_low)) * slack diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py new file mode 100644 index 000000000000..2ed20d501c3f --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Runs dispatch-counter perf workload under LD_PRELOAD and validates wall-time bounds. + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +from perf_cost_model import max_launch_scaling_ratio, model_max_ms + +_MARKER = __import__("re").compile( + r"\[qh-perf\]\s+ballast_mb=(?P\d+)\s+launches=(?P\d+)\s+" + r"wall_ms=(?P[\d.]+)\s+counter=(?P\d+)" +) + + +def parse_marker(text: str) -> dict: + for line in text.splitlines(): + m = _MARKER.search(line) + if m: + return { + k: float(v) if k == "wall_ms" else int(v) + for k, v in m.groupdict().items() + } + raise AssertionError("missing [qh-perf] wall_ms marker") + + +def run_case(testapp: Path, preload: Path, ballast_mb: int, launches: int) -> str: + env = os.environ.copy() + preload = preload.resolve() + if env.get("LD_PRELOAD"): + env["LD_PRELOAD"] = f"{preload}:{env['LD_PRELOAD']}" + else: + env["LD_PRELOAD"] = str(preload) + env.setdefault("ROCPROFILER_TOOL_CONTEXTS", "COUNTER_COLLECTION") + env.setdefault("ROCPROF_COUNTERS", "SQ_WAVES_sum") + env["ROCPROFILER_TOOL_OUTPUT_FILE"] = os.environ.get( + "QH_PERF_JSON", f"/tmp/qh_perf_{launches}.json" + ) + proc = subprocess.run( + [str(testapp.resolve()), str(ballast_mb), str(launches)], + env=env, + capture_output=True, + text=True, + check=False, + ) + out = proc.stdout + proc.stderr + if proc.returncode != 0 or "[qh-perf] PASS" not in out: + raise RuntimeError(f"launches={launches} failed rc={proc.returncode}\n{out}") + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--testapp", type=Path, required=True) + ap.add_argument( + "--preload", type=Path, required=True, help="rocprofiler-sdk-json-tool .so" + ) + ap.add_argument("--ballast-mb", type=int, default=8) + ap.add_argument("--launches-low", type=int, default=8) + ap.add_argument("--launches-high", type=int, default=32) + ap.add_argument("--max-scaling-ratio", type=float, default=6.0) + args = ap.parse_args() + + low_out = run_case(args.testapp, args.preload, args.ballast_mb, args.launches_low) + high_out = run_case(args.testapp, args.preload, args.ballast_mb, args.launches_high) + low_m = parse_marker(low_out) + high_m = parse_marker(high_out) + + for label, m, launches in ( + (f"L={args.launches_low}", low_m, args.launches_low), + (f"L={args.launches_high}", high_m, args.launches_high), + ): + ceiling = model_max_ms(launches) + assert ( + m["wall_ms"] <= ceiling + ), f"{label} wall_ms={m['wall_ms']:.1f} > ceiling {ceiling:.1f} ms" + print(f"[qh-perf-run] {label} wall_ms={m['wall_ms']:.1f} <= {ceiling:.1f} ms") + + ratio = high_m["wall_ms"] / max(low_m["wall_ms"], 0.001) + cap = min( + args.max_scaling_ratio, + max_launch_scaling_ratio(args.launches_low, args.launches_high, 2.0), + ) + assert ratio <= cap, ( + f"scaling ratio {ratio:.2f} > {cap:.2f} " + f"(L={args.launches_low} {low_m['wall_ms']:.1f} ms vs " + f"L={args.launches_high} {high_m['wall_ms']:.1f} ms)" + ) + print(f"[qh-perf-run] PASS scaling ratio={ratio:.2f} <= {cap:.2f}") + + out_json = os.environ.get("QH_PERF_RESULTS_JSON", "") + if out_json: + out_path = Path(out_json) + variant = os.environ.get("QH_PERF_VARIANT", "unknown") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps( + { + "variant": variant, + "ballast_mb": args.ballast_mb, + "launches_low": args.launches_low, + "launches_high": args.launches_high, + "wall_ms_low": low_m["wall_ms"], + "wall_ms_high": high_m["wall_ms"], + "scaling_ratio": ratio, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py new file mode 100644 index 000000000000..cc4570f12587 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Launch-count sweep for dispatch-counter interposition overhead. + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +from perf_cost_model import max_launch_scaling_ratio, model_max_ms +from run_and_validate import parse_marker, run_case + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--testapp", type=Path, required=True) + ap.add_argument("--preload", type=Path, required=True) + ap.add_argument("--ballast-mb", type=int, default=8) + ap.add_argument("--launches", type=int, nargs="+", default=[4, 8, 16, 32]) + ap.add_argument("--slack", type=float, default=2.0) + args = ap.parse_args() + + wall: dict[int, float] = {} + for n in args.launches: + out = run_case(args.testapp, args.preload, args.ballast_mb, n) + wall[n] = parse_marker(out)["wall_ms"] + ceiling = model_max_ms(n) + assert wall[n] <= ceiling, f"L={n} wall_ms={wall[n]:.1f} > {ceiling:.1f} ms" + print(f"[qh-perf-sweep] L={n} wall_ms={wall[n]:.1f} <= {ceiling:.1f} ms") + + ordered = sorted(args.launches) + for i in range(1, len(ordered)): + lo, hi = ordered[i - 1], ordered[i] + ratio = wall[hi] / max(wall[lo], 0.001) + cap = max_launch_scaling_ratio(lo, hi, args.slack) + assert ratio <= cap, f"L={lo}->{hi} ratio={ratio:.2f} > {cap:.2f}" + print(f"[qh-perf-sweep] PASS L={lo}->{hi} ratio={ratio:.2f} <= {cap:.2f}") + + out_path = Path(os.environ.get("QH_PERF_SWEEP_JSON", "")) + if out_path: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps( + { + "variant": os.environ.get("QH_PERF_VARIANT", "unknown"), + "ballast_mb": args.ballast_mb, + "wall_ms": wall, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("[qh-perf-sweep] PASS overall") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f3d750f15565984b684752e36e51b656fb78dffc Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Tue, 25 Aug 2026 02:35:49 +0000 Subject: [PATCH 25/45] test(rocprofiler-sdk): add queue-hooks dispatch perf regression tests Adds LD_PRELOAD dispatch-counter workload and cost-model ctests to guard direct queue-hook vs callback-registry overhead without kernel replay passes. Co-authored-by: Cursor (cherry picked from commit 50a1d31d5ed5be4e38dcc693872b9bc791375c8e) --- projects/rocprofiler-sdk/tests/CMakeLists.txt | 1 + .../tests/queue-hooks-perf/CMakeLists.txt | 84 ++++++++++++ .../tests/queue-hooks-perf/main.cpp | 83 ++++++++++++ .../tests/queue-hooks-perf/perf_cost_model.py | 35 +++++ .../queue-hooks-perf/run_and_validate.py | 123 ++++++++++++++++++ .../queue-hooks-perf/run_launch_sweep.py | 64 +++++++++ 6 files changed, 390 insertions(+) create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py create mode 100644 projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py diff --git a/projects/rocprofiler-sdk/tests/CMakeLists.txt b/projects/rocprofiler-sdk/tests/CMakeLists.txt index 03e40c3d534a..f15667ed1539 100644 --- a/projects/rocprofiler-sdk/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/tests/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(pc_sampling) add_subdirectory(hip-graph-tracing) add_subdirectory(counter-collection) add_subdirectory(counter-collection-per-agent) +add_subdirectory(queue-hooks-perf) add_subdirectory(openmp-tools) add_subdirectory(rocdecode) add_subdirectory(rocjpeg) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt b/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt new file mode 100644 index 000000000000..fa0c96dcc3c3 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/CMakeLists.txt @@ -0,0 +1,84 @@ +# +# Queue interposition perf tests — dispatch counter path (callback registry vs direct hooks). +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +if(NOT CMAKE_HIP_COMPILER) + find_program( + amdclangpp_EXECUTABLE + NAMES amdclang++ + HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm + PATH_SUFFIXES bin llvm/bin NO_CACHE) + mark_as_advanced(amdclangpp_EXECUTABLE) + + if(amdclangpp_EXECUTABLE) + set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}") + endif() +endif() + +project(rocprofiler-sdk-tests-queue-hooks-perf LANGUAGES CXX HIP) + +foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO) + if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "") + set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}") + endif() +endforeach() + +find_package(rocprofiler-sdk REQUIRED) +find_package(Python3 REQUIRED) + +set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP) +add_executable(queue-hooks-perf-testapp) +target_sources(queue-hooks-perf-testapp PRIVATE main.cpp) +target_link_libraries(queue-hooks-perf-testapp PRIVATE Threads::Threads) + +add_test( + NAME test-queue-hooks-perf-scaling + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_and_validate.py --testapp + $ --preload + $ --ballast-mb 8 --launches-low 8 + --launches-high 32 --max-scaling-ratio 6.0) + +set_tests_properties( + test-queue-hooks-perf-scaling + PROPERTIES TIMEOUT + 180 + LABELS + "integration-tests;queue-hooks;queue-hooks-perf" + RESOURCE_LOCK + GPU + DEPENDS + "queue-hooks-perf-testapp;rocprofiler-sdk-json-tool" + ENVIRONMENT + "QH_PERF_RESULTS_JSON=${CMAKE_CURRENT_BINARY_DIR}/qh_perf_scaling.json" + PASS_REGULAR_EXPRESSION + "\\[qh-perf-run\\] PASS" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}") + +add_test( + NAME test-queue-hooks-perf-launch-sweep + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_launch_sweep.py --testapp + $ --preload + $ --ballast-mb 8 --launches 4 8 16 32 + --slack 2.5) + +set_tests_properties( + test-queue-hooks-perf-launch-sweep + PROPERTIES TIMEOUT + 240 + LABELS + "integration-tests;queue-hooks;queue-hooks-perf" + RESOURCE_LOCK + GPU + DEPENDS + "queue-hooks-perf-testapp;rocprofiler-sdk-json-tool" + ENVIRONMENT + "QH_PERF_SWEEP_JSON=${CMAKE_CURRENT_BINARY_DIR}/qh_perf_launch_sweep.json" + PASS_REGULAR_EXPRESSION + "\\[qh-perf-sweep\\] PASS overall" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}") diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp b/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp new file mode 100644 index 000000000000..bb5041da0a62 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/main.cpp @@ -0,0 +1,83 @@ +// MIT License +// +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + +#include + +#include +#include +#include +#include + +#define HIP_CHECK(call) \ + do \ + { \ + hipError_t _err = (call); \ + if(_err != hipSuccess) \ + { \ + fprintf( \ + stderr, "HIP error '%s' at %s:%d\n", hipGetErrorString(_err), __FILE__, __LINE__); \ + return EXIT_FAILURE; \ + } \ + } while(0) + +__global__ void +touch(float* buf, int n, int* counter) +{ + const int stride = blockDim.x * gridDim.x; + for(int i = blockDim.x * blockIdx.x + threadIdx.x; i < n; i += stride) + buf[i] = buf[i] * 1.0001f + 0.001f; + if(blockIdx.x == 0 && threadIdx.x == 0) atomicAdd(counter, 1); +} + +int +main(int argc, char** argv) +{ + int ballast_mb = 8; + int launches = 32; + if(argc > 1) ballast_mb = std::atoi(argv[1]); + if(argc > 2) launches = std::atoi(argv[2]); + if(ballast_mb < 1) ballast_mb = 1; + if(launches < 1) launches = 1; + + const size_t ballast_bytes = static_cast(ballast_mb) * 1024U * 1024U; + const int ballast_elems = static_cast(ballast_bytes / sizeof(float)); + + float* ballast = nullptr; + int* counter = nullptr; + HIP_CHECK(hipMalloc(&ballast, ballast_bytes)); + HIP_CHECK(hipMalloc(&counter, sizeof(int))); + HIP_CHECK(hipMemset(ballast, 0, ballast_bytes)); + HIP_CHECK(hipMemset(counter, 0, sizeof(int))); + + using clock = std::chrono::steady_clock; + const auto start = clock::now(); + for(int i = 0; i < launches; ++i) + { + touch<<<256, 64>>>(ballast, ballast_elems, counter); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + } + const auto end = clock::now(); + const double wall_ms = std::chrono::duration(end - start).count(); + + int counter_h = 0; + HIP_CHECK(hipMemcpy(&counter_h, counter, sizeof(int), hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(ballast)); + HIP_CHECK(hipFree(counter)); + + std::printf("[qh-perf] ballast_mb=%d launches=%d wall_ms=%.3f counter=%d\n", + ballast_mb, + launches, + wall_ms, + counter_h); + + if(counter_h != launches) + { + std::fprintf(stderr, "[qh-perf] FAIL counter=%d expected %d\n", counter_h, launches); + return EXIT_FAILURE; + } + + std::printf("[qh-perf] PASS\n"); + return EXIT_SUCCESS; +} diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py new file mode 100644 index 000000000000..fde2436c6756 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py @@ -0,0 +1,35 @@ +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Cost model for dispatch-counter queue interposition perf tests. +# Architecture-agnostic ceilings for CI regression detection. + +import os + +# Per-dispatch counter instrumentation allowance (ms). +PER_DISPATCH_MS = float(os.environ.get("ROCPROFILER_QH_PER_DISPATCH_MS", "3.0")) + +# Fixed attach / context setup (ms). +FIXED_OVERHEAD_MS = float(os.environ.get("ROCPROFILER_QH_FIXED_OVERHEAD_MS", "80.0")) + +# Slack over linear launch scaling. +OVERHEAD_MARGIN = float(os.environ.get("ROCPROFILER_QH_OVERHEAD_MARGIN", "8.0")) + + +def model_max_ms( + launches: int, + per_dispatch_ms: float | None = None, + fixed_ms: float | None = None, + margin: float | None = None, +) -> float: + per = PER_DISPATCH_MS if per_dispatch_ms is None else per_dispatch_ms + fixed = FIXED_OVERHEAD_MS if fixed_ms is None else fixed_ms + m = OVERHEAD_MARGIN if margin is None else margin + return fixed + launches * per * m + + +def max_launch_scaling_ratio(l_low: int, l_high: int, slack: float = 2.0) -> float: + if l_low <= 0: + return slack * float(l_high) + return (float(l_high) / float(l_low)) * slack diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py new file mode 100644 index 000000000000..2ed20d501c3f --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Runs dispatch-counter perf workload under LD_PRELOAD and validates wall-time bounds. + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +from perf_cost_model import max_launch_scaling_ratio, model_max_ms + +_MARKER = __import__("re").compile( + r"\[qh-perf\]\s+ballast_mb=(?P\d+)\s+launches=(?P\d+)\s+" + r"wall_ms=(?P[\d.]+)\s+counter=(?P\d+)" +) + + +def parse_marker(text: str) -> dict: + for line in text.splitlines(): + m = _MARKER.search(line) + if m: + return { + k: float(v) if k == "wall_ms" else int(v) + for k, v in m.groupdict().items() + } + raise AssertionError("missing [qh-perf] wall_ms marker") + + +def run_case(testapp: Path, preload: Path, ballast_mb: int, launches: int) -> str: + env = os.environ.copy() + preload = preload.resolve() + if env.get("LD_PRELOAD"): + env["LD_PRELOAD"] = f"{preload}:{env['LD_PRELOAD']}" + else: + env["LD_PRELOAD"] = str(preload) + env.setdefault("ROCPROFILER_TOOL_CONTEXTS", "COUNTER_COLLECTION") + env.setdefault("ROCPROF_COUNTERS", "SQ_WAVES_sum") + env["ROCPROFILER_TOOL_OUTPUT_FILE"] = os.environ.get( + "QH_PERF_JSON", f"/tmp/qh_perf_{launches}.json" + ) + proc = subprocess.run( + [str(testapp.resolve()), str(ballast_mb), str(launches)], + env=env, + capture_output=True, + text=True, + check=False, + ) + out = proc.stdout + proc.stderr + if proc.returncode != 0 or "[qh-perf] PASS" not in out: + raise RuntimeError(f"launches={launches} failed rc={proc.returncode}\n{out}") + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--testapp", type=Path, required=True) + ap.add_argument( + "--preload", type=Path, required=True, help="rocprofiler-sdk-json-tool .so" + ) + ap.add_argument("--ballast-mb", type=int, default=8) + ap.add_argument("--launches-low", type=int, default=8) + ap.add_argument("--launches-high", type=int, default=32) + ap.add_argument("--max-scaling-ratio", type=float, default=6.0) + args = ap.parse_args() + + low_out = run_case(args.testapp, args.preload, args.ballast_mb, args.launches_low) + high_out = run_case(args.testapp, args.preload, args.ballast_mb, args.launches_high) + low_m = parse_marker(low_out) + high_m = parse_marker(high_out) + + for label, m, launches in ( + (f"L={args.launches_low}", low_m, args.launches_low), + (f"L={args.launches_high}", high_m, args.launches_high), + ): + ceiling = model_max_ms(launches) + assert ( + m["wall_ms"] <= ceiling + ), f"{label} wall_ms={m['wall_ms']:.1f} > ceiling {ceiling:.1f} ms" + print(f"[qh-perf-run] {label} wall_ms={m['wall_ms']:.1f} <= {ceiling:.1f} ms") + + ratio = high_m["wall_ms"] / max(low_m["wall_ms"], 0.001) + cap = min( + args.max_scaling_ratio, + max_launch_scaling_ratio(args.launches_low, args.launches_high, 2.0), + ) + assert ratio <= cap, ( + f"scaling ratio {ratio:.2f} > {cap:.2f} " + f"(L={args.launches_low} {low_m['wall_ms']:.1f} ms vs " + f"L={args.launches_high} {high_m['wall_ms']:.1f} ms)" + ) + print(f"[qh-perf-run] PASS scaling ratio={ratio:.2f} <= {cap:.2f}") + + out_json = os.environ.get("QH_PERF_RESULTS_JSON", "") + if out_json: + out_path = Path(out_json) + variant = os.environ.get("QH_PERF_VARIANT", "unknown") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps( + { + "variant": variant, + "ballast_mb": args.ballast_mb, + "launches_low": args.launches_low, + "launches_high": args.launches_high, + "wall_ms_low": low_m["wall_ms"], + "wall_ms_high": high_m["wall_ms"], + "scaling_ratio": ratio, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py new file mode 100644 index 000000000000..cc4570f12587 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Launch-count sweep for dispatch-counter interposition overhead. + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +from perf_cost_model import max_launch_scaling_ratio, model_max_ms +from run_and_validate import parse_marker, run_case + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--testapp", type=Path, required=True) + ap.add_argument("--preload", type=Path, required=True) + ap.add_argument("--ballast-mb", type=int, default=8) + ap.add_argument("--launches", type=int, nargs="+", default=[4, 8, 16, 32]) + ap.add_argument("--slack", type=float, default=2.0) + args = ap.parse_args() + + wall: dict[int, float] = {} + for n in args.launches: + out = run_case(args.testapp, args.preload, args.ballast_mb, n) + wall[n] = parse_marker(out)["wall_ms"] + ceiling = model_max_ms(n) + assert wall[n] <= ceiling, f"L={n} wall_ms={wall[n]:.1f} > {ceiling:.1f} ms" + print(f"[qh-perf-sweep] L={n} wall_ms={wall[n]:.1f} <= {ceiling:.1f} ms") + + ordered = sorted(args.launches) + for i in range(1, len(ordered)): + lo, hi = ordered[i - 1], ordered[i] + ratio = wall[hi] / max(wall[lo], 0.001) + cap = max_launch_scaling_ratio(lo, hi, args.slack) + assert ratio <= cap, f"L={lo}->{hi} ratio={ratio:.2f} > {cap:.2f}" + print(f"[qh-perf-sweep] PASS L={lo}->{hi} ratio={ratio:.2f} <= {cap:.2f}") + + out_path = Path(os.environ.get("QH_PERF_SWEEP_JSON", "")) + if out_path: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps( + { + "variant": os.environ.get("QH_PERF_VARIANT", "unknown"), + "ballast_mb": args.ballast_mb, + "wall_ms": wall, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("[qh-perf-sweep] PASS overall") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 52667bfeff7ba2b7c8aa4d0ca405a27bb21ddc23 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 18 Sep 2026 05:48:17 +0000 Subject: [PATCH 26/45] fix(rocprofiler-sdk): move queue hook tags off the callback registry id range The inst_pkt_t tag space is shared between the per-queue callback registry and the queue hooks. QueueController::add_callback allocates strictly positive ids counting up from 1, which is exactly the range the hook tags used, so a packet tagged by a not-yet-migrated service could compare equal to a hook tag. This was inert while the tags were only carried and never inspected, but counters, thread trace and SPM now each select their own packets by comparing against their tag, so the overlap is reachable. Move the tags to a negative range, which the registry cannot produce, and add a static_assert so the invariant survives future additions. Also converts the tags from separate constexpr int64_t values to an unscoped enum with a fixed int64_t underlying type, keeping the implicit conversion to hsa::ClientID so call sites need no casts. (cherry picked from commit b6596f90d5dcc9a9dbecb56d3a9e30af74fbcedc) --- .../hsa/queue_hooks/client_ids.hpp | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp index 815ae5879ee3..b7a45802fef9 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_hooks/client_ids.hpp @@ -30,30 +30,35 @@ namespace hsa { namespace queue_hooks { -// Tags identifying the producer subsystem of each inst_pkt_t entry. -// Only distinctness matters; values are fixed for test-order stability. -// Services are migrated off the per-queue callback registry one at a time; the -// unused ids are reserved so the tag values stay stable across those PRs. +// Tags identifying the producer subsystem of each inst_pkt_t entry, stored in the +// inst_pkt_t pair as an hsa::ClientID. // -// The underlying type is fixed to int64_t to match hsa::ClientID, which is how the -// tag is stored in inst_pkt_t. The enum is deliberately unscoped rather than an -// enum class: unscoped keeps the implicit conversion to ClientID, so the emplace -// sites need no cast, while still grouping the values under one type. +// The underlying type is fixed to int64_t to match hsa::ClientID. The enum is +// deliberately unscoped rather than an enum class: unscoped keeps the implicit +// conversion to ClientID, so the emplace sites need no cast, while still grouping +// the values under one type. // -// These values numerically overlap the per-queue registry's auto-incrementing -// ClientID, which also starts at 1 and is still in use by the services that have -// not been migrated yet. That is inert only because no consumer routes on the tag: -// counters' completed_cb and thread trace's post_kernel_call both identify their -// own packets by pointer lookup or dynamic_cast. Before adding a consumer that -// dispatches on these values, either finish migrating the remaining services off -// the registry or move this enum to a range the registry cannot produce. +// The values are negative because they share the inst_pkt_t tag space with the +// per-queue callback registry, whose QueueController::add_callback hands out +// strictly positive ids counting up from 1. Migrated services (counters, thread +// trace, SPM) select their own packets by comparing against these tags, so a tag +// drawn from the registry's range could match a packet the service does not own. +// Services that have not migrated yet still receive registry ids, so the two +// ranges must stay disjoint until the registry is gone. Any tag added here must +// likewise be negative; -1 is left unused because QueueController::add_callback +// uses it as a local "unassigned" initializer. enum client_id : int64_t { - COUNTERS_CLIENT_ID = 1, - THREAD_TRACE_CLIENT_ID = 2, - PC_SAMPLING_CLIENT_ID = 3, - SPM_CLIENT_ID = 4, + COUNTERS_CLIENT_ID = -1001, + THREAD_TRACE_CLIENT_ID = -1002, + PC_SAMPLING_CLIENT_ID = -1003, + SPM_CLIENT_ID = -1004, }; + +static_assert(COUNTERS_CLIENT_ID < 0 && THREAD_TRACE_CLIENT_ID < 0 && PC_SAMPLING_CLIENT_ID < 0 && + SPM_CLIENT_ID < 0, + "queue hook tags must stay disjoint from the positive ids handed out by " + "QueueController::add_callback"); } // namespace queue_hooks } // namespace hsa } // namespace rocprofiler From 38792f3de8a9bca886ee98beeec107c414404d53 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 18 Sep 2026 06:03:47 +0000 Subject: [PATCH 27/45] style(rocprofiler-sdk): align serializer transition application with the SPM branch Derive the applied serializer state from the post-update refcount instead of the caller's `enable` flag and document why, and format queue_controller.cpp under the clang-format version the repository pins. No behavioral change: for an agent that transitioned, the post-update state and the caller's flag already agreed. Keeps serializer() and update_serialization() byte-identical with the SPM branch so the two migrations do not conflict when they land. (cherry picked from commit 72c897ee396f34ea23386a4d292a9d4044f324a2) --- .../rocprofiler-sdk/hsa/queue_controller.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp index b0d33f419739..2af4c7cefc09 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue_controller.cpp @@ -537,10 +537,9 @@ QueueController::serializer(const Queue* queue) return false; }, [&](auto& m) { - ret = - m.emplace(agent_id, - std::make_shared>()) - .first->second.get(); + ret = m.emplace(agent_id, + std::make_shared>()) + .first->second.get(); if(should_serialize) { ret->wlock([&](auto& serializer) { serializer.enable({}); }); @@ -610,16 +609,19 @@ QueueController::update_serialization(const agent_handle_set_t& agents, bool ena { auto itr = was_enabled.find(agent_id); if(itr == was_enabled.end()) continue; - if(itr->second == state.enabled(agent_id)) continue; + const bool now_enabled = state.enabled(agent_id); + if(itr->second == now_enabled) continue; auto queues = hsa_barrier::queue_map_ptr_t{}; - if(auto it = pd_map.find(agent_id); it != pd_map.end()) - queues = it->second; + if(auto it = pd_map.find(agent_id); it != pd_map.end()) queues = it->second; auto ser_itr = serializers.find(agent_id); if(ser_itr == serializers.end() || !ser_itr->second) continue; + // Derived from the post-update refcount rather than the caller's `enable` + // flag, so the applied state cannot disagree with the count that produced + // the transition. ser_itr->second->wlock([&](auto& serializer) { - if(enable) + if(now_enabled) serializer.enable(queues); else serializer.disable(queues); From d831c5838cb999cc521982dbd9c284e08eb03d6a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 21:28:16 +0000 Subject: [PATCH 28/45] style(rocprofiler-sdk): apply clang-format to the counters queue hook test Already unformatted on the base branch, which fails the Formatting / source check. No functional change. Co-authored-by: Vivek Kale --- .../counters/tests/queue_hooks_test.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 5ec05e5e7b4a..9852cdbfb691 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -177,13 +177,12 @@ TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) TEST(counters_queue_hooks, exit_hook_skips_when_inst_pkt_has_no_counter_client_id) { hsa::inst_pkt_t inst_pkt; - inst_pkt.emplace_back( - std::make_pair(std::make_unique(), - rocprofiler::hsa::queue_hooks::THREAD_TRACE_CLIENT_ID)); + inst_pkt.emplace_back(std::make_pair(std::make_unique(), + rocprofiler::hsa::queue_hooks::THREAD_TRACE_CLIENT_ID)); - auto sess = std::make_shared(); - auto packet = rocprofiler::hsa::packet_data_t{}; - auto fq_pkt = rocprofiler::hsa::rocprofiler_packet{}; + auto sess = std::make_shared(); + auto packet = rocprofiler::hsa::packet_data_t{}; + auto fq_pkt = rocprofiler::hsa::rocprofiler_packet{}; // Must return without touching registered counter contexts (no init required). rocprofiler::counters::kernel_dispatch_phase_exit_hook( From a783e25736ac88a3cdd3c2c40e5e196409d5944c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 22:40:53 +0000 Subject: [PATCH 29/45] fix(rocprofiler-sdk): make the counters queue hooks test compile Two errors, both in counters/tests/queue_hooks_test.cpp: error: reinterpret_cast from 'std::nullptr_t' to 'rocprofiler::hsa::Queue *' is not allowed error: use of undeclared identifier 'set_client_ctx' The test built a Queue& by dereferencing a null pointer to reach the exit hook's early-return path. The hook never dereferences the queue, so take it as a pointer and let the test pass nullptr, matching the PC sampling branch. set_client_ctx exists in the SPM tests but was never carried over here, so add the same reset helper next to get_client_ctx. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/queue_hooks.cpp | 2 +- .../lib/rocprofiler-sdk/counters/queue_hooks.hpp | 4 +++- .../counters/tests/queue_hooks_test.cpp | 16 +++++++++------- .../source/lib/rocprofiler-sdk/hsa/queue.cpp | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index a40fea7f31bf..730c9ea5b95c 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -84,7 +84,7 @@ kernel_dispatch_phase_enter_hook( } void -kernel_dispatch_phase_exit_hook(const hsa::Queue& /*queue*/, +kernel_dispatch_phase_exit_hook(const hsa::Queue* /*queue*/, const hsa::rocprofiler_packet& /*kernel_packet*/, std::shared_ptr& session, hsa::packet_data_t& packet, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp index 6fc716d59258..1a76e7d66b3e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.hpp @@ -55,8 +55,10 @@ kernel_dispatch_phase_enter_hook( // registered dispatch_counter_collection contexts (not only active ones) and calls // each callback's completed_cb; completed_cb self-filters via packet_return_map so // in-flight dispatches still complete after stop_context. +// queue is nullable: the hook never dereferences it, and tests exercise the +// early-return path without an HSA runtime to build a queue from. void -kernel_dispatch_phase_exit_hook(const hsa::Queue& queue, +kernel_dispatch_phase_exit_hook(const hsa::Queue* queue, const hsa::rocprofiler_packet& kernel_packet, std::shared_ptr& session, hsa::packet_data_t& packet, diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index 9852cdbfb691..c5711addb6c7 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -88,6 +88,13 @@ get_client_ctx() return ctx; } +// Reset the shared handle so a later test in this binary starts from a fresh context. +void +set_client_ctx(rocprofiler_context_id_t& ctx) +{ + ctx = rocprofiler_context_id_t{0}; +} + void test_init() { @@ -186,12 +193,7 @@ TEST(counters_queue_hooks, exit_hook_skips_when_inst_pkt_has_no_counter_client_i // Must return without touching registered counter contexts (no init required). rocprofiler::counters::kernel_dispatch_phase_exit_hook( - *reinterpret_cast(nullptr), - fq_pkt, - sess, - packet, - inst_pkt, - rocprofiler::kernel_dispatch::profiling_time{}); + nullptr, fq_pkt, sess, packet, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); SUCCEED(); } @@ -350,7 +352,7 @@ TEST(counters_queue_hooks, stop_context_in_flight_completion_routes_via_hook_pat inst_pkt.emplace_back(std::move(in_flight[i]), hsa::queue_hooks::COUNTERS_CLIENT_ID); rocprofiler::counters::kernel_dispatch_phase_exit_hook( - fq, pkt, sess, packet_data, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); + &fq, pkt, sess, packet_data, inst_pkt, rocprofiler::kernel_dispatch::profiling_time{}); size_t remaining = num_dispatches; cb_info->packet_return_map.rlock([&](const auto& data) { remaining = data.size(); }); diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp index 6e4b5f2d8ad3..3d1fd1d45b7b 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/queue.cpp @@ -171,7 +171,7 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data) // Counter collection completion is migrated off the callback registry (see // WriteInterceptor); invoke it explicitly here. - counters::kernel_dispatch_phase_exit_hook(queue_info_session.queue, + counters::kernel_dispatch_phase_exit_hook(&queue_info_session.queue, packet.kernel_packet, _session, packet, From ebf8cd7ab78c81e83c53e9a68329718b280270af Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 23:48:04 +0000 Subject: [PATCH 30/45] fix(rocprofiler-sdk): construct valid test fixtures in the counters exit hook test Two errors g++ reports against the libstdc++ headers, so they are easy to miss when scanning diagnostics by source path: error: invalid new-expression of abstract class type 'rocprofiler::hsa::AQLPacket' error: use of deleted function 'rocprofiler::hsa::queue_info_session_t::queue_info_session_t()' AQLPacket is abstract; EmptyAQLPacket is the concrete no-op subclass intended for this. queue_info_session_t holds a Queue& so it has no default constructor, and the hook returns before dereferencing the session, so a null shared_ptr keeps the test free of any HSA runtime. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp index c5711addb6c7..aa74cb4a1bed 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/queue_hooks_test.cpp @@ -184,10 +184,12 @@ TEST(counters_queue_hooks, is_any_active_false_when_no_context_active) TEST(counters_queue_hooks, exit_hook_skips_when_inst_pkt_has_no_counter_client_id) { hsa::inst_pkt_t inst_pkt; - inst_pkt.emplace_back(std::make_pair(std::make_unique(), + inst_pkt.emplace_back(std::make_pair(std::make_unique(), rocprofiler::hsa::queue_hooks::THREAD_TRACE_CLIENT_ID)); - auto sess = std::make_shared(); + // The hook returns before it dereferences the session, so a null one keeps this test free of + // any HSA runtime: queue_info_session_t holds a Queue& and cannot be default constructed. + auto sess = std::shared_ptr{}; auto packet = rocprofiler::hsa::packet_data_t{}; auto fq_pkt = rocprofiler::hsa::rocprofiler_packet{}; From 72239053d73d67bd773aa82adedc89280bfa138b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 20 Sep 2026 02:24:27 +0000 Subject: [PATCH 31/45] fix(rocprofiler-sdk): keep the queue-hooks-perf scripts importable on older Python test-queue-hooks-perf-scaling and test-queue-hooks-perf-launch-sweep failed on rhel-8.8, rhel-9.5 and sles-15.6 (ubuntu-22.04 passed, since it ships 3.10): File "tests/queue-hooks-perf/perf_cost_model.py", line 22, in per_dispatch_ms: float | None = None, TypeError: unsupported operand type(s) for |: 'type' and 'NoneType' PEP 604 unions in evaluated annotations need Python 3.10. Use typing.Optional instead, and likewise typing.Dict for the builtin generic in run_launch_sweep, so these scripts import on the oldest interpreter CI runs on. Co-authored-by: Vivek Kale --- .../tests/queue-hooks-perf/perf_cost_model.py | 7 ++++--- .../tests/queue-hooks-perf/run_launch_sweep.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py index fde2436c6756..62ec6a9a21fe 100644 --- a/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/perf_cost_model.py @@ -6,6 +6,7 @@ # Architecture-agnostic ceilings for CI regression detection. import os +from typing import Optional # Per-dispatch counter instrumentation allowance (ms). PER_DISPATCH_MS = float(os.environ.get("ROCPROFILER_QH_PER_DISPATCH_MS", "3.0")) @@ -19,9 +20,9 @@ def model_max_ms( launches: int, - per_dispatch_ms: float | None = None, - fixed_ms: float | None = None, - margin: float | None = None, + per_dispatch_ms: Optional[float] = None, + fixed_ms: Optional[float] = None, + margin: Optional[float] = None, ) -> float: per = PER_DISPATCH_MS if per_dispatch_ms is None else per_dispatch_ms fixed = FIXED_OVERHEAD_MS if fixed_ms is None else fixed_ms diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py index cc4570f12587..b3ec7bbfa683 100644 --- a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_launch_sweep.py @@ -11,6 +11,7 @@ import subprocess import sys from pathlib import Path +from typing import Dict from perf_cost_model import max_launch_scaling_ratio, model_max_ms from run_and_validate import parse_marker, run_case @@ -25,7 +26,7 @@ def main() -> int: ap.add_argument("--slack", type=float, default=2.0) args = ap.parse_args() - wall: dict[int, float] = {} + wall: Dict[int, float] = {} for n in args.launches: out = run_case(args.testapp, args.preload, args.ballast_mb, n) wall[n] = parse_marker(out)["wall_ms"] From 91f368be72bdbb8c379db0fb76abab261d1bbe89 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 20 Sep 2026 04:34:44 +0000 Subject: [PATCH 32/45] fix(rocprofiler-sdk): drop Python 3.7-only subprocess kwargs from the perf harness rhel-8.8 and sles-15.6 run Python 3.6, where subprocess.run rejects both capture_output and text: TypeError: __init__() got an unexpected keyword argument 'capture_output' Use stdout/stderr=PIPE with universal_newlines instead. vermin now reports 3.6 as the minimum required version for all three scripts in this directory, which matches the oldest interpreter in CI. Co-authored-by: Vivek Kale --- .../tests/queue-hooks-perf/run_and_validate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py index 2ed20d501c3f..987bf0c90ade 100644 --- a/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py +++ b/projects/rocprofiler-sdk/tests/queue-hooks-perf/run_and_validate.py @@ -46,8 +46,9 @@ def run_case(testapp: Path, preload: Path, ballast_mb: int, launches: int) -> st proc = subprocess.run( [str(testapp.resolve()), str(ballast_mb), str(launches)], env=env, - capture_output=True, - text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, check=False, ) out = proc.stdout + proc.stderr From 45d2adb7304e0f0764ac37529fd77b06ba622615 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 05:54:17 +0000 Subject: [PATCH 33/45] ci(rocprofiler-sdk): repair code coverage and ROCm release compatibility jobs Both workflows failed before compiling any rocprofiler-sdk source. Code coverage died installing ROCR-Runtime with 'CMake 3.25 or higher is required. You are running version 3.22.1', and release compatibility died resolving find_dependency(NUMA) from hsakmt-config.cmake on the 'latest' ROCm container. This branch predates the fixes for both on develop, so take develop's versions of the two workflows and of the rocprofiler-sdk-util action they depend on. The action change is additive -- python-venv-path defaults to empty and is guarded -- so the continuous integration workflow, which does not pass it, is unaffected. The pinned CMake is held at 3.25 rather than develop's 4.x. The external submodules on this branch (json, pybind11, sqlite, yaml-cpp) still declare cmake_minimum_required below 3.5, which CMake 4 rejects outright, so 4.x would trade one configure failure for another. Co-authored-by: Vivek Kale --- .../actions/rocprofiler-sdk-util/action.yml | 25 ++++++++----- .../resolve-ci-config/action.yml | 10 +++++- .../resolve-ci-config/resolve_ci_config.py | 35 ++++++++++++++++--- .../test_resolve_ci_config.py | 13 ++++++- .../rocprofiler-sdk-code_coverage.yml | 29 ++++++++++----- ...rofiler-sdk-rocm_release_compatibility.yml | 11 +++++- 6 files changed, 97 insertions(+), 26 deletions(-) diff --git a/.github/actions/rocprofiler-sdk-util/action.yml b/.github/actions/rocprofiler-sdk-util/action.yml index c3bec54eb7d9..7dd1e0a8bf64 100644 --- a/.github/actions/rocprofiler-sdk-util/action.yml +++ b/.github/actions/rocprofiler-sdk-util/action.yml @@ -6,6 +6,10 @@ inputs: description: 'Path to ROCm installation' required: false default: '/opt/rocm' + python-venv-path: + description: 'Python virtual environment with the pinned cmake, relative to the workspace' + required: false + default: '' gpu-target: description: 'GPU target (e.g., gfx94X, gfx110X, gfx120X)' required: true @@ -74,13 +78,6 @@ inputs: runs: using: 'composite' steps: - - name: Setup environment variables - shell: bash - run: | - echo "PKG_CONFIG_PATH=/opt/amdgpu/lib64/pkgconfig:/opt/amdgpu/lib/pkgconfig:/opt/amdgpu/lib/$(uname -p)-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ inputs.rocm-path }}/lib64:${{ inputs.rocm-path }}/lib:${{ inputs.rocm-path }}/llvm/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - echo "PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:$HOME/.local/bin:${PATH}" >> $GITHUB_ENV - - name: Setup ccache if: ${{ inputs.enable-ccache == 'true' }} uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18 @@ -102,6 +99,16 @@ runs: echo "Warning: Compiler launcher ${{ inputs.compiler-launcher }} not found or not executable" fi + - name: Setup environment variables + shell: bash + run: | + if [ -n "${{ inputs.python-venv-path }}" ]; then + echo "${GITHUB_WORKSPACE}/${{ inputs.python-venv-path }}/bin" >> $GITHUB_PATH + fi + echo "PKG_CONFIG_PATH=/opt/amdgpu/lib64/pkgconfig:/opt/amdgpu/lib/pkgconfig:/opt/amdgpu/lib/$(uname -p)-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${{ inputs.rocm-path }}/lib64:${{ inputs.rocm-path }}/lib:${{ inputs.rocm-path }}/llvm/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + echo "PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:$HOME/.local/bin:${PATH}" >> $GITHUB_ENV + - name: Build and Install ROCR-Runtime if: ${{ inputs.enable-rocr-build == 'true' }} shell: bash @@ -153,7 +160,7 @@ runs: shell: bash working-directory: projects/rocprofiler-register run: | - export PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:/usr/local/bin:~/.local/bin:$PATH + export PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:${PATH}:/usr/local/bin echo "Install ROCProfiler-Register" cmake -B build-rocprofiler-register \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ @@ -175,7 +182,7 @@ runs: export ROCM_PATH=${{ inputs.rocm-path }} export HIP_PLATFORM=amd export LD_LIBRARY_PATH=${{ inputs.rocm-path }}/lib:${{ inputs.rocm-path }}/lib/rocm_sysdeps/lib:${{ inputs.rocm-path }}/llvm/lib:$LD_LIBRARY_PATH - export PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:/usr/local/bin:~/.local/bin:$PATH + export PATH=${{ inputs.rocm-path }}/bin:${{ inputs.rocm-path }}/llvm/bin:${PATH}:/usr/local/bin # Temporary workaround until hipFILE adds the ROCm sysdeps include path for libmount. if [ -f "${ROCM_PATH}/lib/rocm_sysdeps/include/libmount/libmount.h" ]; then install -D -m 0644 \ diff --git a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/action.yml b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/action.yml index 97c57c0a6019..a5a878ceed99 100644 --- a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/action.yml +++ b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/action.yml @@ -1,5 +1,5 @@ name: Resolve CI config -description: Resolve runner labels from ROCm/therock-ci-config +description: Resolve build and GPU runner labels from ROCm/therock-ci-config inputs: config-path: @@ -38,8 +38,15 @@ inputs: description: Runner label for gfx1151 if the shared CI config cannot be loaded required: false default: linux-gfx1151-gpu-rocm + fallback-linux-build-runner: + description: Linux build runner label if the shared CI config cannot be loaded + required: false + default: aws-linux-scale-rocm-prod outputs: + linux_build_runner: + description: Linux CPU build runner label + value: ${{ steps.runner.outputs.linux_build_runner }} gfx90a_runner: description: Linux gfx90a test runner label value: ${{ steps.runner.outputs.gfx90a_runner }} @@ -89,4 +96,5 @@ runs: FALLBACK_GFX110X_RUNNER: ${{ inputs.fallback-gfx110x-runner }} FALLBACK_GFX120X_RUNNER: ${{ inputs.fallback-gfx120x-runner }} FALLBACK_GFX1151_RUNNER: ${{ inputs.fallback-gfx1151-runner }} + FALLBACK_LINUX_BUILD_RUNNER: ${{ inputs.fallback-linux-build-runner }} run: python3 "${GITHUB_ACTION_PATH}/resolve_ci_config.py" diff --git a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py index 6ce80d21b3d5..4dba6c129615 100644 --- a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py +++ b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py @@ -23,10 +23,12 @@ def warn(message: str) -> None: print(f"::warning::{message}") -def load_gpu_configs(config_path: Path) -> dict[str, dict[str, Any]]: +def load_ci_configs( + config_path: Path, +) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]: if not config_path.exists(): warn("CI config checkout missing. Using fallback runner labels.") - return {} + return {}, {} sys.path.insert(0, str(config_path)) try: @@ -34,13 +36,29 @@ def load_gpu_configs(config_path: Path) -> dict[str, dict[str, Any]]: config = load_config_v1(config_path) all_families = config.get_gpu_families(TRIGGER_TYPES) - return { + gpu_configs = { family: all_families.get(family, {}).get("linux", {}) for family in GPU_FAMILIES } + return gpu_configs, config.build_runners except Exception as exc: warn(f"Failed to load CI config: {exc}. Using fallback runner labels.") - return {} + return {}, {} + + +def load_gpu_configs(config_path: Path) -> dict[str, dict[str, Any]]: + return load_ci_configs(config_path)[0] + + +def get_linux_build_runner(build_runners: dict[str, Any], fallback: str) -> str: + return next( + ( + runner["label"] + for runner in build_runners.get("linux", {}).get("default", []) + if runner.get("label") and runner.get("weight", 0) > 0 + ), + fallback, + ) def set_outputs(outputs: dict[str, str]) -> None: @@ -51,9 +69,16 @@ def set_outputs(outputs: dict[str, str]) -> None: def main() -> None: config_path = Path(os.environ["CI_CONFIG_PATH"]) - gpu_configs = load_gpu_configs(config_path) + gpu_configs, build_runners = load_ci_configs(config_path) outputs: dict[str, str] = {} + fallback_linux_build_runner = os.environ.get("FALLBACK_LINUX_BUILD_RUNNER", "") + linux_build_runner = get_linux_build_runner( + build_runners, fallback_linux_build_runner + ) + outputs["linux_build_runner"] = linux_build_runner + print(f"Using Linux build runner: {linux_build_runner}") + for family in GPU_FAMILIES: linux_config = gpu_configs.get(family, {}) fallback_env = f"FALLBACK_{family.upper()}_RUNNER" diff --git a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py index 3814d71b42e6..ac4fb1db4808 100644 --- a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py +++ b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py @@ -14,7 +14,14 @@ SAMPLE_RUNNER_CONFIG = { "version": "1", - "build_runners": {"linux": {"default": []}}, + "build_runners": { + "linux": { + "default": [ + {"label": "disabled-linux-runner", "weight": 0.0}, + {"label": "configured-linux-runner", "weight": 1.0}, + ] + } + }, "gpu_families": { "presubmit": { "gfx94x": { @@ -224,6 +231,7 @@ def _run_main(self, config=None, env_overrides=None): "FALLBACK_GFX110X_RUNNER": "fallback-gfx110x", "FALLBACK_GFX120X_RUNNER": "fallback-gfx120x", "FALLBACK_GFX1151_RUNNER": "fallback-gfx1151", + "FALLBACK_LINUX_BUILD_RUNNER": "fallback-linux-runner", } if env_overrides: env.update(env_overrides) @@ -240,6 +248,7 @@ def _run_main(self, config=None, env_overrides=None): def test_uses_config_values_when_available(self): outputs = self._run_main(config=SAMPLE_RUNNER_CONFIG) + self.assertEqual(outputs["linux_build_runner"], "configured-linux-runner") self.assertEqual( outputs["gfx94x_runner"], "linux-gfx942-1gpu-ccs-csp-ossci-rocm" ) @@ -256,6 +265,7 @@ def test_uses_config_values_when_available(self): def test_falls_back_to_env_when_config_missing(self): outputs = self._run_main(config=None) + self.assertEqual(outputs["linux_build_runner"], "fallback-linux-runner") self.assertEqual(outputs["gfx94x_runner"], "fallback-gfx94x") self.assertEqual(outputs["gfx950_runner"], "fallback-gfx950") self.assertEqual(outputs["gfx90a_runner"], "fallback-gfx90a") @@ -284,6 +294,7 @@ def test_partial_config_uses_fallback_for_missing_families(self): } outputs = self._run_main(config=partial_config) + self.assertEqual(outputs["linux_build_runner"], "fallback-linux-runner") self.assertEqual(outputs["gfx94x_runner"], "configured-gfx94x") self.assertEqual(outputs["gfx94x_sandbox_runner"], "configured-sandbox") self.assertEqual(outputs["gfx950_runner"], "fallback-gfx950") diff --git a/.github/workflows/rocprofiler-sdk-code_coverage.yml b/.github/workflows/rocprofiler-sdk-code_coverage.yml index 18ed365c6a0b..6f0b1e5b2451 100644 --- a/.github/workflows/rocprofiler-sdk-code_coverage.yml +++ b/.github/workflows/rocprofiler-sdk-code_coverage.yml @@ -47,6 +47,11 @@ concurrency: env: # TODO(jrmadsen): replace LD_RUNPATH_FLAG, GPU_TARGETS, etc. with internal handling in cmake ROCM_PATH: "/opt/rocm" + PYTHON_VENV_PATH: ".venv-rocprofiler-sdk" + # Held at 3.25 rather than develop's 4.x: this branch's external submodules + # (json, pybind11, sqlite, yaml-cpp) still declare cmake_minimum_required + # below 3.5, which CMake 4 rejects outright. + CMAKE_VERSION_DEB: "3.25.0" GPU_TARGETS: "gfx906 gfx908 gfx90a gfx942 gfx950 gfx1030 gfx1100 gfx1101 gfx1102 gfx1201" PATH: "/usr/bin:$PATH" SUBMODULE_PATHS: "projects/rocprofiler-sdk projects/aqlprofile projects/rocprofiler-register projects/rocr-runtime projects/clr projects/hip" @@ -140,6 +145,7 @@ jobs: projects/rocr-runtime projects/clr projects/hip + projects/hrr projects/hipfile submodules: false set-safe-directory: true @@ -167,6 +173,8 @@ jobs: submods-scoped-v1-${{ runner.os }}- - name: Init/Update submodules + timeout-minutes: 15 + continue-on-error: true run: git submodule update --init --recursive --jobs 16 -- ${SUBMODULE_PATHS} - name: Clone ROCDecode @@ -199,12 +207,6 @@ jobs: run: | if [ -d .codecov ]; then cp -r .codecov .codecov.ref; fi - - name: Configure Env - shell: bash - run: | - echo "${PATH}:/usr/local/bin:${HOME}/.local/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:${HOME}/.local/lib" >> $GITHUB_ENV - - name: Install requirements timeout-minutes: 10 shell: bash @@ -212,9 +214,12 @@ jobs: run: | git config --global --add safe.directory '*' apt-get update - apt-get install -y build-essential ccache python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config rpm libzstd-dev - python3 -m pip install -U --user cmake==3.25.0 - python3 -m pip install -U --user -r requirements.txt + apt-get install -y build-essential ccache cmake python3-pip python3-venv gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config rpm libzstd-dev + python3 -m venv --system-site-packages ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }} + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade -r requirements.txt + python3 -m pip install --upgrade cmake==${{ env.CMAKE_VERSION_DEB }} - name: Sync gcov with compilers timeout-minutes: 10 @@ -237,6 +242,7 @@ jobs: uses: ./.github/actions/rocprofiler-sdk-util with: rocm-path: ${{ env.ROCM_PATH }} + python-venv-path: ${{ env.PYTHON_VENV_PATH }} gpu-target: ${{ matrix.system.gpu-target }} compiler-launcher: /usr/bin/ccache enable-rocr-build: ${{ env.ENABLE_ROCR_BUILD }} @@ -254,6 +260,7 @@ jobs: shell: bash working-directory: projects/rocprofiler-sdk run: + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate && python3 ./source/scripts/run-ci.py -B build --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-codecov --build-jobs 16 @@ -275,6 +282,7 @@ jobs: shell: bash working-directory: projects/rocprofiler-sdk run: + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate && find build -type f | egrep '\.gcda$' | xargs rm && python3 ./source/scripts/run-ci.py -B build --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-codecov-tests @@ -297,6 +305,7 @@ jobs: working-directory: projects/rocprofiler-sdk shell: bash run: + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate && find build -type f | egrep '\.gcda$' | xargs rm && python3 ./source/scripts/run-ci.py -B build --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-codecov-samples @@ -329,6 +338,7 @@ jobs: working-directory: projects/rocprofiler-sdk shell: bash run: | + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate echo "PWD: ${PWD}" ls -la @@ -429,6 +439,7 @@ jobs: shell: bash working-directory: projects/rocprofiler-sdk run: | + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate pushd build # # if following fails, there is a test that does not have diff --git a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml index bb993c5d8bae..2619d7f29c4a 100644 --- a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml +++ b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml @@ -41,6 +41,7 @@ concurrency: env: ROCM_PATH: "/opt/rocm" PATH: "/usr/bin:$PATH" + CMAKE_PREFIX_PATH: "/opt/rocm/lib/rocm_sysdeps:/opt/rocm" jobs: rocm-release-compatibility: @@ -80,7 +81,7 @@ jobs: cd projects/rocprofiler-sdk git config --global --add safe.directory '*' apt update - apt install -y wget build-essential cmake g++-11 g++-12 python3-pip libdw-dev libsqlite3-dev libdrm-dev pkg-config + apt install -y wget build-essential g++-11 g++-12 python3-pip libdw-dev libsqlite3-dev libdrm-dev pkg-config # Older ROCm versions don't provide these packages. apt install -y rccl-dev rccl-unittests || true @@ -89,6 +90,14 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12 python3 -m pip install -U --user -r requirements.txt + # Held at 3.25 rather than the 4.x that requirements.txt resolves to: this + # branch's external submodules (json, pybind11, sqlite, yaml-cpp) still declare + # cmake_minimum_required below 3.5, which CMake 4 rejects outright. This must + # run after requirements.txt so the pin is not upgraded away. + python3 -m pip install -U --user cmake==3.25.0 + + # Add pip packages to path + echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: List Files shell: bash From 66a87c0155fd8f4105ba0279717d5cfa88cd21d1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 06:07:12 +0000 Subject: [PATCH 34/45] ci(rocprofiler-sdk): keep ATT quick scan disabled in release compatibility Develop dropped -DROCPROFILER_DISABLE_ATT_QUICK_SCAN=ON because it builds rocprof-trace-decoder, but this branch does not, so rocprofiler-sdk-tool's find_package(rocprof-trace-decoder REQUIRED) fails to configure against every ROCm release container. Restore the flag. Co-authored-by: Vivek Kale --- .github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml index 2619d7f29c4a..cb4bbf6667c7 100644 --- a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml +++ b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml @@ -126,6 +126,7 @@ jobs: -- -DROCPROFILER_DEP_ROCMCORE=ON -DROCPROFILER_BUILD_{TESTS,SAMPLES,DOCS}=OFF + -DROCPROFILER_DISABLE_ATT_QUICK_SCAN=ON -DROCPROFILER_BUILD_AQLPROFILE=${{ matrix.aqlprofile == 'internal' && 'ON' || 'OFF' }} -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DCMAKE_INSTALL_PREFIX="$(realpath /opt/rocm)" From e26e4de98df6f10ac40907f836a9c0ad2cf99ced Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 07:19:29 +0000 Subject: [PATCH 35/45] fix(rocprofiler-sdk): keep HIP ordinals aligned with agents in per-agent scoping test The HIP-device-to-agent mapping pushed a match into gpu_agents and silently moved on when a device had no agent with matching PCI coordinates. Dropping a device shifts every later ordinal, so gpu_agents[0] stops being HIP device 0 and the contexts get scoped to a different physical GPU than the workload runs on. Both positive checks then fail while the negative ones pass trivially, which is what the code coverage job -- the only runner with enough GPUs to actually execute this test rather than skip it -- reported. Stop at the first unmapped ordinal and let main() skip with a message naming the device, and print the mapping and the raw counts the assertions read so a control failure is distinguishable from a scoping bug. Co-authored-by: Vivek Kale --- .../per_agent_scoping_app.cpp | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp index f9b5cd723d51..57fa6cabbc19 100644 --- a/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp +++ b/projects/rocprofiler-sdk/tests/counter-collection-per-agent/per_agent_scoping_app.cpp @@ -91,6 +91,10 @@ rocprofiler_context_id_t ctx_all_second = {.handle = 0}; // reorder HIP ordinals. std::vector gpu_agents = {}; +// HIP ordinal that could not be matched to a rocprofiler agent, or -1 when every device +// mapped. Reported by main() so a skip is distinguishable from a machine with one GPU. +int unmapped_device = -1; + // Intermediate storage used to build the HIP-to-rocprofiler mapping: keeps the // location_id (BDF) and PCI domain alongside the agent id until the mapping is done. struct agent_info_t @@ -254,6 +258,10 @@ tool_init(rocprofiler_client_finalize_t, void*) int device_count = 0; if(hipGetDeviceCount(&device_count) != hipSuccess || device_count < 2) return 0; + // A device with no matching agent must not be skipped silently: dropping it would shift + // every later ordinal, so gpu_agents[i] would stop being HIP device i and the contexts + // would be scoped to a different physical GPU than the workload runs on. Bail instead and + // let main() report the skip. for(int dev = 0; dev < device_count; ++dev) { hipDeviceProp_t prop{}; @@ -261,15 +269,25 @@ tool_init(rocprofiler_client_finalize_t, void*) const auto bus = static_cast(prop.pciBusID); const auto did = static_cast(prop.pciDeviceID); const auto dom = static_cast(prop.pciDomainID); + + bool matched = false; for(const auto& info : all_gpu_agent_info) { if(info.domain == dom && (info.location_id >> 8) == bus && ((info.location_id >> 3) & 0x1Fu) == did) { gpu_agents.push_back(info.id); + matched = true; break; } } + + // Only the first two ordinals are used, so an unmapped device beyond them is harmless. + if(!matched) + { + unmapped_device = dev; + break; + } } if(gpu_agents.size() < 2) return 0; @@ -371,7 +389,12 @@ main() if(device_count < 2 || gpu_agents.size() < 2) { - printf("SKIP: per-agent scoping requires at least 2 GPUs (found %d)\n", device_count); + if(unmapped_device >= 0 && unmapped_device < 2) + printf("SKIP: HIP device %d has no rocprofiler agent with matching PCI " + "coordinates, so per-agent scoping cannot be checked on this machine\n", + unmapped_device); + else + printf("SKIP: per-agent scoping requires at least 2 GPUs (found %d)\n", device_count); return 0; } @@ -406,6 +429,17 @@ main() scoped_ms, kNumStreams, kKernelsPer); + // This test only runs on a multi-GPU machine, so print the mapping and the raw counts the + // assertions below read: a control failure is otherwise indistinguishable from a scoping bug. + printf("hip device 0 -> agent %lu, hip device 1 -> agent %lu\n", + static_cast(agent_0.handle), + static_cast(agent_1.handle)); + printf("unrestricted: dispatches(gpu0)=%zu | scoped: records(gpu1)=%zu dispatches(gpu0)=%zu " + "records(gpu0)=%zu\n", + unrestricted_dispatches_gpu0, + scoped_records_gpu1, + scoped_dispatches_gpu0, + scoped_records_gpu0); // Exact assertions. These hold regardless of how fast the machine is. check(unrestricted_dispatches_gpu0 > 0, From a5bc57cb21bd97f26925d6ed86c2f34de37b827d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 07:40:39 +0000 Subject: [PATCH 36/45] fix(rocprofiler-sdk): close the two context-lifecycle races in stop_context The registered-context registry was walked with no lock from threads that cannot take get_contexts_mutex(). allocate_context() appends to the stable_vector while those walks are in progress, which races on the chunk list, and deregister_client_contexts() destroyed contexts out from under readers that had already taken a pointer. The registry now holds shared_ptr and publishes an immutable snapshot under get_contexts_mutex() on every mutation. Readers load the snapshot instead of the registry, and holding one keeps the contexts it names alive for the duration of the read, so a context dropped mid-read survives until the reader is done with it. stop_context() held get_contexts_mutex() across the service teardown, which contains hsa::queue_controller_sync() -- an unbounded wait on in-flight GPU work. Every context lifecycle operation in the process stalled behind one context's dispatches, and the lock sat on the far side of a wait that the completion path has to get through. Stop is now phased: claim the context under the lock, tear the services down and drain with the lock released, retire the active slot under the lock again. The active slot stays populated across the drain, which the serializer transition depends on, so a set of in-progress stops keeps start_context() and deactivate_client_contexts() from reading that state as a running context. The conflict scan in start_context() moves inside the same lock acquisition as the slot search so the two agree. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/context/context.cpp | 399 +++++++++++++----- .../lib/rocprofiler-sdk/context/context.hpp | 38 ++ 2 files changed, 332 insertions(+), 105 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index f9ec5de4054f..6a9584de6d4a 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -23,6 +23,7 @@ #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/common/container/small_vector.hpp" #include "lib/common/container/stable_vector.hpp" +#include "lib/common/scope_destructor.hpp" #include "lib/common/static_object.hpp" #include "lib/common/synchronized.hpp" #include "lib/common/utility.hpp" @@ -37,23 +38,38 @@ #include #include +#include #include #include #include #include #include -#include #include +#include +#include +#include namespace rocprofiler { namespace context { +// Owns the contexts it names. A reader holding a snapshot keeps them alive for the duration of +// the read, so deregister_client_contexts() dropping the registry's reference cannot destroy a +// context out from under an in-flight completion. The parallel raw-pointer array is what the +// snapshot iterates, so walking one costs no reference counting. +struct registered_context_storage +{ + std::vector> owned = {}; + std::vector view = {}; +}; + namespace { -using reserve_size_t = common::container::reserve_size; -using stable_context_vec_t = common::container::stable_vector, 8>; -using active_context_vec_t = common::container::stable_vector, 8>; +using reserve_size_t = common::container::reserve_size; +using context_ptr_t = std::shared_ptr; +using stable_context_vec_t = common::container::stable_vector; +using active_context_vec_t = common::container::stable_vector, 8>; +using context_snapshot_ptr_t = std::shared_ptr; constexpr auto invalid_client_idx = std::numeric_limits::max(); @@ -97,6 +113,122 @@ get_active_contexts_impl() static auto* _v = new active_context_vec_t{reserve_size_t{active_context_vec_t::chunk_size}}; return *_v; } + +// C++20 replaces these with std::atomic>; the free functions are the C++17 +// spelling and are deprecated there, so select on the feature macro rather than leaving a +// deprecation warning to trip -Werror on a future standard bump. +#if defined(__cpp_lib_atomic_shared_ptr) && __cpp_lib_atomic_shared_ptr >= 201711L +using published_snapshot_t = std::atomic; + +context_snapshot_ptr_t +load_snapshot(published_snapshot_t& _v) +{ + return _v.load(std::memory_order_acquire); +} + +void +store_snapshot(published_snapshot_t& _v, context_snapshot_ptr_t&& _new) +{ + _v.store(std::move(_new), std::memory_order_release); +} +#else +using published_snapshot_t = context_snapshot_ptr_t; + +context_snapshot_ptr_t +load_snapshot(published_snapshot_t& _v) +{ + return std::atomic_load_explicit(&_v, std::memory_order_acquire); +} + +void +store_snapshot(published_snapshot_t& _v, context_snapshot_ptr_t&& _new) +{ + std::atomic_store_explicit(&_v, std::move(_new), std::memory_order_release); +} +#endif + +// Shares the registry's lifetime so that reads after common::destroy_static_objects() see an +// empty snapshot rather than a destroyed one. +published_snapshot_t* +get_published_snapshot() +{ + static auto*& _v = common::static_object::construct(); + return _v; +} + +context_snapshot_ptr_t +current_registered_contexts() +{ + auto* _pub = get_published_snapshot(); + if(!_pub) return {}; + return load_snapshot(*_pub); +} + +// Rebuilds the published snapshot from the registry. Must be called with get_contexts_mutex() +// held, i.e. by whichever function just mutated the registry. +void +publish_registered_contexts() +{ + auto* _pub = get_published_snapshot(); + if(!_pub) return; + + auto _data = std::make_shared(); + + if(auto* _impl = get_registered_contexts_impl()) + { + _data->owned.reserve(_impl->size()); + _data->view.reserve(_impl->size()); + for(const auto& itr : *_impl) + { + if(!itr) continue; + _data->owned.emplace_back(itr); + _data->view.emplace_back(itr.get()); + } + } + + store_snapshot(*_pub, std::move(_data)); +} + +// Snapshots skip deregistered slots, so match on context_idx rather than on position. +context* +find_registered_context(uint64_t handle) +{ + auto _data = current_registered_contexts(); + if(!_data) return nullptr; + + for(const auto& itr : _data->owned) + { + if(itr->context_idx == handle) return itr.get(); + } + return nullptr; +} + +// Context ids whose stop is in progress. stop_context() releases get_contexts_mutex() across the +// GPU drain, so for that window the context is still in the active array with no lock held. This +// set is what keeps start_context() and a second stop_context() from acting on that half-stopped +// state. Guarded by get_contexts_mutex(). +std::unordered_set& +get_stopping_contexts() +{ + static auto _v = std::unordered_set{}; + return _v; +} + +std::condition_variable& +get_contexts_cv() +{ + static auto _v = std::condition_variable{}; + return _v; +} + +// _lk must own get_contexts_mutex(). Waits until no context is mid-stop, which restores the +// property the old single-lock stop_context() had: no other lifecycle operation ever observes a +// context whose services have been torn down but whose active slot is still populated. +void +wait_for_pending_stops(std::unique_lock& _lk) +{ + get_contexts_cv().wait(_lk, []() { return get_stopping_contexts().empty(); }); +} } // namespace std::mutex& @@ -120,22 +252,33 @@ dispatch_counter_collection_service::intersects( return false; } +registered_contexts_snapshot::registered_contexts_snapshot(context_snapshot_ptr_t&& data) +: m_data{std::move(data)} +{ + if(m_data) + { + m_begin = m_data->view.data(); + m_end = m_begin + m_data->view.size(); + } +} + +registered_contexts_snapshot +get_registered_contexts_snapshot() +{ + return registered_contexts_snapshot{current_registered_contexts()}; +} + context_array_t& get_registered_contexts(context_array_t& data, context_filter_t filter) { data.clear(); - if(!get_registered_contexts_impl()) return data; - - auto num_ctx = get_registered_contexts_impl()->size(); - if(num_ctx <= 0) return data; + auto snapshot = get_registered_contexts_snapshot(); + if(snapshot.empty()) return data; - data.reserve(num_ctx); - for(auto& itr : *get_registered_contexts_impl()) + data.reserve(snapshot.size()); + for(const auto* ctx : snapshot) { - if(!itr.has_value()) continue; - - const auto* ctx = &itr.value(); if(!filter || (filter && filter(ctx))) data.emplace_back(ctx); } return data; @@ -231,13 +374,9 @@ allocate_context() // initial context identifier number auto _idx = get_registered_contexts_impl()->size() + get_contexts_offset(); - // make space in registered - get_registered_contexts_impl()->emplace_back(); - // create an entry in the registered - auto& _cfg_v = get_registered_contexts_impl()->back(); - _cfg_v.emplace(); - auto* _cfg = &_cfg_v.value(); + auto& _cfg_v = get_registered_contexts_impl()->emplace_back(std::make_shared()); + auto* _cfg = _cfg_v.get(); // ... if(!_cfg) return std::nullopt; @@ -250,6 +389,8 @@ allocate_context() << " rocprofiler internal error: a context was allocated without an associated tool client " "identifier"; + publish_registered_contexts(); + return rocprofiler_context_id_t{_idx}; } @@ -257,13 +398,10 @@ context* get_mutable_registered_context(rocprofiler_context_id_t id) { if(id.handle < get_contexts_offset()) return nullptr; - if(!get_registered_contexts_impl()) return nullptr; - auto _idx = id.handle - get_contexts_offset(); - if(_idx >= get_registered_contexts_impl()->size()) - return nullptr; - else if(get_registered_contexts_impl()->at(_idx).has_value()) - return &get_registered_contexts_impl()->at(_idx).value(); - return nullptr; + + // Read through the published snapshot rather than indexing the registry: allocate_context() + // may be appending to the registry concurrently. + return find_registered_context(id.handle); } const context* @@ -283,11 +421,6 @@ start_context(rocprofiler_context_id_t context_id) { if(context_id.handle < get_contexts_offset()) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; - if(!get_registered_contexts_impl()) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; - - if((context_id.handle - get_contexts_offset()) >= get_registered_contexts_impl()->size()) - return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; - const auto* cfg = get_registered_context(context_id); if(!cfg) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; @@ -295,47 +428,48 @@ start_context(rocprofiler_context_id_t context_id) if(validate_context(cfg) != ROCPROFILER_STATUS_SUCCESS) return ROCPROFILER_STATUS_ERROR_CONTEXT_INVALID; - auto current_contexts = context_array_t{}; - for(const auto* itr : get_active_contexts(current_contexts)) - { - if(cfg->context_idx == itr->context_idx) - { - return ROCPROFILER_STATUS_SUCCESS; - } - else if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection && - cfg->dispatch_counter_collection->intersects(*itr->dispatch_counter_collection)) - { - // Conflicting context. Two counter-collection contexts can run concurrently as - // long as they target disjoint sets of GPU agents -- the hardware counters they - // program are per-agent, so contexts that never touch the same agent cannot - // contend. A context with no agent restriction claims every agent and therefore - // still conflicts with any other counter-collection context. - return ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT; - } - } - - uint64_t rocp_tot_contexts = get_registered_contexts_impl()->size(); - auto idx = rocp_tot_contexts; - auto& active_contexts = get_active_contexts_impl(); + constexpr auto no_slot = std::numeric_limits::max(); + auto idx = no_slot; + auto& active_contexts = get_active_contexts_impl(); { // hold a lock here to prevent multiple threads from finding the same nullptr slot auto _lk = std::unique_lock{get_contexts_mutex()}; - // try to find a nullptr slot first + + // A context that is mid-stop is still in the active array while its GPU drain runs, so + // starting against that state would either return SUCCESS for a context that is about to + // be cleared, or report a conflict against a context that is on its way out. + wait_for_pending_stops(_lk); + + // The conflict scan reads the same array as the slot search below and has to agree with + // it, so both run under one acquisition of the lock. for(size_t i = 0; i < active_contexts.size(); ++i) { const auto* itr = active_contexts.at(i).load(std::memory_order_relaxed); if(itr == nullptr) { - idx = i; - break; + if(idx == no_slot) idx = i; + continue; } - else if(context_id.handle == itr->context_idx) + + if(cfg->context_idx == itr->context_idx) { return ROCPROFILER_STATUS_SUCCESS; } + + if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection && + cfg->dispatch_counter_collection->intersects(*itr->dispatch_counter_collection)) + { + // Conflicting context. Two counter-collection contexts can run concurrently as + // long as they target disjoint sets of GPU agents -- the hardware counters they + // program are per-agent, so contexts that never touch the same agent cannot + // contend. A context with no agent restriction claims every agent and therefore + // still conflicts with any other counter-collection context. + return ROCPROFILER_STATUS_ERROR_CONTEXT_CONFLICT; + } } + // if no nullptr slot was found, then create one while lock is held - if(idx == rocp_tot_contexts) + if(idx == no_slot) { idx = active_contexts.size(); active_contexts.emplace_back(); @@ -376,72 +510,111 @@ start_context(rocprofiler_context_id_t context_id) rocprofiler_status_t stop_context(rocprofiler_context_id_t idx) { - // hold a lock here to prevent other thread from changing the active contexts array - auto _lk = std::unique_lock{get_contexts_mutex()}; + std::atomic* slot = nullptr; + const context* _expected = nullptr; - // atomically assign the context pointer to NULL so that it is skipped in future - // callbacks - for(auto& itr : get_active_contexts_impl()) + // Phase one, locked: claim the context. Only the search of the active array needs the lock; + // the teardown below must not hold it. { - const context* _expected = itr.load(std::memory_order_acquire); - if(_expected && _expected->context_idx == idx.handle) + auto _lk = std::unique_lock{get_contexts_mutex()}; + + // another thread may already be tearing this context down + wait_for_pending_stops(_lk); + + for(auto& itr : get_active_contexts_impl()) { - // Stop queue-interposed services before clearing the active slot so - // disable_serialization() always runs before dispatches stop being instrumented. - // Clearing the slot first opens the opposite window, in which - // kernel_dispatch_phase_enter_hook sees no active context and a dispatch is submitted - // without serializer packets while the serializer is still enabled. - if(_expected->dispatch_counter_collection) + const context* _ctx = itr.load(std::memory_order_acquire); + if(_ctx && _ctx->context_idx == idx.handle) { - rocprofiler::counters::stop_context(const_cast(_expected)); + slot = &itr; + _expected = _ctx; + break; } + } - if(_expected->dispatch_spm) - rocprofiler::spm::stop_context(const_cast(_expected)); - - if(_expected->device_thread_trace) _expected->device_thread_trace->stop_context(); - if(_expected->dispatch_thread_trace) _expected->dispatch_thread_trace->stop_context(); + if(!slot) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; - bool success = itr.compare_exchange_strong(_expected, nullptr); + get_stopping_contexts().emplace(idx.handle); + } - if(success) - { - auto nactive = get_num_active_contexts().load(std::memory_order_acquire); - if(nactive > 0) get_num_active_contexts().fetch_sub(1, std::memory_order_release); + auto _unclaim = common::scope_destructor{[&idx]() { + { + auto _lk = std::unique_lock{get_contexts_mutex()}; + get_stopping_contexts().erase(idx.handle); + } + get_contexts_cv().notify_all(); + }}; + + // Phase two, unlocked: the service teardowns below call hsa::queue_controller_sync(), an + // unbounded wait on in-flight GPU work. Holding get_contexts_mutex() across it stalls every + // context lifecycle operation in the process behind one context's dispatches, and it puts the + // mutex on the far side of a wait that the completion path has to get through -- so any future + // completion-path read that took the mutex would deadlock rather than merely block. + // + // The active slot stays populated for the whole phase, which is deliberate: + // kernel_dispatch_phase_enter_hook has to keep seeing the context so the serialized -> + // unserialized transition stays coordinated until disable_serialization() has run. Clearing + // the slot first opens the opposite window, in which the enter hook sees no active context and + // a dispatch is submitted without serializer packets while the serializer is still enabled. + // get_stopping_contexts() is what keeps other lifecycle callers from reading that state as a + // running context. + if(_expected->dispatch_counter_collection) + { + rocprofiler::counters::stop_context(const_cast(_expected)); + } - rocprofiler::hsa::queue_interposition:: - notify_queue_interposition_consumer_context_stopped(_expected); + if(_expected->dispatch_spm) rocprofiler::spm::stop_context(const_cast(_expected)); - if(_expected->device_counter_collection) - { - rocprofiler::counters::stop_agent_ctx(const_cast(_expected)); - } + if(_expected->device_thread_trace) _expected->device_thread_trace->stop_context(); + if(_expected->dispatch_thread_trace) _expected->dispatch_thread_trace->stop_context(); -#if ROCPROFILER_SDK_HSA_PC_SAMPLING > 0 - if(_expected->pc_sampler) - { - rocprofiler::pc_sampling::stop_service(_expected); - } -#endif + // Phase three, relocked: retire the slot. The element address is stable across phase two -- + // stable_vector never moves an element once constructed -- so slot is still the entry the + // search found. + { + auto _lk = std::unique_lock{get_contexts_mutex()}; - return ROCPROFILER_STATUS_SUCCESS; - } + if(!slot->compare_exchange_strong(_expected, nullptr)) + { + // deactivate_client_contexts() cleared the slot while phase two ran; the services are + // already stopped, so there is nothing left to retire. + return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; } + + auto nactive = get_num_active_contexts().load(std::memory_order_acquire); + if(nactive > 0) get_num_active_contexts().fetch_sub(1, std::memory_order_release); + } + + // Phase four, unlocked: the remaining services are not queue-interposed, so they only need + // the slot to be cleared first, not the lock to be held. They wait on the GPU too -- + // stop_agent_ctx() waits for its stop packet and pc_sampling::stop_service() drains the + // device buffers -- and the stopping marker above, not the mutex, is what keeps a concurrent + // start_context() from running before they finish. + rocprofiler::hsa::queue_interposition::notify_queue_interposition_consumer_context_stopped( + _expected); + + if(_expected->device_counter_collection) + { + rocprofiler::counters::stop_agent_ctx(const_cast(_expected)); } - return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; // compare exchange failed +#if ROCPROFILER_SDK_HSA_PC_SAMPLING > 0 + if(_expected->pc_sampler) + { + rocprofiler::pc_sampling::stop_service(_expected); + } +#endif + + return ROCPROFILER_STATUS_SUCCESS; } context_id_array_t get_client_contexts(rocprofiler_client_id_t id) { auto _data = context_id_array_t{}; - if(!get_registered_contexts_impl()) return _data; - for(auto& itr : *get_registered_contexts_impl()) + for(const auto* itr : get_registered_contexts_snapshot()) { - if(!itr.has_value()) continue; - if(itr->client_idx == id.handle) { _data.emplace_back(rocprofiler_context_id_t{.handle = itr->context_idx}); @@ -456,10 +629,12 @@ stop_client_contexts(rocprofiler_client_id_t client_id) if(!get_registered_contexts_impl()) return ROCPROFILER_STATUS_ERROR; auto ret = ROCPROFILER_STATUS_SUCCESS; - for(auto& itr : *get_registered_contexts_impl()) - { - if(!itr.has_value()) continue; + // Held across the loop: stop_context() may block on a GPU drain, and a snapshot is what keeps + // the contexts it is iterating alive for that long. + auto snapshot = get_registered_contexts_snapshot(); + for(const auto* itr : snapshot) + { if(itr->client_idx == client_id.handle) { auto status = stop_context(rocprofiler_context_id_t{itr->context_idx}); @@ -475,6 +650,11 @@ void deactivate_client_contexts(rocprofiler_client_id_t client_id) { auto _lk = std::unique_lock{get_contexts_mutex()}; + + // a context mid-stop is still in the active array; let its teardown finish rather than + // clearing the slot underneath it + wait_for_pending_stops(_lk); + for(auto& itr : get_active_contexts_impl()) { const context* itr_v = itr.load(std::memory_order_acquire); @@ -492,9 +672,16 @@ deactivate_client_contexts(rocprofiler_client_id_t client_id) void deregister_client_contexts(rocprofiler_client_id_t client_id) { + if(!get_registered_contexts_impl()) return; + + // Mutates the registry, so it needs the same lock allocate_context() takes. Dropping the + // registry's reference does not destroy the context: any snapshot still naming it holds it + // alive, which is what makes this safe against a read that is already in flight. + auto _lk = std::unique_lock{get_contexts_mutex()}; + for(auto& itr : *get_registered_contexts_impl()) { - if(!itr.has_value()) continue; + if(!itr) continue; if(itr->client_idx == client_id.handle && buffer::get_buffers()) { @@ -505,6 +692,8 @@ deregister_client_contexts(rocprofiler_client_id_t client_id) itr.reset(); } } + + publish_registered_contexts(); } template diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index 8d52e2f8108d..8cce4dd7fb3d 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -217,6 +217,40 @@ stop_context(rocprofiler_context_id_t id); using context_array_t = common::container::small_vector; using context_id_array_t = common::container::small_vector; +// defined in context.cpp +struct registered_context_storage; + +/// \brief An immutable view of the registered contexts, taken at the point of construction. +/// +/// The registry itself may only be walked with get_contexts_mutex() held: allocate_context() +/// appends to it while dispatch and completion paths read it. The completion path cannot take +/// that mutex, because stop_context() waits on the GPU drain and the drain waits on completions, +/// so it reads a snapshot instead. A snapshot also owns the contexts it names, which is what makes +/// it safe against deregister_client_contexts() destroying a context mid-iteration. +class registered_contexts_snapshot +{ +public: + registered_contexts_snapshot() = default; + + const context* const* begin() const { return m_begin; } + const context* const* end() const { return m_end; } + size_t size() const { return m_end - m_begin; } + bool empty() const { return m_begin == m_end; } + +private: + friend registered_contexts_snapshot get_registered_contexts_snapshot(); + + explicit registered_contexts_snapshot(std::shared_ptr&&); + + std::shared_ptr m_data = {}; + const context* const* m_begin = nullptr; + const context* const* m_end = nullptr; +}; + +/// \brief Returns the currently published snapshot of the registered contexts. +registered_contexts_snapshot +get_registered_contexts_snapshot(); + context* get_mutable_registered_context(rocprofiler_context_id_t id); @@ -228,6 +262,10 @@ using context_filter_t = bool (*)(const context*); inline bool default_context_filter(const context* val); +// These read the published snapshot and then drop it, so the pointers they return outlive the +// guarantee that the contexts are alive. That is fine for callers that cannot overlap with +// deregister_client_contexts(); anything on a dispatch or completion path should hold a +// registered_contexts_snapshot for as long as it uses the pointers. context_array_t& get_registered_contexts(context_array_t& data, context_filter_t filter = default_context_filter); From 67f130ccb79f2fc95624aea944cb4db53b78f0eb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 07:40:46 +0000 Subject: [PATCH 37/45] fix(rocprofiler-sdk): read contexts through a snapshot in the counters exit hook kernel_dispatch_phase_exit_hook runs on the HSA async signal handler, so it cannot take get_contexts_mutex(): stop_context() waits on the drain that this completion is part of. It walked the registry unlocked instead, which races with allocate_context(). It now reads the published snapshot, which needs no lock and keeps every context it names alive for the length of the loop. The filter moves inline so the hook no longer builds a vector per completion. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/queue_hooks.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp index 730c9ea5b95c..e7464db13e3d 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/queue_hooks.cpp @@ -107,9 +107,17 @@ kernel_dispatch_phase_exit_hook(const hsa::Queue* /*queue*/, // context from the active list. This is what guarantees a completion that arrives is delivered; // the drain in stop_context is what bounds when completions arrive. Kernel replay needs the // same property, since each pass completes separately. - auto contexts = context::get_registered_contexts(counter_contexts_filter()); + // + // Read through a snapshot rather than the registry itself. This runs on the HSA async signal + // handler, so it cannot take get_contexts_mutex() -- stop_context() waits on the drain that + // this completion is part of -- and the registry is not safe to walk unlocked while + // allocate_context() appends to it. Holding the snapshot also keeps every context it names + // alive for the duration of the loop. + auto contexts = context::get_registered_contexts_snapshot(); for(const auto* ctx : contexts) { + if(!counter_contexts_filter()(ctx)) continue; + for(auto& cb : ctx->dispatch_counter_collection->callbacks) { completed_cb(ctx, cb, session, packet, inst_pkt, dispatch_time); From c1eccc258c0f0a7fa86b7389967a245bcf72ee1c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 08:00:59 +0000 Subject: [PATCH 38/45] ci(rocprofiler-sdk): repair the docs-from-source build The build-docs-from-source job died in 82 ms configuring ROCR-Runtime with 'CMake 3.25 or higher is required. You are running version 3.22.1'. It installs cmake from apt, and this branch predates the container image it runs in, so the distro cmake is now too old for ROCR-Runtime. Same treatment the code coverage and release compatibility jobs already got: install cmake into a venv and hand that venv to the shared util action. Held at 3.25 rather than develop's 4.x because this branch's external submodules still declare cmake_minimum_required below 3.5. Co-authored-by: Vivek Kale --- .github/workflows/rocprofiler-sdk-docs.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rocprofiler-sdk-docs.yml b/.github/workflows/rocprofiler-sdk-docs.yml index 15b2be0d2fc0..ca142eaad31a 100644 --- a/.github/workflows/rocprofiler-sdk-docs.yml +++ b/.github/workflows/rocprofiler-sdk-docs.yml @@ -37,6 +37,11 @@ concurrency: env: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 ROCM_PATH: "/opt/rocm" + PYTHON_VENV_PATH: ".venv-rocprofiler-sdk" + # Held at 3.25 rather than develop's 4.x: this branch's external submodules + # (json, pybind11, sqlite, yaml-cpp) still declare cmake_minimum_required + # below 3.5, which CMake 4 rejects outright. + CMAKE_VERSION_DEB: "3.25.0" ENABLE_ROCR_BUILD: "true" ENABLE_HIP_CLR_BUILD: "true" ENABLE_ROCPROFILER_REGISTER_BUILD: "false" @@ -177,13 +182,18 @@ jobs: working-directory: projects/rocprofiler-sdk/ run: | apt-get update - apt-get install -y cmake gcc g++ libdw-dev libsqlite3-dev rpm - python3 -m pip install -r requirements.txt + apt-get install -y gcc g++ libdw-dev libsqlite3-dev rpm python3-pip python3-venv + python3 -m venv --system-site-packages ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }} + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade -r requirements.txt + python3 -m pip install --upgrade cmake==${{ env.CMAKE_VERSION_DEB }} - name: Build ROCProfiler SDK Dependencies uses: ./.github/actions/rocprofiler-sdk-util with: rocm-path: ${{ env.ROCM_PATH }} + python-venv-path: ${{ env.PYTHON_VENV_PATH }} gpu-target: 'gfx94X' compiler-launcher: /usr/bin/ccache enable-rocr-build: ${{ env.ENABLE_ROCR_BUILD }} @@ -198,6 +208,7 @@ jobs: shell: bash working-directory: projects/rocprofiler-sdk/ run: | + source ${GITHUB_WORKSPACE}/${{ env.PYTHON_VENV_PATH }}/bin/activate CMAKE_PREFIX_PATH=/opt/rocm:${CMAKE_PREFIX_PATH} \ cmake -B build \ -DROCPROFILER_DEP_ROCMCORE=ON \ From 67a7b9366c318ff700f5dc3d872525b369dcd527 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 08:48:02 +0000 Subject: [PATCH 39/45] fix(rocprofiler-sdk): keep a context alive across the unlocked stop phases stop_context() now releases get_contexts_mutex() across the GPU drain, and for that window it was holding nothing but a raw pointer to the context it was tearing down. deregister_client_contexts() takes the same mutex, so it can run in exactly that window and drop the registry's reference -- which was the last one -- destroying the context underneath the drain. The same hole exists in start_context(), whose service starts also run unlocked. Both now hold the registry's shared_ptr for as long as they use the raw pointer, and deregister_client_contexts() waits for a pending stop before retiring a registry entry rather than pulling it out mid-teardown. The retired entries are moved into a local vector and destroyed after the lock is dropped, since a context destructor calls back into HSA and that is not something to run underneath the registry mutex. get_mutable_registered_context() goes back to indexing the registry directly. Routing it through the snapshot was never needed to close either gap -- the snapshot exists for the completion path, which cannot take the mutex -- and it turned an O(1) lookup into a scan for no benefit. start_context()'s slot search likewise goes back to its original shape, so the only change left in it is the wait for pending stops. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/context/context.cpp | 123 +++++++++++------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 6a9584de6d4a..ea0dd9ba0537 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -189,18 +189,18 @@ publish_registered_contexts() store_snapshot(*_pub, std::move(_data)); } -// Snapshots skip deregistered slots, so match on context_idx rather than on position. -context* -find_registered_context(uint64_t handle) +// The registry is index-addressed: allocate_context() derives context_idx from the slot position, +// so a context id maps straight back to its slot. +context_ptr_t +get_registered_context_slot(uint64_t handle) { - auto _data = current_registered_contexts(); - if(!_data) return nullptr; + auto* _impl = get_registered_contexts_impl(); + if(!_impl || handle < get_contexts_offset()) return {}; - for(const auto& itr : _data->owned) - { - if(itr->context_idx == handle) return itr.get(); - } - return nullptr; + auto _idx = handle - get_contexts_offset(); + if(_idx >= _impl->size()) return {}; + + return _impl->at(_idx); } // Context ids whose stop is in progress. stop_context() releases get_contexts_mutex() across the @@ -397,11 +397,7 @@ allocate_context() context* get_mutable_registered_context(rocprofiler_context_id_t id) { - if(id.handle < get_contexts_offset()) return nullptr; - - // Read through the published snapshot rather than indexing the registry: allocate_context() - // may be appending to the registry concurrently. - return find_registered_context(id.handle); + return get_registered_context_slot(id.handle).get(); } const context* @@ -421,43 +417,42 @@ start_context(rocprofiler_context_id_t context_id) { if(context_id.handle < get_contexts_offset()) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; - const auto* cfg = get_registered_context(context_id); + // Held for the rest of the function: the service starts below run without the contexts mutex, + // and deregister_client_contexts() may drop the registry's reference while they do. + auto _owner = context_ptr_t{}; + const context* cfg = nullptr; + { + auto _lk = std::unique_lock{get_contexts_mutex()}; + _owner = get_registered_context_slot(context_id.handle); + cfg = _owner.get(); + } if(!cfg) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; if(validate_context(cfg) != ROCPROFILER_STATUS_SUCCESS) return ROCPROFILER_STATUS_ERROR_CONTEXT_INVALID; - constexpr auto no_slot = std::numeric_limits::max(); - auto idx = no_slot; - auto& active_contexts = get_active_contexts_impl(); + uint64_t rocp_tot_contexts = get_registered_contexts_impl()->size(); + auto idx = rocp_tot_contexts; + auto& active_contexts = get_active_contexts_impl(); { // hold a lock here to prevent multiple threads from finding the same nullptr slot auto _lk = std::unique_lock{get_contexts_mutex()}; // A context that is mid-stop is still in the active array while its GPU drain runs, so - // starting against that state would either return SUCCESS for a context that is about to - // be cleared, or report a conflict against a context that is on its way out. + // scanning against that state would either report a conflict against a context that is on + // its way out, or hand back a slot that the stop is about to clear. wait_for_pending_stops(_lk); - // The conflict scan reads the same array as the slot search below and has to agree with - // it, so both run under one acquisition of the lock. - for(size_t i = 0; i < active_contexts.size(); ++i) + auto current_contexts = context_array_t{}; + for(const auto* itr : get_active_contexts(current_contexts)) { - const auto* itr = active_contexts.at(i).load(std::memory_order_relaxed); - if(itr == nullptr) - { - if(idx == no_slot) idx = i; - continue; - } - if(cfg->context_idx == itr->context_idx) { return ROCPROFILER_STATUS_SUCCESS; } - - if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection && - cfg->dispatch_counter_collection->intersects(*itr->dispatch_counter_collection)) + else if(cfg->dispatch_counter_collection && itr->dispatch_counter_collection && + cfg->dispatch_counter_collection->intersects(*itr->dispatch_counter_collection)) { // Conflicting context. Two counter-collection contexts can run concurrently as // long as they target disjoint sets of GPU agents -- the hardware counters they @@ -468,8 +463,23 @@ start_context(rocprofiler_context_id_t context_id) } } + // try to find a nullptr slot first + for(size_t i = 0; i < active_contexts.size(); ++i) + { + const auto* itr = active_contexts.at(i).load(std::memory_order_relaxed); + if(itr == nullptr) + { + idx = i; + break; + } + else if(context_id.handle == itr->context_idx) + { + return ROCPROFILER_STATUS_SUCCESS; + } + } + // if no nullptr slot was found, then create one while lock is held - if(idx == no_slot) + if(idx == rocp_tot_contexts) { idx = active_contexts.size(); active_contexts.emplace_back(); @@ -482,8 +492,7 @@ start_context(rocprofiler_context_id_t context_id) // atomic swap the pointer into the "active" array used internally const context* _expected = nullptr; - bool success = active_contexts.at(idx).compare_exchange_strong( - _expected, get_registered_context(context_id)); + bool success = active_contexts.at(idx).compare_exchange_strong(_expected, cfg); if(!success) { @@ -512,6 +521,10 @@ stop_context(rocprofiler_context_id_t idx) { std::atomic* slot = nullptr; const context* _expected = nullptr; + // Held for the rest of the function. The teardown below runs without the contexts mutex, so + // without this the registry's reference is the only thing keeping the context alive and + // deregister_client_contexts() could destroy it out from under the drain. + auto _owner = context_ptr_t{}; // Phase one, locked: claim the context. Only the search of the active array needs the lock; // the teardown below must not hold it. @@ -534,6 +547,7 @@ stop_context(rocprofiler_context_id_t idx) if(!slot) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; + _owner = get_registered_context_slot(idx.handle); get_stopping_contexts().emplace(idx.handle); } @@ -674,26 +688,37 @@ deregister_client_contexts(rocprofiler_client_id_t client_id) { if(!get_registered_contexts_impl()) return; - // Mutates the registry, so it needs the same lock allocate_context() takes. Dropping the - // registry's reference does not destroy the context: any snapshot still naming it holds it - // alive, which is what makes this safe against a read that is already in flight. - auto _lk = std::unique_lock{get_contexts_mutex()}; + // Outlives the lock below so the contexts are destroyed without it held: a context destructor + // calls back into HSA, which is not something to do underneath the registry mutex. + auto _retired = std::vector{}; - for(auto& itr : *get_registered_contexts_impl()) { - if(!itr) continue; + // Mutates the registry, so it needs the same lock allocate_context() takes. + auto _lk = std::unique_lock{get_contexts_mutex()}; - if(itr->client_idx == client_id.handle && buffer::get_buffers()) + // a context mid-stop is still being torn down through a raw pointer; retiring its registry + // entry now would pull the object out from under that teardown + wait_for_pending_stops(_lk); + + for(auto& itr : *get_registered_contexts_impl()) { - for(auto& bitr : *buffer::get_buffers()) + if(!itr) continue; + + if(itr->client_idx == client_id.handle && buffer::get_buffers()) { - if(bitr && bitr->context_id == itr->context_idx) bitr.reset(); + for(auto& bitr : *buffer::get_buffers()) + { + if(bitr && bitr->context_id == itr->context_idx) bitr.reset(); + } + // Moved rather than reset: dropping the registry's reference here would destroy + // the context under the lock, and a reader holding an older snapshot still needs + // it to stay alive until that snapshot is released. + _retired.emplace_back(std::move(itr)); } - itr.reset(); } - } - publish_registered_contexts(); + publish_registered_contexts(); + } } template From fb9b88b35d3ecd5a30e89f92126676e6d85a5f3a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 08:52:11 +0000 Subject: [PATCH 40/45] fix(rocprofiler-sdk): make set_dispatch_agents wait out a context stop The phased stop releases get_contexts_mutex() across the GPU drain, so for that window a context is still in the active array with nothing held. set_dispatch_agents() takes that mutex precisely to be atomic with activation state, but the lock alone no longer excludes a stop: it would read a context that is on its way out as a running one and refuse the agent assignment with CONFIGURATION_LOCKED. The wait that start_context() and stop_context() already use is promoted out of the anonymous namespace as context::wait_for_stopping_contexts(), and set_dispatch_agents() takes it after acquiring the mutex. Co-authored-by: Vivek Kale --- .cursor/rules/communication-style.mdc | 34 +++++++++++++++++++ .../lib/rocprofiler-sdk/context/context.cpp | 19 ++++++----- .../lib/rocprofiler-sdk/context/context.hpp | 9 +++++ .../lib/rocprofiler-sdk/counters/core.cpp | 6 ++++ 4 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 .cursor/rules/communication-style.mdc diff --git a/.cursor/rules/communication-style.mdc b/.cursor/rules/communication-style.mdc new file mode 100644 index 000000000000..713db3ec8c7e --- /dev/null +++ b/.cursor/rules/communication-style.mdc @@ -0,0 +1,34 @@ +--- +description: Response format preferences for vkale +alwaysApply: true +--- + +# Response format + +Every response starts with a one-line `**TL;DR:**` giving the outcome, then bullets. +Assume the reader is skimming and may stop after the first two lines. + +- Lead with what happened or what is needed, not with process narration. +- Prefer short bullets over paragraphs. Break long prose into bullets. +- Bold the load-bearing phrase in a bullet when it carries the point. +- Use a short table when comparing more than two things across the same fields. +- Put links inline and make them clickable; never make the reader reconstruct a URL. + +# Content rules + +- State bad news first and plainly: what is broken, unfixed, unverified, or blocked. +- Never describe work as tested unless it was executed. Say explicitly when checks + were static-only, and name what was actually run. +- When claiming a bug is fixed, say how it was verified. A test that passes on both + the old and new code proves nothing — confirm it fails on the old code too. +- Separate "I verified this" from "a tool or subagent reported this." +- End with the decisions that need a human, if any. + +# Boundaries + +- Never merge into `develop`, and never approve a PR. Both are the user's call and + require human review. Merging into a stacked PR's own base branch is allowed. +- Always surface PR links. If PR creation is blocked, provide the compare URL. +- PR descriptions follow `.github/pull_request_template.md`: Motivation, Technical + Details, Issue Tracking, Test Plan, Test Result, Submission Checklist. +- Branch names use the `users/vlkale/-withcursorfixes` convention. diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index ea0dd9ba0537..92aaad408bcc 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -221,15 +221,16 @@ get_contexts_cv() return _v; } -// _lk must own get_contexts_mutex(). Waits until no context is mid-stop, which restores the -// property the old single-lock stop_context() had: no other lifecycle operation ever observes a -// context whose services have been torn down but whose active slot is still populated. +} // namespace + +// Waits until no context is mid-stop, which restores the property the old single-lock +// stop_context() had: no other lifecycle operation ever observes a context whose services have +// been torn down but whose active slot is still populated. void -wait_for_pending_stops(std::unique_lock& _lk) +wait_for_stopping_contexts(std::unique_lock& _lk) { get_contexts_cv().wait(_lk, []() { return get_stopping_contexts().empty(); }); } -} // namespace std::mutex& get_contexts_mutex() @@ -442,7 +443,7 @@ start_context(rocprofiler_context_id_t context_id) // A context that is mid-stop is still in the active array while its GPU drain runs, so // scanning against that state would either report a conflict against a context that is on // its way out, or hand back a slot that the stop is about to clear. - wait_for_pending_stops(_lk); + wait_for_stopping_contexts(_lk); auto current_contexts = context_array_t{}; for(const auto* itr : get_active_contexts(current_contexts)) @@ -532,7 +533,7 @@ stop_context(rocprofiler_context_id_t idx) auto _lk = std::unique_lock{get_contexts_mutex()}; // another thread may already be tearing this context down - wait_for_pending_stops(_lk); + wait_for_stopping_contexts(_lk); for(auto& itr : get_active_contexts_impl()) { @@ -667,7 +668,7 @@ deactivate_client_contexts(rocprofiler_client_id_t client_id) // a context mid-stop is still in the active array; let its teardown finish rather than // clearing the slot underneath it - wait_for_pending_stops(_lk); + wait_for_stopping_contexts(_lk); for(auto& itr : get_active_contexts_impl()) { @@ -698,7 +699,7 @@ deregister_client_contexts(rocprofiler_client_id_t client_id) // a context mid-stop is still being torn down through a raw pointer; retiring its registry // entry now would pull the object out from under that teardown - wait_for_pending_stops(_lk); + wait_for_stopping_contexts(_lk); for(auto& itr : *get_registered_contexts_impl()) { diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index 8cce4dd7fb3d..c1c4c376c2ad 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -306,6 +306,15 @@ deregister_client_contexts(rocprofiler_client_id_t id); std::mutex& get_contexts_mutex(); +// Blocks until no context stop is in progress. stop_context() releases get_contexts_mutex() +// across the GPU drain, so holding the mutex alone is not enough to exclude a stop: for that +// window the context is still in the active array with nothing held. Anything that reads +// activation state to decide whether a context is running has to wait here first. +// +// _lk must own get_contexts_mutex(); it is released while waiting and re-acquired on return. +void +wait_for_stopping_contexts(std::unique_lock& _lk); + inline bool default_context_filter(const context* val) { diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp index 848ba3eb1be0..576752a2ab46 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/core.cpp @@ -226,6 +226,12 @@ set_dispatch_agents(rocprofiler_context_id_t context_id, // set for serialization) and then race with the unordered-set assignment below while // dispatch hooks read it. auto _lk = std::unique_lock{context::get_contexts_mutex()}; + + // A context that is mid-stop is still in the active array while its GPU drain runs, and that + // drain happens with the mutex released, so the lock alone does not exclude it. Without this + // the scan below would read a context that is on its way out as a running one. + context::wait_for_stopping_contexts(_lk); + for(const auto* itr : context::get_active_contexts()) { if(itr && itr->context_idx == ctx_p->context_idx) From d007ad423d1e6691cb0aeb8b3ae28952475de8fa Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 08:57:57 +0000 Subject: [PATCH 41/45] fix(rocprofiler-sdk): tie the context snapshot's lifetime to the registry's The registry and the snapshot published from it were two separate static objects. common::destroy_static_objects() runs registered destructors in reverse construction order, and construction order here is whichever was touched first: get_registered_contexts() reaches the snapshot, while allocate_context() reaches the registry, so either can come first depending on the tool. When the snapshot is constructed first it is destroyed last, and by then it holds the only remaining reference to every context -- so the contexts are destroyed after the statics registered between the two, including the HSA tables that ~agent_callback_data calls hsa_signal_destroy_fn through. Both now live in one struct behind a single static object, where the snapshot is destroyed before the registry by the ordinary reverse-member rule. Contexts are then released at the same point in teardown as before the snapshot existed. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/context/context.cpp | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 92aaad408bcc..fb0b16537efe 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -92,14 +92,6 @@ get_client_index() return _v; } -stable_context_vec_t*& -get_registered_contexts_impl() -{ - static auto*& _v = common::static_object::construct( - reserve_size_t{stable_context_vec_t::chunk_size}); - return _v; -} - auto& get_num_active_contexts() { @@ -147,13 +139,38 @@ store_snapshot(published_snapshot_t& _v, context_snapshot_ptr_t&& _new) } #endif -// Shares the registry's lifetime so that reads after common::destroy_static_objects() see an -// empty snapshot rather than a destroyed one. +// The registry and the snapshot published from it live in one static object rather than two. +// common::destroy_static_objects() tears static objects down in reverse construction order, so two +// objects would be ordered by whichever happened to be touched first -- and if the snapshot +// outlived the registry it would be holding the last reference to every context, destroying them +// after the HSA tables that a context destructor calls into are already gone. As members, the +// snapshot is destroyed before the registry by the ordinary reverse-member rule. +struct registry_state +{ + stable_context_vec_t registry = + stable_context_vec_t{reserve_size_t{stable_context_vec_t::chunk_size}}; + published_snapshot_t published = {}; +}; + +registry_state* +get_registry_state() +{ + static auto*& _v = common::static_object::construct(); + return _v; +} + +stable_context_vec_t* +get_registered_contexts_impl() +{ + auto* _state = get_registry_state(); + return (_state) ? &_state->registry : nullptr; +} + published_snapshot_t* get_published_snapshot() { - static auto*& _v = common::static_object::construct(); - return _v; + auto* _state = get_registry_state(); + return (_state) ? &_state->published : nullptr; } context_snapshot_ptr_t From 10b7d5eafba3670b26493ab813f933d28f6c9392 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 09:51:18 +0000 Subject: [PATCH 42/45] chore: drop personal editor rules file from the branch Committed by accident alongside the set_dispatch_agents fix. The repository tracks no .cursor content, so this does not belong in a product change. Co-authored-by: Vivek Kale --- .cursor/rules/communication-style.mdc | 34 --------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .cursor/rules/communication-style.mdc diff --git a/.cursor/rules/communication-style.mdc b/.cursor/rules/communication-style.mdc deleted file mode 100644 index 713db3ec8c7e..000000000000 --- a/.cursor/rules/communication-style.mdc +++ /dev/null @@ -1,34 +0,0 @@ ---- -description: Response format preferences for vkale -alwaysApply: true ---- - -# Response format - -Every response starts with a one-line `**TL;DR:**` giving the outcome, then bullets. -Assume the reader is skimming and may stop after the first two lines. - -- Lead with what happened or what is needed, not with process narration. -- Prefer short bullets over paragraphs. Break long prose into bullets. -- Bold the load-bearing phrase in a bullet when it carries the point. -- Use a short table when comparing more than two things across the same fields. -- Put links inline and make them clickable; never make the reader reconstruct a URL. - -# Content rules - -- State bad news first and plainly: what is broken, unfixed, unverified, or blocked. -- Never describe work as tested unless it was executed. Say explicitly when checks - were static-only, and name what was actually run. -- When claiming a bug is fixed, say how it was verified. A test that passes on both - the old and new code proves nothing — confirm it fails on the old code too. -- Separate "I verified this" from "a tool or subagent reported this." -- End with the decisions that need a human, if any. - -# Boundaries - -- Never merge into `develop`, and never approve a PR. Both are the user's call and - require human review. Merging into a stacked PR's own base branch is allowed. -- Always surface PR links. If PR creation is blocked, provide the compare URL. -- PR descriptions follow `.github/pull_request_template.md`: Motivation, Technical - Details, Issue Tracking, Test Plan, Test Result, Submission Checklist. -- Branch names use the `users/vlkale/-withcursorfixes` convention. From a0116036e0415ea30ed5d7c3452a903cd7a03b74 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 09:52:12 +0000 Subject: [PATCH 43/45] docs(rocprofiler-sdk): correct the lost-exchange comment in stop_context The stopping marker holds deactivate_client_contexts() off until phase four returns, so it can no longer clear the slot mid-teardown. Say that, instead of describing a race the marker already rules out. Co-authored-by: Vivek Kale --- .../source/lib/rocprofiler-sdk/context/context.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index fb0b16537efe..45b3ce42a63a 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -608,8 +608,9 @@ stop_context(rocprofiler_context_id_t idx) if(!slot->compare_exchange_strong(_expected, nullptr)) { - // deactivate_client_contexts() cleared the slot while phase two ran; the services are - // already stopped, so there is nothing left to retire. + // Not reachable as written: deactivate_client_contexts() is the only other writer of + // this slot and the stopping marker holds it off until phase four returns. Kept + // because the pre-phasing code bailed out the same way on a lost exchange. return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; } From 4c03247fca3843c38d45494d6eb9bc4b5f8a74c4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 21 Sep 2026 17:04:29 +0000 Subject: [PATCH 44/45] fix(rocprofiler-sdk): guard the device-counting signal teardown against HSA shutdown agent_callback_data's destructor reached through hsa::get_core_table() unconditionally. Nothing guarantees the owning context is destroyed while HSA is still up, so this faults when teardown runs after the runtime has been shut down. hsa::active_signals::destroy() already guards the identical call this way. Co-authored-by: Vivek Kale --- .../lib/rocprofiler-sdk/counters/device_counting.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/device_counting.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/device_counting.cpp index 22ed73bbaae0..7c8755c1a23c 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/device_counting.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/device_counting.cpp @@ -698,7 +698,17 @@ device_counting_service_hsa_registration() agent_callback_data::~agent_callback_data() { - if(completion.handle != 0) hsa::get_core_table()->hsa_signal_destroy_fn(completion); + if(completion.handle == 0) return; + + // Guarded the same way as hsa::active_signals::destroy(). The owning context is not + // guaranteed to be destroyed while HSA is still up -- teardown can reach here after the + // runtime has been shut down, and the table pointer is also null under unit test -- so + // reaching through the table unconditionally faults instead of leaking a signal. + const auto* _core_table = hsa::get_core_table(); + if(hsa::get_hsa_ref_count() > 0 && _core_table && _core_table->hsa_signal_destroy_fn) + { + _core_table->hsa_signal_destroy_fn(completion); + } } } // namespace counters } // namespace rocprofiler From e31df6b91373e43f12a0b29630ebf1ff5701dea3 Mon Sep 17 00:00:00 2001 From: "Vivek, Kale" Date: Mon, 21 Sep 2026 20:03:51 +0000 Subject: [PATCH 45/45] chore(rocprofiler-sdk): mark #8891 as stacked on #11970 Marker commit only; no file changes. It keeps #8891 open as a distinct pull request now that its base is users/vkale/remove-callbacks-counters--with-cursor-fixes, which already contains every commit on this branch.