Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/rocprofiler-sdk-code_coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ jobs:
git config --global --add safe.directory '*'
apt-get update
apt-get install -y build-essential ccache python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config rpm libzstd-dev
python3 -m pip install -U --user cmake==3.24.0
python3 -m pip install -U --user cmake==3.25.0
python3 -m pip install -U --user -r requirements.txt

- name: Sync gcov with compilers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ jobs:
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12
python3 -m pip install -U --user -r requirements.txt
python3 -m pip install -U --user cmake==3.24.0
python3 -m pip install -U --user cmake==3.25.0

- name: Build ROCProfiler SDK Dependencies
uses: ./.github/actions/rocprofiler-sdk-util
Expand Down Expand Up @@ -540,7 +540,7 @@ jobs:
apt-get install -y gcc-${{ env.GCC_COMPILER_VERSION }} g++-${{ env.GCC_COMPILER_VERSION }}
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_COMPILER_VERSION }} 100 --slave /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_COMPILER_VERSION }} --slave /usr/bin/gcov gcov /usr/bin/gcov-${{ env.GCC_COMPILER_VERSION }}
python3 -m pip install -U --user -r requirements.txt
python3 -m pip install -U --user cmake==3.24.0
python3 -m pip install -U --user cmake==3.25.0

- name: Enable PC Sampling
if: ${{ contains(matrix.system.gpu, 'mi200') || contains(matrix.system.gpu, 'mi300a') }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,9 @@ get_doorbell_tls()
return _v;
}

using async_signal_task_t = std::function<void()>;
using async_signal_task_vector_t = std::vector<async_signal_task_t>;

inline void
publish_submitted_packets(QueueState* state, uint64_t submit_pos)
{
Expand Down Expand Up @@ -294,7 +297,29 @@ async_signal_handler_exists()
{
return common::static_object<internal_threading::task_group_t>::get();
}
} // namespace

size_t
get_async_signal_handler_thread_count()
{
constexpr auto fallback_thread_count = int64_t{4};

const auto gpu_thread_count = common::get_env("GPU_MAX_HW_QUEUES", fallback_thread_count);
const auto thread_count =
common::get_env("ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS", gpu_thread_count);

if(thread_count < 1)
{
ROCP_WARNING << "ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS/GPU_MAX_HW_QUEUES resolved to "
<< thread_count << "; using 1 async signal handler thread";
return 1;
}

return static_cast<size_t>(thread_count);
}

namespace
{
internal_threading::task_group_t*
get_async_signal_handler()
{
Expand All @@ -312,8 +337,7 @@ get_async_signal_handler()
static auto*& _v =
common::static_object<internal_threading::task_group_t>::construct_via_function(
static_cast<create_task_group_fn_t>(&internal_threading::create_task_group),
common::get_env("ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS",
common::get_env("GPU_MAX_HW_QUEUES", 4)));
get_async_signal_handler_thread_count());

return _v;
}
Expand Down Expand Up @@ -427,14 +451,15 @@ async_signal_handler(hsa_signal_t completion_signal,
}

// Local kernel-dispatch tracing path: swaps in pooled completion signals,
// runs KERNEL_DISPATCH_ENQUEUE tracer hooks, and enqueues a completion-signal
// waiter on the async signal handler pool. Strict 1:1 packet forwarding; does
// runs KERNEL_DISPATCH_ENQUEUE tracer hooks, and prepares a completion-signal
// waiter for the async signal handler pool. Strict 1:1 packet forwarding; does
// not insert PM4 packets. Distinct from Queue::WriteInterceptor (legacy path).
void
write_interceptor(Queue* queue,
const void* packets,
uint64_t pkt_count,
hsa_amd_queue_intercept_packet_writer writer)
hsa_amd_queue_intercept_packet_writer writer,
async_signal_task_vector_t* deferred_async_tasks)
{
using callback_record_t = packet_data_t::callback_record_t;
using packet_vector_t = common::container::small_vector<rocprofiler_packet, 512>;
Expand Down Expand Up @@ -519,7 +544,7 @@ write_interceptor(Queue* queue,

using packet_writer_fn_t = std::function<void(packet_vector_t &&)>;

auto process_packet_batch = [&queue, &corr_id, tracing_data_v](
auto process_packet_batch = [&queue, &corr_id, tracing_data_v, deferred_async_tasks](
const rocprofiler_packet* _packets,
uint64_t _num_packets,
const packet_writer_fn_t& _writer) {
Expand Down Expand Up @@ -675,32 +700,45 @@ write_interceptor(Queue* queue,
_info_session.packet_data.emplace_back(std::move(_packet_data));
}

auto last_completion_signal = null_signal;
auto current_signal_value = hsa_signal_value_t{0};
auto _shared_info_session = std::shared_ptr<queue_info_session_t>{};

if(!_info_session.packet_data.empty())
{
auto last_completion_signal = _info_session.packet_data.back().completion_signal;
last_completion_signal = _info_session.packet_data.back().completion_signal;

ROCP_FATAL_IF(last_completion_signal == null_signal)
<< "invalid completion signal in the last packet of the batch";

auto current_signal_value =
current_signal_value =
get_core_table()->hsa_signal_load_scacquire_fn(last_completion_signal);

ROCP_INFO << fmt::format(
" Enqueued batch with completion signal {{.handle={}}} with value {}",
last_completion_signal.handle,
current_signal_value);

auto _shared_info_session =
std::make_shared<queue_info_session_t>(std::move(_info_session));
get_async_signal_handler()->async(
[_signal_v = last_completion_signal,
_expected_signal_v = current_signal_value,
_session_v = std::move(_shared_info_session)]() mutable {
async_signal_handler(_signal_v, _expected_signal_v, std::move(_session_v));
});
_shared_info_session = std::make_shared<queue_info_session_t>(std::move(_info_session));
}

// Copy packets into the real queue before creating the completion waiter. The caller
// defers the actual async enqueue until after it publishes the final doorbell.
_writer(std::move(transformed_packets));

if(_shared_info_session)
{
auto _task = [_signal_v = last_completion_signal,
_expected_signal_v = current_signal_value,
_session_v = std::move(_shared_info_session)]() mutable {
async_signal_handler(_signal_v, _expected_signal_v, std::move(_session_v));
};

if(deferred_async_tasks)
deferred_async_tasks->emplace_back(std::move(_task));
else
get_async_signal_handler()->async(std::move(_task));
}
};

ROCP_TRACE_IF(pkt_count > 1) << fmt::format(
Expand All @@ -719,7 +757,8 @@ process_doorbell_impl(const queue_state_ptr_t& state,
{
if(!state) return;

auto* state_ptr = state.get();
auto* state_ptr = state.get();
auto deferred_async_tasks = async_signal_task_vector_t{};

// gate_lock serializes doorbell processing; producers never take it, so no deadlock.
std::unique_lock<std::mutex> lock{state_ptr->gate_lock};
Expand Down Expand Up @@ -792,8 +831,11 @@ process_doorbell_impl(const queue_state_ptr_t& state,
{
// call local write_interceptor directly instead of heavyweight
// Queue::invoke_write_interceptor
write_interceptor(
const_cast<Queue*>(queue), source_snapshot, pkt_count, ring_buffer_writer);
write_interceptor(const_cast<Queue*>(queue),
source_snapshot,
pkt_count,
ring_buffer_writer,
&deferred_async_tasks);
}
else
{
Expand Down Expand Up @@ -827,6 +869,13 @@ process_doorbell_impl(const queue_state_ptr_t& state,
tls.ring_doorbell = nullptr;
tls.last_published_submit_pos = 0;
tls.state = nullptr;

// Arm completion waiters only after the final doorbell is visible
// so they can never wait on unpublished packets.
lock.unlock();

for(auto& itr : deferred_async_tasks)
get_async_signal_handler()->async(std::move(itr));
}

std::shared_ptr<QueueState>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <hsa/hsa_api_trace.h>

#include <atomic>
#include <cstddef>
#include <functional>
#include <memory>
#include <mutex>
Expand Down Expand Up @@ -158,6 +159,9 @@ cas_write_index_impl(QueueState* state,
uint64_t
load_write_index_impl(const QueueState* state, std::memory_order order = std::memory_order_relaxed);

size_t
get_async_signal_handler_thread_count();

/// Type alias for doorbell function callback
using doorbell_fn_t = std::function<void(hsa_signal_t, hsa_signal_value_t)>;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
// SOFTWARE.

#include "lib/rocprofiler-sdk/hsa/queue_interposition.hpp"
#include "lib/common/environment.hpp"

#include <gtest/gtest.h>

Expand Down Expand Up @@ -172,6 +173,33 @@ TEST(queue_interposition, load_write_index_returns_virtual_wptr)
EXPECT_EQ(load_write_index_impl(&state), 99u);
}

TEST(queue_interposition, async_signal_handler_thread_count_uses_gpu_queue_count)
{
common::env_store env(
{{"GPU_MAX_HW_QUEUES", "7", 1}, {"ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS", "", 1}});
ASSERT_TRUE(env.push());

EXPECT_EQ(get_async_signal_handler_thread_count(), 7u);
}

TEST(queue_interposition, async_signal_handler_thread_count_clamps_zero_gpu_queue_count)
{
common::env_store env(
{{"GPU_MAX_HW_QUEUES", "0", 1}, {"ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS", "", 1}});
ASSERT_TRUE(env.push());

EXPECT_EQ(get_async_signal_handler_thread_count(), 1u);
}

TEST(queue_interposition, async_signal_handler_thread_count_override_takes_precedence)
{
common::env_store env(
{{"GPU_MAX_HW_QUEUES", "0", 1}, {"ROCPROFILER_ASYNC_SIGNAL_HANDLER_THREADS", "3", 1}});
ASSERT_TRUE(env.push());

EXPECT_EQ(get_async_signal_handler_thread_count(), 3u);
}

namespace
{
hsa_kernel_dispatch_packet_t*
Expand Down
Loading