From 022daefbdb9d3b3bd75d7c397af35a2780b21738 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Fri, 11 Nov 2022 01:51:49 +0000 Subject: [PATCH 01/26] WIP --- .../onnxruntime/core/common/profiler_common.h | 60 ++++++- .../core/providers/cuda/cupti_manager.cc | 148 ++++++++++++++++++ .../core/providers/cuda/cupti_manager.h | 73 +++++++++ .../core/providers/rocm/roctracer_manager.cc | 21 --- .../core/providers/rocm/roctracer_manager.h | 32 +--- 5 files changed, 281 insertions(+), 53 deletions(-) create mode 100644 onnxruntime/core/providers/cuda/cupti_manager.cc create mode 100644 onnxruntime/core/providers/cuda/cupti_manager.h diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index 07d6cc101b31c..d9d3f12c5d0d5 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -10,6 +10,63 @@ namespace onnxruntime { namespace profiling { +class ProfilerActivityBuffer { + public: + ProfilerActivityBuffer() + : data_(nullptr), size_(0) {} + + ProfilerActivityBuffer(const char* data, size_t size) + : data_(std::make_unique(size)), size_(size) { + memcpy(data_.get(), data, size); + } + + ProfilerActivityBuffer(const ProfilerActivityBuffer& other) + : ProfilerActivityBuffer(other.data_.get(), other.size_) {} + + ProfilerActivityBuffer(ProfilerActivityBuffer&& other) + : ProfilerActivityBuffer() { + std::swap(data_, other.data_); + std::swap(size_, other.size_); + } + + ProfilerActivityBuffer& operator=(const ProfilerActivityBuffer& other) { + if (&other == this) { + return *this; + } + + size_ = other.size_; + data_ = std::make_unique(other.size_); + memcpy(data_.get(), other.data_.get(), size_); + return *this; + } + + ProfilerActivityBuffer& operator=(ProfilerActivityBuffer&& other) { + if (&other == this) { + return *this; + } + std::swap(data_, other.data_); + std::swap(size_, other.size_); + return *this; + } + + // accessors + char* GetData() { return data_.get(); } + const char* GetData() const { return data_.get(); } + size_t GetSize() const { return size_; } + + static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size) { + ProfilerActivityBuffer res{}; + res.data_ = data; + res.size_ = size; + return res; + } + + private: + std::unique_ptr data_; + size_t size_; +}; + + enum EventCategory { SESSION_EVENT = 0, NODE_EVENT, @@ -23,7 +80,8 @@ static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { "Session", "Node", "Kernel", - "Api"}; + "Api" +}; // Timing record for all events. struct EventRecord { diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc new file mode 100644 index 0000000000000..b61a67fbb8774 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -0,0 +1,148 @@ +#include "cupti_manager.h" + +namespace onnxruntime { +namespace profiling { + +CUPTIManager& CUPTIManager::GetInstance() { + static CUPTIManager instance; + return instance; +} + +CUPTIManager::~CUPTIManager() { + StopLogging(); + Clear(); +} + +uint64_t CUPTIManager::RegisterClient() { + std::lock_guard lock(cupti_manager_mutex_); + auto res = next_client_id_++; + per_client_events_by_ext_correlation_.insert({res, {}}); + ++num_active_clients_; + return res; +} + +void CUPTIManager::DeregisterClient(uint64_t client_handle) { + std::lock_guard lock(cupti_manager_mutex_); + per_client_events_by_ext_correlation_.erase(client_handle); + --num_active_clients_; + if (num_active_clients_ == 0) { + StopLogging(); + } +} + +void CUPTIManager::StartLogging() { + std::lock_guard lock(cupti_manager_mutex_); + if (logging_enabled_) { + return; + } + if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS && + cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { + logging_enabled_ = true; + } else { + StopLogging(); + logging_enabled_ = false; + } +} + +void CUPTIManager::StopLogging() { + std::lock_guard lock(cupti_manager_mutex_); + cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION); + cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL); + cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY); + cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DRIVER); + cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME); + logging_enabled_ = false; +} + +void CUPTIManager::Clear() { + unprocessed_activity_buffers_.clear(); + unique_correlation_id_to_client_offset_.clear(); + per_client_events_by_ext_correlation_.clear(); + cupti_correlation_to_unique_correlation_.clear(); +} + +bool CUPTIManager::PushCorrelation(uint64_t client_handle, + uint64_t external_correlation_id, + TimePoint origin) { + std::lock_guard lock(cupti_manager_mutex_); + if (!logging_enabled_) { + return false; + } + if (per_client_events_by_ext_correlation_.find(client_handle) == + per_client_events_by_ext_correlation_.end()) { + return false; + } + + // external_correlation_id is simply the timestamp of this event, + // relative to profiling_start_time. i.e., it was computed as: + // external_correlation_id = + // std::chrono::duration_cast(event_start_time - profiling_start_time).count() + // + // Because of the relative nature of the external_correlation_id, the same + // external_correlation_id can be reused across different clients, which then makes it + // impossible to recover the client from the external_correlation_id, which in turn + // makes it impossible to map events (which are tagged with external_correlation_id) to clients. + // + // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) + // as follows: + // unique_cid = + // external_correlation_id + + // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() + // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. + // + // Of course, clients expect lists of events to be returned (on a call to Consume()), that are + // still keyed on the external_correlation_id that they've specified here, so we need to remember the + // offset to be subtracted + + uint64_t offset = + std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); + auto unique_cid = external_correlation_id + offset; + + cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, unique_cid); + + unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); + return true; +} + +void CUPTIManager::PopCorrelation(uint64_t& popped_correlation_id) { + popped_correlation_id = 0; + std::lock_guard lock(cupti_manager_mutex_); + if (!logging_enabled_) { + return; + } + + uint64_t unique_cid; + cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &unique_cid); + // lookup the offset and subtract it before returning popped_external_correlation_id to the client + auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + return; + } + popped_correlation_id = unique_cid - client_it->second.second; +} + +void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { + uint8_t* bfr = (uint8_t*)malloc(kActivityBufferSize + kActivityBufferAlignSize); + *size = kActivityBufferSize; + *buffer = AlignBuffer(bfr, kActivityBufferAlignSize); + *maxNumRecords = 0; +} + +void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { + auto instance = GetInstance(); + std::lock_guard lock(instance.unprocessed_activity_buffers_lock_); + instance.unprocessed_activity_buffers_.emplace_back( + CUPTIActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size); + ); +} + +void CUPTIManager::Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { + +} + +} // namespace profiling +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h new file mode 100644 index 0000000000000..f70dc75ea1990 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -0,0 +1,73 @@ +#pragma once + + +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) + +#include +#include +#include + +#include + +#include "core/platform/ort_mutex.h" +#include "core/common/profiler_common.h" +#include "core/common/inlined_containers.h" + +namespace onnxruntime { +namespace profiling { + +using CUPTIActivityBuffer = ProfilerActivityBuffer; + +class CUPTIManager final +{ +public: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); + ~CUPTIManager(); + static CUPTIManager& GetInstance(); + uint64_t RegisterClient(); + void DeregisterClient(uint64_t client_handle); + + void StartLogging(); + void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); + bool PushCorrelation(uint64_t client_handle, uint64_t external_correlation_id, TimePoint profiling_start_time); + void PopCorrelation(uint64_t& popped_correlation_id); + +private: + static constexpr size_t kActivityBufferSize = 32 * 1024; + static constexpr size_t kActivityBufferAlignSize = 8; + static constexpr void* AlignBuffer(buffer, align) { + return (((uintptr_t)(buffer) & ((align)-1)) + ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) + : (buffer)); + } + + CUPTIManager() = default; + static void CUPTIAPI BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); + static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size); + void StopLogging(); + void Clear(); + + std::mutex unprocessed_activity_buffers_lock_; + std::vector unprocessed_activity_buffers_; + std::mutex activity_buffer_processor_mutex_; + std::mutex callback_mutex_; + std::mutex cupti_manager_mutex_; + uint64_t next_client_id_ = 1; + uint64_t num_active_clients_ = 0; + bool logging_enabled_ = false; + + // Keyed on unique_correlation_id -> (client_id/client_handle, offset) + // unique_correlation_id - offset == external_correlation_id + InlinedHashMap> unique_correlation_id_to_client_offset_; + + // Keyed on cupti_correlation_id -> unique_correlation_id + InlinedHashMap cupti_correlation_to_unique_correlation_; + + // client_id/client_handle -> external_correlation_id -> events + InlinedHashMap> per_client_events_by_ext_correlation_; +}; /* class CUPTIManager*/ + +#endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ + +} /* namespace profiling */ +} /* namespace onnxruntime */ diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index e02abd93af3be..adde49629913f 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -22,27 +22,6 @@ const std::vector RoctracerManager::hip_api_calls_to_trace = { "hipExtModuleLaunchKernel", }; -// Implementation of RoctracerActivityBuffer -RoctracerActivityBuffer& RoctracerActivityBuffer::operator=(const RoctracerActivityBuffer& other) { - if (&other == this) { - return *this; - } - - size_ = other.size_; - data_ = std::make_unique(other.size_); - memcpy(data_.get(), other.data_.get(), size_); - return *this; -} - -RoctracerActivityBuffer& RoctracerActivityBuffer::operator=(RoctracerActivityBuffer&& other) { - if (&other == this) { - return *this; - } - std::swap(data_, other.data_); - std::swap(size_, other.size_); - return *this; -} - // Implementation of RoctracerManager RoctracerManager& RoctracerManager::GetInstance() { static RoctracerManager instance; diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index 31ba862d5d65c..8d8f005621497 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -18,37 +18,7 @@ namespace onnxruntime { namespace profiling { -class RoctracerActivityBuffer { - public: - RoctracerActivityBuffer() - : data_(nullptr), size_(0) {} - - RoctracerActivityBuffer(const char* data, size_t size) - : data_(std::make_unique(size)), size_(size) { - memcpy(data_.get(), data, size); - } - - RoctracerActivityBuffer(const RoctracerActivityBuffer& other) - : RoctracerActivityBuffer(other.data_.get(), other.size_) {} - - RoctracerActivityBuffer(RoctracerActivityBuffer&& other) - : RoctracerActivityBuffer() { - std::swap(data_, other.data_); - std::swap(size_, other.size_); - } - - RoctracerActivityBuffer& operator=(const RoctracerActivityBuffer& other); - RoctracerActivityBuffer& operator=(RoctracerActivityBuffer&& other); - - // accessors - char* GetData() { return data_.get(); } - const char* GetData() const { return data_.get(); } - size_t GetSize() const { return size_; } - - private: - std::unique_ptr data_; - size_t size_; -}; +using RoctracerActivityBuffer = ProfilerActivityBuffer; struct ApiCallRecord { uint32_t domain_; From e04b80fb5a27f8804580fc9a72cb84e37f3d2a58 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 15 Nov 2022 23:26:37 +0000 Subject: [PATCH 02/26] WIP --- .../onnxruntime/core/common/profiler_common.h | 253 +++++++++++++++++- .../core/providers/cuda/cupti_manager.cc | 189 +++++++------ .../core/providers/cuda/cupti_manager.h | 38 +-- .../core/providers/rocm/roctracer_manager.cc | 191 ++----------- .../core/providers/rocm/roctracer_manager.h | 54 +--- 5 files changed, 399 insertions(+), 326 deletions(-) diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index d9d3f12c5d0d5..ed24c6d057301 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -3,9 +3,12 @@ #pragma once -#include "core/common/common.h" -#include +#include #include +#include + +#include "core/common/common.h" +#include "core/common/inlined_containers.h" namespace onnxruntime { namespace profiling { @@ -56,7 +59,7 @@ class ProfilerActivityBuffer { static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size) { ProfilerActivityBuffer res{}; - res.data_ = data; + res.data_.reset(data); res.size_ = size; return res; } @@ -66,6 +69,243 @@ class ProfilerActivityBuffer { size_t size_; }; +class GPUTracerManager +{ +public: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); + virtual ~GPUTracerManager() {} + + virtual uint64_t RegisterClient() { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto res = next_client_id_++; + per_client_events_by_ext_correlation_.insert({res, {}}); + ++num_active_clients_; + return res; + } + return 0; + } + + virtual void DeregisterClient(uint64_t client_handle) { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + per_client_events_by_ext_correlation_.erase(it); + --num_active_clients_; + if (num_active_clients_ == 0) { + StopLogging(); + } + } + } + + virtual void StartLogging() = 0; + virtual void Consume(uint64_t client_handle, const TimePoint& start_time, std::mp& events) { + events.clear(); + { + // Flush any pending activity records before starting + // to process the accumulated activity records. + std::lock_guard lock_manager(manager_instance_mutex_); + FlushActivities(); + } + + std::vector activity_buffers; + { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + std::swap(unprocessed_activity_buffers_, activity_buffers); + unprocessed_activity_buffers_.clear(); + } + + { + // Ensure that at most one thread is working through the activity buffers at any time. + std::lock_guard lock_two(activity_buffer_processor_mutex_); + ProcessActivityBuffers(activity_buffers, start_time); + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + std::swap(events, it->second); + } + } + + virtual bool PushCorrelation(uint64_t client_handle, + uint64_t external_correlation_id, + TimePoint profiling_start_time) { + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return false; + } + + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + // not a registered client, do nothing + return false; + } + + uint64_t offset; + auto unique_cid = GetUniqueCorrelationId(client_handle, external_correlation_id, profiling_start_time, offset); + unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); + return PushUniqueCorrelation(unique_cid); + } + + virtual void PopCorrelation(uint64_t& popped_correlation_id) { + popped_correlation_id = 0; + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + uint64_t unique_cid; + PopUniqueCorrelation(unique_cid); + // lookup the offset and subtract it before returning popped_external_correlation_id to the client + auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + popped_external_correlation_id = 0; + return; + } + popped_external_correlation_id = unique_cid - client_it->second.second; + } + + void PopCorrelation() { + uint64_t unused; + PopCorrelation(unused); + } + +protected: + GPUTracerManager() {} + + void EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer) { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + unprocessed_activity_buffers_.emplace_back(std::move(buffer)); + } + + // Requires: manager_instance_mutex_ must be held + virtual void Clear() { + unprocessed_activity_buffers_.clear(); + unique_correlation_id_to_client_offset_.clear(); + per_client_events_by_ext_correlation_.clear(); + tracer_correlation_to_unique_correlation_.clear(); + } + + virtual void StopLogging() = 0; + virtual void ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time) = 0; + + virtual bool PushUniqueCorrelation(uint64_t unique_cid) = 0; + virtual void PopUniqueCorrelation(uint64_t& popped_unique_cid) = 0; + virtual void FlushActivities() = 0; + + Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { + auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + return nullptr; + } + + // See the comments on the GetUniqueCorrelationId method for an explanation of + // of this offset computation and why it's required. + auto const& client_handle_offset = client_it->second; + auto external_correlation = unique_correlation_id - client_handle_offset.second; + + auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; + return &event_list; + } + + uint64_t GetUniqueCorrelationId(uint64_t client_handle, + uint64_t external_correlation_id, + TimePoint profiling_start_time, + uint64_t& offset) { + // external_correlation_id is simply the timestamp of this event, + // relative to profiling_start_time. i.e., it was computed as: + // external_correlation_id = + // std::chrono::duration_cast(event_start_time - profiling_start_time).count() + // + // Because of the relative nature of the external_correlation_id, the same + // external_correlation_id can be reused across different clients, which then makes it + // impossible to recover the client from the external_correlation_id, which in turn + // makes it impossible to map events (which are tagged with external_correlation_id) to clients. + // + // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) + // as follows: + // unique_cid = + // external_correlation_id + + // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() + // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. + // + // Of course, clients expect lists of events to be returned (on a call to Consume()), that are + // still keyed on the external_correlation_id that they've specified here, so we need to remember the + // offset to be subtracted + + offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); + auto unique_cid = external_correlation_id + offset; + return unique_cid; + } + + // Not thread-safe: subclasses must ensure mutual-exclusion when calling this method + void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) + { + auto it = tracer_correlation_to_unique_correlation_.find(tracer_correlation_id); + if (it == tracer_correlation_to_unique_correlation_.end()) { + // We're yet to receive a mapping to unique_correlation_id for this tracer_correlation_id + DeferEventMapping(std::move(event), tracer_correlation_id); + return; + } + auto unique_correlation_id = it->second; + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->emplace_back(std::move(event)); + } + } + + // Not thread-safe: subclasses must ensure mutual-exclusion when calling this method + void MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->insert(p_event_list->end(), + std::make_move_iterator(events.begin()), + std::make_move_iterator(events.end())); + } + } + + void DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id) { + events_pending_client_mapping_[tracer_correlation_id].emplace_back(std::move(event)); + } + + void NotifyOnCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id) { + tracer_correlation_to_unique_correlation_[tracer_correlation_id] = unique_correlation_id; + auto pending_it = events_pending_client_mapping_.find(tracer_correlation_id); + if (pending_it == events_pending_client_mapping_.end()) { + return; + } + // Map the pending events to the right client + MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); + events_pending_client_mapping_.erase(pending_it); + } + + std::mutex manager_instance_mutex_; + uint64_t next_client_id_ = 1; + uint64_t num_active_clients_ = 0; + bool logging_enabled_ = false; + std::mutex unprocessed_activity_buffers_mutex_; + std::mutex activity_buffer_processor_mutex_; + + // Unprocessed activity buffers + std::vector unprocessed_activity_buffers_; + + // Keyed on unique_correlation_id -> (client_id/client_handle, offset) + // unique_correlation_id - offset == external_correlation_id + InlinedHashMap> unique_correlation_id_to_client_offset_; + + // Keyed on tracer_correlation_id -> unique_correlation_id + InlinedHashMap tracer_correlation_to_unique_correlation_; + + // client_id/client_handle -> external_correlation_id -> events + InlinedHashMap> per_client_events_by_ext_correlation_; + + // Keyed on tracer correlation_id, keeps track of activity records + // for which we haven't established the external_correlation_id yet. + InlinedHashMap> events_pending_client_mapping_; +}; enum EventCategory { SESSION_EVENT = 0, @@ -146,5 +386,12 @@ class EpProfiler { std::string demangle(const char* name); std::string demangle(const std::string& name); +// Convert a pointer to a hex string +static inline std::string PointerToHexString(const void* ptr) { + std::ostringstream sstr; + sstr << std::hex << ptr; + return sstr.str(); +} + } // namespace profiling } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index b61a67fbb8774..ebefd67d13027 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -3,35 +3,44 @@ namespace onnxruntime { namespace profiling { +static inline const char* GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { + switch (kind) { + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: + return "MemcpyHostToDevice"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH: + return "MemcpyDeviceToHost"; + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA: + return "MemcpyHostToDeviceArray"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH: + return "MemcpyDeviceArrayToHost"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA: + return "MemcpyDeviceArrayToDeviceArray"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD: + return "MemcpyDeviceArrayToDevice"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA: + return "MemcpyDeviceToDeviceArray"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD: + return "MemcpyDeviceToDevice"; + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH: + return "MemcpyHostToHost"; + default: + break; + } + return ""; +} + CUPTIManager& CUPTIManager::GetInstance() { static CUPTIManager instance; return instance; } CUPTIManager::~CUPTIManager() { - StopLogging(); - Clear(); -} - -uint64_t CUPTIManager::RegisterClient() { - std::lock_guard lock(cupti_manager_mutex_); - auto res = next_client_id_++; - per_client_events_by_ext_correlation_.insert({res, {}}); - ++num_active_clients_; - return res; -} - -void CUPTIManager::DeregisterClient(uint64_t client_handle) { - std::lock_guard lock(cupti_manager_mutex_); - per_client_events_by_ext_correlation_.erase(client_handle); - --num_active_clients_; - if (num_active_clients_ == 0) { - StopLogging(); - } + StopLogging(); + Clear(); } void CUPTIManager::StartLogging() { - std::lock_guard lock(cupti_manager_mutex_); + std::lock_guard lock(manager_instance_mutex_); if (logging_enabled_) { return; } @@ -58,71 +67,87 @@ void CUPTIManager::StopLogging() { logging_enabled_ = false; } -void CUPTIManager::Clear() { - unprocessed_activity_buffers_.clear(); - unique_correlation_id_to_client_offset_.clear(); - per_client_events_by_ext_correlation_.clear(); - cupti_correlation_to_unique_correlation_.clear(); +bool CUPTIManager::PushUniqueCorrelation(uint64_t unique_cid) { + return cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, unique_cid) == CUPTI_SUCCESS; } -bool CUPTIManager::PushCorrelation(uint64_t client_handle, - uint64_t external_correlation_id, - TimePoint origin) { - std::lock_guard lock(cupti_manager_mutex_); - if (!logging_enabled_) { - return false; - } - if (per_client_events_by_ext_correlation_.find(client_handle) == - per_client_events_by_ext_correlation_.end()) { - return false; +void CUPTIManager::PopUniqueCorrelation(uint64_t& popped_unique_cid) { + if (cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &popped_unique_cid) != CUPTI_SUCCESS) { + popped_unique_cid = 0; } - - // external_correlation_id is simply the timestamp of this event, - // relative to profiling_start_time. i.e., it was computed as: - // external_correlation_id = - // std::chrono::duration_cast(event_start_time - profiling_start_time).count() - // - // Because of the relative nature of the external_correlation_id, the same - // external_correlation_id can be reused across different clients, which then makes it - // impossible to recover the client from the external_correlation_id, which in turn - // makes it impossible to map events (which are tagged with external_correlation_id) to clients. - // - // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) - // as follows: - // unique_cid = - // external_correlation_id + - // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() - // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. - // - // Of course, clients expect lists of events to be returned (on a call to Consume()), that are - // still keyed on the external_correlation_id that they've specified here, so we need to remember the - // offset to be subtracted - - uint64_t offset = - std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); - auto unique_cid = external_correlation_id + offset; - - cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, unique_cid); - - unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); - return true; } -void CUPTIManager::PopCorrelation(uint64_t& popped_correlation_id) { - popped_correlation_id = 0; - std::lock_guard lock(cupti_manager_mutex_); - if (!logging_enabled_) { - return; - } +void CUPTIManager::FlushActivities() { + cuptiActivityFlushAll(1); +} - uint64_t unique_cid; - cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &unique_cid); - // lookup the offset and subtract it before returning popped_external_correlation_id to the client - auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - return; - } - popped_correlation_id = unique_cid - client_it->second.second; +void CUPTIManager::ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time) { + auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); + for (auto const& buffer : buffers) { + auto size = buffer.GetSize(); + if (size == 0) { + continue; + } + CUpti_Activity* record = nullptr; + CUptiResult status; + do { + EventRecord event; + status = cuptiActivityGetNextRecord(buffer.GetData(), size, &record); + if (status == CUPTI_SUCCESS) { + if (CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL == record->kind || + CUPTI_ACTIVITY_KIND_KERNEL == record->kind) { + CUpti_ActivityKernel8* kernel = (CUpti_ActivityKernel8*)record; + std::unordered_map args { + {"stream", std::to_string(kernel->streamId)}, + {"grid_x", std::to_string(kernel->gridX)}, + {"grid_y", std::to_string(kernel->gridY)}, + {"grid_z", std::to_string(kernel->gridZ)}, + {"block_x", std::to_string(kernel->blockX)}, + {"block_y", std::to_string(kernel->blockY)}, + {"block_z", std::to_string(kernel->blockZ)}, + }; + + std::string name{kernel->name}; + + new (&event) EventRecord { + /* cat = */ EventCategory::KERNEL_EVENT, + /* pid = */ -1, + /* tid = */ -1, + /* name = */ std::move(name), + /* ts = */ (int64_t)(kernel->start - start_time_ns) / 1000, + /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, + /* args = */ std::move(args) + }; + MapEventToClient(record->correlationId, std::move(Event)); + } else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) { + CUpti_ActivityMemcpy3* mmcpy = (CUpti_ActivityMemcpy3*)record; + std::string name{GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind)}; + std::unordered_map args { + {"stream", std::to_string(mmcpy->streamId)}, + {"grid_x", "-1"}, + {"grid_y", "-1"}, + {"grid_z", "-1"}, + {"block_x", "-1"}, + {"block_y", "-1"}, + {"block_z", "-1"}, + }; + new (&event) EventRecord { + /* cat = */ EventCategory::KERNEL_EVENT, + /* pid = */ -1, + /* tid = */ -1, + /* name = */ std::move(name), + /* ts = */ (int64_t)(kernel->start - start_time_ns) / 1000, + /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, + /* args = */ std::move(args)}; + MapEventToClient(record->correlationId, std::move(Event)); + } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { + auto correlation = reinterpret_cast(record); + NotifyOnCorrelation(correlation->correlationId, correlation->externalId); + } + } + } (status == CUPTI_SUCCESS); /* do */ + } /* for */ } void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { @@ -136,13 +161,9 @@ void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer auto instance = GetInstance(); std::lock_guard lock(instance.unprocessed_activity_buffers_lock_); instance.unprocessed_activity_buffers_.emplace_back( - CUPTIActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size); + CUPTIActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size) ); } -void CUPTIManager::Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { - -} - } // namespace profiling } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index f70dc75ea1990..8d576d3f6553e 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -18,19 +18,21 @@ namespace profiling { using CUPTIActivityBuffer = ProfilerActivityBuffer; -class CUPTIManager final +class CUPTIManager : public GPUTracerManager { public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); ~CUPTIManager(); static CUPTIManager& GetInstance(); - uint64_t RegisterClient(); - void DeregisterClient(uint64_t client_handle); + void StartLogging() override; - void StartLogging(); - void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); - bool PushCorrelation(uint64_t client_handle, uint64_t external_correlation_id, TimePoint profiling_start_time); - void PopCorrelation(uint64_t& popped_correlation_id); +protected: + bool PushUniqueCorrelation(uint64_t unique_cid) override; + void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; + void StopLogging() override; + void ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time) override; + void FlushActivities() override; private: static constexpr size_t kActivityBufferSize = 32 * 1024; @@ -42,29 +44,9 @@ class CUPTIManager final } CUPTIManager() = default; + static void CUPTIAPI BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size); - void StopLogging(); - void Clear(); - - std::mutex unprocessed_activity_buffers_lock_; - std::vector unprocessed_activity_buffers_; - std::mutex activity_buffer_processor_mutex_; - std::mutex callback_mutex_; - std::mutex cupti_manager_mutex_; - uint64_t next_client_id_ = 1; - uint64_t num_active_clients_ = 0; - bool logging_enabled_ = false; - - // Keyed on unique_correlation_id -> (client_id/client_handle, offset) - // unique_correlation_id - offset == external_correlation_id - InlinedHashMap> unique_correlation_id_to_client_offset_; - - // Keyed on cupti_correlation_id -> unique_correlation_id - InlinedHashMap cupti_correlation_to_unique_correlation_; - - // client_id/client_handle -> external_correlation_id -> events - InlinedHashMap> per_client_events_by_ext_correlation_; }; /* class CUPTIManager*/ #endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index adde49629913f..b6da61fa0cfd5 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -32,25 +32,8 @@ RoctracerManager::~RoctracerManager() { StopLogging(); } -uint64_t RoctracerManager::RegisterClient() { - std::lock_guard lock(roctracer_manager_mutex_); - auto res = next_client_id_++; - per_client_events_by_ext_correlation_.insert({res, {}}); - ++num_active_clients_; - return res; -} - -void RoctracerManager::DeregisterClient(uint64_t client_handle) { - std::lock_guard lock(roctracer_manager_mutex_); - per_client_events_by_ext_correlation_.erase(client_handle); - --num_active_clients_; - if (num_active_clients_ == 0) { - StopLogging(); - } -} - void RoctracerManager::StartLogging() { - std::lock_guard lock(roctracer_manager_mutex_); + std::lock_guard lock(manager_instance_mutex_); if (logging_enabled_) { return; } @@ -83,16 +66,7 @@ void RoctracerManager::StartLogging() { logging_enabled_ = true; } -// Requires: roctracer_manager_mutex_ must be held -void RoctracerManager::Clear() { - unprocessed_activity_buffers_.clear(); - api_call_args_.clear(); - unique_correlation_id_to_client_offset_.clear(); - roctracer_correlation_to_unique_correlation_.clear(); - per_client_events_by_ext_correlation_.clear(); -} - -// Requires: roctracer_manager_mutex_ must be held +// Requires: manager_instance_mutex_ must be held void RoctracerManager::StopLogging() { if (!logging_enabled_) { return; @@ -109,97 +83,16 @@ void RoctracerManager::StopLogging() { Clear(); } -void RoctracerManager::Consume(uint64_t client_handle, const TimePoint& start_time, - std::map& events) { - events.clear(); - { - // Flush any pending activity records before starting - // to process the accumulated activity records. - std::lock_guard lock_manager(roctracer_manager_mutex_); - roctracer_flush_activity(); - } - - std::vector activity_buffers; - { - std::lock_guard lock(unprocessed_activity_buffers_lock_); - std::swap(unprocessed_activity_buffers_, activity_buffers); - unprocessed_activity_buffers_.clear(); - } - - { - // Ensure that at most one thread is working through the activity buffers at any time. - std::lock_guard lock_two(activity_buffer_processor_mutex_); - ProcessActivityBuffers(activity_buffers, start_time); - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - std::swap(events, it->second); - } -} - -bool RoctracerManager::PushCorrelation(uint64_t client_handle, - uint64_t external_correlation_id, - TimePoint profiling_start_time) { - std::lock_guard lock(roctracer_manager_mutex_); - - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - // not a registered client, do nothing - return false; - } - - // external_correlation_id is simply the timestamp of this event, - // relative to profiling_start_time. i.e., it was computed as: - // external_correlation_id = - // std::chrono::duration_cast(event_start_time - profiling_start_time).count() - // - // Because of the relative nature of the external_correlation_id, the same - // external_correlation_id can be reused across different clients, which then makes it - // impossible to recover the client from the external_correlation_id, which in turn - // makes it impossible to map events (which are tagged with external_correlation_id) to clients. - // - // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) - // as follows: - // unique_cid = - // external_correlation_id + - // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() - // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. - // - // Of course, clients expect lists of events to be returned (on a call to Consume()), that are - // still keyed on the external_correlation_id that they've specified here, so we need to remember the - // offset to be subtracted - - uint64_t offset = - std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); - auto unique_cid = external_correlation_id + offset; - roctracer_activity_push_external_correlation_id(unique_cid); - - unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); - return true; -} - -void RoctracerManager::PopCorrelation(uint64_t& popped_external_correlation_id) { - std::lock_guard lock(roctracer_manager_mutex_); - uint64_t unique_cid; - roctracer_activity_pop_external_correlation_id(&unique_cid); - // lookup the offset and subtract it before returning popped_external_correlation_id to the client - auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - popped_external_correlation_id = 0; - return; - } - popped_external_correlation_id = unique_cid - client_it->second.second; +RoctracerManager::Clear() { + GPUTracerManager::Clear(); + api_call_args_.clear(); } void RoctracerManager::ActivityCallback(const char* begin, const char* end, void* arg) { size_t size = end - begin; RoctracerActivityBuffer activity_buffer{reinterpret_cast(begin), size}; auto& instance = GetInstance(); - { - std::lock_guard lock(instance.unprocessed_activity_buffers_lock_); - instance.unprocessed_activity_buffers_.emplace_back(std::move(activity_buffer)); - } + instance.EnqueueActivityBuffer(std::move(activity_buffer)); } void RoctracerManager::ApiCallback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) { @@ -214,7 +107,7 @@ void RoctracerManager::ApiCallback(uint32_t domain, uint32_t cid, const void* ca auto& instance = GetInstance(); { - std::lock_guard lock(instance.api_call_args_lock_); + std::lock_guard lock(instance.api_call_args_mutex_); auto& record = instance.api_call_args_[data->correlation_id]; record.domain_ = domain; record.cid_ = cid; @@ -222,10 +115,18 @@ void RoctracerManager::ApiCallback(uint32_t domain, uint32_t cid, const void* ca } } -static inline std::string PointerToHexString(const void* ptr) { - std::ostringstream sstr; - sstr << std::hex << ptr; - return sstr.str(); +bool RoctracerManager::PushUniqueCorrelation(uint64_t unique_cid) { + return roctracer_activity_push_external_correlation_id(unique_cid) == ROCTRACER_STATUS_SUCCESS; +} + +void RoctracerManager::PopUniqueCorrelation(uint64_t& popped_unique_cid) { + if (roctracer_activity_pop_external_correlation_id(&popped_unique_cid) != ROCTRACER_STATUS_SUCCESS) { + popped_unique_cid = 0; + } +} + +void RoctracerManager::FlushActivities() { + roctracer_flush_activity(); } static inline std::string MemcpyKindToString(hipMemcpyKind kind) { @@ -348,38 +249,6 @@ bool RoctracerManager::CreateEventForActivityRecord(const roctracer_record_t* re return true; } -Events* RoctracerManager::GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { - auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - // :-( well, we tried really, really hard to map this event to a client. - return nullptr; - } - - // See the comments on the PushCorrelation method for an explanation of - // of this offset computation and why it's required. - auto const& client_handle_offset = client_it->second; - auto external_correlation = unique_correlation_id - client_handle_offset.second; - - auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; - return &event_list; -} - -void RoctracerManager::MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->insert(p_event_list->end(), - std::make_move_iterator(events.begin()), - std::make_move_iterator(events.end())); - } -} - -void RoctracerManager::MapEventToClient(uint64_t unique_correlation_id, EventRecord&& event) { - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->emplace_back(std::move(event)); - } -} - void RoctracerManager::ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) { auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); @@ -390,18 +259,7 @@ void RoctracerManager::ProcessActivityBuffers(const std::vectordomain == ACTIVITY_DOMAIN_EXT_API) { - roctracer_correlation_to_unique_correlation_[current_record->correlation_id] = current_record->external_id; - - // check for any events pending client mapping on this correlation - auto pending_it = events_pending_client_mapping_.find(current_record->correlation_id); - if (pending_it == events_pending_client_mapping_.end()) { - continue; - } - - // we have one or more pending events, map them to the client - MapEventsToClient(current_record->external_id, std::move(pending_it->second)); - events_pending_client_mapping_.erase(pending_it); - // no additional events to be mapped for this record + NotifyOnCorrelation(current_record->correlation_id, current_record->external_id) continue; } else if (current_record->domain == ACTIVITY_DOMAIN_HIP_OPS) { if (current_record->op == 1 && current_record->kind == HipOpMarker) { @@ -426,15 +284,8 @@ void RoctracerManager::ProcessActivityBuffers(const std::vectorcorrelation_id); - if (ext_corr_it == roctracer_correlation_to_unique_correlation_.end()) { - // defer the processing of this event - events_pending_client_mapping_[current_record->correlation_id].emplace_back(std::move(event)); - continue; - } - MapEventToClient(ext_corr_it->second, std::move(event)); + MapEventToClient(current_record->correlation_id, std::move(event)); } } } diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index 8d8f005621497..a7a5e4098905f 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -26,67 +26,39 @@ struct ApiCallRecord { hip_api_data_t api_data_{}; }; -class RoctracerManager { +class RoctracerManager : public GPUTracerManager { public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RoctracerManager); ~RoctracerManager(); - static RoctracerManager& GetInstance(); + void StartLogging() override; - uint64_t RegisterClient(); - void DeregisterClient(uint64_t client_handle); - - void StartLogging(); - void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); - - bool PushCorrelation(uint64_t client_handle, uint64_t external_correlation_id, TimePoint profiling_start_time); - void PopCorrelation(uint64_t& popped_correlation_id); - bool PopCorrelation(); + protected: + bool PushUniqueCorrelation(uint64_t unique_cid) override; + void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; + void StopLogging() override; + void ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time) override; + void Clear() override; + void FlushActivities() override; private: RoctracerManager() = default; static void ActivityCallback(const char* begin, const char* end, void* arg); static void ApiCallback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg); - void ProcessActivityBuffers(const std::vector& buffers, - const TimePoint& start_time); bool CreateEventForActivityRecord(const roctracer_record_t* record, uint64_t start_time_ns, const ApiCallRecord& call_record, EventRecord& event); - Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id); - void MapEventToClient(uint64_t external_correlation_id, EventRecord&& event); - void MapEventsToClient(uint64_t external_correlation_id, Events&& events); - void StopLogging(); - void Clear(); + // Some useful constants for processing activity buffers static constexpr uint32_t HipOpMarker = 4606; - std::mutex unprocessed_activity_buffers_lock_; - std::vector unprocessed_activity_buffers_; - std::mutex activity_buffer_processor_mutex_; - std::mutex api_call_args_lock_; + std::mutex api_call_args_mutex_; InlinedHashMap api_call_args_; - // Keyed on unique_correlation_id -> (client_id/client_handle, offset) - // unique_correlation_id - offset == external_correlation_id - InlinedHashMap> unique_correlation_id_to_client_offset_; - - // Keyed on roctracer_correlation_id -> unique_correlation_id - InlinedHashMap roctracer_correlation_to_unique_correlation_; - - // client_id/client_handle -> external_correlation_id -> events - InlinedHashMap> per_client_events_by_ext_correlation_; - uint64_t next_client_id_ = 1; - uint64_t num_active_clients_ = 0; - bool logging_enabled_ = false; - std::mutex roctracer_manager_mutex_; - - // Keyed on roctracer correlation_id, keeps track of activity records - // for which we haven't established the external_correlation_id yet. - InlinedHashMap> events_pending_client_mapping_; - // The api calls to track static const std::vector hip_api_calls_to_trace; -}; +}; /* class RoctracerManager */ } /* end namespace profiling */ } /* end namespace onnxruntime*/ From d8f6c63ebd211cf0289f3648791d3bcd70112aee Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Wed, 16 Nov 2022 00:37:00 +0000 Subject: [PATCH 03/26] Fixes --- .../onnxruntime/core/common/profiler_common.h | 195 +++++++++--------- .../core/providers/cuda/cupti_manager.cc | 25 ++- .../core/providers/cuda/cupti_manager.h | 5 +- 3 files changed, 107 insertions(+), 118 deletions(-) diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index ed24c6d057301..8d4652625c1a1 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -69,6 +69,71 @@ class ProfilerActivityBuffer { size_t size_; }; +enum EventCategory { + SESSION_EVENT = 0, + NODE_EVENT, + KERNEL_EVENT, + API_EVENT, + EVENT_CATEGORY_MAX +}; + +// Event descriptions for the above session events. +static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { + "Session", + "Node", + "Kernel", + "Api" +}; + +// Timing record for all events. +struct EventRecord { + EventRecord() = default; + EventRecord(EventCategory category, + int process_id, + int thread_id, + std::string&& event_name, + long long time_stamp, + long long duration, + std::unordered_map&& event_args) + : cat(category), + pid(process_id), + tid(thread_id), + name(std::move(event_name)), + ts(time_stamp), + dur(duration), + args(std::move(event_args)) {} + + EventRecord(EventCategory category, + int process_id, + int thread_id, + const std::string& event_name, + long long time_stamp, + long long duration, + const std::unordered_map& event_args) + : cat(category), + pid(process_id), + tid(thread_id), + name(event_name), + ts(time_stamp), + dur(duration), + args(event_args) {} + + EventRecord(const EventRecord& other) = default; + EventRecord(EventRecord&& other) = default; + EventRecord& operator=(const EventRecord& other) = default; + EventRecord& operator=(EventRecord&& other) = default; + + EventCategory cat = EventCategory::API_EVENT; + int pid = -1; + int tid = -1; + std::string name{}; + long long ts = 0; + long long dur = 0; + std::unordered_map args{}; +}; + +using Events = std::vector; + class GPUTracerManager { public: @@ -102,7 +167,7 @@ class GPUTracerManager } virtual void StartLogging() = 0; - virtual void Consume(uint64_t client_handle, const TimePoint& start_time, std::mp& events) { + virtual void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { events.clear(); { // Flush any pending activity records before starting @@ -111,7 +176,7 @@ class GPUTracerManager FlushActivities(); } - std::vector activity_buffers; + std::vector activity_buffers; { std::lock_guard lock(unprocessed_activity_buffers_mutex_); std::swap(unprocessed_activity_buffers_, activity_buffers); @@ -144,14 +209,33 @@ class GPUTracerManager return false; } - uint64_t offset; - auto unique_cid = GetUniqueCorrelationId(client_handle, external_correlation_id, profiling_start_time, offset); + // external_correlation_id is simply the timestamp of this event, + // relative to profiling_start_time. i.e., it was computed as: + // external_correlation_id = + // std::chrono::duration_cast(event_start_time - profiling_start_time).count() + // + // Because of the relative nature of the external_correlation_id, the same + // external_correlation_id can be reused across different clients, which then makes it + // impossible to recover the client from the external_correlation_id, which in turn + // makes it impossible to map events (which are tagged with external_correlation_id) to clients. + // + // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) + // as follows: + // unique_cid = + // external_correlation_id + + // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() + // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. + // + // Of course, clients expect lists of events to be returned (on a call to Consume()), that are + // still keyed on the external_correlation_id that they've specified here, so we need to remember the + // offset to be subtracted + uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); + auto unique_cid = external_correlation_id + offset; unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); return PushUniqueCorrelation(unique_cid); } - virtual void PopCorrelation(uint64_t& popped_correlation_id) { - popped_correlation_id = 0; + virtual void PopCorrelation(uint64_t& popped_external_correlation_id) { std::lock_guard lock(manager_instance_mutex_); if (!logging_enabled_) { return; @@ -189,7 +273,7 @@ class GPUTracerManager } virtual void StopLogging() = 0; - virtual void ProcessActivityBuffers(const std::vector& buffers, + virtual void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) = 0; virtual bool PushUniqueCorrelation(uint64_t unique_cid) = 0; @@ -211,36 +295,6 @@ class GPUTracerManager return &event_list; } - uint64_t GetUniqueCorrelationId(uint64_t client_handle, - uint64_t external_correlation_id, - TimePoint profiling_start_time, - uint64_t& offset) { - // external_correlation_id is simply the timestamp of this event, - // relative to profiling_start_time. i.e., it was computed as: - // external_correlation_id = - // std::chrono::duration_cast(event_start_time - profiling_start_time).count() - // - // Because of the relative nature of the external_correlation_id, the same - // external_correlation_id can be reused across different clients, which then makes it - // impossible to recover the client from the external_correlation_id, which in turn - // makes it impossible to map events (which are tagged with external_correlation_id) to clients. - // - // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) - // as follows: - // unique_cid = - // external_correlation_id + - // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() - // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. - // - // Of course, clients expect lists of events to be returned (on a call to Consume()), that are - // still keyed on the external_correlation_id that they've specified here, so we need to remember the - // offset to be subtracted - - offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); - auto unique_cid = external_correlation_id + offset; - return unique_cid; - } - // Not thread-safe: subclasses must ensure mutual-exclusion when calling this method void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) { @@ -305,72 +359,7 @@ class GPUTracerManager // Keyed on tracer correlation_id, keeps track of activity records // for which we haven't established the external_correlation_id yet. InlinedHashMap> events_pending_client_mapping_; -}; - -enum EventCategory { - SESSION_EVENT = 0, - NODE_EVENT, - KERNEL_EVENT, - API_EVENT, - EVENT_CATEGORY_MAX -}; - -// Event descriptions for the above session events. -static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { - "Session", - "Node", - "Kernel", - "Api" -}; - -// Timing record for all events. -struct EventRecord { - EventRecord() = default; - EventRecord(EventCategory category, - int process_id, - int thread_id, - std::string&& event_name, - long long time_stamp, - long long duration, - std::unordered_map&& event_args) - : cat(category), - pid(process_id), - tid(thread_id), - name(std::move(event_name)), - ts(time_stamp), - dur(duration), - args(std::move(event_args)) {} - - EventRecord(EventCategory category, - int process_id, - int thread_id, - const std::string& event_name, - long long time_stamp, - long long duration, - const std::unordered_map& event_args) - : cat(category), - pid(process_id), - tid(thread_id), - name(event_name), - ts(time_stamp), - dur(duration), - args(event_args) {} - - EventRecord(const EventRecord& other) = default; - EventRecord(EventRecord&& other) = default; - EventRecord& operator=(const EventRecord& other) = default; - EventRecord& operator=(EventRecord&& other) = default; - - EventCategory cat = EventCategory::API_EVENT; - int pid = -1; - int tid = -1; - std::string name{}; - long long ts = 0; - long long dur = 0; - std::unordered_map args{}; -}; - -using Events = std::vector; +}; /* class GPUTracerManager */ //Execution Provider Profiler class EpProfiler { diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index ebefd67d13027..6bea2d2a7054a 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -58,7 +58,7 @@ void CUPTIManager::StartLogging() { } void CUPTIManager::StopLogging() { - std::lock_guard lock(cupti_manager_mutex_); + std::lock_guard lock(manager_instance_mutex_); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY); @@ -93,11 +93,11 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector CUptiResult status; do { EventRecord event; - status = cuptiActivityGetNextRecord(buffer.GetData(), size, &record); + status = cuptiActivityGetNextRecord(reinterpret_cast(const_cast(buffer.GetData())), size, &record); if (status == CUPTI_SUCCESS) { if (CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL == record->kind || CUPTI_ACTIVITY_KIND_KERNEL == record->kind) { - CUpti_ActivityKernel8* kernel = (CUpti_ActivityKernel8*)record; + CUpti_ActivityKernel3* kernel = (CUpti_ActivityKernel3*)record; std::unordered_map args { {"stream", std::to_string(kernel->streamId)}, {"grid_x", std::to_string(kernel->gridX)}, @@ -119,9 +119,9 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, /* args = */ std::move(args) }; - MapEventToClient(record->correlationId, std::move(Event)); + MapEventToClient(kernel->correlationId, std::move(event)); } else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) { - CUpti_ActivityMemcpy3* mmcpy = (CUpti_ActivityMemcpy3*)record; + CUpti_ActivityMemcpy* mmcpy = (CUpti_ActivityMemcpy*)record; std::string name{GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind)}; std::unordered_map args { {"stream", std::to_string(mmcpy->streamId)}, @@ -137,16 +137,16 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector /* pid = */ -1, /* tid = */ -1, /* name = */ std::move(name), - /* ts = */ (int64_t)(kernel->start - start_time_ns) / 1000, - /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, + /* ts = */ (int64_t)(mmcpy->start - start_time_ns) / 1000, + /* dur = */ (int64_t)(mmcpy->end - mmcpy->start) / 1000, /* args = */ std::move(args)}; - MapEventToClient(record->correlationId, std::move(Event)); + MapEventToClient(mmcpy->correlationId, std::move(event)); } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { auto correlation = reinterpret_cast(record); NotifyOnCorrelation(correlation->correlationId, correlation->externalId); } } - } (status == CUPTI_SUCCESS); /* do */ + } while (status == CUPTI_SUCCESS); } /* for */ } @@ -158,10 +158,9 @@ void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size } void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { - auto instance = GetInstance(); - std::lock_guard lock(instance.unprocessed_activity_buffers_lock_); - instance.unprocessed_activity_buffers_.emplace_back( - CUPTIActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size) + auto& instance = GetInstance(); + instance.EnqueueActivityBuffer( + ProfilerActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size) ); } diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 8d576d3f6553e..36a5f58c500d3 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -30,14 +30,15 @@ class CUPTIManager : public GPUTracerManager bool PushUniqueCorrelation(uint64_t unique_cid) override; void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; void StopLogging() override; - void ProcessActivityBuffers(const std::vector& buffers, + void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) override; void FlushActivities() override; private: static constexpr size_t kActivityBufferSize = 32 * 1024; static constexpr size_t kActivityBufferAlignSize = 8; - static constexpr void* AlignBuffer(buffer, align) { + + static constexpr uint8_t* AlignBuffer(uint8_t* buffer, int align) { return (((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) : (buffer)); From b28c38a17b08bdb44900359481cffcb270a65f04 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Wed, 16 Nov 2022 18:51:36 +0000 Subject: [PATCH 04/26] Complete implementation of profiler --- .../onnxruntime/core/common/profiler_common.h | 47 +++++ .../core/providers/cuda/cuda_profiler.cc | 199 +++--------------- .../core/providers/cuda/cuda_profiler.h | 49 +---- .../core/providers/cuda/cupti_manager.cc | 13 +- .../core/providers/cuda/cupti_manager.h | 5 +- .../core/providers/rocm/rocm_profiler.cc | 40 +--- .../core/providers/rocm/rocm_profiler.h | 5 +- 7 files changed, 96 insertions(+), 262 deletions(-) diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index 8d4652625c1a1..c774ae7f57173 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -371,6 +371,53 @@ class EpProfiler { virtual void Stop(uint64_t){}; // called after op stop, accept an id as argument to identify the op }; +// Base class for a GPU profiler +class GPUProfilerBase : public EpProfiler { +protected: + GPUProfilerBase() = default; + + void MergeEvents(std::map& events_to_merge, Events& events) { + Events merged_events; + + auto event_iter = std::make_move_iterator(events.begin()); + auto event_end = std::make_move_iterator(events.end()); + for (auto& map_iter : events_to_merge) { + auto ts = static_cast(map_iter.first); + while (event_iter != event_end && event_iter->ts < ts) { + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + // find the last event with the same timestamp. + while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { + ++event_iter; + } + + if (event_iter != event_end && event_iter->ts == ts) { + uint64_t increment = 1; + for (auto& evt : map_iter.second) { + evt.args["op_name"] = event_iter->args["op_name"]; + + // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. + // So, we adjust the timestamp here to something sensible. + evt.ts = event_iter->ts + increment; + ++increment; + } + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + merged_events.insert(merged_events.end(), + std::make_move_iterator(map_iter.second.begin()), + std::make_move_iterator(map_iter.second.end())); + } + + // move any remaining events + merged_events.insert(merged_events.end(), event_iter, event_end); + std::swap(events, merged_events); + } +}; + // Demangle C++ symbols std::string demangle(const char* name); std::string demangle(const std::string& name); diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 01048451748f2..81849cea4f701 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -8,206 +8,63 @@ #include #include "core/common/profiler_common.h" +#include "cupti_manager.h" namespace onnxruntime { namespace profiling { -auto KEVENT = onnxruntime::profiling::KERNEL_EVENT; -std::atomic_flag CudaProfiler::enabled{0}; -std::vector CudaProfiler::stats; -std::unordered_map CudaProfiler::id_map; +// audupa: Debugging only, delete before merging +// #define CUDA_VERSION 11600 -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +// auto KEVENT = onnxruntime::profiling::KERNEL_EVENT; +// std::atomic_flag CudaProfiler::enabled{0}; +// std::vector CudaProfiler::stats; +// std::unordered_map CudaProfiler::id_map; -#define BUF_SIZE (32 * 1024) -#define ALIGN_SIZE (8) -#define ALIGN_BUFFER(buffer, align) \ - (((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) : (buffer)) -#define DUR(s, e) ((e - s) / 1000) - -static const char* GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { - switch (kind) { - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: - return "MemcpyHostToDevice"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH: - return "MemcpyDeviceToHost"; - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA: - return "MemcpyHostToDeviceArray"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH: - return "MemcpyDeviceArrayToHost"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA: - return "MemcpyDeviceArrayToDeviceArray"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD: - return "MemcpyDeviceArrayToDevice"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA: - return "MemcpyDeviceToDeviceArray"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD: - return "MemcpyDeviceToDevice"; - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH: - return "MemcpyHostToHost"; - default: - break; - } - return ""; -} +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -void CUPTIAPI CudaProfiler::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { - uint8_t* bfr = (uint8_t*)malloc(BUF_SIZE + ALIGN_SIZE); - *size = BUF_SIZE; - *buffer = ALIGN_BUFFER(bfr, ALIGN_SIZE); - *maxNumRecords = 0; +CudaProfiler::CudaProfiler() { + auto& manager = CUPTIManager::GetInstance(); + client_handle_ = manager.RegisterClient(); } -void CUPTIAPI CudaProfiler::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t validSize) { - CUptiResult status; - CUpti_Activity* record = NULL; - if (validSize > 0) { - do { - status = cuptiActivityGetNextRecord(buffer, validSize, &record); - if (status == CUPTI_SUCCESS) { - if (CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL == record->kind) { - CUpti_ActivityKernel3* kernel = (CUpti_ActivityKernel3*)record; - stats.push_back({kernel->name, kernel->streamId, - kernel->gridX, kernel->gridY, kernel->gridZ, - kernel->blockX, kernel->blockY, kernel->blockZ, - static_cast(kernel->start), - static_cast(kernel->end), - kernel->correlationId}); - } else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) { - CUpti_ActivityMemcpy3* mmcpy = (CUpti_ActivityMemcpy3*)record; - stats.push_back({GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind), - mmcpy->streamId, -1, -1, -1, -1, -1, -1, - static_cast(mmcpy->start), - static_cast(mmcpy->end), - mmcpy->correlationId}); - } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { - auto correlation = reinterpret_cast(record); - id_map.insert({correlation->correlationId, correlation->externalId}); - } - } else if (status == CUPTI_ERROR_MAX_LIMIT_REACHED) { - break; - } - } while (1); - } - free(buffer); +CudaProfiler::~CudaProfiler() { + auto& manager = CUPTIManager::GetInstance(); + manager.DeregisterClient(client_handle_); } -bool CudaProfiler::StartProfiling(TimePoint /* profiling_start_time */) { - if (!enabled.test_and_set()) { - if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS && - cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { - initialized_ = true; - return true; - } else { - DisableEvents(); - enabled.clear(); - return false; - } - } - return false; +bool CudaProfiler::StartProfiling(TimePoint profiling_start_time) { + auto& manager = CUPTIManager::GetInstance(); + manager.StartLogging(); + profiling_start_time_ = profiling_start_time; + return true; } void CudaProfiler::EndProfiling(TimePoint start_time, Events& events) { - std::map> event_map; - if (initialized_) { - DisableEvents(); - cuptiActivityFlushAll(1); - int64_t profiling_start = std::chrono::duration_cast(start_time.time_since_epoch()).count(); - for (const auto& stat : stats) { - std::initializer_list> args = {{"op_name", ""}, - {"stream", std::to_string(stat.stream_)}, - {"grid_x", std::to_string(stat.grid_x_)}, - {"grid_y", std::to_string(stat.grid_y_)}, - {"grid_z", std::to_string(stat.grid_z_)}, - {"block_x", std::to_string(stat.block_x_)}, - {"block_y", std::to_string(stat.block_y_)}, - {"block_z", std::to_string(stat.block_z_)}}; - EventRecord event{ - KEVENT, -1, -1, demangle(stat.name_), DUR(profiling_start, stat.start_), DUR(stat.start_, stat.stop_), {args.begin(), args.end()}}; - auto ts = id_map[stat.correlation_id]; - if (event_map.find(ts) == event_map.end()) { - event_map.insert({ts, {event}}); - } else { - event_map[ts].push_back(std::move(event)); - } - } - auto insert_iter = events.begin(); - for (auto& map_iter : event_map) { - auto ts = static_cast(map_iter.first); - while (insert_iter != events.end() && insert_iter->ts < ts) { - insert_iter++; - } - if (insert_iter != events.end() && insert_iter->ts == ts) { - for (auto& evt_iter : map_iter.second) { - evt_iter.args["op_name"] = insert_iter->args["op_name"]; - } - insert_iter = events.insert(insert_iter + 1, map_iter.second.begin(), map_iter.second.end()); - } else { - insert_iter = events.insert(insert_iter, map_iter.second.begin(), map_iter.second.end()); - } - while (insert_iter != events.end() && insert_iter->cat == EventCategory::KERNEL_EVENT) { - insert_iter++; - } - } - cuptiFinalize(); - Clear(); - } //if initialized -} - -CudaProfiler::~CudaProfiler() { - if (initialized_) { - DisableEvents(); - cuptiFinalize(); - Clear(); - } + auto& manager = CUPTIManager::GetInstance(); + std::map event_map; + manager.Consume(client_handle_, start_time, event_map); + MergeEvents(event_map, events); } void CudaProfiler::Start(uint64_t id) { - if (initialized_) { - cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, id); - } + auto& manager = CUPTIManager::GetInstance(); + manager.PushCorrelation(client_handle_, id, profiling_start_time_); } void CudaProfiler::Stop(uint64_t) { - if (initialized_) { - uint64_t last_id{0}; - cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &last_id); - } -} - -void CudaProfiler::DisableEvents() { - cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION); - cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL); - cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY); - cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DRIVER); - cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME); -} - -void CudaProfiler::Clear() { - if (initialized_) { - id_map.clear(); - stats.clear(); - initialized_ = false; - enabled.clear(); - } + auto& manager = CUPTIManager::GetInstance(); + manager.PopCorrelation(); } #else // for cuda 10.x, no profiling -void CUPTIAPI CudaProfiler::BufferRequested(uint8_t**, size_t*, size_t*) {} -void CUPTIAPI CudaProfiler::BufferCompleted(CUcontext, uint32_t, uint8_t*, size_t, size_t) {} -bool CudaProfiler::StartProfiling() { return false; } +bool CudaProfiler::StartProfiling(TimePoint) { return false; } void CudaProfiler::EndProfiling(TimePoint, Events&) {} CudaProfiler::~CudaProfiler() {} void CudaProfiler::Start(uint64_t) {} void CudaProfiler::Stop(uint64_t) {} -void CudaProfiler::DisableEvents() {} -void CudaProfiler::Clear() {} #endif diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index d98d53d9bc17b..d09c79dd0d65e 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -1,64 +1,31 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/common/profiler_common.h" - #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) - -#include "core/platform/ort_mutex.h" -#include #include #include #include +#include "core/common/profiler_common.h" + namespace onnxruntime { namespace profiling { using Events = std::vector; -class CudaProfiler final : public EpProfiler { +class CudaProfiler final : public GPUProfilerBase { public: - CudaProfiler() = default; - CudaProfiler(const CudaProfiler&) = delete; - CudaProfiler& operator=(const CudaProfiler&) = delete; - CudaProfiler(CudaProfiler&& cuda_profiler) noexcept { - initialized_ = cuda_profiler.initialized_; - cuda_profiler.initialized_ = false; - } - CudaProfiler& operator=(CudaProfiler&& cuda_profiler) noexcept { - initialized_ = cuda_profiler.initialized_; - cuda_profiler.initialized_ = false; - return *this; - } + CudaProfiler(); + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudaProfiler); ~CudaProfiler(); bool StartProfiling(TimePoint profiling_start_time) override; void EndProfiling(TimePoint start_time, Events& events) override; void Start(uint64_t) override; void Stop(uint64_t) override; - private: - static void CUPTIAPI BufferRequested(uint8_t**, size_t*, size_t*); - static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t*, size_t, size_t); - struct KernelStat { - std::string name_ = {}; - uint32_t stream_ = 0; - int32_t grid_x_ = 0; - int32_t grid_y_ = 0; - int32_t grid_z_ = 0; - int32_t block_x_ = 0; - int32_t block_y_ = 0; - int32_t block_z_ = 0; - int64_t start_ = 0; - int64_t stop_ = 0; - uint32_t correlation_id = 0; - }; - static std::atomic_flag enabled; - static std::vector stats; - static std::unordered_map id_map; - - void DisableEvents(); - void Clear(); - bool initialized_ = false; +private: + uint64_t client_handle_ = 0; + TimePoint profiling_start_time_{}; }; } // namespace profiling diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index 6bea2d2a7054a..54177079fccaf 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -3,7 +3,7 @@ namespace onnxruntime { namespace profiling { -static inline const char* GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { +static inline std::string GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { switch (kind) { case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: return "MemcpyHostToDevice"; @@ -68,11 +68,13 @@ void CUPTIManager::StopLogging() { } bool CUPTIManager::PushUniqueCorrelation(uint64_t unique_cid) { - return cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, unique_cid) == CUPTI_SUCCESS; + auto res = cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, unique_cid); + return res == CUPTI_SUCCESS; } void CUPTIManager::PopUniqueCorrelation(uint64_t& popped_unique_cid) { - if (cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &popped_unique_cid) != CUPTI_SUCCESS) { + auto res = cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &popped_unique_cid); + if (res != CUPTI_SUCCESS) { popped_unique_cid = 0; } } @@ -139,8 +141,9 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector /* name = */ std::move(name), /* ts = */ (int64_t)(mmcpy->start - start_time_ns) / 1000, /* dur = */ (int64_t)(mmcpy->end - mmcpy->start) / 1000, - /* args = */ std::move(args)}; - MapEventToClient(mmcpy->correlationId, std::move(event)); + /* args = */ std::move(args) + }; + MapEventToClient(mmcpy->correlationId, std::move(event)); } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { auto correlation = reinterpret_cast(record); NotifyOnCorrelation(correlation->correlationId, correlation->externalId); diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 36a5f58c500d3..7f46ca8c0a1ec 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -1,6 +1,5 @@ #pragma once - #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) #include @@ -9,9 +8,9 @@ #include -#include "core/platform/ort_mutex.h" -#include "core/common/profiler_common.h" #include "core/common/inlined_containers.h" +#include "core/common/profiler_common.h" + namespace onnxruntime { namespace profiling { diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.cc b/onnxruntime/core/providers/rocm/rocm_profiler.cc index 6fb0221a378c9..ae81d7b81fbd5 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.cc +++ b/onnxruntime/core/providers/rocm/rocm_profiler.cc @@ -33,45 +33,7 @@ void RocmProfiler::EndProfiling(TimePoint start_time, Events& events) { auto& manager = RoctracerManager::GetInstance(); std::map event_map; manager.Consume(client_handle_, start_time, event_map); - - Events merged_events; - - auto event_iter = std::make_move_iterator(events.begin()); - auto event_end = std::make_move_iterator(events.end()); - for (auto& map_iter : event_map) { - auto ts = static_cast(map_iter.first); - while (event_iter != event_end && event_iter->ts < ts) { - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - // find the last event with the same timestamp. - while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { - ++event_iter; - } - - if (event_iter != event_end && event_iter->ts == ts) { - uint64_t increment = 1; - for (auto& evt : map_iter.second) { - evt.args["op_name"] = event_iter->args["op_name"]; - - // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. - // So, we adjust the timestamp here to something sensible. - evt.ts = event_iter->ts + increment; - ++increment; - } - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - merged_events.insert(merged_events.end(), - std::make_move_iterator(map_iter.second.begin()), - std::make_move_iterator(map_iter.second.end())); - } - - // move any remaining events - merged_events.insert(merged_events.end(), event_iter, event_end); - std::swap(events, merged_events); + MergeEvents(event_map, events); } void RocmProfiler::Start(uint64_t id) { diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.h b/onnxruntime/core/providers/rocm/rocm_profiler.h index 40516a184c795..44cf2d348836f 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.h +++ b/onnxruntime/core/providers/rocm/rocm_profiler.h @@ -12,11 +12,10 @@ namespace profiling { using Events = std::vector; -class RocmProfiler final : public EpProfiler { +class RocmProfiler final : public GPUProfilerBase { public: RocmProfiler(); - RocmProfiler(const RocmProfiler&) = delete; - RocmProfiler& operator=(const RocmProfiler&) = delete; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RocmProfiler); ~RocmProfiler(); bool StartProfiling(TimePoint profiling_start_time) override; void EndProfiling(TimePoint start_time, Events& events) override; From b62edc24d29c14cd38e836405cf400cb0255dd47 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Wed, 16 Nov 2022 22:10:40 +0000 Subject: [PATCH 05/26] Refactor code common to GPU profilers into separate CUs --- .../core/common/gpu_profiler_common.h | 121 ++++++ .../onnxruntime/core/common/profiler_common.h | 345 +----------------- .../core/common/gpu_profiler_common.cc | 303 +++++++++++++++ onnxruntime/core/common/profiler.cc | 2 +- .../core/providers/cuda/cuda_profiler.cc | 4 +- .../core/providers/cuda/cuda_profiler.h | 3 +- .../core/providers/cuda/cupti_manager.cc | 23 +- .../core/providers/cuda/cupti_manager.h | 13 +- .../core/providers/rocm/rocm_profiler.cc | 1 - .../core/providers/rocm/rocm_profiler.h | 2 +- .../core/providers/rocm/roctracer_manager.cc | 21 +- .../core/providers/rocm/roctracer_manager.h | 7 +- .../providers/shared_library/provider_api.h | 6 +- 13 files changed, 459 insertions(+), 392 deletions(-) create mode 100644 include/onnxruntime/core/common/gpu_profiler_common.h create mode 100644 onnxruntime/core/common/gpu_profiler_common.cc diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h new file mode 100644 index 0000000000000..1a2fa36603d5c --- /dev/null +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -0,0 +1,121 @@ +#pragma once + +#include "core/common/profiler_common.h" +#include "core/common/inlined_containers.h" + +#include +#include +#include +#include +#include +#include + + +namespace onnxruntime { +namespace profiling { + +class ProfilerActivityBuffer { + public: + ProfilerActivityBuffer(); + ProfilerActivityBuffer(const char* data, size_t size); + ProfilerActivityBuffer(const ProfilerActivityBuffer& other); + ProfilerActivityBuffer(ProfilerActivityBuffer&& other); + ProfilerActivityBuffer& operator=(const ProfilerActivityBuffer& other); + ProfilerActivityBuffer& operator=(ProfilerActivityBuffer&& other); + + // accessors + char* GetData() { return data_.get(); } + const char* GetData() const { return data_.get(); } + size_t GetSize() const { return size_; } + + static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size); + + private: + std::unique_ptr data_; + size_t size_; +}; /* end class ProfilerActivityBuffer */ + +class GPUTracerManager +{ +public: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); + virtual ~GPUTracerManager() {} + + virtual uint64_t RegisterClient(); + virtual void DeregisterClient(uint64_t client_handle); + + void StartLogging(); + void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); + bool PushCorrelation(uint64_t client_handle, + uint64_t external_correlation_id, + TimePoint profiling_start_time); + void PopCorrelation(uint64_t& popped_external_correlation_id); + void PopCorrelation(); + +protected: + GPUTracerManager() = default; + + // Functional API to be implemented by subclasses + virtual bool OnStartLogging() = 0; + virtual void OnStopLogging() = 0; + virtual void ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time) = 0; + virtual bool PushUniqueCorrelation(uint64_t unique_cid) = 0; + virtual void PopUniqueCorrelation(uint64_t& popped_unique_cid) = 0; + virtual void FlushActivities() = 0; + + // Service API for subclasses + void EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer); + // To be called by subclasses only from ProcessActivityBuffers + void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event); + // To be called by subclasses only from ProcessActivityBuffers + void NotifyNewCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id); + +private: + void StopLogging(); + void Clear(); + Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id); + void MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events); + void DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id); + + std::mutex manager_instance_mutex_; + uint64_t next_client_id_ = 1; + uint64_t num_active_clients_ = 0; + bool logging_enabled_ = false; + std::mutex unprocessed_activity_buffers_mutex_; + std::mutex activity_buffer_processor_mutex_; + + // Unprocessed activity buffers + std::vector unprocessed_activity_buffers_; + + // Keyed on unique_correlation_id -> (client_id/client_handle, offset) + // unique_correlation_id - offset == external_correlation_id + InlinedHashMap> unique_correlation_id_to_client_offset_; + + // Keyed on tracer_correlation_id -> unique_correlation_id + InlinedHashMap tracer_correlation_to_unique_correlation_; + + // client_id/client_handle -> external_correlation_id -> events + InlinedHashMap> per_client_events_by_ext_correlation_; + + // Keyed on tracer correlation_id, keeps track of activity records + // for which we haven't established the external_correlation_id yet. + InlinedHashMap> events_pending_client_mapping_; +}; /* class GPUTracerManager */ + +// Base class for a GPU profiler +class GPUProfilerBase : public EpProfiler { +protected: + GPUProfilerBase() = default; + void MergeEvents(std::map& events_to_merge, Events& events); +}; /* class GPUProfilerBase */ + +// Convert a pointer to a hex string +static inline std::string PointerToHexString(const void* ptr) { + std::ostringstream sstr; + sstr << std::hex << ptr; + return sstr.str(); +} + +} /* end namespace profiling */ +} /* end namespace onnxruntime */ diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index c774ae7f57173..7c615f7d4a613 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -3,72 +3,14 @@ #pragma once -#include +#include "core/common/common.h" + #include #include -#include "core/common/common.h" -#include "core/common/inlined_containers.h" - namespace onnxruntime { namespace profiling { -class ProfilerActivityBuffer { - public: - ProfilerActivityBuffer() - : data_(nullptr), size_(0) {} - - ProfilerActivityBuffer(const char* data, size_t size) - : data_(std::make_unique(size)), size_(size) { - memcpy(data_.get(), data, size); - } - - ProfilerActivityBuffer(const ProfilerActivityBuffer& other) - : ProfilerActivityBuffer(other.data_.get(), other.size_) {} - - ProfilerActivityBuffer(ProfilerActivityBuffer&& other) - : ProfilerActivityBuffer() { - std::swap(data_, other.data_); - std::swap(size_, other.size_); - } - - ProfilerActivityBuffer& operator=(const ProfilerActivityBuffer& other) { - if (&other == this) { - return *this; - } - - size_ = other.size_; - data_ = std::make_unique(other.size_); - memcpy(data_.get(), other.data_.get(), size_); - return *this; - } - - ProfilerActivityBuffer& operator=(ProfilerActivityBuffer&& other) { - if (&other == this) { - return *this; - } - std::swap(data_, other.data_); - std::swap(size_, other.size_); - return *this; - } - - // accessors - char* GetData() { return data_.get(); } - const char* GetData() const { return data_.get(); } - size_t GetSize() const { return size_; } - - static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size) { - ProfilerActivityBuffer res{}; - res.data_.reset(data); - res.size_ = size; - return res; - } - - private: - std::unique_ptr data_; - size_t size_; -}; - enum EventCategory { SESSION_EVENT = 0, NODE_EVENT, @@ -78,7 +20,7 @@ enum EventCategory { }; // Event descriptions for the above session events. -static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { +static constexpr const char* event_category_names_[EVENT_CATEGORY_MAX] = { "Session", "Node", "Kernel", @@ -134,233 +76,6 @@ struct EventRecord { using Events = std::vector; -class GPUTracerManager -{ -public: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); - virtual ~GPUTracerManager() {} - - virtual uint64_t RegisterClient() { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto res = next_client_id_++; - per_client_events_by_ext_correlation_.insert({res, {}}); - ++num_active_clients_; - return res; - } - return 0; - } - - virtual void DeregisterClient(uint64_t client_handle) { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - per_client_events_by_ext_correlation_.erase(it); - --num_active_clients_; - if (num_active_clients_ == 0) { - StopLogging(); - } - } - } - - virtual void StartLogging() = 0; - virtual void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { - events.clear(); - { - // Flush any pending activity records before starting - // to process the accumulated activity records. - std::lock_guard lock_manager(manager_instance_mutex_); - FlushActivities(); - } - - std::vector activity_buffers; - { - std::lock_guard lock(unprocessed_activity_buffers_mutex_); - std::swap(unprocessed_activity_buffers_, activity_buffers); - unprocessed_activity_buffers_.clear(); - } - - { - // Ensure that at most one thread is working through the activity buffers at any time. - std::lock_guard lock_two(activity_buffer_processor_mutex_); - ProcessActivityBuffers(activity_buffers, start_time); - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - std::swap(events, it->second); - } - } - - virtual bool PushCorrelation(uint64_t client_handle, - uint64_t external_correlation_id, - TimePoint profiling_start_time) { - std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return false; - } - - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - // not a registered client, do nothing - return false; - } - - // external_correlation_id is simply the timestamp of this event, - // relative to profiling_start_time. i.e., it was computed as: - // external_correlation_id = - // std::chrono::duration_cast(event_start_time - profiling_start_time).count() - // - // Because of the relative nature of the external_correlation_id, the same - // external_correlation_id can be reused across different clients, which then makes it - // impossible to recover the client from the external_correlation_id, which in turn - // makes it impossible to map events (which are tagged with external_correlation_id) to clients. - // - // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) - // as follows: - // unique_cid = - // external_correlation_id + - // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() - // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. - // - // Of course, clients expect lists of events to be returned (on a call to Consume()), that are - // still keyed on the external_correlation_id that they've specified here, so we need to remember the - // offset to be subtracted - uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); - auto unique_cid = external_correlation_id + offset; - unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); - return PushUniqueCorrelation(unique_cid); - } - - virtual void PopCorrelation(uint64_t& popped_external_correlation_id) { - std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return; - } - uint64_t unique_cid; - PopUniqueCorrelation(unique_cid); - // lookup the offset and subtract it before returning popped_external_correlation_id to the client - auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - popped_external_correlation_id = 0; - return; - } - popped_external_correlation_id = unique_cid - client_it->second.second; - } - - void PopCorrelation() { - uint64_t unused; - PopCorrelation(unused); - } - -protected: - GPUTracerManager() {} - - void EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer) { - std::lock_guard lock(unprocessed_activity_buffers_mutex_); - unprocessed_activity_buffers_.emplace_back(std::move(buffer)); - } - - // Requires: manager_instance_mutex_ must be held - virtual void Clear() { - unprocessed_activity_buffers_.clear(); - unique_correlation_id_to_client_offset_.clear(); - per_client_events_by_ext_correlation_.clear(); - tracer_correlation_to_unique_correlation_.clear(); - } - - virtual void StopLogging() = 0; - virtual void ProcessActivityBuffers(const std::vector& buffers, - const TimePoint& start_time) = 0; - - virtual bool PushUniqueCorrelation(uint64_t unique_cid) = 0; - virtual void PopUniqueCorrelation(uint64_t& popped_unique_cid) = 0; - virtual void FlushActivities() = 0; - - Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { - auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - return nullptr; - } - - // See the comments on the GetUniqueCorrelationId method for an explanation of - // of this offset computation and why it's required. - auto const& client_handle_offset = client_it->second; - auto external_correlation = unique_correlation_id - client_handle_offset.second; - - auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; - return &event_list; - } - - // Not thread-safe: subclasses must ensure mutual-exclusion when calling this method - void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) - { - auto it = tracer_correlation_to_unique_correlation_.find(tracer_correlation_id); - if (it == tracer_correlation_to_unique_correlation_.end()) { - // We're yet to receive a mapping to unique_correlation_id for this tracer_correlation_id - DeferEventMapping(std::move(event), tracer_correlation_id); - return; - } - auto unique_correlation_id = it->second; - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->emplace_back(std::move(event)); - } - } - - // Not thread-safe: subclasses must ensure mutual-exclusion when calling this method - void MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->insert(p_event_list->end(), - std::make_move_iterator(events.begin()), - std::make_move_iterator(events.end())); - } - } - - void DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id) { - events_pending_client_mapping_[tracer_correlation_id].emplace_back(std::move(event)); - } - - void NotifyOnCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id) { - tracer_correlation_to_unique_correlation_[tracer_correlation_id] = unique_correlation_id; - auto pending_it = events_pending_client_mapping_.find(tracer_correlation_id); - if (pending_it == events_pending_client_mapping_.end()) { - return; - } - // Map the pending events to the right client - MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); - events_pending_client_mapping_.erase(pending_it); - } - - std::mutex manager_instance_mutex_; - uint64_t next_client_id_ = 1; - uint64_t num_active_clients_ = 0; - bool logging_enabled_ = false; - std::mutex unprocessed_activity_buffers_mutex_; - std::mutex activity_buffer_processor_mutex_; - - // Unprocessed activity buffers - std::vector unprocessed_activity_buffers_; - - // Keyed on unique_correlation_id -> (client_id/client_handle, offset) - // unique_correlation_id - offset == external_correlation_id - InlinedHashMap> unique_correlation_id_to_client_offset_; - - // Keyed on tracer_correlation_id -> unique_correlation_id - InlinedHashMap tracer_correlation_to_unique_correlation_; - - // client_id/client_handle -> external_correlation_id -> events - InlinedHashMap> per_client_events_by_ext_correlation_; - - // Keyed on tracer correlation_id, keeps track of activity records - // for which we haven't established the external_correlation_id yet. - InlinedHashMap> events_pending_client_mapping_; -}; /* class GPUTracerManager */ - //Execution Provider Profiler class EpProfiler { public: @@ -371,63 +86,9 @@ class EpProfiler { virtual void Stop(uint64_t){}; // called after op stop, accept an id as argument to identify the op }; -// Base class for a GPU profiler -class GPUProfilerBase : public EpProfiler { -protected: - GPUProfilerBase() = default; - - void MergeEvents(std::map& events_to_merge, Events& events) { - Events merged_events; - - auto event_iter = std::make_move_iterator(events.begin()); - auto event_end = std::make_move_iterator(events.end()); - for (auto& map_iter : events_to_merge) { - auto ts = static_cast(map_iter.first); - while (event_iter != event_end && event_iter->ts < ts) { - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - // find the last event with the same timestamp. - while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { - ++event_iter; - } - - if (event_iter != event_end && event_iter->ts == ts) { - uint64_t increment = 1; - for (auto& evt : map_iter.second) { - evt.args["op_name"] = event_iter->args["op_name"]; - - // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. - // So, we adjust the timestamp here to something sensible. - evt.ts = event_iter->ts + increment; - ++increment; - } - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - merged_events.insert(merged_events.end(), - std::make_move_iterator(map_iter.second.begin()), - std::make_move_iterator(map_iter.second.end())); - } - - // move any remaining events - merged_events.insert(merged_events.end(), event_iter, event_end); - std::swap(events, merged_events); - } -}; - // Demangle C++ symbols std::string demangle(const char* name); std::string demangle(const std::string& name); -// Convert a pointer to a hex string -static inline std::string PointerToHexString(const void* ptr) { - std::ostringstream sstr; - sstr << std::hex << ptr; - return sstr.str(); -} - } // namespace profiling } // namespace onnxruntime diff --git a/onnxruntime/core/common/gpu_profiler_common.cc b/onnxruntime/core/common/gpu_profiler_common.cc new file mode 100644 index 0000000000000..37f37798c1e5e --- /dev/null +++ b/onnxruntime/core/common/gpu_profiler_common.cc @@ -0,0 +1,303 @@ +#include "core/common/gpu_profiler_common.h" + +namespace onnxruntime { +namespace profiling { + +// Implementation of ProfilerActivityBuffer +ProfilerActivityBuffer::ProfilerActivityBuffer() + : data_(nullptr), size_(0) {} + +ProfilerActivityBuffer::ProfilerActivityBuffer(const char* data, size_t size) + : data_(std::make_unique(size)), size_(size) { + memcpy(data_.get(), data, size); +} + +ProfilerActivityBuffer::ProfilerActivityBuffer(const ProfilerActivityBuffer& other) + : ProfilerActivityBuffer(other.data_.get(), other.size_) {} + +ProfilerActivityBuffer::ProfilerActivityBuffer(ProfilerActivityBuffer&& other) + : ProfilerActivityBuffer() { + std::swap(data_, other.data_); + std::swap(size_, other.size_); +} + +ProfilerActivityBuffer& ProfilerActivityBuffer::operator=(const ProfilerActivityBuffer& other) { + if (&other == this) { + return *this; + } + + size_ = other.size_; + data_ = std::make_unique(other.size_); + memcpy(data_.get(), other.data_.get(), size_); + return *this; +} + +ProfilerActivityBuffer& ProfilerActivityBuffer::operator=(ProfilerActivityBuffer&& other) { + if (&other == this) { + return *this; + } + std::swap(data_, other.data_); + std::swap(size_, other.size_); + return *this; +} + +ProfilerActivityBuffer ProfilerActivityBuffer::CreateFromPreallocatedBuffer(char* data, size_t size) { + ProfilerActivityBuffer res{}; + res.data_.reset(data); + res.size_ = size; + return res; +} + + +// Implementation of GPUTracerManager +uint64_t GPUTracerManager::RegisterClient() { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto res = next_client_id_++; + per_client_events_by_ext_correlation_.insert({res, {}}); + ++num_active_clients_; + return res; + } + return 0; +} + +void GPUTracerManager::DeregisterClient(uint64_t client_handle) { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + per_client_events_by_ext_correlation_.erase(it); + --num_active_clients_; + if (num_active_clients_ == 0) { + StopLogging(); + } + } +} + +void GPUTracerManager::StartLogging() { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + return; + } + + logging_enabled_ = OnStartLogging(); +} + +void GPUTracerManager::StopLogging() { + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + OnStopLogging(); + logging_enabled_ = false; + Clear(); +} + +void GPUTracerManager::Consume(uint64_t client_handle, + const TimePoint& start_time, + std::map& events) { + events.clear(); + { + // Flush any pending activity records before starting + // to process the accumulated activity records. + std::lock_guard lock_manager(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + + FlushActivities(); + } + + std::vector activity_buffers; + { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + std::swap(unprocessed_activity_buffers_, activity_buffers); + unprocessed_activity_buffers_.clear(); + } + + { + // Ensure that at most one thread is working through the activity buffers at any time. + std::lock_guard lock_two(activity_buffer_processor_mutex_); + ProcessActivityBuffers(activity_buffers, start_time); + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + std::swap(events, it->second); + } +} + +bool GPUTracerManager::PushCorrelation(uint64_t client_handle, + uint64_t external_correlation_id, + TimePoint profiling_start_time) { + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return false; + } + + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + // not a registered client, do nothing + return false; + } + + // external_correlation_id is simply the timestamp of this event, + // relative to profiling_start_time. i.e., it was computed as: + // external_correlation_id = + // std::chrono::duration_cast(event_start_time - profiling_start_time).count() + // + // Because of the relative nature of the external_correlation_id, the same + // external_correlation_id can be reused across different clients, which then makes it + // impossible to recover the client from the external_correlation_id, which in turn + // makes it impossible to map events (which are tagged with external_correlation_id) to clients. + // + // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) + // as follows: + // unique_cid = + // external_correlation_id + + // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() + // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. + // + // Of course, clients expect lists of events to be returned (on a call to Consume()), that are + // still keyed on the external_correlation_id that they've specified here, so we need to remember the + // offset to be subtracted + uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); + auto unique_cid = external_correlation_id + offset; + unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); + return PushUniqueCorrelation(unique_cid); +} + +void GPUTracerManager::PopCorrelation(uint64_t& popped_external_correlation_id) { + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + uint64_t unique_cid; + PopUniqueCorrelation(unique_cid); + // lookup the offset and subtract it before returning popped_external_correlation_id to the client + auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + popped_external_correlation_id = 0; + return; + } + popped_external_correlation_id = unique_cid - client_it->second.second; +} + +void GPUTracerManager::PopCorrelation() { + uint64_t unused; + PopCorrelation(unused); +} + +void GPUTracerManager::EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer) { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + unprocessed_activity_buffers_.emplace_back(std::move(buffer)); +} + +void GPUTracerManager::Clear() { + unprocessed_activity_buffers_.clear(); + unique_correlation_id_to_client_offset_.clear(); + per_client_events_by_ext_correlation_.clear(); + tracer_correlation_to_unique_correlation_.clear(); + events_pending_client_mapping_.clear(); +} + +Events* GPUTracerManager::GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { + auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + return nullptr; + } + + // See the comments on the GetUniqueCorrelationId method for an explanation of + // of this offset computation and why it's required. + auto const& client_handle_offset = client_it->second; + auto external_correlation = unique_correlation_id - client_handle_offset.second; + + auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; + return &event_list; +} + +void GPUTracerManager::MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) { + auto it = tracer_correlation_to_unique_correlation_.find(tracer_correlation_id); + if (it == tracer_correlation_to_unique_correlation_.end()) { + // We're yet to receive a mapping to unique_correlation_id for this tracer_correlation_id + DeferEventMapping(std::move(event), tracer_correlation_id); + return; + } + auto unique_correlation_id = it->second; + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->emplace_back(std::move(event)); + } +} + +void GPUTracerManager::MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->insert(p_event_list->end(), + std::make_move_iterator(events.begin()), + std::make_move_iterator(events.end())); + } +} + +void GPUTracerManager::DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id) { + events_pending_client_mapping_[tracer_correlation_id].emplace_back(std::move(event)); +} + +void GPUTracerManager::NotifyNewCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id) { + tracer_correlation_to_unique_correlation_[tracer_correlation_id] = unique_correlation_id; + auto pending_it = events_pending_client_mapping_.find(tracer_correlation_id); + if (pending_it == events_pending_client_mapping_.end()) { + return; + } + // Map the pending events to the right client + MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); + events_pending_client_mapping_.erase(pending_it); +} + + +// Implementation of GPUProfileBase +void GPUProfilerBase::MergeEvents(std::map& events_to_merge, Events& events) { + Events merged_events; + + auto event_iter = std::make_move_iterator(events.begin()); + auto event_end = std::make_move_iterator(events.end()); + for (auto& map_iter : events_to_merge) { + auto ts = static_cast(map_iter.first); + while (event_iter != event_end && event_iter->ts < ts) { + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + // find the last event with the same timestamp. + while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { + ++event_iter; + } + + if (event_iter != event_end && event_iter->ts == ts) { + uint64_t increment = 1; + for (auto& evt : map_iter.second) { + evt.args["op_name"] = event_iter->args["op_name"]; + + // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. + // So, we adjust the timestamp here to something sensible. + evt.ts = event_iter->ts + increment; + ++increment; + } + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + merged_events.insert(merged_events.end(), + std::make_move_iterator(map_iter.second.begin()), + std::make_move_iterator(map_iter.second.end())); + } + + // move any remaining events + merged_events.insert(merged_events.end(), event_iter, event_end); + std::swap(events, merged_events); +} + +} /* end namespace profiling */ +} /* end namespace onnxruntime */ diff --git a/onnxruntime/core/common/profiler.cc b/onnxruntime/core/common/profiler.cc index 89dc5a667a8fb..413b34fda5561 100644 --- a/onnxruntime/core/common/profiler.cc +++ b/onnxruntime/core/common/profiler.cc @@ -124,7 +124,7 @@ std::string Profiler::EndProfiling() { for (size_t i = 0; i < events_.size(); ++i) { auto& rec = events_[i]; - profile_stream_ << R"({"cat" : ")" << event_categor_names_[rec.cat] << "\","; + profile_stream_ << R"({"cat" : ")" << event_category_names_[rec.cat] << "\","; profile_stream_ << "\"pid\" :" << rec.pid << ","; profile_stream_ << "\"tid\" :" << rec.tid << ","; profile_stream_ << "\"dur\" :" << rec.dur << ","; diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 81849cea4f701..47bd2b23c7e4a 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -2,13 +2,13 @@ // Licensed under the MIT License. #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) -#include "cuda_profiler.h" #include #include #include -#include "core/common/profiler_common.h" #include "cupti_manager.h" +#include "cuda_profiler.h" + namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index d09c79dd0d65e..50f1e619afcc1 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -5,10 +5,9 @@ #include #include -#include "core/common/profiler_common.h" +#include "core/common/gpu_profiler_common.h" namespace onnxruntime { - namespace profiling { using Events = std::vector; diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index 54177079fccaf..632a73c6c3b8b 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -34,37 +34,28 @@ CUPTIManager& CUPTIManager::GetInstance() { return instance; } -CUPTIManager::~CUPTIManager() { - StopLogging(); - Clear(); -} +CUPTIManager::~CUPTIManager() {} -void CUPTIManager::StartLogging() { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - return; - } +bool CUPTIManager::OnStartLogging() { if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS && cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS && cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) == CUPTI_SUCCESS && cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS && cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS && cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { - logging_enabled_ = true; + return true; } else { - StopLogging(); - logging_enabled_ = false; + OnStopLogging(); + return false; } } -void CUPTIManager::StopLogging() { - std::lock_guard lock(manager_instance_mutex_); +void CUPTIManager::OnStopLogging() { cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DRIVER); cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME); - logging_enabled_ = false; } bool CUPTIManager::PushUniqueCorrelation(uint64_t unique_cid) { @@ -146,7 +137,7 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector MapEventToClient(mmcpy->correlationId, std::move(event)); } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { auto correlation = reinterpret_cast(record); - NotifyOnCorrelation(correlation->correlationId, correlation->externalId); + NotifyNewCorrelation(correlation->correlationId, correlation->externalId); } } } while (status == CUPTI_SUCCESS); diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 7f46ca8c0a1ec..0b57397415ad0 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -8,8 +8,9 @@ #include +#include "core/common/gpu_profiler_common.h" #include "core/common/inlined_containers.h" -#include "core/common/profiler_common.h" + namespace onnxruntime { @@ -23,12 +24,12 @@ class CUPTIManager : public GPUTracerManager ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); ~CUPTIManager(); static CUPTIManager& GetInstance(); - void StartLogging() override; protected: bool PushUniqueCorrelation(uint64_t unique_cid) override; void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; - void StopLogging() override; + bool OnStartLogging() override; + void OnStopLogging() override; void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) override; void FlushActivities() override; @@ -37,6 +38,12 @@ class CUPTIManager : public GPUTracerManager static constexpr size_t kActivityBufferSize = 32 * 1024; static constexpr size_t kActivityBufferAlignSize = 8; + // TODO: Is this even needed? malloc() is required to return + // a memory block that meets the alignment requirements for _any_ data type. + // On any platform that supports an 8-byte datatype (double? long long?) + // this means that malloc() already returns memory aligned at + // _at least_ 8 byte boundaries, rendering this additional alignment + // redundant? static constexpr uint8_t* AlignBuffer(uint8_t* buffer, int align) { return (((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.cc b/onnxruntime/core/providers/rocm/rocm_profiler.cc index ae81d7b81fbd5..7ec591eadfa63 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.cc +++ b/onnxruntime/core/providers/rocm/rocm_profiler.cc @@ -5,7 +5,6 @@ #include #include -#include "core/common/profiler_common.h" #include "core/providers/rocm/rocm_profiler.h" #include "core/providers/rocm/roctracer_manager.h" diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.h b/onnxruntime/core/providers/rocm/rocm_profiler.h index 44cf2d348836f..37fe9b53a2105 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.h +++ b/onnxruntime/core/providers/rocm/rocm_profiler.h @@ -3,7 +3,7 @@ #include #include -#include "core/common/profiler_common.h" +#include "core/common/gpu_profiler_common.h" #if defined(USE_ROCM) && defined(ENABLE_ROCM_PROFILING) diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index b6da61fa0cfd5..4277e5026239e 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -32,12 +32,7 @@ RoctracerManager::~RoctracerManager() { StopLogging(); } -void RoctracerManager::StartLogging() { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - return; - } - +void RoctracerManager::OnStartLogging() { // The following line shows up in all the samples, I do not know // what the point is, but without it, the roctracer APIs don't work. roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); @@ -66,25 +61,13 @@ void RoctracerManager::StartLogging() { logging_enabled_ = true; } -// Requires: manager_instance_mutex_ must be held void RoctracerManager::StopLogging() { - if (!logging_enabled_) { - return; - } - roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_API); roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS); roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); roctracer_stop(); roctracer_flush_activity(); roctracer_close_pool(); - - logging_enabled_ = false; - Clear(); -} - -RoctracerManager::Clear() { - GPUTracerManager::Clear(); api_call_args_.clear(); } @@ -259,7 +242,7 @@ void RoctracerManager::ProcessActivityBuffers(const std::vectordomain == ACTIVITY_DOMAIN_EXT_API) { - NotifyOnCorrelation(current_record->correlation_id, current_record->external_id) + NotifyNewCorrelation(current_record->correlation_id, current_record->external_id) continue; } else if (current_record->domain == ACTIVITY_DOMAIN_HIP_OPS) { if (current_record->op == 1 && current_record->kind == HipOpMarker) { diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index a7a5e4098905f..8d5a71a58064a 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -12,7 +12,7 @@ #include #include -#include "core/common/profiler_common.h" +#include "core/common/gpu_profiler_common.h" #include "core/common/inlined_containers.h" namespace onnxruntime { @@ -31,15 +31,14 @@ class RoctracerManager : public GPUTracerManager { ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RoctracerManager); ~RoctracerManager(); static RoctracerManager& GetInstance(); - void StartLogging() override; protected: bool PushUniqueCorrelation(uint64_t unique_cid) override; void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; - void StopLogging() override; + void OnStopLogging() override; + void OnStartLogging() override; void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) override; - void Clear() override; void FlushActivities() override; private: diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index b60ef5be4a28f..1d2ad932f4ae7 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -261,10 +261,14 @@ std::string GetEnvironmentVar(const std::string& var_name); namespace profiling { + class GPUTracerManager; + class ProfilerActivityBuffer; + class GPUProfilerBase; + std::string demangle(const char* name); std::string demangle(const std::string& name); -}; +} /* namespace profiling */ namespace logging { From 9e6c3563805ea6fd51a43b460136f7317ae25469 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 17 Nov 2022 00:52:53 +0000 Subject: [PATCH 06/26] Fixes --- include/onnxruntime/core/common/gpu_profiler_common.h | 2 +- onnxruntime/core/common/gpu_profiler_common.cc | 2 ++ onnxruntime/core/providers/cuda/cuda_profiler.cc | 6 ------ onnxruntime/core/providers/rocm/roctracer_manager.cc | 11 +++++------ onnxruntime/core/providers/rocm/roctracer_manager.h | 2 +- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 1a2fa36603d5c..abb5924a9b78f 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -39,7 +39,7 @@ class GPUTracerManager { public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); - virtual ~GPUTracerManager() {} + virtual ~GPUTracerManager(); virtual uint64_t RegisterClient(); virtual void DeregisterClient(uint64_t client_handle); diff --git a/onnxruntime/core/common/gpu_profiler_common.cc b/onnxruntime/core/common/gpu_profiler_common.cc index 37f37798c1e5e..2cd33e867e6e3 100644 --- a/onnxruntime/core/common/gpu_profiler_common.cc +++ b/onnxruntime/core/common/gpu_profiler_common.cc @@ -50,6 +50,8 @@ ProfilerActivityBuffer ProfilerActivityBuffer::CreateFromPreallocatedBuffer(char // Implementation of GPUTracerManager +GPUTracerManager::~GPUTracerManager() {} + uint64_t GPUTracerManager::RegisterClient() { std::lock_guard lock(manager_instance_mutex_); if (logging_enabled_) { diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 47bd2b23c7e4a..4198ec7d7882b 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -11,17 +11,11 @@ namespace onnxruntime { - namespace profiling { // audupa: Debugging only, delete before merging // #define CUDA_VERSION 11600 -// auto KEVENT = onnxruntime::profiling::KERNEL_EVENT; -// std::atomic_flag CudaProfiler::enabled{0}; -// std::vector CudaProfiler::stats; -// std::unordered_map CudaProfiler::id_map; - #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 CudaProfiler::CudaProfiler() { diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index 4277e5026239e..7512d05236e7d 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -28,11 +28,9 @@ RoctracerManager& RoctracerManager::GetInstance() { return instance; } -RoctracerManager::~RoctracerManager() { - StopLogging(); -} +RoctracerManager::~RoctracerManager() {} -void RoctracerManager::OnStartLogging() { +bool RoctracerManager::OnStartLogging() { // The following line shows up in all the samples, I do not know // what the point is, but without it, the roctracer APIs don't work. roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); @@ -59,9 +57,10 @@ void RoctracerManager::OnStartLogging() { roctracer_start(); logging_enabled_ = true; + return true; } -void RoctracerManager::StopLogging() { +void RoctracerManager::OnStopLogging() { roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_API); roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS); roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); @@ -242,7 +241,7 @@ void RoctracerManager::ProcessActivityBuffers(const std::vectordomain == ACTIVITY_DOMAIN_EXT_API) { - NotifyNewCorrelation(current_record->correlation_id, current_record->external_id) + NotifyNewCorrelation(current_record->correlation_id, current_record->external_id); continue; } else if (current_record->domain == ACTIVITY_DOMAIN_HIP_OPS) { if (current_record->op == 1 && current_record->kind == HipOpMarker) { diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index 8d5a71a58064a..db2402f125651 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -36,7 +36,7 @@ class RoctracerManager : public GPUTracerManager { bool PushUniqueCorrelation(uint64_t unique_cid) override; void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; void OnStopLogging() override; - void OnStartLogging() override; + bool OnStartLogging() override; void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) override; void FlushActivities() override; From d1783f3c0794cad5289e14da859e3de8eab337db Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 17 Nov 2022 01:00:30 +0000 Subject: [PATCH 07/26] Fixes --- .../core/providers/rocm/roctracer_manager.cc | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index 7512d05236e7d..20884f98cda4f 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -30,33 +30,49 @@ RoctracerManager& RoctracerManager::GetInstance() { RoctracerManager::~RoctracerManager() {} +#define ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(expr_) \ +do { \ + if (expr_ != ROCTRACER_STATUS_SUCESS) { \ + OnStopLogging(); \ + return false; \ + } \ +} while (false) + bool RoctracerManager::OnStartLogging() { // The following line shows up in all the samples, I do not know // what the point is, but without it, the roctracer APIs don't work. - roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); + + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( + roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr) != ROCTRACER_STATUS_SUCCESS + ); roctracer_properties_t hcc_cb_properties; memset(&hcc_cb_properties, 0, sizeof(roctracer_properties_t)); hcc_cb_properties.buffer_size = kActivityBufferSize; hcc_cb_properties.buffer_callback_fun = ActivityCallback; - roctracer_open_pool(&hcc_cb_properties); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_open_pool(&hcc_cb_properties)); // Enable selective activity and API callbacks for the HIP APIs - roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); - roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_API); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API)); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_API)); for (auto const& logged_api : hip_api_calls_to_trace) { uint32_t cid = 0; - roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, logged_api.c_str(), &cid, nullptr); - roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, cid, ApiCallback, nullptr); - roctracer_enable_op_activity(ACTIVITY_DOMAIN_HIP_API, cid); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( + roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, logged_api.c_str(), &cid, nullptr) + ); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( + roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, cid, ApiCallback, nullptr) + ); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( + roctracer_enable_op_activity(ACTIVITY_DOMAIN_HIP_API, cid) + ); } // Enable activity logging in the HIP_OPS/HCC_OPS domain. - roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS); + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)); - roctracer_start(); - logging_enabled_ = true; + ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_start()); return true; } From 779f78e4f1d88ab6f334d2f25c98274ddaf7df1d Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 17 Nov 2022 01:01:43 +0000 Subject: [PATCH 08/26] Fix typo --- onnxruntime/core/providers/rocm/roctracer_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index 20884f98cda4f..7cbf111c0e1d2 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -32,7 +32,7 @@ RoctracerManager::~RoctracerManager() {} #define ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(expr_) \ do { \ - if (expr_ != ROCTRACER_STATUS_SUCESS) { \ + if (expr_ != ROCTRACER_STATUS_SUCCESS) { \ OnStopLogging(); \ return false; \ } \ From be8bb8dfc103e41b7ac38508d29e8a18a1af6136 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 17 Nov 2022 01:09:47 +0000 Subject: [PATCH 09/26] Add guards --- onnxruntime/core/providers/cuda/cuda_profiler.h | 6 +++++- onnxruntime/core/providers/cuda/cupti_manager.cc | 4 ++++ onnxruntime/core/providers/cuda/cupti_manager.h | 4 ---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 50f1e619afcc1..8b75482c238ca 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -1,6 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. + +#pragma once + #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) + #include #include #include @@ -36,7 +40,7 @@ namespace onnxruntime { namespace profiling { -class CudaProfiler final : public EpProfiler { +class CudaProfiler final : public GPUProfilerBase { public: bool StartProfiling(TimePoint) override { return true; } void EndProfiling(TimePoint, Events&) override{}; diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index 632a73c6c3b8b..3afbbd40a12be 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -3,6 +3,8 @@ namespace onnxruntime { namespace profiling { +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) + static inline std::string GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { switch (kind) { case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: @@ -158,5 +160,7 @@ void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer ); } +#endif /* defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ + } // namespace profiling } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 0b57397415ad0..6bf76648025ea 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -11,13 +11,9 @@ #include "core/common/gpu_profiler_common.h" #include "core/common/inlined_containers.h" - - namespace onnxruntime { namespace profiling { -using CUPTIActivityBuffer = ProfilerActivityBuffer; - class CUPTIManager : public GPUTracerManager { public: From fa85c1f677905ea8df4cabc6fb091fc62cbdf979 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 17 Nov 2022 02:29:06 +0000 Subject: [PATCH 10/26] Fixes --- include/onnxruntime/core/common/gpu_profiler_common.h | 6 +++--- onnxruntime/core/common/gpu_profiler_common.cc | 2 -- onnxruntime/core/providers/cuda/cupti_manager.cc | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index abb5924a9b78f..572d48c8f6af6 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -39,10 +39,10 @@ class GPUTracerManager { public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); - virtual ~GPUTracerManager(); + virtual ~GPUTracerManager() {} - virtual uint64_t RegisterClient(); - virtual void DeregisterClient(uint64_t client_handle); + uint64_t RegisterClient(); + void DeregisterClient(uint64_t client_handle); void StartLogging(); void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); diff --git a/onnxruntime/core/common/gpu_profiler_common.cc b/onnxruntime/core/common/gpu_profiler_common.cc index 2cd33e867e6e3..37f37798c1e5e 100644 --- a/onnxruntime/core/common/gpu_profiler_common.cc +++ b/onnxruntime/core/common/gpu_profiler_common.cc @@ -50,8 +50,6 @@ ProfilerActivityBuffer ProfilerActivityBuffer::CreateFromPreallocatedBuffer(char // Implementation of GPUTracerManager -GPUTracerManager::~GPUTracerManager() {} - uint64_t GPUTracerManager::RegisterClient() { std::lock_guard lock(manager_instance_mutex_); if (logging_enabled_) { diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index 3afbbd40a12be..c584367a588b3 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -76,7 +76,7 @@ void CUPTIManager::FlushActivities() { cuptiActivityFlushAll(1); } -void CUPTIManager::ProcessActivityBuffers(const std::vector& buffers, +void CUPTIManager::ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) { auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); for (auto const& buffer : buffers) { From 097280983bd2723c75cbc51c6af8262c91d5de48 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Fri, 18 Nov 2022 20:23:14 +0000 Subject: [PATCH 11/26] Reimplement using CRTP/inline classes to avoid linker challenges --- .../core/common/gpu_profiler_common.h | 347 ++++++++++++++++-- .../core/common/gpu_profiler_common.cc | 303 --------------- .../core/providers/cuda/cupti_manager.h | 15 +- .../core/providers/rocm/roctracer_manager.h | 15 +- .../providers/shared_library/provider_api.h | 4 - 5 files changed, 330 insertions(+), 354 deletions(-) delete mode 100644 onnxruntime/core/common/gpu_profiler_common.cc diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 572d48c8f6af6..28fd363070a70 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -4,6 +4,7 @@ #include "core/common/inlined_containers.h" #include +#include #include #include #include @@ -14,69 +15,310 @@ namespace onnxruntime { namespace profiling { +// The classes in this header are implemented as template/inline classes +// to avoid having to export symbols from the main onnxruntime shared library +// to ExecutionProvider (EP) shared libraries. +// More context: The main onnxruntime shared library is optimized for size +// using --gc-sections during link time to ensure that any unreferenced code +// is not retained. This poses a problem in using a design pattern where the +// (abstract) base class is implemented in the main onnxruntime shared library, +// but (concrete) subclasses are implemented in EP shared libraries. Now, because +// EP shared libraries are loaded at runtime (as of 11/2022), there will be no +// references to the base class symbols when the main onnxruntime shared library +// is compiled. Thus, the base class symbols will not be included in the +// main onnxruntime shared library. This manifests in being unable to load +// EP shared libs (because the base class symbols referenced by derived +// classes are missing). +// We solve this by implementing base classes that are common to all GPU profilers +// inline in this header. + class ProfilerActivityBuffer { public: - ProfilerActivityBuffer(); - ProfilerActivityBuffer(const char* data, size_t size); - ProfilerActivityBuffer(const ProfilerActivityBuffer& other); - ProfilerActivityBuffer(ProfilerActivityBuffer&& other); - ProfilerActivityBuffer& operator=(const ProfilerActivityBuffer& other); - ProfilerActivityBuffer& operator=(ProfilerActivityBuffer&& other); + ProfilerActivityBuffer() + : data_(nullptr), size_(0) {} + + ProfilerActivityBuffer(const char* data, size_t size) + : data_(std::make_unique(size)), size_(size) { + memcpy(data_.get(), data, size_); + } + + ProfilerActivityBuffer(const ProfilerActivityBuffer& other) + : ProfilerActivityBuffer(other.GetData(), other.GetSize()) {} + + ProfilerActivityBuffer(ProfilerActivityBuffer&& other) + : ProfilerActivityBuffer() { + std::swap(data_, other.data_); + std::swap(size_, other.size_); + } + + ProfilerActivityBuffer& operator=(const ProfilerActivityBuffer& other) { + if (&other == this) { + return *this; + } + + new (this) ProfilerActivityBuffer {other}; + return *this; + } + + ProfilerActivityBuffer& operator=(ProfilerActivityBuffer&& other) { + if (&other == this) { + return *this; + } + + new (this) ProfilerActivityBuffer {std::move(other)}; + return *this; + } + + static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size) { + ProfilerActivityBuffer res{}; + res.data_.reset(data); + res.size_ = size; + return res; + } // accessors char* GetData() { return data_.get(); } const char* GetData() const { return data_.get(); } size_t GetSize() const { return size_; } - static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size); - private: std::unique_ptr data_; size_t size_; }; /* end class ProfilerActivityBuffer */ +template class GPUTracerManager { public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); virtual ~GPUTracerManager() {} - uint64_t RegisterClient(); - void DeregisterClient(uint64_t client_handle); + uint64_t RegisterClient() { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto res = next_client_id_++; + per_client_events_by_ext_correlation_.insert({res, {}}); + ++num_active_clients_; + return res; + } + return 0; + } + + void DeregisterClient(uint64_t client_handle) { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + per_client_events_by_ext_correlation_.erase(it); + --num_active_clients_; + if (num_active_clients_ == 0) { + StopLogging(); + } + } + } + + void StartLogging() { + std::lock_guard lock(manager_instance_mutex_); + if (logging_enabled_) { + return; + } + + auto this_as_derived = static_cast(this); + logging_enabled_ = this_as_derived->OnStartLogging(); + } + + void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { + auto this_as_derived = static_cast(this); + events.clear(); + { + // Flush any pending activity records before starting + // to process the accumulated activity records. + std::lock_guard lock_manager(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + + this_as_derived->FlushActivities(); + } + + std::vector activity_buffers; + { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + std::swap(unprocessed_activity_buffers_, activity_buffers); + unprocessed_activity_buffers_.clear(); + } + + { + // Ensure that at most one thread is working through the activity buffers at any time. + std::lock_guard lock_two(activity_buffer_processor_mutex_); + this_as_derived->ProcessActivityBuffers(activity_buffers, start_time); + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + std::swap(events, it->second); + } + } - void StartLogging(); - void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events); bool PushCorrelation(uint64_t client_handle, uint64_t external_correlation_id, - TimePoint profiling_start_time); - void PopCorrelation(uint64_t& popped_external_correlation_id); - void PopCorrelation(); + TimePoint profiling_start_time) { + auto this_as_derived = static_cast(this); + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return false; + } + + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + // not a registered client, do nothing + return false; + } + + // external_correlation_id is simply the timestamp of this event, + // relative to profiling_start_time. i.e., it was computed as: + // external_correlation_id = + // std::chrono::duration_cast(event_start_time - profiling_start_time).count() + // + // Because of the relative nature of the external_correlation_id, the same + // external_correlation_id can be reused across different clients, which then makes it + // impossible to recover the client from the external_correlation_id, which in turn + // makes it impossible to map events (which are tagged with external_correlation_id) to clients. + // + // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) + // as follows: + // unique_cid = + // external_correlation_id + + // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() + // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. + // + // Of course, clients expect lists of events to be returned (on a call to Consume()), that are + // still keyed on the external_correlation_id that they've specified here, so we need to remember the + // offset to be subtracted + uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); + auto unique_cid = external_correlation_id + offset; + unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); + return this_as_derived->PushUniqueCorrelation(unique_cid); + } + + void PopCorrelation(uint64_t& popped_external_correlation_id) { + auto this_as_derived = static_cast(this); + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + uint64_t unique_cid; + this_as_derived->PopUniqueCorrelation(unique_cid); + // lookup the offset and subtract it before returning popped_external_correlation_id to the client + auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + popped_external_correlation_id = 0; + return; + } + popped_external_correlation_id = unique_cid - client_it->second.second; + } + + void PopCorrelation() { + uint64_t unused; + PopCorrelation(unused); + } protected: GPUTracerManager() = default; +#if 0 // Functional API to be implemented by subclasses - virtual bool OnStartLogging() = 0; - virtual void OnStopLogging() = 0; - virtual void ProcessActivityBuffers(const std::vector& buffers, - const TimePoint& start_time) = 0; - virtual bool PushUniqueCorrelation(uint64_t unique_cid) = 0; - virtual void PopUniqueCorrelation(uint64_t& popped_unique_cid) = 0; - virtual void FlushActivities() = 0; - - // Service API for subclasses - void EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer); + // Included here only for documentation purposes + bool OnStartLogging(); + void OnStopLogging(); + void ProcessActivityBuffers(const std::vector& buffers, + const TimePoint& start_time); + bool PushUniqueCorrelation(uint64_t unique_cid); + void PopUniqueCorrelation(uint64_t& popped_unique_cid); + void FlushActivities(); +#endif + + void EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer) { + std::lock_guard lock(unprocessed_activity_buffers_mutex_); + unprocessed_activity_buffers_.emplace_back(std::move(buffer)); + } + // To be called by subclasses only from ProcessActivityBuffers - void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event); + void MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) { + auto it = tracer_correlation_to_unique_correlation_.find(tracer_correlation_id); + if (it == tracer_correlation_to_unique_correlation_.end()) { + // We're yet to receive a mapping to unique_correlation_id for this tracer_correlation_id + DeferEventMapping(std::move(event), tracer_correlation_id); + return; + } + auto unique_correlation_id = it->second; + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->emplace_back(std::move(event)); + } + } + // To be called by subclasses only from ProcessActivityBuffers - void NotifyNewCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id); + void NotifyNewCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id) { + tracer_correlation_to_unique_correlation_[tracer_correlation_id] = unique_correlation_id; + auto pending_it = events_pending_client_mapping_.find(tracer_correlation_id); + if (pending_it == events_pending_client_mapping_.end()) { + return; + } + // Map the pending events to the right client + MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); + events_pending_client_mapping_.erase(pending_it); + } private: - void StopLogging(); - void Clear(); - Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id); - void MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events); - void DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id); + void StopLogging() { + auto this_as_derived = static_cast(this); + std::lock_guard lock(manager_instance_mutex_); + if (!logging_enabled_) { + return; + } + this_as_derived->OnStopLogging(); + logging_enabled_ = false; + Clear(); + } + + void Clear() { + unprocessed_activity_buffers_.clear(); + unique_correlation_id_to_client_offset_.clear(); + per_client_events_by_ext_correlation_.clear(); + tracer_correlation_to_unique_correlation_.clear(); + events_pending_client_mapping_.clear(); + } + + Events* GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { + auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); + if (client_it == unique_correlation_id_to_client_offset_.end()) { + return nullptr; + } + + // See the comments on the GetUniqueCorrelationId method for an explanation of + // of this offset computation and why it's required. + auto const& client_handle_offset = client_it->second; + auto external_correlation = unique_correlation_id - client_handle_offset.second; + + auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; + return &event_list; + } + + void MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { + auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); + if (p_event_list != nullptr) { + p_event_list->insert(p_event_list->end(), + std::make_move_iterator(events.begin()), + std::make_move_iterator(events.end())); + } + } + + void DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id) { + events_pending_client_mapping_[tracer_correlation_id].emplace_back(std::move(event)); + } std::mutex manager_instance_mutex_; uint64_t next_client_id_ = 1; @@ -107,7 +349,46 @@ class GPUTracerManager class GPUProfilerBase : public EpProfiler { protected: GPUProfilerBase() = default; - void MergeEvents(std::map& events_to_merge, Events& events); + void MergeEvents(std::map& events_to_merge, Events& events) { + Events merged_events; + + auto event_iter = std::make_move_iterator(events.begin()); + auto event_end = std::make_move_iterator(events.end()); + for (auto& map_iter : events_to_merge) { + auto ts = static_cast(map_iter.first); + while (event_iter != event_end && event_iter->ts < ts) { + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + // find the last event with the same timestamp. + while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { + ++event_iter; + } + + if (event_iter != event_end && event_iter->ts == ts) { + uint64_t increment = 1; + for (auto& evt : map_iter.second) { + evt.args["op_name"] = event_iter->args["op_name"]; + + // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. + // So, we adjust the timestamp here to something sensible. + evt.ts = event_iter->ts + increment; + ++increment; + } + merged_events.emplace_back(*event_iter); + ++event_iter; + } + + merged_events.insert(merged_events.end(), + std::make_move_iterator(map_iter.second.begin()), + std::make_move_iterator(map_iter.second.end())); + } + + // move any remaining events + merged_events.insert(merged_events.end(), event_iter, event_end); + std::swap(events, merged_events); + } }; /* class GPUProfilerBase */ // Convert a pointer to a hex string diff --git a/onnxruntime/core/common/gpu_profiler_common.cc b/onnxruntime/core/common/gpu_profiler_common.cc deleted file mode 100644 index 37f37798c1e5e..0000000000000 --- a/onnxruntime/core/common/gpu_profiler_common.cc +++ /dev/null @@ -1,303 +0,0 @@ -#include "core/common/gpu_profiler_common.h" - -namespace onnxruntime { -namespace profiling { - -// Implementation of ProfilerActivityBuffer -ProfilerActivityBuffer::ProfilerActivityBuffer() - : data_(nullptr), size_(0) {} - -ProfilerActivityBuffer::ProfilerActivityBuffer(const char* data, size_t size) - : data_(std::make_unique(size)), size_(size) { - memcpy(data_.get(), data, size); -} - -ProfilerActivityBuffer::ProfilerActivityBuffer(const ProfilerActivityBuffer& other) - : ProfilerActivityBuffer(other.data_.get(), other.size_) {} - -ProfilerActivityBuffer::ProfilerActivityBuffer(ProfilerActivityBuffer&& other) - : ProfilerActivityBuffer() { - std::swap(data_, other.data_); - std::swap(size_, other.size_); -} - -ProfilerActivityBuffer& ProfilerActivityBuffer::operator=(const ProfilerActivityBuffer& other) { - if (&other == this) { - return *this; - } - - size_ = other.size_; - data_ = std::make_unique(other.size_); - memcpy(data_.get(), other.data_.get(), size_); - return *this; -} - -ProfilerActivityBuffer& ProfilerActivityBuffer::operator=(ProfilerActivityBuffer&& other) { - if (&other == this) { - return *this; - } - std::swap(data_, other.data_); - std::swap(size_, other.size_); - return *this; -} - -ProfilerActivityBuffer ProfilerActivityBuffer::CreateFromPreallocatedBuffer(char* data, size_t size) { - ProfilerActivityBuffer res{}; - res.data_.reset(data); - res.size_ = size; - return res; -} - - -// Implementation of GPUTracerManager -uint64_t GPUTracerManager::RegisterClient() { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto res = next_client_id_++; - per_client_events_by_ext_correlation_.insert({res, {}}); - ++num_active_clients_; - return res; - } - return 0; -} - -void GPUTracerManager::DeregisterClient(uint64_t client_handle) { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - per_client_events_by_ext_correlation_.erase(it); - --num_active_clients_; - if (num_active_clients_ == 0) { - StopLogging(); - } - } -} - -void GPUTracerManager::StartLogging() { - std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - return; - } - - logging_enabled_ = OnStartLogging(); -} - -void GPUTracerManager::StopLogging() { - std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return; - } - OnStopLogging(); - logging_enabled_ = false; - Clear(); -} - -void GPUTracerManager::Consume(uint64_t client_handle, - const TimePoint& start_time, - std::map& events) { - events.clear(); - { - // Flush any pending activity records before starting - // to process the accumulated activity records. - std::lock_guard lock_manager(manager_instance_mutex_); - if (!logging_enabled_) { - return; - } - - FlushActivities(); - } - - std::vector activity_buffers; - { - std::lock_guard lock(unprocessed_activity_buffers_mutex_); - std::swap(unprocessed_activity_buffers_, activity_buffers); - unprocessed_activity_buffers_.clear(); - } - - { - // Ensure that at most one thread is working through the activity buffers at any time. - std::lock_guard lock_two(activity_buffer_processor_mutex_); - ProcessActivityBuffers(activity_buffers, start_time); - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - std::swap(events, it->second); - } -} - -bool GPUTracerManager::PushCorrelation(uint64_t client_handle, - uint64_t external_correlation_id, - TimePoint profiling_start_time) { - std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return false; - } - - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - // not a registered client, do nothing - return false; - } - - // external_correlation_id is simply the timestamp of this event, - // relative to profiling_start_time. i.e., it was computed as: - // external_correlation_id = - // std::chrono::duration_cast(event_start_time - profiling_start_time).count() - // - // Because of the relative nature of the external_correlation_id, the same - // external_correlation_id can be reused across different clients, which then makes it - // impossible to recover the client from the external_correlation_id, which in turn - // makes it impossible to map events (which are tagged with external_correlation_id) to clients. - // - // To address these difficulties, we construct a new correlation_id (let's call it unique_cid) - // as follows: - // unique_cid = - // external_correlation_id + - // std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count() - // now, unique_cid is monotonically increasing with time, so it can be used to reliably map events to clients. - // - // Of course, clients expect lists of events to be returned (on a call to Consume()), that are - // still keyed on the external_correlation_id that they've specified here, so we need to remember the - // offset to be subtracted - uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); - auto unique_cid = external_correlation_id + offset; - unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); - return PushUniqueCorrelation(unique_cid); -} - -void GPUTracerManager::PopCorrelation(uint64_t& popped_external_correlation_id) { - std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return; - } - uint64_t unique_cid; - PopUniqueCorrelation(unique_cid); - // lookup the offset and subtract it before returning popped_external_correlation_id to the client - auto client_it = unique_correlation_id_to_client_offset_.find(unique_cid); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - popped_external_correlation_id = 0; - return; - } - popped_external_correlation_id = unique_cid - client_it->second.second; -} - -void GPUTracerManager::PopCorrelation() { - uint64_t unused; - PopCorrelation(unused); -} - -void GPUTracerManager::EnqueueActivityBuffer(ProfilerActivityBuffer&& buffer) { - std::lock_guard lock(unprocessed_activity_buffers_mutex_); - unprocessed_activity_buffers_.emplace_back(std::move(buffer)); -} - -void GPUTracerManager::Clear() { - unprocessed_activity_buffers_.clear(); - unique_correlation_id_to_client_offset_.clear(); - per_client_events_by_ext_correlation_.clear(); - tracer_correlation_to_unique_correlation_.clear(); - events_pending_client_mapping_.clear(); -} - -Events* GPUTracerManager::GetEventListForUniqueCorrelationId(uint64_t unique_correlation_id) { - auto client_it = unique_correlation_id_to_client_offset_.find(unique_correlation_id); - if (client_it == unique_correlation_id_to_client_offset_.end()) { - return nullptr; - } - - // See the comments on the GetUniqueCorrelationId method for an explanation of - // of this offset computation and why it's required. - auto const& client_handle_offset = client_it->second; - auto external_correlation = unique_correlation_id - client_handle_offset.second; - - auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; - return &event_list; -} - -void GPUTracerManager::MapEventToClient(uint64_t tracer_correlation_id, EventRecord&& event) { - auto it = tracer_correlation_to_unique_correlation_.find(tracer_correlation_id); - if (it == tracer_correlation_to_unique_correlation_.end()) { - // We're yet to receive a mapping to unique_correlation_id for this tracer_correlation_id - DeferEventMapping(std::move(event), tracer_correlation_id); - return; - } - auto unique_correlation_id = it->second; - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->emplace_back(std::move(event)); - } -} - -void GPUTracerManager::MapEventsToClient(uint64_t unique_correlation_id, std::vector&& events) { - auto p_event_list = GetEventListForUniqueCorrelationId(unique_correlation_id); - if (p_event_list != nullptr) { - p_event_list->insert(p_event_list->end(), - std::make_move_iterator(events.begin()), - std::make_move_iterator(events.end())); - } -} - -void GPUTracerManager::DeferEventMapping(EventRecord&& event, uint64_t tracer_correlation_id) { - events_pending_client_mapping_[tracer_correlation_id].emplace_back(std::move(event)); -} - -void GPUTracerManager::NotifyNewCorrelation(uint64_t tracer_correlation_id, uint64_t unique_correlation_id) { - tracer_correlation_to_unique_correlation_[tracer_correlation_id] = unique_correlation_id; - auto pending_it = events_pending_client_mapping_.find(tracer_correlation_id); - if (pending_it == events_pending_client_mapping_.end()) { - return; - } - // Map the pending events to the right client - MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); - events_pending_client_mapping_.erase(pending_it); -} - - -// Implementation of GPUProfileBase -void GPUProfilerBase::MergeEvents(std::map& events_to_merge, Events& events) { - Events merged_events; - - auto event_iter = std::make_move_iterator(events.begin()); - auto event_end = std::make_move_iterator(events.end()); - for (auto& map_iter : events_to_merge) { - auto ts = static_cast(map_iter.first); - while (event_iter != event_end && event_iter->ts < ts) { - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - // find the last event with the same timestamp. - while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { - ++event_iter; - } - - if (event_iter != event_end && event_iter->ts == ts) { - uint64_t increment = 1; - for (auto& evt : map_iter.second) { - evt.args["op_name"] = event_iter->args["op_name"]; - - // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. - // So, we adjust the timestamp here to something sensible. - evt.ts = event_iter->ts + increment; - ++increment; - } - merged_events.emplace_back(*event_iter); - ++event_iter; - } - - merged_events.insert(merged_events.end(), - std::make_move_iterator(map_iter.second.begin()), - std::make_move_iterator(map_iter.second.end())); - } - - // move any remaining events - merged_events.insert(merged_events.end(), event_iter, event_end); - std::swap(events, merged_events); -} - -} /* end namespace profiling */ -} /* end namespace onnxruntime */ diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 6bf76648025ea..58f4333593dfd 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -14,21 +14,22 @@ namespace onnxruntime { namespace profiling { -class CUPTIManager : public GPUTracerManager +class CUPTIManager : public GPUTracerManager { + friend class GPUTracerManager; public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); ~CUPTIManager(); static CUPTIManager& GetInstance(); protected: - bool PushUniqueCorrelation(uint64_t unique_cid) override; - void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; - bool OnStartLogging() override; - void OnStopLogging() override; + bool PushUniqueCorrelation(uint64_t unique_cid); + void PopUniqueCorrelation(uint64_t& popped_unique_cid); + bool OnStartLogging(); + void OnStopLogging(); void ProcessActivityBuffers(const std::vector& buffers, - const TimePoint& start_time) override; - void FlushActivities() override; + const TimePoint& start_time); + void FlushActivities(); private: static constexpr size_t kActivityBufferSize = 32 * 1024; diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index db2402f125651..03356af60171d 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -26,20 +26,21 @@ struct ApiCallRecord { hip_api_data_t api_data_{}; }; -class RoctracerManager : public GPUTracerManager { +class RoctracerManager : public GPUTracerManager { + friend class GPUTracerManager; public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RoctracerManager); ~RoctracerManager(); static RoctracerManager& GetInstance(); protected: - bool PushUniqueCorrelation(uint64_t unique_cid) override; - void PopUniqueCorrelation(uint64_t& popped_unique_cid) override; - void OnStopLogging() override; - bool OnStartLogging() override; + bool PushUniqueCorrelation(uint64_t unique_cid); + void PopUniqueCorrelation(uint64_t& popped_unique_cid); + void OnStopLogging(); + bool OnStartLogging(); void ProcessActivityBuffers(const std::vector& buffers, - const TimePoint& start_time) override; - void FlushActivities() override; + const TimePoint& start_time); + void FlushActivities(); private: RoctracerManager() = default; diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 1d2ad932f4ae7..6d0a672166fb4 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -261,10 +261,6 @@ std::string GetEnvironmentVar(const std::string& var_name); namespace profiling { - class GPUTracerManager; - class ProfilerActivityBuffer; - class GPUProfilerBase; - std::string demangle(const char* name); std::string demangle(const std::string& name); From 2b485227220ce632317d10332ae7ec64e5c375d8 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Fri, 18 Nov 2022 21:56:24 +0000 Subject: [PATCH 12/26] Minor fixes. Works fine on CUDA --- .../core/common/gpu_profiler_common.h | 37 +++++++++---------- .../core/providers/cuda/cupti_manager.cc | 2 +- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 28fd363070a70..f204f6433eb9e 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -95,27 +95,22 @@ class GPUTracerManager uint64_t RegisterClient() { std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto res = next_client_id_++; - per_client_events_by_ext_correlation_.insert({res, {}}); - ++num_active_clients_; - return res; - } - return 0; + auto res = next_client_id_++; + per_client_events_by_ext_correlation_.insert({res, {}}); + ++num_active_clients_; + return res; } void DeregisterClient(uint64_t client_handle) { std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { - auto it = per_client_events_by_ext_correlation_.find(client_handle); - if (it == per_client_events_by_ext_correlation_.end()) { - return; - } - per_client_events_by_ext_correlation_.erase(it); - --num_active_clients_; - if (num_active_clients_ == 0) { - StopLogging(); - } + auto it = per_client_events_by_ext_correlation_.find(client_handle); + if (it == per_client_events_by_ext_correlation_.end()) { + return; + } + per_client_events_by_ext_correlation_.erase(it); + --num_active_clients_; + if (num_active_clients_ == 0 && logging_enabled_) { + StopLogging(); } } @@ -273,9 +268,9 @@ class GPUTracerManager } private: + // Requires: manager_instance_mutex_ should be held void StopLogging() { auto this_as_derived = static_cast(this); - std::lock_guard lock(manager_instance_mutex_); if (!logging_enabled_) { return; } @@ -284,6 +279,7 @@ class GPUTracerManager Clear(); } + // Requires: manager_instance_mutex_ should be held void Clear() { unprocessed_activity_buffers_.clear(); unique_correlation_id_to_client_offset_.clear(); @@ -370,11 +366,12 @@ class GPUProfilerBase : public EpProfiler { uint64_t increment = 1; for (auto& evt : map_iter.second) { evt.args["op_name"] = event_iter->args["op_name"]; + evt.args["parent_name"] = event_iter->name; - // roctracer doesn't use Jan 1 1970 as an epoch for its timestamps. + // Tracers may not use Jan 1 1970 as an epoch for timestamps. // So, we adjust the timestamp here to something sensible. evt.ts = event_iter->ts + increment; - ++increment; + increment += evt.dur; } merged_events.emplace_back(*event_iter); ++event_iter; diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index c584367a588b3..d9f15302b1656 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -103,7 +103,7 @@ void CUPTIManager::ProcessActivityBuffers(const std::vectorblockZ)}, }; - std::string name{kernel->name}; + std::string name {demangle(kernel->name)}; new (&event) EventRecord { /* cat = */ EventCategory::KERNEL_EVENT, From 760d5892c56a898b2f4a9821aa6a58ed2c59bb24 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Sat, 19 Nov 2022 00:49:27 +0000 Subject: [PATCH 13/26] Fixes for ROCm --- cmake/onnxruntime_rocm_hipify.cmake | 2 ++ .../onnxruntime/core/common/gpu_profiler_common.h | 2 +- .../core/providers/rocm/roctracer_manager.cc | 13 +++++-------- onnxruntime/core/providers/rocm/roctracer_manager.h | 4 +--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmake/onnxruntime_rocm_hipify.cmake b/cmake/onnxruntime_rocm_hipify.cmake index 3b76c5a80e7b1..76d6e3d220c60 100644 --- a/cmake/onnxruntime_rocm_hipify.cmake +++ b/cmake/onnxruntime_rocm_hipify.cmake @@ -151,6 +151,8 @@ set(provider_excluded_files "cuda_utils.cu" "cudnn_common.cc" "cudnn_common.h" + "cupti_manager.cc" + "cupti_manager.h" "fpgeneric.cu" "gpu_data_transfer.cc" "gpu_data_transfer.h" diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index f204f6433eb9e..9cb43135c7112 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -263,7 +263,7 @@ class GPUTracerManager return; } // Map the pending events to the right client - MapEventsToClient(tracer_correlation_id, std::move(pending_it->second)); + MapEventsToClient(unique_correlation_id, std::move(pending_it->second)); events_pending_client_mapping_.erase(pending_it); } diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index 7cbf111c0e1d2..1b425331aeaef 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -34,17 +34,14 @@ RoctracerManager::~RoctracerManager() {} do { \ if (expr_ != ROCTRACER_STATUS_SUCCESS) { \ OnStopLogging(); \ - return false; \ + return false; \ } \ } while (false) bool RoctracerManager::OnStartLogging() { // The following line shows up in all the samples, I do not know // what the point is, but without it, the roctracer APIs don't work. - - ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( - roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr) != ROCTRACER_STATUS_SUCCESS - ); + roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); roctracer_properties_t hcc_cb_properties; memset(&hcc_cb_properties, 0, sizeof(roctracer_properties_t)); @@ -72,7 +69,7 @@ bool RoctracerManager::OnStartLogging() { // Enable activity logging in the HIP_OPS/HCC_OPS domain. ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)); - ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(roctracer_start()); + roctracer_start(); return true; } @@ -88,7 +85,7 @@ void RoctracerManager::OnStopLogging() { void RoctracerManager::ActivityCallback(const char* begin, const char* end, void* arg) { size_t size = end - begin; - RoctracerActivityBuffer activity_buffer{reinterpret_cast(begin), size}; + ProfilerActivityBuffer activity_buffer{reinterpret_cast(begin), size}; auto& instance = GetInstance(); instance.EnqueueActivityBuffer(std::move(activity_buffer)); } @@ -247,7 +244,7 @@ bool RoctracerManager::CreateEventForActivityRecord(const roctracer_record_t* re return true; } -void RoctracerManager::ProcessActivityBuffers(const std::vector& buffers, +void RoctracerManager::ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) { auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index 03356af60171d..0c571aeb8b78b 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -18,8 +18,6 @@ namespace onnxruntime { namespace profiling { -using RoctracerActivityBuffer = ProfilerActivityBuffer; - struct ApiCallRecord { uint32_t domain_; uint32_t cid_; @@ -38,7 +36,7 @@ class RoctracerManager : public GPUTracerManager { void PopUniqueCorrelation(uint64_t& popped_unique_cid); void OnStopLogging(); bool OnStartLogging(); - void ProcessActivityBuffers(const std::vector& buffers, + void ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time); void FlushActivities(); From a77116304d3f16087a1acf5d408c24d9b04189ce Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Sat, 19 Nov 2022 01:15:21 +0000 Subject: [PATCH 14/26] Ran clang-format on all changed source files --- .../core/common/gpu_profiler_common.h | 32 ++-- .../onnxruntime/core/common/profiler_common.h | 3 +- .../core/providers/cuda/cuda_profiler.cc | 1 - .../core/providers/cuda/cuda_profiler.h | 2 +- .../core/providers/cuda/cupti_manager.cc | 175 +++++++++--------- .../core/providers/cuda/cupti_manager.h | 50 ++--- .../core/providers/rocm/roctracer_manager.cc | 21 +-- .../core/providers/rocm/roctracer_manager.h | 2 +- 8 files changed, 138 insertions(+), 148 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 9cb43135c7112..4482c54f08fff 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -11,7 +11,6 @@ #include #include - namespace onnxruntime { namespace profiling { @@ -35,18 +34,18 @@ namespace profiling { class ProfilerActivityBuffer { public: ProfilerActivityBuffer() - : data_(nullptr), size_(0) {} + : data_(nullptr), size_(0) {} ProfilerActivityBuffer(const char* data, size_t size) - : data_(std::make_unique(size)), size_(size) { + : data_(std::make_unique(size)), size_(size) { memcpy(data_.get(), data, size_); } ProfilerActivityBuffer(const ProfilerActivityBuffer& other) - : ProfilerActivityBuffer(other.GetData(), other.GetSize()) {} + : ProfilerActivityBuffer(other.GetData(), other.GetSize()) {} ProfilerActivityBuffer(ProfilerActivityBuffer&& other) - : ProfilerActivityBuffer() { + : ProfilerActivityBuffer() { std::swap(data_, other.data_); std::swap(size_, other.size_); } @@ -56,7 +55,7 @@ class ProfilerActivityBuffer { return *this; } - new (this) ProfilerActivityBuffer {other}; + new (this) ProfilerActivityBuffer{other}; return *this; } @@ -65,7 +64,7 @@ class ProfilerActivityBuffer { return *this; } - new (this) ProfilerActivityBuffer {std::move(other)}; + new (this) ProfilerActivityBuffer{std::move(other)}; return *this; } @@ -87,9 +86,8 @@ class ProfilerActivityBuffer { }; /* end class ProfilerActivityBuffer */ template -class GPUTracerManager -{ -public: +class GPUTracerManager { + public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GPUTracerManager); virtual ~GPUTracerManager() {} @@ -117,7 +115,7 @@ class GPUTracerManager void StartLogging() { std::lock_guard lock(manager_instance_mutex_); if (logging_enabled_) { - return; + return; } auto this_as_derived = static_cast(this); @@ -220,7 +218,7 @@ class GPUTracerManager PopCorrelation(unused); } -protected: + protected: GPUTracerManager() = default; #if 0 @@ -267,12 +265,12 @@ class GPUTracerManager events_pending_client_mapping_.erase(pending_it); } -private: + private: // Requires: manager_instance_mutex_ should be held void StopLogging() { auto this_as_derived = static_cast(this); if (!logging_enabled_) { - return; + return; } this_as_derived->OnStopLogging(); logging_enabled_ = false; @@ -343,7 +341,7 @@ class GPUTracerManager // Base class for a GPU profiler class GPUProfilerBase : public EpProfiler { -protected: + protected: GPUProfilerBase() = default; void MergeEvents(std::map& events_to_merge, Events& events) { Events merged_events; @@ -378,8 +376,8 @@ class GPUProfilerBase : public EpProfiler { } merged_events.insert(merged_events.end(), - std::make_move_iterator(map_iter.second.begin()), - std::make_move_iterator(map_iter.second.end())); + std::make_move_iterator(map_iter.second.begin()), + std::make_move_iterator(map_iter.second.end())); } // move any remaining events diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index 7c615f7d4a613..46b3e5248d1be 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -24,8 +24,7 @@ static constexpr const char* event_category_names_[EVENT_CATEGORY_MAX] = { "Session", "Node", "Kernel", - "Api" -}; + "Api"}; // Timing record for all events. struct EventRecord { diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 4198ec7d7882b..a38f7b276ba31 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -9,7 +9,6 @@ #include "cupti_manager.h" #include "cuda_profiler.h" - namespace onnxruntime { namespace profiling { diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 8b75482c238ca..5bc233ac3dc27 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -26,7 +26,7 @@ class CudaProfiler final : public GPUProfilerBase { void Start(uint64_t) override; void Stop(uint64_t) override; -private: + private: uint64_t client_handle_ = 0; TimePoint profiling_start_time_{}; }; diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index d9f15302b1656..dab7a58252cb6 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -39,17 +39,17 @@ CUPTIManager& CUPTIManager::GetInstance() { CUPTIManager::~CUPTIManager() {} bool CUPTIManager::OnStartLogging() { - if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS && - cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS && - cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { - return true; - } else { - OnStopLogging(); - return false; - } + if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS && + cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS && + cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { + return true; + } else { + OnStopLogging(); + return false; + } } void CUPTIManager::OnStopLogging() { @@ -67,9 +67,9 @@ bool CUPTIManager::PushUniqueCorrelation(uint64_t unique_cid) { void CUPTIManager::PopUniqueCorrelation(uint64_t& popped_unique_cid) { auto res = cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &popped_unique_cid); - if (res != CUPTI_SUCCESS) { - popped_unique_cid = 0; - } + if (res != CUPTI_SUCCESS) { + popped_unique_cid = 0; + } } void CUPTIManager::FlushActivities() { @@ -78,89 +78,86 @@ void CUPTIManager::FlushActivities() { void CUPTIManager::ProcessActivityBuffers(const std::vector& buffers, const TimePoint& start_time) { - auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); - for (auto const& buffer : buffers) { - auto size = buffer.GetSize(); - if (size == 0) { - continue; + auto start_time_ns = std::chrono::duration_cast(start_time.time_since_epoch()).count(); + for (auto const& buffer : buffers) { + auto size = buffer.GetSize(); + if (size == 0) { + continue; + } + CUpti_Activity* record = nullptr; + CUptiResult status; + do { + EventRecord event; + status = cuptiActivityGetNextRecord(reinterpret_cast(const_cast(buffer.GetData())), size, &record); + if (status == CUPTI_SUCCESS) { + if (CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL == record->kind || + CUPTI_ACTIVITY_KIND_KERNEL == record->kind) { + CUpti_ActivityKernel3* kernel = (CUpti_ActivityKernel3*)record; + std::unordered_map args{ + {"stream", std::to_string(kernel->streamId)}, + {"grid_x", std::to_string(kernel->gridX)}, + {"grid_y", std::to_string(kernel->gridY)}, + {"grid_z", std::to_string(kernel->gridZ)}, + {"block_x", std::to_string(kernel->blockX)}, + {"block_y", std::to_string(kernel->blockY)}, + {"block_z", std::to_string(kernel->blockZ)}, + }; + + std::string name{demangle(kernel->name)}; + + new (&event) EventRecord{ + /* cat = */ EventCategory::KERNEL_EVENT, + /* pid = */ -1, + /* tid = */ -1, + /* name = */ std::move(name), + /* ts = */ (int64_t)(kernel->start - start_time_ns) / 1000, + /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, + /* args = */ std::move(args)}; + MapEventToClient(kernel->correlationId, std::move(event)); + } else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) { + CUpti_ActivityMemcpy* mmcpy = (CUpti_ActivityMemcpy*)record; + std::string name{GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind)}; + std::unordered_map args{ + {"stream", std::to_string(mmcpy->streamId)}, + {"grid_x", "-1"}, + {"grid_y", "-1"}, + {"grid_z", "-1"}, + {"block_x", "-1"}, + {"block_y", "-1"}, + {"block_z", "-1"}, + }; + new (&event) EventRecord{ + /* cat = */ EventCategory::KERNEL_EVENT, + /* pid = */ -1, + /* tid = */ -1, + /* name = */ std::move(name), + /* ts = */ (int64_t)(mmcpy->start - start_time_ns) / 1000, + /* dur = */ (int64_t)(mmcpy->end - mmcpy->start) / 1000, + /* args = */ std::move(args)}; + MapEventToClient(mmcpy->correlationId, std::move(event)); + } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { + auto correlation = reinterpret_cast(record); + NotifyNewCorrelation(correlation->correlationId, correlation->externalId); } - CUpti_Activity* record = nullptr; - CUptiResult status; - do { - EventRecord event; - status = cuptiActivityGetNextRecord(reinterpret_cast(const_cast(buffer.GetData())), size, &record); - if (status == CUPTI_SUCCESS) { - if (CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL == record->kind || - CUPTI_ACTIVITY_KIND_KERNEL == record->kind) { - CUpti_ActivityKernel3* kernel = (CUpti_ActivityKernel3*)record; - std::unordered_map args { - {"stream", std::to_string(kernel->streamId)}, - {"grid_x", std::to_string(kernel->gridX)}, - {"grid_y", std::to_string(kernel->gridY)}, - {"grid_z", std::to_string(kernel->gridZ)}, - {"block_x", std::to_string(kernel->blockX)}, - {"block_y", std::to_string(kernel->blockY)}, - {"block_z", std::to_string(kernel->blockZ)}, - }; - - std::string name {demangle(kernel->name)}; - - new (&event) EventRecord { - /* cat = */ EventCategory::KERNEL_EVENT, - /* pid = */ -1, - /* tid = */ -1, - /* name = */ std::move(name), - /* ts = */ (int64_t)(kernel->start - start_time_ns) / 1000, - /* dur = */ (int64_t)(kernel->end - kernel->start) / 1000, - /* args = */ std::move(args) - }; - MapEventToClient(kernel->correlationId, std::move(event)); - } else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) { - CUpti_ActivityMemcpy* mmcpy = (CUpti_ActivityMemcpy*)record; - std::string name{GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind)}; - std::unordered_map args { - {"stream", std::to_string(mmcpy->streamId)}, - {"grid_x", "-1"}, - {"grid_y", "-1"}, - {"grid_z", "-1"}, - {"block_x", "-1"}, - {"block_y", "-1"}, - {"block_z", "-1"}, - }; - new (&event) EventRecord { - /* cat = */ EventCategory::KERNEL_EVENT, - /* pid = */ -1, - /* tid = */ -1, - /* name = */ std::move(name), - /* ts = */ (int64_t)(mmcpy->start - start_time_ns) / 1000, - /* dur = */ (int64_t)(mmcpy->end - mmcpy->start) / 1000, - /* args = */ std::move(args) - }; - MapEventToClient(mmcpy->correlationId, std::move(event)); - } else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) { - auto correlation = reinterpret_cast(record); - NotifyNewCorrelation(correlation->correlationId, correlation->externalId); - } - } - } while (status == CUPTI_SUCCESS); - } /* for */ + } + } while (status == CUPTI_SUCCESS); + } /* for */ } void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { - uint8_t* bfr = (uint8_t*)malloc(kActivityBufferSize + kActivityBufferAlignSize); - *size = kActivityBufferSize; - *buffer = AlignBuffer(bfr, kActivityBufferAlignSize); - *maxNumRecords = 0; + uint8_t* bfr = (uint8_t*)malloc(kActivityBufferSize + kActivityBufferAlignSize); + *size = kActivityBufferSize; + *buffer = AlignBuffer(bfr, kActivityBufferAlignSize); + *maxNumRecords = 0; } void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { - auto& instance = GetInstance(); - instance.EnqueueActivityBuffer( - ProfilerActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size) - ); + auto& instance = GetInstance(); + instance.EnqueueActivityBuffer( + ProfilerActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size)); } #endif /* defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ -} // namespace profiling -} // namespace onnxruntime +} // namespace profiling +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 58f4333593dfd..ab50a1d395024 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -14,15 +14,15 @@ namespace onnxruntime { namespace profiling { -class CUPTIManager : public GPUTracerManager -{ - friend class GPUTracerManager; -public: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); - ~CUPTIManager(); - static CUPTIManager& GetInstance(); - -protected: +class CUPTIManager : public GPUTracerManager { + friend class GPUTracerManager; + + public: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CUPTIManager); + ~CUPTIManager(); + static CUPTIManager& GetInstance(); + + protected: bool PushUniqueCorrelation(uint64_t unique_cid); void PopUniqueCorrelation(uint64_t& popped_unique_cid); bool OnStartLogging(); @@ -31,26 +31,26 @@ class CUPTIManager : public GPUTracerManager const TimePoint& start_time); void FlushActivities(); -private: - static constexpr size_t kActivityBufferSize = 32 * 1024; - static constexpr size_t kActivityBufferAlignSize = 8; - - // TODO: Is this even needed? malloc() is required to return - // a memory block that meets the alignment requirements for _any_ data type. - // On any platform that supports an 8-byte datatype (double? long long?) - // this means that malloc() already returns memory aligned at - // _at least_ 8 byte boundaries, rendering this additional alignment - // redundant? - static constexpr uint8_t* AlignBuffer(uint8_t* buffer, int align) { - return (((uintptr_t)(buffer) & ((align)-1)) + private: + static constexpr size_t kActivityBufferSize = 32 * 1024; + static constexpr size_t kActivityBufferAlignSize = 8; + + // TODO: Is this even needed? malloc() is required to return + // a memory block that meets the alignment requirements for _any_ data type. + // On any platform that supports an 8-byte datatype (double? long long?) + // this means that malloc() already returns memory aligned at + // _at least_ 8 byte boundaries, rendering this additional alignment + // redundant? + static constexpr uint8_t* AlignBuffer(uint8_t* buffer, int align) { + return (((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) : (buffer)); - } + } - CUPTIManager() = default; + CUPTIManager() = default; - static void CUPTIAPI BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); - static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size); + static void CUPTIAPI BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); + static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size); }; /* class CUPTIManager*/ #endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.cc b/onnxruntime/core/providers/rocm/roctracer_manager.cc index 1b425331aeaef..6c17fbfc09643 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.cc +++ b/onnxruntime/core/providers/rocm/roctracer_manager.cc @@ -31,12 +31,12 @@ RoctracerManager& RoctracerManager::GetInstance() { RoctracerManager::~RoctracerManager() {} #define ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL(expr_) \ -do { \ - if (expr_ != ROCTRACER_STATUS_SUCCESS) { \ - OnStopLogging(); \ - return false; \ - } \ -} while (false) + do { \ + if (expr_ != ROCTRACER_STATUS_SUCCESS) { \ + OnStopLogging(); \ + return false; \ + } \ + } while (false) bool RoctracerManager::OnStartLogging() { // The following line shows up in all the samples, I do not know @@ -56,14 +56,11 @@ bool RoctracerManager::OnStartLogging() { for (auto const& logged_api : hip_api_calls_to_trace) { uint32_t cid = 0; ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( - roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, logged_api.c_str(), &cid, nullptr) - ); + roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, logged_api.c_str(), &cid, nullptr)); ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( - roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, cid, ApiCallback, nullptr) - ); + roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, cid, ApiCallback, nullptr)); ROCTRACER_STATUS_RETURN_FALSE_ON_FAIL( - roctracer_enable_op_activity(ACTIVITY_DOMAIN_HIP_API, cid) - ); + roctracer_enable_op_activity(ACTIVITY_DOMAIN_HIP_API, cid)); } // Enable activity logging in the HIP_OPS/HCC_OPS domain. diff --git a/onnxruntime/core/providers/rocm/roctracer_manager.h b/onnxruntime/core/providers/rocm/roctracer_manager.h index 0c571aeb8b78b..dda0a905185e1 100644 --- a/onnxruntime/core/providers/rocm/roctracer_manager.h +++ b/onnxruntime/core/providers/rocm/roctracer_manager.h @@ -26,6 +26,7 @@ struct ApiCallRecord { class RoctracerManager : public GPUTracerManager { friend class GPUTracerManager; + public: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RoctracerManager); ~RoctracerManager(); @@ -47,7 +48,6 @@ class RoctracerManager : public GPUTracerManager { bool CreateEventForActivityRecord(const roctracer_record_t* record, uint64_t start_time_ns, const ApiCallRecord& call_record, EventRecord& event); - // Some useful constants for processing activity buffers static constexpr uint32_t HipOpMarker = 4606; From 4d5542d9279e0af7fbdf501868d84aae86ba285e Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Sat, 19 Nov 2022 02:14:06 +0000 Subject: [PATCH 15/26] Fix memory leak --- onnxruntime/core/providers/cuda/cupti_manager.cc | 7 ++++--- onnxruntime/core/providers/cuda/cupti_manager.h | 13 ------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index dab7a58252cb6..e40f5ebe7d254 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -145,10 +145,11 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector(buf); } void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index ab50a1d395024..dd7d41fc63b32 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -33,19 +33,6 @@ class CUPTIManager : public GPUTracerManager { private: static constexpr size_t kActivityBufferSize = 32 * 1024; - static constexpr size_t kActivityBufferAlignSize = 8; - - // TODO: Is this even needed? malloc() is required to return - // a memory block that meets the alignment requirements for _any_ data type. - // On any platform that supports an 8-byte datatype (double? long long?) - // this means that malloc() already returns memory aligned at - // _at least_ 8 byte boundaries, rendering this additional alignment - // redundant? - static constexpr uint8_t* AlignBuffer(uint8_t* buffer, int align) { - return (((uintptr_t)(buffer) & ((align)-1)) - ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) - : (buffer)); - } CUPTIManager() = default; From 2253df5abe5a03545718671e68333b5815e71f17 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Sat, 19 Nov 2022 02:16:53 +0000 Subject: [PATCH 16/26] Fix --- onnxruntime/core/providers/cuda/cupti_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index e40f5ebe7d254..fc275cc867fc3 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -147,7 +147,7 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector(buf); } From 67479d5226cbf3c56f0f3caf07479dc0d7f5216c Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Sat, 19 Nov 2022 02:19:26 +0000 Subject: [PATCH 17/26] semicolon --- onnxruntime/core/providers/cuda/cupti_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index fc275cc867fc3..d24b375c0f0aa 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -146,7 +146,7 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector(buf); From 67a5ef2c4a794bccbef95b31d98476dfdd25ac40 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 21:57:49 +0000 Subject: [PATCH 18/26] Review rework + bug fix + refactor common code into base class for GPU profilers --- .../core/common/gpu_profiler_common.h | 115 +++++++++++++----- .../core/providers/cuda/cuda_profiler.cc | 47 +------ .../core/providers/cuda/cuda_profiler.h | 21 +--- .../core/providers/cuda/cupti_manager.cc | 16 ++- .../core/providers/cuda/cupti_manager.h | 6 +- .../core/providers/rocm/rocm_profiler.cc | 26 ---- .../core/providers/rocm/rocm_profiler.h | 12 +- .../providers/shared_library/provider_api.h | 2 +- 8 files changed, 112 insertions(+), 133 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 4482c54f08fff..1a87b428a1569 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -68,9 +68,9 @@ class ProfilerActivityBuffer { return *this; } - static ProfilerActivityBuffer CreateFromPreallocatedBuffer(char* data, size_t size) { + static ProfilerActivityBuffer CreateFromPreallocatedBuffer(std::unique_ptr&& buffer_ptr, size_t size) { ProfilerActivityBuffer res{}; - res.data_.reset(data); + res.data_ = std::move(buffer_ptr); res.size_ = size; return res; } @@ -107,19 +107,19 @@ class GPUTracerManager { } per_client_events_by_ext_correlation_.erase(it); --num_active_clients_; - if (num_active_clients_ == 0 && logging_enabled_) { + if (num_active_clients_ == 0 && tracing_enabled_) { StopLogging(); } } void StartLogging() { std::lock_guard lock(manager_instance_mutex_); - if (logging_enabled_) { + if (tracing_enabled_) { return; } auto this_as_derived = static_cast(this); - logging_enabled_ = this_as_derived->OnStartLogging(); + tracing_enabled_ = this_as_derived->OnStartLogging(); } void Consume(uint64_t client_handle, const TimePoint& start_time, std::map& events) { @@ -129,7 +129,7 @@ class GPUTracerManager { // Flush any pending activity records before starting // to process the accumulated activity records. std::lock_guard lock_manager(manager_instance_mutex_); - if (!logging_enabled_) { + if (!tracing_enabled_) { return; } @@ -155,19 +155,19 @@ class GPUTracerManager { } } - bool PushCorrelation(uint64_t client_handle, + void PushCorrelation(uint64_t client_handle, uint64_t external_correlation_id, TimePoint profiling_start_time) { auto this_as_derived = static_cast(this); std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { - return false; + if (!tracing_enabled_) { + return; } auto it = per_client_events_by_ext_correlation_.find(client_handle); if (it == per_client_events_by_ext_correlation_.end()) { // not a registered client, do nothing - return false; + return; } // external_correlation_id is simply the timestamp of this event, @@ -193,13 +193,13 @@ class GPUTracerManager { uint64_t offset = std::chrono::duration_cast(profiling_start_time.time_since_epoch()).count(); auto unique_cid = external_correlation_id + offset; unique_correlation_id_to_client_offset_[unique_cid] = std::make_pair(client_handle, offset); - return this_as_derived->PushUniqueCorrelation(unique_cid); + this_as_derived->PushUniqueCorrelation(unique_cid); } void PopCorrelation(uint64_t& popped_external_correlation_id) { auto this_as_derived = static_cast(this); std::lock_guard lock(manager_instance_mutex_); - if (!logging_enabled_) { + if (!tracing_enabled_) { return; } uint64_t unique_cid; @@ -269,11 +269,11 @@ class GPUTracerManager { // Requires: manager_instance_mutex_ should be held void StopLogging() { auto this_as_derived = static_cast(this); - if (!logging_enabled_) { + if (!tracing_enabled_) { return; } this_as_derived->OnStopLogging(); - logging_enabled_ = false; + tracing_enabled_ = false; Clear(); } @@ -296,7 +296,6 @@ class GPUTracerManager { // of this offset computation and why it's required. auto const& client_handle_offset = client_it->second; auto external_correlation = unique_correlation_id - client_handle_offset.second; - auto& event_list = per_client_events_by_ext_correlation_[client_handle_offset.first][external_correlation]; return &event_list; } @@ -317,7 +316,7 @@ class GPUTracerManager { std::mutex manager_instance_mutex_; uint64_t next_client_id_ = 1; uint64_t num_active_clients_ = 0; - bool logging_enabled_ = false; + bool tracing_enabled_ = false; std::mutex unprocessed_activity_buffers_mutex_; std::mutex activity_buffer_processor_mutex_; @@ -340,9 +339,12 @@ class GPUTracerManager { }; /* class GPUTracerManager */ // Base class for a GPU profiler +template class GPUProfilerBase : public EpProfiler { protected: GPUProfilerBase() = default; + virtual ~GPUProfilerBase() {} + void MergeEvents(std::map& events_to_merge, Events& events) { Events merged_events; @@ -350,29 +352,52 @@ class GPUProfilerBase : public EpProfiler { auto event_end = std::make_move_iterator(events.end()); for (auto& map_iter : events_to_merge) { auto ts = static_cast(map_iter.first); - while (event_iter != event_end && event_iter->ts < ts) { + + // find the last occurence of a matching timestamp, + // if one exists + while (event_iter != event_end && + (event_iter->ts < ts || + (event_iter->ts == ts && + (event_iter + 1) != event_end && + (event_iter + 1)->ts == ts))) { merged_events.emplace_back(*event_iter); ++event_iter; } - // find the last event with the same timestamp. - while (event_iter != event_end && event_iter->ts == ts && (event_iter + 1)->ts == ts) { - ++event_iter; - } + uint64_t increment; + uint64_t last_ts; + bool copy_op_names = false; + std::string op_name; + std::string parent_name; + // Tracers may not use Jan 1 1970 as an epoch for timestamps. + // So, we need to adjust the timestamp to something sensible. if (event_iter != event_end && event_iter->ts == ts) { - uint64_t increment = 1; - for (auto& evt : map_iter.second) { - evt.args["op_name"] = event_iter->args["op_name"]; - evt.args["parent_name"] = event_iter->name; - - // Tracers may not use Jan 1 1970 as an epoch for timestamps. - // So, we adjust the timestamp here to something sensible. - evt.ts = event_iter->ts + increment; - increment += evt.dur; - } + // In this particular case we have located a parent event -- in the main event stream -- + // for the GPU events. We use the timestamps from that event to adjust timestamps + last_ts = event_iter->ts; + increment = 1; + copy_op_names = true; + op_name = event_iter->args["op_name"]; + parent_name = event_iter->name; merged_events.emplace_back(*event_iter); ++event_iter; + } else { + // No parent event, let's just set the timestamp based on the + // timestamp of the call to EpProfiler->Start() + last_ts = ts; + increment = 1; + } + + for (auto& evt : map_iter.second) { + if (copy_op_names) { + // If we have found a matching parent event, + // then inherit some names from the parent. + evt.args["op_name"] = op_name; + evt.args["parent_name"] = parent_name; + } + evt.ts = last_ts + increment; + increment += evt.dur; } merged_events.insert(merged_events.end(), @@ -384,6 +409,34 @@ class GPUProfilerBase : public EpProfiler { merged_events.insert(merged_events.end(), event_iter, event_end); std::swap(events, merged_events); } + + uint64_t client_handle_; + TimePoint profiling_start_time_; + +public: + virtual bool StartProfiling(TimePoint profiling_start_time) override { + auto& manager = TManager::GetInstance(); + manager.StartLogging(); + profiling_start_time_ = profiling_start_time; + return true; + } + + virtual void EndProfiling(TimePoint start_time, Events& events) override { + auto& manager = TManager::GetInstance(); + std::map event_map; + manager.Consume(client_handle_, start_time, event_map); + MergeEvents(event_map, events); + } + + virtual void Start(uint64_t id) override { + auto& manager = TManager::GetInstance(); + manager.PushCorrelation(client_handle_, id, profiling_start_time_); + } + + virtual void Stop(uint64_t) override { + auto& manager = TManager::GetInstance(); + manager.PopCorrelation(); + } }; /* class GPUProfilerBase */ // Convert a pointer to a hex string diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index a38f7b276ba31..cadf8bc668841 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -1,22 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 #include #include #include -#include "cupti_manager.h" #include "cuda_profiler.h" namespace onnxruntime { namespace profiling { -// audupa: Debugging only, delete before merging -// #define CUDA_VERSION 11600 - -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 - CudaProfiler::CudaProfiler() { auto& manager = CUPTIManager::GetInstance(); client_handle_ = manager.RegisterClient(); @@ -27,40 +21,7 @@ CudaProfiler::~CudaProfiler() { manager.DeregisterClient(client_handle_); } -bool CudaProfiler::StartProfiling(TimePoint profiling_start_time) { - auto& manager = CUPTIManager::GetInstance(); - manager.StartLogging(); - profiling_start_time_ = profiling_start_time; - return true; -} - -void CudaProfiler::EndProfiling(TimePoint start_time, Events& events) { - auto& manager = CUPTIManager::GetInstance(); - std::map event_map; - manager.Consume(client_handle_, start_time, event_map); - MergeEvents(event_map, events); -} - -void CudaProfiler::Start(uint64_t id) { - auto& manager = CUPTIManager::GetInstance(); - manager.PushCorrelation(client_handle_, id, profiling_start_time_); -} - -void CudaProfiler::Stop(uint64_t) { - auto& manager = CUPTIManager::GetInstance(); - manager.PopCorrelation(); -} - -#else // for cuda 10.x, no profiling - -bool CudaProfiler::StartProfiling(TimePoint) { return false; } -void CudaProfiler::EndProfiling(TimePoint, Events&) {} -CudaProfiler::~CudaProfiler() {} -void CudaProfiler::Start(uint64_t) {} -void CudaProfiler::Stop(uint64_t) {} - -#endif +} // namespace onnxruntime +} // namespace profiling -} // namespace profiling -} // namespace onnxruntime -#endif +#endif // defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 5bc233ac3dc27..4247218303342 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -3,44 +3,33 @@ #pragma once -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 #include #include #include #include "core/common/gpu_profiler_common.h" +#include "cupti_manager.h" namespace onnxruntime { namespace profiling { -using Events = std::vector; - -class CudaProfiler final : public GPUProfilerBase { +class CudaProfiler final : public GPUProfilerBase { public: CudaProfiler(); ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudaProfiler); ~CudaProfiler(); - bool StartProfiling(TimePoint profiling_start_time) override; - void EndProfiling(TimePoint start_time, Events& events) override; - void Start(uint64_t) override; - void Stop(uint64_t) override; - - private: - uint64_t client_handle_ = 0; - TimePoint profiling_start_time_{}; }; } // namespace profiling } // namespace onnxruntime -#else - +#else /* !defined(USE_CUDA) || !defined(ENABLE_CUDA_PROFILING) || !defined(CUDA_VERSION) || CUDA_VERSION < 11000 */ namespace onnxruntime { - namespace profiling { -class CudaProfiler final : public GPUProfilerBase { +class CudaProfiler final : public EpProfiler { public: bool StartProfiling(TimePoint) override { return true; } void EndProfiling(TimePoint, Events&) override{}; diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index d24b375c0f0aa..67ada92ac3df4 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -1,9 +1,11 @@ #include "cupti_manager.h" +#include + namespace onnxruntime { namespace profiling { -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 static inline std::string GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) { switch (kind) { @@ -146,7 +148,13 @@ void CUPTIManager::ProcessActivityBuffers(const std::vector(buf); @@ -154,8 +162,10 @@ void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { auto& instance = GetInstance(); + std::unique_ptr buffer_ptr; + buffer_ptr.reset(buffer); instance.EnqueueActivityBuffer( - ProfilerActivityBuffer::CreateFromPreallocatedBuffer(reinterpret_cast(buffer), valid_size)); + ProfilerActivityBuffer::CreateFromPreallocatedBuffer(std::move(buffer_ptr), valid_size)); } #endif /* defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index dd7d41fc63b32..3100fae34575a 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 #include #include @@ -40,7 +40,7 @@ class CUPTIManager : public GPUTracerManager { static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size); }; /* class CUPTIManager*/ -#endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ - } /* namespace profiling */ } /* namespace onnxruntime */ + +#endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.cc b/onnxruntime/core/providers/rocm/rocm_profiler.cc index 7ec591eadfa63..de52f512c5229 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.cc +++ b/onnxruntime/core/providers/rocm/rocm_profiler.cc @@ -6,7 +6,6 @@ #include #include "core/providers/rocm/rocm_profiler.h" -#include "core/providers/rocm/roctracer_manager.h" namespace onnxruntime { namespace profiling { @@ -21,31 +20,6 @@ RocmProfiler::~RocmProfiler() { manager.DeregisterClient(client_handle_); } -bool RocmProfiler::StartProfiling(TimePoint profiling_start_time) { - auto& manager = RoctracerManager::GetInstance(); - manager.StartLogging(); - profiling_start_time_ = profiling_start_time; - return true; -} - -void RocmProfiler::EndProfiling(TimePoint start_time, Events& events) { - auto& manager = RoctracerManager::GetInstance(); - std::map event_map; - manager.Consume(client_handle_, start_time, event_map); - MergeEvents(event_map, events); -} - -void RocmProfiler::Start(uint64_t id) { - auto& manager = RoctracerManager::GetInstance(); - manager.PushCorrelation(client_handle_, id, profiling_start_time_); -} - -void RocmProfiler::Stop(uint64_t id) { - auto& manager = RoctracerManager::GetInstance(); - uint64_t unused; - manager.PopCorrelation(unused); -} - } // namespace profiling } // namespace onnxruntime #endif diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.h b/onnxruntime/core/providers/rocm/rocm_profiler.h index 37fe9b53a2105..e1603ad0c6314 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.h +++ b/onnxruntime/core/providers/rocm/rocm_profiler.h @@ -4,6 +4,7 @@ #include #include "core/common/gpu_profiler_common.h" +#include "roctracer_manager.h" #if defined(USE_ROCM) && defined(ENABLE_ROCM_PROFILING) @@ -12,19 +13,11 @@ namespace profiling { using Events = std::vector; -class RocmProfiler final : public GPUProfilerBase { +class RocmProfiler final : public GPUProfilerBase { public: RocmProfiler(); ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RocmProfiler); ~RocmProfiler(); - bool StartProfiling(TimePoint profiling_start_time) override; - void EndProfiling(TimePoint start_time, Events& events) override; - void Start(uint64_t) override; - void Stop(uint64_t) override; - - private: - uint64_t client_handle_; - TimePoint profiling_start_time_; }; } // namespace profiling @@ -33,7 +26,6 @@ class RocmProfiler final : public GPUProfilerBase { #else namespace onnxruntime { - namespace profiling { class RocmProfiler final : public EpProfiler { diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 6d0a672166fb4..6876d94052231 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -264,7 +264,7 @@ namespace profiling { std::string demangle(const char* name); std::string demangle(const std::string& name); -} /* namespace profiling */ +} // namespace profiling namespace logging { From 2437ea095e572ffdece018327977adef8bdcb8be Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 22:11:02 +0000 Subject: [PATCH 19/26] Fixes --- onnxruntime/core/providers/cuda/cuda_profiler.h | 3 +++ onnxruntime/core/providers/rocm/rocm_profiler.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 4247218303342..a5cba46d6b337 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -31,6 +31,9 @@ namespace profiling { class CudaProfiler final : public EpProfiler { public: + CudaProfiler() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudaProfiler); + ~CudaProfiler() {} bool StartProfiling(TimePoint) override { return true; } void EndProfiling(TimePoint, Events&) override{}; void Start(uint64_t) override{}; diff --git a/onnxruntime/core/providers/rocm/rocm_profiler.h b/onnxruntime/core/providers/rocm/rocm_profiler.h index e1603ad0c6314..070cca570f481 100644 --- a/onnxruntime/core/providers/rocm/rocm_profiler.h +++ b/onnxruntime/core/providers/rocm/rocm_profiler.h @@ -30,6 +30,9 @@ namespace profiling { class RocmProfiler final : public EpProfiler { public: + RocmProfiler() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RocmProfiler); + ~RocmProfiler() {} bool StartProfiling(TimePoint) override { return true; } void EndProfiling(TimePoint, Events&) override{}; void Start(uint64_t) override{}; From 7f357f71946d5059401aa1c3a461721fafcbd931 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 22:36:01 +0000 Subject: [PATCH 20/26] Fixes --- onnxruntime/core/providers/cuda/cuda_profiler.cc | 12 ++++++++++-- onnxruntime/core/providers/cuda/cuda_profiler.h | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index cadf8bc668841..05faed1e9e05b 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 #include #include @@ -21,7 +22,14 @@ CudaProfiler::~CudaProfiler() { manager.DeregisterClient(client_handle_); } +#else /* #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ + +CudaProfiler::CudaProfiler() {} +CudaProfiler::~CudaProfiler() {} + +#endif /* #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ + } // namespace onnxruntime } // namespace profiling -#endif // defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#endif /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index a5cba46d6b337..91b7df52c6576 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -3,7 +3,7 @@ #pragma once -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) #include #include @@ -25,7 +25,8 @@ class CudaProfiler final : public GPUProfilerBase { } // namespace profiling } // namespace onnxruntime -#else /* !defined(USE_CUDA) || !defined(ENABLE_CUDA_PROFILING) || !defined(CUDA_VERSION) || CUDA_VERSION < 11000 */ +#else /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ + namespace onnxruntime { namespace profiling { From 293bdbfecccbea6baa25c7a341cbd24ad5af15df Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 22:47:31 +0000 Subject: [PATCH 21/26] Fix conditional compilation mess --- .../core/providers/cuda/cuda_profiler.cc | 14 ++++---------- .../core/providers/cuda/cuda_profiler.h | 19 ++++++++----------- .../core/providers/cuda/cupti_manager.h | 7 ++++++- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 05faed1e9e05b..4419e0be02b21 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 + #include #include @@ -12,6 +11,8 @@ namespace onnxruntime { namespace profiling { +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 + CudaProfiler::CudaProfiler() { auto& manager = CUPTIManager::GetInstance(); client_handle_ = manager.RegisterClient(); @@ -22,14 +23,7 @@ CudaProfiler::~CudaProfiler() { manager.DeregisterClient(client_handle_); } -#else /* #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ - -CudaProfiler::CudaProfiler() {} -CudaProfiler::~CudaProfiler() {} - -#endif /* #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ +#endif /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ } // namespace onnxruntime } // namespace profiling - -#endif /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 91b7df52c6576..1f015cab7ab2f 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -3,8 +3,6 @@ #pragma once -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) - #include #include #include @@ -15,6 +13,11 @@ namespace onnxruntime { namespace profiling { +// Do not move this check for CUDA_VERSION above #include "cupti_manager.h" +// the CUDA_VERSION macro is defined in cupti.h, which in turn is included +// by cupti_manager.h +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 + class CudaProfiler final : public GPUProfilerBase { public: CudaProfiler(); @@ -22,13 +25,7 @@ class CudaProfiler final : public GPUProfilerBase { ~CudaProfiler(); }; -} // namespace profiling -} // namespace onnxruntime - -#else /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ - -namespace onnxruntime { -namespace profiling { +#else /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ class CudaProfiler final : public EpProfiler { public: @@ -41,7 +38,7 @@ class CudaProfiler final : public EpProfiler { void Stop(uint64_t) override{}; }; +#endif + } // namespace profiling } // namespace onnxruntime - -#endif diff --git a/onnxruntime/core/providers/cuda/cupti_manager.h b/onnxruntime/core/providers/cuda/cupti_manager.h index 3100fae34575a..9bc6815242b87 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.h +++ b/onnxruntime/core/providers/cuda/cupti_manager.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) #include #include @@ -8,6 +8,10 @@ #include +// Do not move the check for CUDA_VERSION above #include +// the macros are defined in cupti.h +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 + #include "core/common/gpu_profiler_common.h" #include "core/common/inlined_containers.h" @@ -43,4 +47,5 @@ class CUPTIManager : public GPUTracerManager { } /* namespace profiling */ } /* namespace onnxruntime */ +#endif /* #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ #endif /* #if defined (USE_CUDA) && defined(ENABLE_CUDA_PROFILING) */ From c26af7e486574afe587ffdbe01993a5a3e4b40e0 Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 22:55:06 +0000 Subject: [PATCH 22/26] Fix --- onnxruntime/core/providers/cuda/cupti_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cupti_manager.cc b/onnxruntime/core/providers/cuda/cupti_manager.cc index 67ada92ac3df4..5a03855b2f524 100644 --- a/onnxruntime/core/providers/cuda/cupti_manager.cc +++ b/onnxruntime/core/providers/cuda/cupti_manager.cc @@ -154,7 +154,7 @@ void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size // In the BufferCompleted callback, we pass the returned buffer into a ProfilerActivityBuffer // object, which then assumes ownership of the buffer. RAII semantics then delete/free the // buffer whenever the ProfilerActivityBuffer is destroyed. - auto buf = new char[kActivityBufferSize] + auto buf = new char[kActivityBufferSize]; *size = kActivityBufferSize; *maxNumRecords = 0; *buffer = reinterpret_cast(buf); @@ -163,7 +163,7 @@ void CUPTIAPI CUPTIManager::BufferRequested(uint8_t** buffer, size_t* size, size void CUPTIAPI CUPTIManager::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t valid_size) { auto& instance = GetInstance(); std::unique_ptr buffer_ptr; - buffer_ptr.reset(buffer); + buffer_ptr.reset(reinterpret_cast(buffer)); instance.EnqueueActivityBuffer( ProfilerActivityBuffer::CreateFromPreallocatedBuffer(std::move(buffer_ptr), valid_size)); } From c0afba1ff1fbf9cf3fb10d42c69d2e9d7a566b3f Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Tue, 29 Nov 2022 23:34:01 +0000 Subject: [PATCH 23/26] Ran clang-format on all changed files --- include/onnxruntime/core/common/gpu_profiler_common.h | 6 +++--- onnxruntime/core/providers/cuda/cuda_profiler.cc | 5 ++--- .../core/providers/shared_library/provider_api.h | 10 +++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index 1a87b428a1569..e3482d51226a5 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -358,8 +358,8 @@ class GPUProfilerBase : public EpProfiler { while (event_iter != event_end && (event_iter->ts < ts || (event_iter->ts == ts && - (event_iter + 1) != event_end && - (event_iter + 1)->ts == ts))) { + (event_iter + 1) != event_end && + (event_iter + 1)->ts == ts))) { merged_events.emplace_back(*event_iter); ++event_iter; } @@ -413,7 +413,7 @@ class GPUProfilerBase : public EpProfiler { uint64_t client_handle_; TimePoint profiling_start_time_; -public: + public: virtual bool StartProfiling(TimePoint profiling_start_time) override { auto& manager = TManager::GetInstance(); manager.StartLogging(); diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index 4419e0be02b21..492eff158b585 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - #include #include #include @@ -25,5 +24,5 @@ CudaProfiler::~CudaProfiler() { #endif /* #if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 */ -} // namespace onnxruntime -} // namespace profiling +} // namespace profiling +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 6876d94052231..5df59de207ec3 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -261,15 +261,15 @@ std::string GetEnvironmentVar(const std::string& var_name); namespace profiling { - std::string demangle(const char* name); - std::string demangle(const std::string& name); +std::string demangle(const char* name); +std::string demangle(const std::string& name); -} // namespace profiling +} // namespace profiling namespace logging { - unsigned int GetThreadId(); - unsigned int GetProcessId(); +unsigned int GetThreadId(); +unsigned int GetProcessId(); struct Category { static const char* onnxruntime; ///< General output From 4b9cecd034cb900e6a296175ac6dbdcb0074be6b Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Wed, 7 Dec 2022 22:40:59 +0000 Subject: [PATCH 24/26] Rework timestamp adjustment logic based on review feedback --- .../core/common/gpu_profiler_common.h | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index e3482d51226a5..ce86b77ebc925 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -351,6 +351,10 @@ class GPUProfilerBase : public EpProfiler { auto event_iter = std::make_move_iterator(events.begin()); auto event_end = std::make_move_iterator(events.end()); for (auto& map_iter : events_to_merge) { + if (map_iter.second.empty()) { + continue; + } + auto ts = static_cast(map_iter.first); // find the last occurence of a matching timestamp, @@ -364,8 +368,7 @@ class GPUProfilerBase : public EpProfiler { ++event_iter; } - uint64_t increment; - uint64_t last_ts; + int64_t origin_ts; bool copy_op_names = false; std::string op_name; std::string parent_name; @@ -375,8 +378,7 @@ class GPUProfilerBase : public EpProfiler { if (event_iter != event_end && event_iter->ts == ts) { // In this particular case we have located a parent event -- in the main event stream -- // for the GPU events. We use the timestamps from that event to adjust timestamps - last_ts = event_iter->ts; - increment = 1; + origin_ts = event_iter->ts + 1; copy_op_names = true; op_name = event_iter->args["op_name"]; parent_name = event_iter->name; @@ -385,10 +387,17 @@ class GPUProfilerBase : public EpProfiler { } else { // No parent event, let's just set the timestamp based on the // timestamp of the call to EpProfiler->Start() - last_ts = ts; - increment = 1; + origin_ts = ts; } + // calculate the offset from the origin to the + // first kernel event. Subsequent kernel event + // timestamps will have this offset subtracted from + // them to maintain relative timing between + // kernel events, while still roughly reconciling + // with the Jan 1 1970 epoch. + auto offset_from_origin = origin_ts - map_iter.second[0].ts; + for (auto& evt : map_iter.second) { if (copy_op_names) { // If we have found a matching parent event, @@ -396,8 +405,8 @@ class GPUProfilerBase : public EpProfiler { evt.args["op_name"] = op_name; evt.args["parent_name"] = parent_name; } - evt.ts = last_ts + increment; - increment += evt.dur; + + evt.ts += offset_from_origin; } merged_events.insert(merged_events.end(), From dfc3320b2615aca81ee892e4deca0da544788e1c Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Thu, 8 Dec 2022 23:20:26 +0000 Subject: [PATCH 25/26] Fix rename usage in dependency --- winml/adapter/winml_adapter_environment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winml/adapter/winml_adapter_environment.cpp b/winml/adapter/winml_adapter_environment.cpp index 81789aaaa628c..bb56a8f3a4ce8 100644 --- a/winml/adapter/winml_adapter_environment.cpp +++ b/winml/adapter/winml_adapter_environment.cpp @@ -28,7 +28,7 @@ class WinmlAdapterLoggingWrapper : public LoggingWrapper { if (profiling_function_) { OrtProfilerEventRecord ort_event_record = {}; ort_event_record.category_ = static_cast(event_record.cat); - ort_event_record.category_name_ = onnxruntime::profiling::event_categor_names_[event_record.cat]; + ort_event_record.category_name_ = onnxruntime::profiling::event_category_names_[event_record.cat]; ort_event_record.duration_ = event_record.dur; ort_event_record.event_name_ = event_record.name.c_str(); ort_event_record.execution_provider_ = (event_record.cat == onnxruntime::profiling::EventCategory::NODE_EVENT) ? event_record.args["provider"].c_str() : nullptr; From 4334a9f7a5e07bb7138264f6173329e71ad06b9e Mon Sep 17 00:00:00 2001 From: Abhishek Udupa Date: Fri, 9 Dec 2022 00:48:43 +0000 Subject: [PATCH 26/26] Fix typo --- include/onnxruntime/core/common/gpu_profiler_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/onnxruntime/core/common/gpu_profiler_common.h b/include/onnxruntime/core/common/gpu_profiler_common.h index ce86b77ebc925..dc27a1521ddcc 100644 --- a/include/onnxruntime/core/common/gpu_profiler_common.h +++ b/include/onnxruntime/core/common/gpu_profiler_common.h @@ -357,7 +357,7 @@ class GPUProfilerBase : public EpProfiler { auto ts = static_cast(map_iter.first); - // find the last occurence of a matching timestamp, + // find the last occurrence of a matching timestamp, // if one exists while (event_iter != event_end && (event_iter->ts < ts ||