diff --git a/projects/rocr-runtime/libhsakmt/include/impl/wddm/device.h b/projects/rocr-runtime/libhsakmt/include/impl/wddm/device.h index 49499514168c..3f006036fd36 100644 --- a/projects/rocr-runtime/libhsakmt/include/impl/wddm/device.h +++ b/projects/rocr-runtime/libhsakmt/include/impl/wddm/device.h @@ -64,6 +64,7 @@ #include "impl/wddm/types.h" #include "impl/wddm/gpu_memory.h" #include "impl/wddm/cmd_util.h" +#include "hsakmt/hsakmttypes.h" namespace wsl { namespace thunk { @@ -252,6 +253,10 @@ class WDDMDevice { private: bool Escape(void* priv_data, uint32_t priv_size, bool hw_access) const; NTSTATUS ParseDeviceInfo(void); + uint64_t AllocateCwsrSize(uint64_t* out_ctx_size = nullptr, uint64_t* out_debug_size = nullptr) const; + void FillCwsrHeader(void* cpu_addr, uint64_t ctx_save_restore_size, + uint64_t debug_memory_size, uint32_t num_xcc, + volatile HSAint64* error_reason, HSAuint32 error_event_id); void DestroyDeviceInfo(void); bool CreateDevice(void); bool DestroyDevice(void); diff --git a/projects/rocr-runtime/libhsakmt/include/impl/wddm/queue.h b/projects/rocr-runtime/libhsakmt/include/impl/wddm/queue.h index 1a9838ff97cc..f4429d1e48c1 100644 --- a/projects/rocr-runtime/libhsakmt/include/impl/wddm/queue.h +++ b/projects/rocr-runtime/libhsakmt/include/impl/wddm/queue.h @@ -141,6 +141,12 @@ class WDDMQueue { std::atomic* ring_rptr = nullptr; uint32_t aql_doorbell_offset_ = 0; //!< Doorbell offset for this AQL queue + + bool needs_cwsr_ = true; //!< false for SDMA queues (mirrors Linux handle_concrete_asic() SDMA early-return) + GpuMemoryHandle cwsr_mem_ = nullptr; //!< CWSR (Context Wave Save/Restore) memory allocation + D3DKMT_HANDLE cwsr_mem_handle_ = 0; //!< KMT allocation handle of CWSR region (passed as CwsrMemHandle) + volatile int64_t* error_reason_ = nullptr; //!< ErrorReason payload ptr (QueueResource::ErrorReason) + HSAuint32 error_event_id_ = 0; //!< ErrorEventId from HsaEvent::EventId (0 if no event) }; class ComputeQueue : public WDDMQueue { @@ -153,7 +159,8 @@ class ComputeQueue : public WDDMQueue { volatile int64_t *error_addr, uint32_t cmdbuf_size, uint32_t engine, - bool use_hws = true); + bool use_hws = true, + HSAuint32 event_id = 0); ~ComputeQueue(); diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/queues.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/queues.cpp index af76744c3e8d..f8fb999b8d0f 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/queues.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/queues.cpp @@ -84,7 +84,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueueV2(HSAuint32 NodeId, HSAKMT_STATUS result; CHECK_DXG_OPEN(); - assert(Event == nullptr); #if defined(__linux__) // Defer resolving libhsa-runtime64 symbols until a queue is actually needed. @@ -117,13 +116,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueueV2(HSAuint32 NodeId, uint32_t cmdbuf_size = device_->GetCmdbufSize(); uint32_t queue_engine = device_->GetComputeEngine(); bool use_hws = device_->IsHwsEnabled(queue_engine); + HSAuint32 event_id = Event ? Event->EventId : 0; auto queue_ = new wsl::thunk::ComputeQueue( device_, QueueAddress, pkg_num, reinterpret_cast *>( QueueResource->Queue_write_ptr_aql), reinterpret_cast *>( QueueResource->Queue_read_ptr_aql), - QueueResource->ErrorReason, cmdbuf_size, queue_engine, use_hws); + QueueResource->ErrorReason, cmdbuf_size, queue_engine, use_hws, event_id); QueueResource->QueueId = reinterpret_cast(queue_); // for doorbell_signal.hardware_doorbell_ptr diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/device.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/device.cpp index 9abb16a680ff..646775f7b14d 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/device.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/device.cpp @@ -874,10 +874,180 @@ bool WDDMDevice::SubmitToSwQueue(WDDMQueue *queue, uint64_t command_addr, return true; } +// Compute the CWSR (Context Wave Save/Restore) region size for this device. +// +// Mirrors the Linux KFD calculation in update_ctx_save_restore_size() (queues.c). +// The region must hold, per XCC: +// - HsaUserContextSaveAreaHeader (save-area header) +// - Control stack (wave_num * bytes_per_wave + 8), page-aligned +// - WG data (VGPR + SGPR + LDS + HW-regs per CU), page-aligned +// multiplied by num_xcc for multi-XCC devices. +// +// Constants ported from queues.c / dxg/queues.cpp: +// CNTL_STACK_BYTES_PER_WAVE : gfx10+ = 12, older = 8 +// WG_CONTEXT_DATA_SIZE_PER_CU : vgpr_size + SGPR_SIZE_PER_CU + LDS + HWREG_SIZE_PER_CU +// SGPR_SIZE_PER_CU: gfx10+ = 0x5000 (20 KB), older = 0x4000 (16 KB) [matches KMD CalCwsrSaveAreaSize] +// HWREG_SIZE_PER_CU: gfx10+ = 0x1400 (5 KB), older = 0x1000 (4 KB) [matches KMD CalCwsrSaveAreaSize] +// vgpr_size: all gfx10/gfx11/gfx12 = 256KB/CU (0x40000) [matches KMD CalCwsrSaveAreaSize] +// DEBUGGER_BYTES_PER_WAVE = 32, DEBUGGER_BYTES_ALIGN = 64 +uint64_t WDDMDevice::AllocateCwsrSize(uint64_t* out_ctx_size, uint64_t* out_debug_size) const { + const uint32_t page_size = 4096; + const uint32_t debugger_bytes_per_wave = 32; + const uint32_t debugger_bytes_align = 64; + + const int major = device_info_.major; + const uint32_t num_xcc = device_info_.num_xcc ? device_info_.num_xcc : 1; + // compute_unit_count in device_info_ is the total CU count across all XCCs. + const uint32_t cu_num = device_info_.compute_unit_count / num_xcc; + // wave_per_cu from device_info_ is per-CU (40 for Navi10/12/14, 32 for Navi2x+). + // KMD CalCwsrSaveAreaSize() split: gfx10+ (FAMILY_NV) vs pre-gfx10 + const bool is_gfx10_plus = (major >= 10); + + const uint32_t wave_per_cu = device_info_.wave_per_cu ? device_info_.wave_per_cu : 32; + uint32_t wave_num = cu_num * wave_per_cu; + // Pre-Navi (gfx9): Linux queues.c caps wave_num at NumShaderBanks/NumArrays*512. + // Matches get_num_waves() in queues.c for gfxv < GFX_VERSION_NAVI10. + // num_shader_engine is total across all XCCs (wkmi.cpp scales it by num_xcc for gfx9.4), + // so divide by num_xcc to get per-XCC SE count, matching cu_num which is also per-XCC. + if (!is_gfx10_plus) { + const uint32_t num_se = device_info_.num_shader_engine / num_xcc; + const uint32_t num_sa = device_info_.shader_array_per_shader_engine; + if (num_se > 0 && num_sa > 0) + wave_num = std::min(wave_num, (num_se / num_sa) * 512u); + } + + // Control stack: gfx10+ = 12 bytes/wave, older = 8 bytes/wave + const uint32_t bytes_per_wave = is_gfx10_plus ? 12 : 8; + uint32_t ctl_stack_size = wave_num * bytes_per_wave + 8; + // gfx10.x (Navi) HW control stack RAM is physically limited to 0x7000 bytes. + // Matches Linux queues.c: if ((gfxv & 0x3f0000) == 0xA0000) ctl_stack_size = MIN(..., 0x7000) + // major == 10 covers the full gfx10.x family (Navi10/12/14, Navi21/22/23/24). + if (major == 10) + ctl_stack_size = std::min(ctl_stack_size, 0x7000u); + + // Sizes from KMD kdx/src/RunListMgr.cpp CalCwsrSaveAreaSize(): + // gfx10+: sgpr=0x5000, hwreg=0x1400, vgpr=0x40000 + // pre-gfx10: sgpr=0x4000, hwreg=0x1000, vgpr=0x40000 + const uint32_t vgpr_size_per_cu = 0x40000; + const uint32_t sgpr_size_per_cu = is_gfx10_plus ? 0x5000 : 0x4000; + const uint32_t hwreg_size_per_cu = is_gfx10_plus ? 0x1400 : 0x1000; + // LDS size stored in device_info_.lds_size (bytes) + const uint32_t lds_size_bytes = device_info_.lds_size; + const uint32_t wg_size_per_cu = vgpr_size_per_cu + sgpr_size_per_cu + lds_size_bytes + hwreg_size_per_cu; + const uint32_t wg_data_size = cu_num * wg_size_per_cu; + + // Align sizes to page boundary + auto page_align_up = [&](uint64_t x) -> uint64_t { + return (x + page_size - 1) & ~(uint64_t)(page_size - 1); + }; + auto align_up = [&](uint64_t x, uint32_t align) -> uint64_t { + return (x + align - 1) & ~(uint64_t)(align - 1); + }; + + // ctx_save_restore_size per XCC (header + ctl_stack + wg_data) + uint64_t ctx_size = page_align_up(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size) + + page_align_up(wg_data_size); + + // debug_memory_size per XCC + uint64_t debug_size = align_up(wave_num * debugger_bytes_per_wave, debugger_bytes_align); + + if (out_ctx_size) *out_ctx_size = ctx_size; + if (out_debug_size) *out_debug_size = debug_size; + + uint64_t total = page_align_up((ctx_size + debug_size) * num_xcc); + + return total; +} + +// Initialize HsaUserContextSaveAreaHeader for each XCC in the CWSR region. +// +// Mirrors Linux's fill_cwsr_header() in queues.c. The CWSR allocation is +// divided into equal-sized per-XCC slots of ctx_save_restore_size bytes, each +// beginning with an HsaUserContextSaveAreaHeader. This must be called after +// the CPU-accessible system memory is allocated so the runtime and debugger can +// locate the control stack, wave state, and debug areas on context save. +// +// Layout per XCC slot (offsets relative to slot base): +// [0] HsaUserContextSaveAreaHeader +// [ctl_stack_offset] Control stack (ctl_stack_size bytes, page-aligned) +// [wg_data_offset] Wave/WG state (wg_data_size bytes, page-aligned) +// [debug_offset] Debugger area (debug_memory_size bytes, 64-byte aligned) +// +// DebugOffset in each slot's header is relative to that slot's own base address, +// not the start of the allocation. It points forward to the debug area at the end +// of the last XCC slot: (NumXcc - i) * ctx_save_restore_size from slot i's base. +void WDDMDevice::FillCwsrHeader(void* cpu_addr, uint64_t ctx_save_restore_size, + uint64_t debug_memory_size, uint32_t num_xcc, + volatile HSAint64* error_reason, HSAuint32 error_event_id) { + for (uint32_t i = 0; i < num_xcc; i++) { + auto* header = reinterpret_cast( + static_cast(cpu_addr) + i * ctx_save_restore_size); + + // ErrorEventId: the EventId of the HsaEvent passed at queue creation by the + // runtime (HSA_EVENTTYPE_SIGNAL, shared across all queues on the agent). + // Mirrors Linux fill_cwsr_header(): Event ? Event->EventId : 0. + header->ErrorEventId = error_event_id; + + // ErrorReason mirrors Linux fill_cwsr_header(): pointer to the HSA signal + // payload used by the runtime to report the error reason bitmask on + // queue exception. Sourced from QueueResource->ErrorReason, stored on the + // queue as error_reason_ and passed through here. + header->ErrorReason = error_reason; + + // DebugOffset is from this XCC's slot base to the debug area of the *last* + // XCC slot, matching fill_cwsr_header(): + // header->DebugOffset = (NumXcc - i) * ctx_save_restore_size + header->DebugOffset = static_cast((num_xcc - i) * ctx_save_restore_size); + header->DebugSize = static_cast(debug_memory_size * num_xcc); + + // ControlStackOffset/Size and WaveStateOffset/Size describe where the + // saved control stack and wave state ended up inside this XCC's slot. + // They are written by the kernel (KFD/KMD) during AMDKFD_IOC_GET_QUEUE_WAVE_STATE + // after preemption; see struct kfd_context_save_area_header::wave_state. + // Zero is the correct initial value — no context has been saved yet. + // rocdbgapi reads these fields (queue.cpp) only after a context save has + // occurred, so zeroing here is safe and matches Linux fill_cwsr_header(). + header->ControlStackOffset = 0; + header->ControlStackSize = 0; + header->WaveStateOffset = 0; + header->WaveStateSize = 0; + header->Reserved1 = 0; + } +} + bool WDDMDevice::CreateHwQueue(WDDMQueue *queue) { void *priv_data; int priv_size; + // Allocate CWSR (Context Wave Save/Restore) region in system (GTT) memory. + // Matches Linux: anonymous mmap + register_svm_range(alwaysMapped=true). + // locked keeps pages pinned (HSA_SVM_FLAG_GPU_ALWAYS_MAPPED equivalent). + // The allocation handle is passed to KMD via UMDKMDIF_CREATEHWQUEUE_PRIVATE_DATA::CwsrMemHandle. + // SDMA queues skip CWSR — mirrors Linux handle_concrete_asic() which returns + // early for KFD_IOC_QUEUE_TYPE_SDMA/SDMA_XGMI before allocating ctx_save_restore. + if (queue->cwsr_mem_ == nullptr && queue->needs_cwsr_) { + uint64_t ctx_save_restore_size = 0; + uint64_t debug_memory_size = 0; + GpuMemoryCreateInfo cwsr_create_info{}; + cwsr_create_info.domain = Wkmi::kSystem; + cwsr_create_info.size = AllocateCwsrSize(&ctx_save_restore_size, &debug_memory_size); + + GpuMemory *cwsr_gpu_mem = nullptr; + ErrorCode cwsr_code = CreateGpuMemory(cwsr_create_info, &cwsr_gpu_mem); + if (cwsr_code != ErrorCode::Success) { + pr_err("CWSR memory allocation failed\n"); + return false; + } + queue->cwsr_mem_ = cwsr_gpu_mem->GetGpuMemoryHandle(); + queue->cwsr_mem_handle_ = cwsr_gpu_mem->KmtHandle(); + + // Initialise the per-XCC HsaUserContextSaveAreaHeader in the CPU-visible + // system memory, mirroring fill_cwsr_header() in Linux queues.c. + const uint32_t num_xcc = device_info_.num_xcc ? device_info_.num_xcc : 1; + FillCwsrHeader(cwsr_gpu_mem->CpuAddress(), ctx_save_restore_size, + debug_memory_size, num_xcc, queue->error_reason_, queue->error_event_id_); + } + priv_size = Wkmi::GetHwQueuePrivDataSize(); priv_data = malloc(priv_size); assert(priv_data); @@ -897,7 +1067,8 @@ bool WDDMDevice::CreateHwQueue(WDDMQueue *queue) { } Wkmi::FillinHwQueuePrivData(priv_data, FwManagedGfxState, queue->prio, is_aql, queue->cmdbuf_addr, queue->cmdbuf_size, reinterpret_cast(queue->ring_wptr), - reinterpret_cast(queue->ring_rptr), resource, &doorbell_loc); + reinterpret_cast(queue->ring_rptr), resource, &doorbell_loc, + queue->cwsr_mem_handle_); D3DKMT_CREATEHWQUEUE createHwQueue = {0}; createHwQueue.hHwContext = queue->context; @@ -909,6 +1080,11 @@ bool WDDMDevice::CreateHwQueue(WDDMQueue *queue) { if (ret != STATUS_SUCCESS) { pr_err("fail %x\n", ret); free(priv_data); + if (queue->cwsr_mem_ != nullptr) { + delete GpuMemory::Convert(queue->cwsr_mem_); + queue->cwsr_mem_ = nullptr; + queue->cwsr_mem_handle_ = 0; + } return false; } if (doorbell_loc != nullptr) { @@ -935,6 +1111,13 @@ bool WDDMDevice::DestroyHwQueue(WDDMQueue *queue) { return false; } + if (queue->cwsr_mem_ != nullptr) { + auto cwsr_gpu_mem = GpuMemory::Convert(queue->cwsr_mem_); + delete cwsr_gpu_mem; + queue->cwsr_mem_ = nullptr; + queue->cwsr_mem_handle_ = 0; + } + return true; } diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/queue.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/queue.cpp index c37f47db5db2..4db852bbe5d3 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/queue.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/queue.cpp @@ -248,7 +248,7 @@ void ComputeQueue::AqlToPm4Thread(ComputeQueue* queue) { ComputeQueue::ComputeQueue(WDDMDevice* device, void* ring, uint64_t ring_size, std::atomic* _ring_wptr, std::atomic* _ring_rptr, volatile int64_t* error_addr, uint32_t cmdbuf_size, uint32_t engine, - bool use_hws) + bool use_hws, HSAuint32 event_id) : WDDMQueue( device, (use_hws && device->IsAqlSupported()) ? reinterpret_cast(ring) : 0, (use_hws && device->IsAqlSupported()) ? ring_size * 64 : cmdbuf_size, engine, use_hws), @@ -271,6 +271,8 @@ ComputeQueue::ComputeQueue(WDDMDevice* device, void* ring, uint64_t ring_size, scratch_base_(nullptr) { ring_wptr = _ring_wptr; ring_rptr = _ring_rptr; + error_reason_ = error_addr; + error_event_id_ = event_id; amd_queue_rocr_ = (amd_queue_v2_t*)((char*)ring_rptr - offsetof(amd_queue_t, read_dispatch_id)); amd_queue_memory_ = GetGpuMemoryFromAddress(amd_queue_rocr_); native_aql_ = use_hws && device->IsAqlSupported(); @@ -1147,6 +1149,7 @@ SDMAQueue::SDMAQueue(WDDMDevice* device, void* ring, uint64_t cmdbuf_size, uint3 thread_stop_(false), ib_size(0), ib_start_addr(0) { + needs_cwsr_ = false; bool ret = device->CreateQueue(this); assert(ret);