diff --git a/projects/rocr-runtime/libhsakmt/include/impl/wddm/gpu_memory.h b/projects/rocr-runtime/libhsakmt/include/impl/wddm/gpu_memory.h index 9cafa058774d..1d9595b0a2b3 100644 --- a/projects/rocr-runtime/libhsakmt/include/impl/wddm/gpu_memory.h +++ b/projects/rocr-runtime/libhsakmt/include/impl/wddm/gpu_memory.h @@ -44,6 +44,7 @@ #include #include +#include #include "util/utils.h" #include "impl/wddm/types.h" #include "impl/wddm/thunks.h" @@ -259,6 +260,14 @@ class GpuMemory { ErrorCode MapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0); ErrorCode UnmapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0); + + // This allocation's view on dev. System and user memory can create a peer view + // at the owner's VA; the owner keeps that reservation. *created is set when a + // new peer view was added. + GpuMemory* ViewOn(WDDMDevice* dev, bool create, bool* created); + void DropPeer(GpuMemory* peer); + void ReleasePeers(); + GpuMemory* Owner() const { return owner_; } ErrorCode MapMemoryToVirtualAddress(bool create_phys_mem = true); ErrorCode MakeResident(); ErrorCode Evict(); @@ -286,11 +295,12 @@ class GpuMemory { protected: explicit GpuMemory(WDDMDevice *device); -private: - uint64_t AdjustSize(gpusize size) const; private: friend class WDDMDevice; + uint64_t AdjustSize(gpusize size) const; + ErrorCode MapFromPeer(GpuMemory* owner); + WDDMDevice *const device_; GpuMemoryDesc desc_; @@ -303,6 +313,9 @@ class GpuMemory { int mem_fd_; // IPC sigal's sys mem fd + GpuMemory* owner_ = nullptr; // set when this object is a peer view + std::vector peers_; // peer views; only the owner has entries + bool is_phymem_created = false; // status of physical memory allocation bool is_sysmem_locked_ = false; // kSystem allocation pinned via D3DKMTLock2 diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/libdrm.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/libdrm.cpp index 03af80e574d1..3289a185fc93 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/libdrm.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/libdrm.cpp @@ -120,12 +120,13 @@ HSAKMTAPI int amdgpu_bo_import(amdgpu_device_handle dev, } wsl::thunk::WDDMDevice *pDevice = reinterpret_cast(dev); + HSAuint32 node_id = static_cast(pDevice->NodeId()); wsl::thunk::GpuMemoryHandle mem_handle; bool is_ipc_memfd = is_ipc_sysmemfd(shared_handle); bool alloc_va = is_ipc_memfd; // kmt handle importer is false for dma_buf_fd - HSAKMT_STATUS ret = import_dmabuf_fd(shared_handle, pDevice->NodeId(), alloc_va, is_ipc_memfd, + HSAKMT_STATUS ret = import_dmabuf_fd(shared_handle, 1, &node_id, alloc_va, is_ipc_memfd, &mem_handle, false); if (ret == HSAKMT_STATUS_SUCCESS) { //use GpuMemory object handle as drm buf handle diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/librocdxg.h b/projects/rocr-runtime/libhsakmt/src/dxg/librocdxg.h index 3287fc08ecf5..807175372d27 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/librocdxg.h +++ b/projects/rocr-runtime/libhsakmt/src/dxg/librocdxg.h @@ -36,6 +36,7 @@ #endif #include #include +#include #include "hsakmt/hsakmt.h" #if defined(__linux__) #include "hsakmt/hsakmt_drm.h" @@ -70,8 +71,7 @@ struct hsakmtRuntime { system_heap_space_start_(0), system_heap_space_size_(0), handle_aperture_start_(0), - handle_aperture_size_(0), - default_node(1) {} + handle_aperture_size_(0) {} void HeapInit(); void HeapFini(); @@ -113,7 +113,8 @@ struct hsakmtRuntime { unsigned long dxg_open_count; size_t max_single_alloc_size; - uint32_t default_node; + /* GPU HSA nodes used when the caller does not name a node. */ + std::vector default_node; D3DKMT_DRIVERVERSION wddm_version = static_cast(0); /* local heap means bo's backend is vram of all GPUs */ @@ -305,7 +306,8 @@ uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id); bool is_ipc_sysmemfd(uint64_t fd); -HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, uint32_t NodeId, bool alloc_va, bool is_ipc_memfd, +HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, HSAuint64 NumberOfNodes, HSAuint32* NodeArray, + bool alloc_va, bool is_ipc_memfd, wsl::thunk::GpuMemoryHandle* GpuMemHandle, bool is_kmt_handle, uint64_t size_hint = 0); diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/memory.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/memory.cpp index f66d0b982514..595c4013f597 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/memory.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/memory.cpp @@ -35,6 +35,7 @@ #endif #include #include +#include #include "impl/wddm/gpu_memory.h" #include "util/simple_heap.h" #include "util/os.h" @@ -200,7 +201,12 @@ HSAKMT_STATUS hsaKmtAllocMemoryAlignInternal(HSAuint32 PreferredNode, } else *MemoryAddress = nullptr; - uint32_t node = (PreferredNode < CpuNodes()) ? dxg_runtime->default_node : PreferredNode; + uint32_t node = PreferredNode; + if (PreferredNode < CpuNodes()) { + if (dxg_runtime->default_node.empty()) + return HSAKMT_STATUS_ERROR; + node = dxg_runtime->default_node[0]; + } wsl::thunk::WDDMDevice *dev = get_wddmdev(node); if (!dev) return HSAKMT_STATUS_ERROR; @@ -374,6 +380,7 @@ HSAKMT_STATUS hsaKmtFreeMemoryInternal(void *MemoryAddress, #endif it->second.dmabuf_fd = -1; } + gpu_mem->ReleasePeers(); allocation_map_->erase(it); } @@ -536,7 +543,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodesExt(HSAuint64 Graphic #if defined(__linux__) if (is_ipc_sysmemfd(GraphicsResourceHandle)) { - GraphicsResourceInfo->NodeId = dxg_runtime->default_node; + if (dxg_runtime->default_node.empty()) + return HSAKMT_STATUS_ERROR; + GraphicsResourceInfo->NodeId = dxg_runtime->default_node[0]; struct stat st; if (fstat(static_cast(GraphicsResourceHandle), &st) == 0) @@ -548,9 +557,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodesExt(HSAuint64 Graphic #endif if (NumberOfNodes == 0) { + if (dxg_runtime->default_node.empty()) + return HSAKMT_STATUS_ERROR; RegisterFlags.ui32.requiresVAddr = 0; - NumberOfNodes = 1; - NodeArray = (HSAuint32*)&(dxg_runtime->default_node); + NumberOfNodes = dxg_runtime->default_node.size(); + NodeArray = dxg_runtime->default_node.data(); } else { RegisterFlags.ui32.requiresVAddr = 1; @@ -559,7 +570,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodesExt(HSAuint64 Graphic pr_debug("number of nodes %" PRIu64 "\n", NumberOfNodes); wsl::thunk::GpuMemoryHandle mem_handle; - ret = import_dmabuf_fd(GraphicsResourceHandle, NodeArray[0], RegisterFlags.ui32.requiresVAddr, + ret = import_dmabuf_fd(GraphicsResourceHandle, NumberOfNodes, NodeArray, + RegisterFlags.ui32.requiresVAddr, false, &mem_handle, RegisterFlags.ui32.kmtHandle, GraphicsResourceInfo->SizeHintInBytes); if (ret != HSAKMT_STATUS_SUCCESS) { @@ -621,13 +633,39 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtExportDMABufHandle(void *MemoryAddress, } -HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, uint32_t NodeId, bool alloc_va, bool is_ipc_memfd, +static HSAKMT_STATUS map_to_nodes(wsl::thunk::GpuMemory* mem, HSAuint64 count, + HSAuint32* nodes) { + std::vector added; + + for (HSAuint64 i = 0; i < count; ++i) { + auto* dev = get_wddmdev(nodes[i]); + bool created = false; + auto* view = dev ? mem->ViewOn(dev, true, &created) : nullptr; + if (view == nullptr) { + pr_err("failed to map memory to node %u\n", nodes[i]); + for (auto* peer : added) + mem->DropPeer(peer); + return HSAKMT_STATUS_ERROR; + } + if (created) + added.push_back(view); + } + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, HSAuint64 NumberOfNodes, HSAuint32* NodeArray, + bool alloc_va, bool is_ipc_memfd, wsl::thunk::GpuMemoryHandle* GpuMemHandle, bool is_kmt_handle, uint64_t size_hint) { CHECK_DXG_OPEN(); + if (NumberOfNodes == 0 || NodeArray == nullptr) + return HSAKMT_STATUS_ERROR; + *GpuMemHandle = nullptr; - wsl::thunk::WDDMDevice* dev = get_wddmdev(NodeId); + wsl::thunk::WDDMDevice* dev = get_wddmdev(NodeArray[0]); + if (!dev) + return HSAKMT_STATUS_ERROR; wsl::thunk::GpuMemory *gpu_mem = nullptr; wsl::thunk::GpuMemoryCreateInfo create_info{}; create_info.dmabuf_fd = DMABufFd; @@ -662,6 +700,8 @@ HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, uint32_t NodeId, bool alloc_va wsl::thunk::GpuMemory *conflict_mem = wsl::thunk::GpuMemory::Convert(it->second.handle); conflict_mem->IncSharedReference(); *GpuMemHandle = it->second.handle; + if (NumberOfNodes > 1) + return map_to_nodes(conflict_mem, NumberOfNodes, NodeArray); return HSAKMT_STATUS_SUCCESS; } else if (code != ErrorCode::Success) { pr_err("fail to import fd, ret %d\n", (int)code); @@ -676,6 +716,15 @@ HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, uint32_t NodeId, bool alloc_va *GpuMemHandle = gpu_mem->GetGpuMemoryHandle(); + if (NumberOfNodes > 1) { + auto ret = map_to_nodes(gpu_mem, NumberOfNodes, NodeArray); + if (ret != HSAKMT_STATUS_SUCCESS) { + delete gpu_mem; + *GpuMemHandle = nullptr; + return ret; + } + } + std::lock_guard gard(*allocation_map_lock_); /* * the gpu_mem->Flags() need convert back from GpuMemoryCreateFlags to @@ -684,7 +733,7 @@ HSAKMT_STATUS import_dmabuf_fd(uint64_t DMABufFd, uint32_t NodeId, bool alloc_va (*allocation_map_)[MemoryAddress] = Allocation( *GpuMemHandle, MemoryAddress, (uint64_t)MemoryAddress, gpu_mem->Size(), false, nullptr, gpu_mem->ClientSize(), - NodeId, gpu_mem->Flags()); + NodeArray[0], gpu_mem->Flags()); return HSAKMT_STATUS_SUCCESS; } @@ -806,15 +855,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress) { HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, HSAuint64 MemorySizeInBytes, HSAuint64 *AlternateVAGPU) { - - HSAuint64 NumberOfNodes = 1; - HSAuint32 NodeArray[] = {dxg_runtime->default_node}; HsaMemFlags MemFlags; MemFlags.Value = 0; MemFlags.ui32.CoarseGrain = 1; return hsaKmtMapMemoryToGPUNodes(MemoryAddress, MemorySizeInBytes, AlternateVAGPU, - MemFlags, NumberOfNodes, NodeArray); + MemFlags, dxg_runtime->default_node.size(), dxg_runtime->default_node.data()); } HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( @@ -826,6 +872,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( pr_err("FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; } + if (NumberOfNodes == 0 || NodeArray == nullptr) + return HSAKMT_STATUS_ERROR; uint64_t start = rocr::AlignDown((uint64_t)MemoryAddress, 4096); uint64_t end = @@ -870,7 +918,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( // GTT/Local mem if (it->size >= MemorySizeInBytes) { *AlternateVAGPU = (uint64_t)MemoryAddress; - return HSAKMT_STATUS_SUCCESS; + return map_to_nodes(gpu_mem, NumberOfNodes, NodeArray); } else { return HSAKMT_STATUS_ERROR; } @@ -880,10 +928,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( // Reuse the user-pointer mapping already registered for this address. auto *allocation = FindAllocation(MemoryAddress, aligned_size); if (allocation != nullptr && allocation->userptr) { - wsl::thunk::GpuMemory::Convert(allocation->handle)->IncMappingCount(); + auto* mapped = wsl::thunk::GpuMemory::Convert(allocation->handle); *AlternateVAGPU = (uintptr_t)allocation->gpu_addr + ((uintptr_t)MemoryAddress - (uintptr_t)allocation->cpu_addr); - return HSAKMT_STATUS_SUCCESS; + auto status = map_to_nodes(mapped, NumberOfNodes, NodeArray); + if (status == HSAKMT_STATUS_SUCCESS) + mapped->IncMappingCount(); + return status; } } @@ -911,9 +962,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( return HSAKMT_STATUS_ERROR; } + auto ret = map_to_nodes(gpu_mem, NumberOfNodes, NodeArray); + if (ret != HSAKMT_STATUS_SUCCESS) { + delete gpu_mem; + return ret; + } + { std::lock_guard guard(*allocation_map_lock_); - (*allocation_map_)[MemoryAddress] = + (*allocation_map_)[MemoryAddress] = Allocation(handle, aligned_ptr, addr, aligned_size, true, MemoryAddress, MemorySizeInBytes); (*allocation_map_)[(void *)addr] = @@ -958,14 +1015,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) { // IPC mem wsl::thunk::GpuMemoryDescFlags flags; flags.reserved = gpu_mem->Flags(); - if (flags.is_imported_vram_ipc && - !gpu_mem->IsSharedFromSameProcess()) { + if (flags.is_imported_vram_ipc) { + if (gpu_mem->IsSharedFromSameProcess()) + return HSAKMT_STATUS_SUCCESS; + auto code = gpu_mem->UnmapGpuVirtualAddress(gpu_mem->GpuAddress(), gpu_mem->Size()); if (code != ErrorCode::Success) return HSAKMT_STATUS_ERROR; gpu_mem->Evict(); - - return HSAKMT_STATUS_SUCCESS; + if (!gpu_mem->GetDevice()->WaitOnPagingFenceFromCpu()) + return HSAKMT_STATUS_ERROR; } if (it->second.userptr) { @@ -976,6 +1035,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) { } return HSAKMT_STATUS_SUCCESS; } + + gpu_mem->ReleasePeers(); } return HSAKMT_STATUS_SUCCESS; @@ -1148,11 +1209,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtHandleImport(const HsaHandleImportDesc* import_des wsl::thunk::GpuMemoryHandle mem_handle; wsl::thunk::WDDMDevice *pDevice = reinterpret_cast(import_desc->device_handle); + HSAuint32 node_id = static_cast(pDevice->NodeId()); bool is_ipc_memfd = is_ipc_sysmemfd(import_desc->dmabuf_fd); bool alloc_va = is_ipc_memfd; // kmt handle importer is false for dma_buf_fd - HSAKMT_STATUS ret = import_dmabuf_fd(import_desc->dmabuf_fd, pDevice->NodeId(), alloc_va, is_ipc_memfd, + HSAKMT_STATUS ret = import_dmabuf_fd(import_desc->dmabuf_fd, 1, &node_id, alloc_va, is_ipc_memfd, &mem_handle, false); if (ret == HSAKMT_STATUS_SUCCESS) { //use GpuMemory object handle as drm buf handle @@ -1202,32 +1264,51 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMemoryVaMap(HsaMemoryObjectHandle Handle, HsaMemoryMapFlags flags, HSAuint32 NodeId) { CHECK_DXG_OPEN(); - (void)NodeId; + (void)flags; wsl::thunk::GpuMemory* gpu_mem = reinterpret_cast(Handle); assert(gpu_mem != nullptr); - if (gpu_mem->GpuAddress() == addr) { + wsl::thunk::WDDMDevice* target = get_wddmdev(NodeId); + if (!target) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + // Page-table update is submitted to NodeId. That is the paging fence this + // call advances, so wait on the target device. + wsl::thunk::GpuMemory* view = gpu_mem; + if (gpu_mem->GetDevice() != target) { + std::lock_guard guard(*allocation_map_lock_); + auto* owner = gpu_mem->Owner() ? gpu_mem->Owner() : gpu_mem; + view = owner->ViewOn(target, false, nullptr); + if (view == nullptr) { + if (owner->GpuAddress() != addr) + return HSAKMT_STATUS_ERROR; + view = owner->ViewOn(target, true, nullptr); + if (view == nullptr) + return HSAKMT_STATUS_ERROR; + // ViewOn already mapped this VA and waited on the target. + return HSAKMT_STATUS_SUCCESS; + } + } + + if (view->GpuAddress() == addr) { pr_info("bo is mapped already\n"); return HSAKMT_STATUS_SUCCESS; - } else if (gpu_mem->GpuAddress()) { + } else if (view->GpuAddress()) { pr_err("amdgpu_bo_va_op: GPU memory already mapped at %p, but requested to map at %p\n", - reinterpret_cast(gpu_mem->GpuAddress()), reinterpret_cast(addr)); + reinterpret_cast(view->GpuAddress()), reinterpret_cast(addr)); return HSAKMT_STATUS_ERROR; } - auto code = gpu_mem->MapGpuVirtualAddress(static_cast(addr), size, offset); + auto code = view->MapGpuVirtualAddress(static_cast(addr), size, offset); if (code != ErrorCode::Success) return HSAKMT_STATUS_ERROR; - code = gpu_mem->MakeResident(); + code = view->MakeResident(); if (code != ErrorCode::Success) return HSAKMT_STATUS_ERROR; - // Wait on paging fence to ensure map and residency are completed. MakeResident() - // updates the same paging fence as Map(), hence it's safe to wait for the last one. - if (!gpu_mem->GetDevice()->WaitOnPagingFenceFromCpu()) { + if (!target->WaitOnPagingFenceFromCpu()) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } @@ -1236,19 +1317,28 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMemoryVaUnmap(HsaMemoryObjectHandle Handle, HSAuint64 offset, HSAuint64 size, HSAuint64 addr, HSAuint32 NodeId) { CHECK_DXG_OPEN(); - (void)NodeId; wsl::thunk::GpuMemory* gpu_mem = reinterpret_cast(Handle); assert(gpu_mem != nullptr); - auto code = gpu_mem->UnmapGpuVirtualAddress(static_cast(addr), size, offset); + wsl::thunk::WDDMDevice* target = get_wddmdev(NodeId); + if (!target) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + wsl::thunk::GpuMemory* view = gpu_mem; + if (gpu_mem->GetDevice() != target) { + std::lock_guard guard(*allocation_map_lock_); + view = gpu_mem->ViewOn(target, false, nullptr); + if (view == nullptr) + return HSAKMT_STATUS_SUCCESS; + } + + auto code = view->UnmapGpuVirtualAddress(static_cast(addr), size, offset); if (code != ErrorCode::Success) return HSAKMT_STATUS_ERROR; - // Wait on paging fence to ensure unmap is completed. Evict() doesn't - // update the fence. - if (!gpu_mem->GetDevice()->WaitOnPagingFenceFromCpu()) { + // Unmap updates the target device's paging fence. Evict() does not. + if (!target->WaitOnPagingFenceFromCpu()) return HSAKMT_STATUS_ERROR; - } - gpu_mem->Evict(); + view->Evict(); return HSAKMT_STATUS_SUCCESS; } diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/openclose.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/openclose.cpp index 8c1032518301..37619839017a 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/openclose.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/openclose.cpp @@ -721,16 +721,6 @@ static HSAKMT_STATUS init_vars_from_env(void) { dxg_runtime->sdma_gpu_poll_ = (envvar[0] == '1' && envvar[1] == '\0'); } - // Set default GPU node based on ROCR_VISIBLE_DEVICES. Extract the first numeric value - // from the environment variable and add 1 to convert to 1-indexed node ID. - if ((envvar = getenv("ROCR_VISIBLE_DEVICES")) != nullptr) { - std::string devices(envvar); - size_t first_num_pos = devices.find_first_of("0123456789"); - if (first_num_pos != std::string::npos) { - dxg_runtime->default_node = std::stoi(devices.substr(first_num_pos)); - } - } - // Determine SVM (Shared Virtual Memory) API support. Enabled by default unless // HSA_USE_SVM is explicitly set to "0". Currently forced to false. envvar = getenv("HSA_USE_SVM"); diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/topology.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/topology.cpp index 7ab01f36f926..f2f33b7428ae 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/topology.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/topology.cpp @@ -854,8 +854,10 @@ HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties& props) { dxg_topology->num_sysfs_nodes = dxg_topology->numa_node_count_ + num_adapters; dxg_runtime->HeapInit(); props.NumNodes = dxg_topology->num_sysfs_nodes; - // Update default GPU node to account CPU nodes - dxg_runtime->default_node = dxg_topology->numa_node_count_; + // Every enumerated GPU is a default map target. + dxg_runtime->default_node.clear(); + for (uint32_t i = 0; i < num_adapters; ++i) + dxg_runtime->default_node.push_back(dxg_topology->numa_node_count_ + i); return HSAKMT_STATUS_SUCCESS; } diff --git a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/gpu_memory.cpp b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/gpu_memory.cpp index f048356e86b8..f22e93c9dc53 100644 --- a/projects/rocr-runtime/libhsakmt/src/dxg/wddm/gpu_memory.cpp +++ b/projects/rocr-runtime/libhsakmt/src/dxg/wddm/gpu_memory.cpp @@ -54,13 +54,17 @@ GpuMemory::GpuMemory(WDDMDevice *device) : device_(device) { } GpuMemory::~GpuMemory() { + ReleasePeers(); // Release the Lock2 pin before tearing down the allocation handles, since // DestroyAllocation invalidates the handles Unlock2 needs. UnlockSystemMemory(); - FreeGpuVirtualAddress(GpuAddress(), Size()); + // A peer view maps the owner's VA. The owner frees that reservation. + if (owner_ == nullptr) { + FreeGpuVirtualAddress(GpuAddress(), Size()); + if (desc_.handle_ape_addr > 0) + dxg_runtime->HandleApertureFree(desc_.handle_ape_addr); + } FreePhysicalMemory(); - if (desc_.handle_ape_addr > 0) - dxg_runtime->HandleApertureFree(desc_.handle_ape_addr); } ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) { @@ -872,6 +876,85 @@ ErrorCode GpuMemory::ImportPhysicalAllocHandle(const GpuMemoryCreateInfo& create return finalize_import(shared_info_ptr); } +GpuMemory* GpuMemory::ViewOn(WDDMDevice* dev, bool create, bool* created) { + if (owner_ != nullptr) + return owner_->ViewOn(dev, create, created); + if (created) + *created = false; + if (dev == device_) + return this; + + for (auto* peer : peers_) { + if (peer->GetDevice() == dev) + return peer; + } + if (!create) + return nullptr; + + auto* peer = new GpuMemory(dev); + if (peer->MapFromPeer(this) != ErrorCode::Success) { + delete peer; + return nullptr; + } + peers_.push_back(peer); + if (created) + *created = true; + return peer; +} + +void GpuMemory::DropPeer(GpuMemory* peer) { + for (auto it = peers_.begin(); it != peers_.end(); ++it) { + if (*it == peer) { + peers_.erase(it); + break; + } + } + + if (peer->IsPhysicalCreated()) { + if (peer->UnmapGpuVirtualAddress(peer->GpuAddress(), peer->Size()) != ErrorCode::Success) + pr_err("peer unmap failed\n"); + peer->Evict(); + } + delete peer; +} + +void GpuMemory::ReleasePeers() { + while (!peers_.empty()) + DropPeer(peers_.back()); +} + +ErrorCode GpuMemory::MapFromPeer(GpuMemory* owner) { + if (owner->GetDevice() == device_ || (!owner->IsSystem() && !owner->IsUserMemory())) + return ErrorCode::UnSupported; + + owner_ = owner; + desc_ = owner->desc_; + num_allocations_ = owner->NumChunks(); + if (num_allocations_ > 1) + alloc_handles_ptr_ = new WinAllocationHandle[num_allocations_]; + memset(alloc_handles_ptr_, 0, num_allocations_ * sizeof(WinAllocationHandle)); + + auto code = CreatePhysicalMemory(); + if (code != ErrorCode::Success) + return code; + + code = MapGpuVirtualAddress(GpuAddress(), Size()); + if (code != ErrorCode::Success) + return code; + + code = MakeResident(); + if (code != ErrorCode::Success) { + UnmapGpuVirtualAddress(GpuAddress(), Size()); + return code; + } + + if (!device_->WaitOnPagingFenceFromCpu()) { + UnmapGpuVirtualAddress(GpuAddress(), Size()); + return ErrorCode::Unknown; + } + return ErrorCode::Success; +} + ErrorCode GpuMemory::ImportPhysicalHandle(const GpuMemoryCreateInfo& create_info, gpusize* gpu_addr) { const auto dmabuf_fd = create_info.dmabuf_fd;