From eef6b27096f0d095f12d8fcd79b8f8cc5a9fe6dd Mon Sep 17 00:00:00 2001 From: huangxiaobai-ydmy <202200820046@mail.sdu.edu.cn> Date: Tue, 27 Oct 2026 11:41:07 +0000 Subject: [PATCH 1/3] [Bugfix][Store] Open CUDA IPC handles on the caller's device Dummy-client exports record the exporter's CUDA device ordinal, which is relative to that process's CUDA_VISIBLE_DEVICES. Opening with cudaSetDevice(handle.device_id) maps every worker's buffer onto the real client's GPU 0 when CVD layouts differ. Match TENT NVLinkTransport: open on the current device with lazy peer access and remember that local ordinal for close. Fixes #4980 Signed-off-by: YDMY_Kirito <202200820046@mail.sdu.edu.cn> (Rebased onto main after #5030; packaging commits dropped.) --- mooncake-store/src/device/cuda_ipc_buffer.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mooncake-store/src/device/cuda_ipc_buffer.cpp b/mooncake-store/src/device/cuda_ipc_buffer.cpp index 317dcc040e..09eec5ff3c 100644 --- a/mooncake-store/src/device/cuda_ipc_buffer.cpp +++ b/mooncake-store/src/device/cuda_ipc_buffer.cpp @@ -178,14 +178,15 @@ tl::expected CudaIpcBufferMapping::Open( static_assert(sizeof(cudaIpcMemHandle_t) == kCudaIpcHandleSize, "Unexpected CUDA IPC handle size"); - if (handle.size == 0 || handle.device_id < 0 || - AddOverflows(handle.offset, handle.size)) { + // handle.device_id is the exporter's ordinal under its own + // CUDA_VISIBLE_DEVICES. Do not require or switch to it here; opening on + // the caller's current device matches TENT NVLinkTransport (#4980). + if (handle.size == 0 || AddOverflows(handle.offset, handle.size)) { return tl::unexpected(ErrorCode::INVALID_PARAMS); } - int current_device = -1; - cudaGetDevice(¤t_device); - if (cudaSetDevice(handle.device_id) != cudaSuccess) { + int open_device = -1; + if (cudaGetDevice(&open_device) != cudaSuccess || open_device < 0) { ClearCudaError(); return tl::unexpected(ErrorCode::INTERNAL_ERROR); } @@ -193,6 +194,10 @@ tl::expected CudaIpcBufferMapping::Open( cudaIpcMemHandle_t ipc_handle{}; std::memcpy(&ipc_handle, handle.handle.data(), sizeof(ipc_handle)); + // Open on the CALLER's current device. Never cudaSetDevice(handle.device_id): + // that ordinal is relative to the peer's CUDA_VISIBLE_DEVICES and may map + // to a different physical GPU (or an invalid index) in this process. + // cudaIpcMemLazyEnablePeerAccess enables peer access as needed. void *base = nullptr; cudaError_t ret = cudaIpcOpenMemHandle(&base, ipc_handle, cudaIpcMemLazyEnablePeerAccess); @@ -209,14 +214,13 @@ tl::expected CudaIpcBufferMapping::Open( ret = cudaErrorInvalidValue; } } - if (current_device >= 0) cudaSetDevice(current_device); if (ret != cudaSuccess || base == nullptr) { ClearCudaError(); return tl::unexpected(ErrorCode::INVALID_PARAMS); } auto *ptr = static_cast(base) + handle.offset; - return CudaIpcBufferMapping(base, ptr, handle.device_id); + return CudaIpcBufferMapping(base, ptr, open_device); #else (void)handle; return tl::unexpected(ErrorCode::INVALID_PARAMS); From aa4cf580a8211b86f821045877d020e4a023f546 Mon Sep 17 00:00:00 2001 From: hantenglong <202200820046@mail.sdu.edu.cn> Date: Wed, 28 Oct 2026 00:54:24 +0000 Subject: [PATCH 2/3] [Bugfix][Store] Open CUDA IPC via exporter PCI bus ID Exporter ordinals are relative to CUDA_VISIBLE_DEVICES and cannot be reused by the real client. Carry cudaDeviceGetPCIBusId in the handle and resolve with cudaDeviceGetByPCIBusId on open; keep device_id for logging and open on the caller device only for legacy empty pci_bus_id handles. (Rebased onto main after #5030; packaging commits dropped.) --- .../include/device/cuda_ipc_buffer_handle.h | 8 ++- mooncake-store/src/device/cuda_ipc_buffer.cpp | 51 +++++++++++++++---- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mooncake-store/include/device/cuda_ipc_buffer_handle.h b/mooncake-store/include/device/cuda_ipc_buffer_handle.h index 21c6951172..e86c338829 100644 --- a/mooncake-store/include/device/cuda_ipc_buffer_handle.h +++ b/mooncake-store/include/device/cuda_ipc_buffer_handle.h @@ -15,7 +15,12 @@ struct CudaIpcBufferHandle { std::array handle{}; uint64_t offset = 0; uint64_t size = 0; + // Exporter-local ordinal under its CUDA_VISIBLE_DEVICES; keep for logging + // / backward compatibility. Prefer pci_bus_id when opening. int32_t device_id = -1; + // Stable GPU identity across CUDA_VISIBLE_DEVICES remapping + // (cudaDeviceGetPCIBusId). Empty means a legacy exporter. + std::string pci_bus_id; }; struct CudaIpcShmBufferRef { @@ -38,7 +43,8 @@ struct CudaIpcReadRequest { } // namespace mooncake -YLT_REFL(mooncake::CudaIpcBufferHandle, handle, offset, size, device_id); +YLT_REFL(mooncake::CudaIpcBufferHandle, handle, offset, size, device_id, + pci_bus_id); YLT_REFL(mooncake::CudaIpcShmBufferRef, ptr, size); YLT_REFL(mooncake::CudaIpcWriteRequest, key, metadata, payload); YLT_REFL(mooncake::CudaIpcReadRequest, key, destination, source_offset, size); diff --git a/mooncake-store/src/device/cuda_ipc_buffer.cpp b/mooncake-store/src/device/cuda_ipc_buffer.cpp index 09eec5ff3c..1aa3e4f446 100644 --- a/mooncake-store/src/device/cuda_ipc_buffer.cpp +++ b/mooncake-store/src/device/cuda_ipc_buffer.cpp @@ -103,6 +103,13 @@ tl::expected ExportCudaIpcBuffer( cudaIpcMemHandle_t ipc_handle{}; cudaError_t ret = cudaIpcGetMemHandle(&ipc_handle, (void *)base_ptr); + + // PCI bus ID survives CUDA_VISIBLE_DEVICES remapping across processes + // (#4980). Prefer this over attr.device when the peer opens the handle. + char pci_bus_id[32] = {}; + const cudaError_t pci_ret = + cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), attr.device); + if (current_device >= 0) cudaSetDevice(current_device); if (ret != cudaSuccess) { ClearCudaError(); @@ -114,6 +121,11 @@ tl::expected ExportCudaIpcBuffer( exported.offset = offset; exported.size = payload_size; exported.device_id = attr.device; + if (pci_ret == cudaSuccess && pci_bus_id[0] != '\0') { + exported.pci_bus_id = pci_bus_id; + } else { + ClearCudaError(); + } return exported; #else (void)ptr; @@ -178,26 +190,44 @@ tl::expected CudaIpcBufferMapping::Open( static_assert(sizeof(cudaIpcMemHandle_t) == kCudaIpcHandleSize, "Unexpected CUDA IPC handle size"); - // handle.device_id is the exporter's ordinal under its own - // CUDA_VISIBLE_DEVICES. Do not require or switch to it here; opening on - // the caller's current device matches TENT NVLinkTransport (#4980). + // Never cudaSetDevice(handle.device_id): that ordinal is relative to the + // peer's CUDA_VISIBLE_DEVICES and may map to a different physical GPU + // (or an invalid index) in this process (#4980). Prefer pci_bus_id. if (handle.size == 0 || AddOverflows(handle.offset, handle.size)) { return tl::unexpected(ErrorCode::INVALID_PARAMS); } - int open_device = -1; - if (cudaGetDevice(&open_device) != cudaSuccess || open_device < 0) { + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess || + previous_device < 0) { ClearCudaError(); return tl::unexpected(ErrorCode::INTERNAL_ERROR); } + int open_device = previous_device; + bool switched_device = false; + if (!handle.pci_bus_id.empty()) { + if (cudaDeviceGetByPCIBusId(&open_device, handle.pci_bus_id.c_str()) != + cudaSuccess) { + ClearCudaError(); + // Exporter GPU is not visible under this process's + // CUDA_VISIBLE_DEVICES. + return tl::unexpected(ErrorCode::INVALID_PARAMS); + } + if (open_device != previous_device) { + if (cudaSetDevice(open_device) != cudaSuccess) { + ClearCudaError(); + return tl::unexpected(ErrorCode::INTERNAL_ERROR); + } + switched_device = true; + } + } + // Legacy handles without pci_bus_id: open on the caller's current device + // (same as TENT NVLinkTransport). Do not trust handle.device_id. + cudaIpcMemHandle_t ipc_handle{}; std::memcpy(&ipc_handle, handle.handle.data(), sizeof(ipc_handle)); - // Open on the CALLER's current device. Never cudaSetDevice(handle.device_id): - // that ordinal is relative to the peer's CUDA_VISIBLE_DEVICES and may map - // to a different physical GPU (or an invalid index) in this process. - // cudaIpcMemLazyEnablePeerAccess enables peer access as needed. void *base = nullptr; cudaError_t ret = cudaIpcOpenMemHandle(&base, ipc_handle, cudaIpcMemLazyEnablePeerAccess); @@ -214,6 +244,9 @@ tl::expected CudaIpcBufferMapping::Open( ret = cudaErrorInvalidValue; } } + if (switched_device) { + if (cudaSetDevice(previous_device) != cudaSuccess) ClearCudaError(); + } if (ret != cudaSuccess || base == nullptr) { ClearCudaError(); return tl::unexpected(ErrorCode::INVALID_PARAMS); From cd9a3421e004376f7368ac82ddaba729d83bfa41 Mon Sep 17 00:00:00 2001 From: hantenglong <202200820046@mail.sdu.edu.cn> Date: Wed, 28 Oct 2026 01:03:23 +0000 Subject: [PATCH 3/3] [Bugfix][Store] clang-format CudaIpcBufferMapping::Open (Rebased onto main after #5030; packaging commits dropped.) --- mooncake-store/src/device/cuda_ipc_buffer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mooncake-store/src/device/cuda_ipc_buffer.cpp b/mooncake-store/src/device/cuda_ipc_buffer.cpp index 1aa3e4f446..a919073bf7 100644 --- a/mooncake-store/src/device/cuda_ipc_buffer.cpp +++ b/mooncake-store/src/device/cuda_ipc_buffer.cpp @@ -198,8 +198,7 @@ tl::expected CudaIpcBufferMapping::Open( } int previous_device = -1; - if (cudaGetDevice(&previous_device) != cudaSuccess || - previous_device < 0) { + if (cudaGetDevice(&previous_device) != cudaSuccess || previous_device < 0) { ClearCudaError(); return tl::unexpected(ErrorCode::INTERNAL_ERROR); }