diff --git a/projects/rccl-tests/src/gin_rocshmem_gda_factory_test.cu b/projects/rccl-tests/src/gin_rocshmem_gda_factory_test.cu index 15e321384b55..1b7cc7531f96 100644 --- a/projects/rccl-tests/src/gin_rocshmem_gda_factory_test.cu +++ b/projects/rccl-tests/src/gin_rocshmem_gda_factory_test.cu @@ -22,7 +22,7 @@ #include #include -#include +#include #define HIP_CHECK(cmd) do { \ hipError_t e = (cmd); \ @@ -54,7 +54,9 @@ __global__ void gin_put_kernel(rocshmem::QueuePair **qps, rocshmem::ActiveWFInfo wf_info(peer, rocshmem::ThreadScope::thread); printf("gin_put_kernel: peer=%d dst=%p src=%p bytes=%zu rkey=0x%x lkey=0x%x\n", peer, dst, src, nbytes, dst_rkey, src_lkey); - qps[peer]->put_nbi(dst, dst_rkey, src, src_lkey, nbytes, wf_info); + uintptr_t dst_raddr = reinterpret_cast(dst); + uintptr_t src_laddr = reinterpret_cast(src); + qps[peer]->put_nbi(dst_raddr, dst_rkey, src_laddr, src_lkey, nbytes, wf_info); printf("gin_put_kernel: put posted, calling quiet\n"); qps[peer]->quiet(wf_info); printf("gin_put_kernel: quiet done\n"); @@ -69,7 +71,8 @@ __global__ void gin_atomic_kernel(rocshmem::QueuePair **qps, rocshmem::ActiveWFInfo wf_info(peer, rocshmem::ThreadScope::thread); printf("gin_atomic_kernel: peer=%d addr=%p rkey=0x%x val=%lld\n", peer, remote_addr, rkey, (long long)value); - qps[peer]->atomic_add(remote_addr, rkey, value, wf_info); + uintptr_t raddr = reinterpret_cast(remote_addr); + qps[peer]->atomic_add(raddr, rkey, value, wf_info); printf("gin_atomic_kernel: atomic posted, calling quiet\n"); qps[peer]->quiet(wf_info); printf("gin_atomic_kernel: quiet done\n"); @@ -85,11 +88,15 @@ __global__ void gin_put_signal_kernel(rocshmem::QueuePair **qps, int peer) { if (threadIdx.x == 0 && blockIdx.x == 0) { rocshmem::ActiveWFInfo wf_info(peer, rocshmem::ThreadScope::thread); + static constexpr auto post_options = rocshmem::PostOpt{rocshmem::RingDB}; printf("gin_put_signal: peer=%d put dst=%p rkey=0x%x src=%p lkey=0x%x bytes=%zu\n", peer, dst, dst_rkey, src, src_lkey, nbytes); - qps[peer]->put_nbi(dst, dst_rkey, src, src_lkey, nbytes, wf_info, /*ring_db=*/false); + uintptr_t dst_raddr = reinterpret_cast(dst); + uintptr_t src_laddr = reinterpret_cast(src); + uintptr_t signal_raddr = reinterpret_cast(signal_addr); + qps[peer]->put_nbi(dst_raddr, dst_rkey, src_laddr, src_lkey, nbytes, wf_info, post_options); printf("gin_put_signal: atomic signal=%p rkey=0x%x\n", signal_addr, signal_rkey); - qps[peer]->atomic_add(signal_addr, signal_rkey, 1, wf_info); + qps[peer]->atomic_add(signal_raddr, signal_rkey, 1, wf_info); printf("gin_put_signal: calling quiet\n"); qps[peer]->quiet(wf_info); printf("gin_put_signal: done\n"); diff --git a/projects/rccl/CHANGELOG.md b/projects/rccl/CHANGELOG.md index ad9a3f5164dd..ffcd0b80bb00 100644 --- a/projects/rccl/CHANGELOG.md +++ b/projects/rccl/CHANGELOG.md @@ -14,6 +14,7 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: * Multiple GIN proxy progress threads via `NCCL_GIN_PROXY_NTHREADS`. * Enabled hierarchical Copy Engine `ncclAllGather` and `ncclAlltoAll` on multi-node communicators: the intra-node phase runs over SDMA/CE across the LSA team and the inter-node phase over one-sided RMA through the CPU (GIN) proxy. The path is off by default and takes two opt-ins: `NCCL_CTA_POLICY=2` (equivalently `config.CTAPolicy = NCCL_CTA_POLICY_ZERO`), and symmetric registration of both the send and the receive buffer through `ncclCommWindowRegister`. On architectures other than gfx1250 (MI450) a third is needed, `NCCL_CUMEM_ENABLE=1`, because symmetric memory depends on cuMem and cuMem auto-detection enables itself only on gfx1250. `NCCL_GIN_ENABLE`, `NCCL_NUM_RMA_CTX`, `NCCL_WIN_ENABLE` and `NCCL_DMABUF_ENABLE` need no change, since their defaults already qualify; the inter-node rail draws from the internal RMA context pool (`NCCL_NUM_RMA_INT_CTX`, default 4) rather than the user-addressable contexts, so `numRmaCtx` only has to stay above zero. Selection also requires an available GIN backend and the same number of ranks on every node; communicators that do not qualify keep using the kernel path. Validated for correctness and path selection from 1 to 32 nodes. * Added `ncclGinFenceLevel` barrier semantics on the Anvil SDMA GIN backend (`NCCL_GIN_TYPE=7`). `gin.get`, `flushAsync` and `wait` are implemented instead of trapping, and a barrier's signal is now ordered behind the payload on the peer's SDMA queue, so `Put`, `Get` and the default `Put | Get` fences drain what they promise. `Put` covers a rank's puts to itself, and `ncclGinAllContexts` fences every GIN context. Single-node validated on gfx950. +* Removed GIN rocSHMEM GDA dependency on rocSHMEM GDA bitcode: GIN rocSHMEM GDA is now header-only. The CMake `roc::rccl` target provides the required include directories in its `INSTALL_INTERFACE` property. ### Changed * **Breaking: `NCCL_GIN_TYPE` values for AMD backends are not compatible with 2.30.7.** NCCL 2.31 inserted EFA GDA at value 5, so rocSHMEM GDA moved 5→6 and Anvil SDMA moved 6→7. The IB proxy remains `2`. Jobs that still set `NCCL_GIN_TYPE=6` now select rocSHMEM GDA, not Anvil SDMA. See `src/gin/README.md`. diff --git a/projects/rccl/cmake/DeviceLinker.cmake b/projects/rccl/cmake/DeviceLinker.cmake index 549a365c70e6..8670af49d78e 100644 --- a/projects/rccl/cmake/DeviceLinker.cmake +++ b/projects/rccl/cmake/DeviceLinker.cmake @@ -287,6 +287,9 @@ foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) if((ENABLE_ROCSHMEM OR ENABLE_ROCSHMEM_GIN) AND TARGET rocshmem_static) add_dependencies(${_dev_target} rocshmem_static) endif() + if(ENABLE_ROCSHMEM_GIN AND TARGET copy_rocshmem_headers) + add_dependencies(${_dev_target} copy_rocshmem_headers) + endif() # Pass rocshmem device bitcode to per-kernel compiles so rocshmem device # symbols resolve during the per-arch device.elf link step. # ENABLE_ROCSHMEM: rocshmem_n_pes, alltoall_wg, etc. @@ -1101,13 +1104,7 @@ if(GENERATE_SYM_KERNELS) add_custom_command( OUTPUT ${_qp_bc} - COMMAND ${_llvm_link} - ${_bc_dir}/queue_pair.bc - ${_bc_dir}/queue_pair_mlx5.bc - ${_bc_dir}/queue_pair_bnxt.bc - ${_bc_dir}/queue_pair_ionic.bc - ${_cm_bc} - -o ${_qp_raw} + COMMAND ${_llvm_link} ${_cm_bc} -o ${_qp_raw} COMMAND ${_llvm_dis} -o ${_qp_raw}.ll ${_qp_raw} COMMAND grep -v -E "@llvm[.]compiler[.]used|@__hip_cuid_" ${_qp_raw}.ll > ${_qp_raw}.clean.ll @@ -1164,6 +1161,9 @@ add_custom_target(device_linker_build ALL DEPENDS ${COMMON_FAT_OBJ} ${ONERANK_FAT_OBJ} ${COLLECTIVES_FAT_OBJ} ${DIAG_P2P_FAT_OBJ} ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${DDA_ALLTOALL_IPC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL128_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL128_FAT_OBJ} ${CE_REDUCE_FAT_OBJS} ${SYM_FAT_OBJS} ${GIN_ALLTOALL_SDMA_FAT_OBJ} ${GIN_ALLREDUCE_SDMA_FAT_OBJ} ${RCCL_EP_FAT_OBJ} ${DEVICE_ELF_SYMLINKS} ) add_dependencies(device_linker_build hipify_all copy_nccl_device_headers) +if(ENABLE_ROCSHMEM_GIN AND TARGET copy_rocshmem_headers) + add_dependencies(device_linker_build copy_rocshmem_headers) +endif() if((ENABLE_ROCSHMEM OR ENABLE_ROCSHMEM_GIN) AND TARGET rocshmem_static) # The fat objects above include GIN device headers, which pull in rocSHMEM # headers installed to ext/rocshmem/include by the ExternalProject. @@ -1202,3 +1202,6 @@ set(DEVICE_LINKER_OBJECTS # =========================================================================== add_custom_target(device_ir DEPENDS ${ALL_IR_FILES}) add_dependencies(device_ir hipify_all copy_nccl_device_headers) +if(ENABLE_ROCSHMEM_GIN AND TARGET copy_rocshmem_headers) + add_dependencies(device_ir copy_rocshmem_headers) +endif() diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index 8108c182769b..558aafcce106 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -98,6 +98,10 @@ function(add_rocshmem_targets) # ----------------------------------------------------------------- if(ROCSHMEM_INSTALL_DIR) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + # Set rocshmem_static_ROOT; variable read on enter, so must be set BEFORE find_package() + if(NOT rocshmem_static_ROOT) + set(rocshmem_static_ROOT "${ROCSHMEM_INSTALL_DIR}") + endif() find_package(rocshmem_static) if(rocshmem_static_FOUND) set(ROCSHMEM_INCLUDE_DIR "${ROCSHMEM_INCLUDE_DIR}" PARENT_SCOPE) @@ -158,3 +162,23 @@ function(add_rocshmem_targets) set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" PARENT_SCOPE) endfunction() + +# copy files in the list-valued variable ${FILE_LIST_VAR} from ${SRC_DIR} to ${DST_DIR} +# appends the destination file location to the list-valued variable ${OUTPUT_LIST_VAR} +function(copy_files FILE_LIST_VAR SRC_DIR DST_DIR OUTPUT_LIST_VAR) + foreach(file_name ${${FILE_LIST_VAR}}) + set(src_file "${SRC_DIR}/${file_name}") + set(dst_file "${DST_DIR}/${file_name}") + get_filename_component(dst_file_dir "${dst_file}" DIRECTORY) + add_custom_command( + OUTPUT "${dst_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${dst_file_dir}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${src_file}" "${dst_file}" + DEPENDS "${src_file}" + COMMENT "Copying ${src_file} -> ${dst_file}" + VERBATIM + ) + list(APPEND ${OUTPUT_LIST_VAR} "${dst_file}") + endforeach() + set(${OUTPUT_LIST_VAR} ${${OUTPUT_LIST_VAR}} PARENT_SCOPE) +endfunction() diff --git a/projects/rccl/src/CMakeLists.txt b/projects/rccl/src/CMakeLists.txt index d0368abbccbb..6e9b7687890a 100644 --- a/projects/rccl/src/CMakeLists.txt +++ b/projects/rccl/src/CMakeLists.txt @@ -265,7 +265,6 @@ set(SRC_FILES include/nccl_device/gin/proxy/gin_proxy_device_host_common.h include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h - include/nccl_device/gin/rocshmem_gda/queue_pair_device.h include/nccl_device/gin_barrier.h include/nccl_device/impl/barrier__funcs.h include/nccl_device/impl/barrier__types.h @@ -594,6 +593,13 @@ if(ENABLE_ROCSHMEM_GIN) src/algorithms/gin/sdma/gin_alltoall_sdma.cu src/algorithms/gin/sdma/gin_all_reduce_sdma.cu ) + list(APPEND SRC_FILES + src/include/nccl_device/gin/rocshmem_gda/log.hpp + src/include/nccl_device/gin/rocshmem_gda/util.hpp + src/include/nccl_device/gin/rocshmem_gda/containers/free_list.hpp + src/include/nccl_device/gin/rocshmem_gda/gda/ibv_core.hpp + src/include/nccl_device/gin/rocshmem_gda/rocshmem/rocshmem_common.hpp + ) endif() if(USE_AMDSMI) @@ -1055,6 +1061,90 @@ if (ENABLE_ROCSHMEM OR ENABLE_ROCSHMEM_GIN) endif() endif() endif() + +if (ENABLE_ROCSHMEM_GIN) + # Ensure we see our copied headers before rocSHMEM's + target_include_directories(rccl BEFORE PRIVATE ${PROJECT_BINARY_DIR}/include/nccl_device/gin/rocshmem_gda) + + # Need to add the GIN GDA directory to the public interface: + # internal rocSHMEM headers assume that the source directory is in the include search path + target_include_directories(rccl BEFORE INTERFACE + $ + $ + ) + + # Add target for copying the rocSHMEM headers + add_custom_target(copy_rocshmem_headers) + add_dependencies(rccl copy_rocshmem_headers) + + if (ROCSHMEM_INCLUDE_DIR) + # Copy rocSHMEM public include headers (primarily rocshmem_config.h) into RCCL + set(ROCSHMEM_INCLUDE_HEADERS rocshmem/rocshmem_config.h) + copy_files( + ROCSHMEM_INCLUDE_HEADERS + "${ROCSHMEM_INCLUDE_DIR}" + "${PROJECT_BINARY_DIR}/include/nccl_device/gin/rocshmem_gda" + GIN_GDA_ROCSHMEM_INCLUDE_HEADERS + ) + add_custom_target(copy_rocshmem_include_headers DEPENDS ${GIN_GDA_ROCSHMEM_INCLUDE_HEADERS}) + # Ensure rocSHMEM is fully built/installed before copying installed headers + if (TARGET rocshmem_ext) + add_dependencies(copy_rocshmem_include_headers rocshmem_ext) + endif() + add_dependencies(copy_rocshmem_headers copy_rocshmem_include_headers) + endif() + + if (ROCSHMEM_SOURCE_DIR) + # Copy rocSHMEM QueuePair device headers into RCCL + set(ROCSHMEM_SOURCE_HEADERS + # Queue Pair core + gda/queue_pair/queue_pair_common.hpp + gda/queue_pair/queue_pair_option.hpp + gda/queue_pair/queue_pair_interface.hpp + gda/queue_pair/queue_pair_shmem.hpp + gda/queue_pair/queue_pair_device.hpp + gda/queue_pair_provider.hpp + # GDA_MUX (Multiplexing provider) + gda/queue_pair_mux.hpp + # GDA_IONIC (AMD Pensando IONIC RDMA provider) + gda/ionic/queue_pair_ionic.hpp + gda/ionic/provider_gda_ionic.hpp + gda/ionic/ionic_dv.h + gda/ionic/ionic_fw.h + # GDA_BNXT (Broadcom BNXT RDMA provider) + gda/bnxt/queue_pair_bnxt.hpp + gda/bnxt/provider_gda_bnxt.hpp + gda/bnxt/bnxt_re_dv.h + gda/bnxt/bnxt_re_hsi.h + # GDA_MLX5 (Mellanox MLX5 RDMA provider) + gda/mlx5/queue_pair_mlx5.hpp + gda/mlx5/provider_gda_mlx5.hpp + gda/mlx5/mlx5dv_core.hpp + gda/mlx5/mlx5_ifc_core.hpp + # GDA_QUEUEPAIR_MOCK (QueuePair mock object) + gda/queue_pair/queue_pair_mock.hpp + # rocSHMEM internals / miscellaneous + gda/endian.hpp + gda/gda_enums.hpp + bit.hpp + constmem.hpp + constants.hpp + ) + copy_files( + ROCSHMEM_SOURCE_HEADERS + "${ROCSHMEM_SOURCE_DIR}/src" + "${PROJECT_BINARY_DIR}/include/nccl_device/gin/rocshmem_gda" + GIN_GDA_ROCSHMEM_SOURCE_HEADERS + ) + add_custom_target(copy_rocshmem_source_headers DEPENDS ${GIN_GDA_ROCSHMEM_SOURCE_HEADERS}) + add_dependencies(copy_rocshmem_headers copy_rocshmem_source_headers) + endif() + + # Add rule to install ibvcore.h so that it can be found by ibv_core.hpp + rocm_install(FILES ${HIPIFY_DIR}/src/include/ibvcore.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nccl_device/gin/rocshmem_gda) +endif() + if (ENABLE_ROCSHMEM) target_link_libraries(rccl PRIVATE ${IBVERBS}) endif() diff --git a/projects/rccl/src/gin/gin_rocshmem_gda_factory.cc b/projects/rccl/src/gin/gin_rocshmem_gda_factory.cc index f9a2dda85e8a..6c0fe87c2a7b 100644 --- a/projects/rccl/src/gin/gin_rocshmem_gda_factory.cc +++ b/projects/rccl/src/gin/gin_rocshmem_gda_factory.cc @@ -39,12 +39,13 @@ */ #include "gin/gin_rocshmem_gda_factory.h" -#include +#include +#include #include -#include #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ #include #include #include +#include #include #include #include @@ -116,11 +118,11 @@ struct gin_dest_info { struct rocshmem_gin_qp_set { // GPU QP initialization — must be a member to access QueuePair private fields via friend - int initialize_gpu_qp(QueuePair* gpu_qp, int idx); + int initialize_host_gpu_qp(QueuePair* host_gpu_qp, int idx); int nRanks; int myRank; - int provider; // GDAProvider enum + GDAProvider provider; // GDAProvider enum // NIC state (owned by this set) GinNicDevice nic; @@ -150,7 +152,10 @@ struct rocshmem_gin_qp_set { QueuePair* host_qps = nullptr; QueuePair* gpu_qps = nullptr; - uint32_t inline_threshold = 8; + // Dummy atomic fetch location + uint64_t fetch_atomic_dummy; + struct ibv_mr* fetch_atomic_dummy_mr = nullptr; + uint32_t sq_size = envvar::gda::sq_size; }; @@ -232,17 +237,19 @@ static int gin_memory_lock_to_fine_grain(void* ptr, size_t size, void** gpu_ptr, // NIC discovery and IB device open /////////////////////////////////////////////////////////////////////////////// -static int gin_detect_provider(struct ibv_device_attr* attr) { +static GDAProvider gin_detect_provider(struct ibv_device_attr* attr) { + switch (static_cast(attr->vendor_id)) { #if defined(GDA_BNXT) - if (attr->vendor_id == GDA_BNXT_VENDOR_ID) return GDAProvider::BNXT; + case gda::vendor_id::BNXT: return GDAProvider::BNXT; #endif #if defined(GDA_IONIC) - if (attr->vendor_id == GDA_IONIC_VENDOR_ID) return GDAProvider::IONIC; + case gda::vendor_id::IONIC: return GDAProvider::IONIC; #endif #if defined(GDA_MLX5) - if (attr->vendor_id == GDA_MLX5_VENDOR_ID) return GDAProvider::MLX5; + case gda::vendor_id::MLX5: return GDAProvider::MLX5; #endif - return -1; + } + return GDAProvider::UNSET; } static int gin_open_ib_device(rocshmem_gin_qp_set* set) { @@ -290,8 +297,8 @@ static int gin_open_ib_device(rocshmem_gin_qp_set* set) { continue; } - int provider = gin_detect_provider(&dev_attr); - if (provider < 0) { + GDAProvider provider = gin_detect_provider(&dev_attr); + if (provider == GDAProvider::UNSET) { LOG_WARN("GIN QP factory: device %s vendor_id=0x%x not recognized as IONIC/BNXT/MLX5, skipping", ibv.get_device_name(dev_list[d]), dev_attr.vendor_id); ibv.close_device(ctx); @@ -542,7 +549,7 @@ static int gin_create_cqs(rocshmem_gin_qp_set* set) { cq_attr.parent_domain = set->nic.pd_uxdma[i & 1]; struct ibv_cq_ex* cq_ex = nullptr; if (set->ionic_dv.create_cq_ex) cq_ex = set->ionic_dv.create_cq_ex(set->nic.context, &cq_attr, &ionic_cq_attr); - if (!cq_ex) cq_ex = ibv_create_cq_ex(set->nic.context, &cq_attr); + if (!cq_ex) cq_ex = ibv.create_cq_ex(set->nic.context, &cq_attr); if (!cq_ex) return -1; set->ibv_cqs[i] = ibv.cq_ex_to_cq(cq_ex); if (!set->ibv_cqs[i]) return -1; @@ -665,7 +672,7 @@ static int gin_create_qps(rocshmem_gin_qp_set* set) { attr.cap.max_send_wr = set->sq_size; attr.cap.max_send_sge = 1; attr.cap.max_recv_sge = 1; - attr.cap.max_inline_data = set->inline_threshold; + attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; attr.sq_sig_all = 0; attr.qp_type = IBV_QPT_RC; attr.comp_mask = IBV_QP_INIT_ATTR_PD; @@ -698,7 +705,7 @@ static int gin_create_qps(rocshmem_gin_qp_set* set) { ib_qp_attr.cap.max_recv_wr = 0; ib_qp_attr.cap.max_send_sge = 1; ib_qp_attr.cap.max_recv_sge = 0; - ib_qp_attr.cap.max_inline_data = set->inline_threshold; + ib_qp_attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; ib_qp_attr.qp_type = IBV_QPT_RC; ib_qp_attr.sq_sig_all = 0; @@ -792,7 +799,6 @@ static int gin_create_qps(rocshmem_gin_qp_set* set) { case GDAProvider::MLX5: { set->mlx5_qps.resize(n); - set->inline_threshold = sizeof(gda_mlx5_wqe_inline_data::data); for (int i = 0; i < n; i++) { int err = set->mlx5dv.create_qp(set->mlx5_qps[i], set->nic.context, set->nic.pd_orig, set->sq_size); @@ -928,7 +934,7 @@ static int gin_modify_qps_rtr_to_rts(rocshmem_gin_qp_set* set, struct gin_dest_i // Does NOT set lkey/rkey (GIN uses put_nbi for per-buffer keys). /////////////////////////////////////////////////////////////////////////////// -int rocshmem_gin_qp_set::initialize_gpu_qp(QueuePair* gpu_qp, int idx) { +int rocshmem_gin_qp_set::initialize_host_gpu_qp(QueuePair* host_gpu_qp, int idx) { switch (this->provider) { #if defined(GDA_IONIC) case GDAProvider::IONIC: @@ -951,22 +957,27 @@ int rocshmem_gin_qp_set::initialize_gpu_qp(QueuePair* gpu_qp, int idx) { ionic_dv_cq dvcq; ionic_dv.get_cq(&dvcq, ibv_cqs[idx], udma_idx); - gpu_qp->cq_dbreg = &gpu_db_ptr[dvctx.cq_qtype]; - gpu_qp->cq_dbval = dvcq.q.db_val; - gpu_qp->cq_mask = dvcq.q.mask; - gpu_qp->ionic_cq_buf = reinterpret_cast(dvcq.q.ptr); - ionic_dv_qp dvqp; ionic_dv.get_qp(&dvqp, ibv_qps[idx]); - gpu_qp->sq_dbreg = &gpu_db_ptr[dvctx.sq_qtype]; - gpu_qp->sq_dbval = dvqp.sq.db_val; - gpu_qp->sq_mask = dvqp.sq.mask; - gpu_qp->ionic_sq_buf = reinterpret_cast(dvqp.sq.ptr); + // QueuePairIONIC constructor arguments + uint32_t qpn = ibv_qps[idx]->qp_num; + + ionic_v1_wqe* sq_buf = reinterpret_cast(dvqp.sq.ptr); + uint64_t* sq_dbreg = &gpu_db_ptr[dvctx.sq_qtype]; + uint64_t sq_dbval = dvqp.sq.db_val; + uint16_t sq_mask = dvqp.sq.mask; + + ionic_v1_cqe* cq_buf = reinterpret_cast(dvcq.q.ptr); + uint64_t* cq_dbreg = &gpu_db_ptr[dvctx.cq_qtype]; + uint64_t cq_dbval = dvcq.q.db_val; + uint16_t cq_mask = dvcq.q.mask; - gpu_qp->qp_num = ibv_qps[idx]->qp_num; - gpu_qp->inline_threshold = 32; - // lkey/rkey left at 0 — GIN uses put_nbi + // QueuePair is either QueuePairIONIC or QueuePairMux + new (host_gpu_qp) + QueuePair{QueuePairIONIC{qpn, &fetch_atomic_dummy, fetch_atomic_dummy_mr->lkey, + ionic_device_sq{sq_buf, sq_dbreg, sq_dbval, sq_mask}, + ionic_device_cq{cq_buf, cq_dbreg, cq_dbval, cq_mask}}}; return 0; } #endif @@ -992,37 +1003,40 @@ int rocshmem_gin_qp_set::initialize_gpu_qp(QueuePair* gpu_qp, int idx) { err = this->bnxt_re_dv.init_obj(&dv_obj, BNXT_RE_DV_OBJ_CQ); if (err) return -1; - memset(&gpu_qp->bnxt_cq, 0, sizeof(bnxt_device_cq)); - gpu_qp->bnxt_cq.buf = this->bnxt_scqs[idx].buf; - gpu_qp->bnxt_cq.depth = this->bnxt_scqs[idx].depth; - gpu_qp->bnxt_cq.id = dv_cq.cqn; - - // Export SQ - memset(&gpu_qp->bnxt_sq, 0, sizeof(bnxt_device_sq)); - gpu_qp->bnxt_sq.buf = this->bnxt_qps[idx].sq_buf; - gpu_qp->bnxt_sq.depth = this->bnxt_qps[idx].mem_info.sq_slots; - gpu_qp->bnxt_sq.id = this->ibv_qps[idx]->qp_num; - gpu_qp->bnxt_sq.msntbl = this->bnxt_qps[idx].msntbl; - gpu_qp->bnxt_sq.msn_tbl_sz = this->bnxt_qps[idx].msn_tbl_sz; - gpu_qp->bnxt_sq.psn_sz_log2 = std::log2(this->bnxt_qps[idx].mem_info.sq_psn_sz); - gpu_qp->bnxt_sq.mtu = 128 << this->nic.portinfo.active_mtu; // ibv_mtu enum: 1=256, 2=512, 3=1024, 4=2048, 5=4096 - // Export doorbell - if (hipHostRegister(this->bnxt_qps[idx].db_region_attr->dbr, getpagesize(), hipHostRegisterDefault) != hipSuccess) + void* gpu_dbr_ptr = nullptr; + if (hipHostRegister(bnxt_qps[idx].db_region_attr->dbr, getpagesize(), hipHostRegisterDefault) != hipSuccess) return -1; - if (hipHostGetDevicePointer((void**)&gpu_qp->bnxt_dbr, bnxt_qps[idx].db_region_attr->dbr, 0) != hipSuccess) + if (hipHostGetDevicePointer(&gpu_dbr_ptr, bnxt_qps[idx].db_region_attr->dbr, 0) != hipSuccess) return -1; - gpu_qp->qp_num = this->ibv_qps[idx]->qp_num; - gpu_qp->inline_threshold = inline_threshold; + // QueuePairBNXT constructor arguments + uint32_t qpn = ibv_qps[idx]->qp_num; + + uint64_t* dbr = static_cast(gpu_dbr_ptr); + + void* sq_buf = bnxt_qps[idx].sq_buf; + uint32_t sq_depth = bnxt_qps[idx].mem_info.sq_slots; + void* msntbl = bnxt_qps[idx].msntbl; + uint32_t msn_tbl_sz = bnxt_qps[idx].msn_tbl_sz; + uint32_t psn_sz_log2 = bit_log2(bnxt_qps[idx].mem_info.sq_psn_sz); + uint32_t mtu = 128 << nic.portinfo.active_mtu; // ibv_mtu enum: 1=256, 2=512, 3=1024, 4=2048, 5=4096 + + void* cq_buf = bnxt_scqs[idx].buf; + uint32_t cq_depth = bnxt_scqs[idx].depth; + + // QueuePair is either QueuePairBNXT or QueuePairMux + new (host_gpu_qp) + QueuePair{QueuePairBNXT{qpn, &fetch_atomic_dummy, fetch_atomic_dummy_mr->lkey, dbr, + bnxt_device_sq{sq_buf, sq_depth, msntbl, msn_tbl_sz, + psn_sz_log2, mtu}, + bnxt_device_cq{cq_buf, cq_depth}}}; LOG_TRACE("gin_qp init[%d]: qp_num=%u sq.buf=%p sq.depth=%u sq.id=%u " "cq.buf=%p cq.depth=%u cq.id=%u dbr=%p (host=%p) " - "msntbl=%p msn_tbl_sz=%u psn_sz_log2=%u mtu=%lu inline=%u", - idx, gpu_qp->qp_num, gpu_qp->bnxt_sq.buf, gpu_qp->bnxt_sq.depth, gpu_qp->bnxt_sq.id, - gpu_qp->bnxt_cq.buf, gpu_qp->bnxt_cq.depth, gpu_qp->bnxt_cq.id, (void*)gpu_qp->bnxt_dbr, - (void*)this->bnxt_qps[idx].db_region_attr->dbr, gpu_qp->bnxt_sq.msntbl, gpu_qp->bnxt_sq.msn_tbl_sz, - gpu_qp->bnxt_sq.psn_sz_log2, (unsigned long)gpu_qp->bnxt_sq.mtu, gpu_qp->inline_threshold); + "msntbl=%p msn_tbl_sz=%u psn_sz_log2=%u mtu=%u", + idx, qpn, sq_buf, sq_depth, qpn, cq_buf, cq_depth, dv_cq.cqn, dbr, + bnxt_qps[idx].db_region_attr->dbr, msntbl, msn_tbl_sz, psn_sz_log2, mtu); return 0; } #endif @@ -1034,18 +1048,28 @@ int rocshmem_gin_qp_set::initialize_gpu_qp(QueuePair* gpu_qp, int idx) { int hip_dev_id = -1; if (hipGetDevice(&hip_dev_id) != hipSuccess) return -1; - gpu_qp->mlx5_cq = gda_mlx5_device_cq((mlx5_cqe64*)qp.cq, qp.cq_dbrec); - void* gpu_db_ptr = nullptr; if (gin_memory_lock_to_fine_grain(qp.uar->reg_addr, MLX5_DB_BLUEFLAME_BUFFER_SIZE, &gpu_db_ptr, hip_dev_id) != 0) return -1; - gpu_qp->mlx5_sq = gda_mlx5_device_sq{(gda_mlx5_wqe*)qp.sq, &qp.qp_dbrec[MLX5_SND_DBR], - (gda_mlx5_doorbell*)gpu_db_ptr, qp.sq_depth}; + // QueuePairMLX5 constructor arguments + uint32_t qpn = qp.qpn; + + gda_mlx5_wqe* sq_buf = reinterpret_cast(qp.sq); + // qp.dbrec points to two __be32 values: RQ dbrec at MLX5_RCV_DBR and SQ dbrec at MLX5_SND_DBR + __be32* sq_dbrec = &qp.qp_dbrec[MLX5_SND_DBR]; + gda_mlx5_doorbell* sq_db = reinterpret_cast(gpu_db_ptr); + uint16_t sq_depth = static_cast(qp.sq_depth); + + mlx5_cqe64* cq_buf = reinterpret_cast(qp.cq); + __be32* cq_dbrec = qp.cq_dbrec; + + // QueuePair is either QueuePairMLX5 or QueuePairMux + new (host_gpu_qp) + QueuePair{QueuePairMLX5{qpn, &fetch_atomic_dummy, fetch_atomic_dummy_mr->lkey, + gda_mlx5_device_sq{sq_buf, sq_dbrec, sq_db, sq_depth}, + gda_mlx5_device_cq{cq_buf, cq_dbrec}}}; - gpu_qp->qp_num = qp.qpn; - gpu_qp->inline_threshold = inline_threshold; - // lkey/rkey: for MLX5, keys are big-endian. Left at 0 for GIN. return 0; } #endif @@ -1180,15 +1204,19 @@ int rocshmem_gin_create_qps(int nRanks, int myRank, int (*allgather)(void* ctx, set->host_qps = (QueuePair*)malloc(qp_size); if (!set->host_qps) goto fail; - for (int i = 0; i < nRanks; i++) { - new (&set->host_qps[i]) QueuePair(set->nic.pd_orig, set->provider); - if (hipMemcpy(&set->gpu_qps[i], &set->host_qps[i], sizeof(QueuePair), hipMemcpyHostToDevice) != hipSuccess) - goto fail; + // Register dummy fetch atomic location to return atomic fetch values (never read) + set->fetch_atomic_dummy_mr = ibv.reg_mr(set->nic.pd_orig, &set->fetch_atomic_dummy, + sizeof(uint64_t), IBV_ACCESS_LOCAL_WRITE); + if (!set->fetch_atomic_dummy_mr) goto fail; + for (int i = 0; i < nRanks; i++) { // 9. Initialize GPU-specific state (doorbells, CQ/SQ buffers) - if (set->initialize_gpu_qp(&set->gpu_qps[i], i) != 0) goto fail; + if (set->initialize_host_gpu_qp(&set->host_qps[i], i) != 0) goto fail; } + if (hipMemcpy(set->gpu_qps, set->host_qps, qp_size, hipMemcpyHostToDevice) != hipSuccess) + goto fail; + // Build array of QueuePair pointers for the GPU context QueuePair** host_ptrs = (QueuePair**)malloc(nRanks * sizeof(QueuePair*)); for (int i = 0; i < nRanks; i++) host_ptrs[i] = &set->gpu_qps[i]; @@ -1222,12 +1250,12 @@ void rocshmem_gin_destroy_qps(rocshmem_gin_qp_set_t qp_set) { if (!qp_set) return; // Destroy QueuePair objects - if (qp_set->host_qps) { - for (int i = 0; i < qp_set->nRanks; i++) qp_set->host_qps[i].~QueuePair(); - free(qp_set->host_qps); - } + if (qp_set->host_qps) free(qp_set->host_qps); if (qp_set->gpu_qps) (void)hipFree(qp_set->gpu_qps); + // Unregister dummy fetch atomic location + if (qp_set->fetch_atomic_dummy_mr) wrap_ibv_dereg_mr(qp_set->fetch_atomic_dummy_mr); + #if defined(GDA_BNXT) if (qp_set->provider == GDAProvider::BNXT) { for (size_t i = 0; i < qp_set->bnxt_qps.size(); i++) { @@ -1307,8 +1335,8 @@ int rocshmem_gin_reg_mr(rocshmem_gin_qp_set_t qp_set, void* addr, size_t size, i if (!mr) return -1; *out_mr = mr; - *out_lkey = mr->lkey; - *out_rkey = mr->rkey; + *out_lkey = QueuePair::to_provider_endianness(mr->lkey); + *out_rkey = QueuePair::to_provider_endianness(mr->rkey); return 0; } @@ -1364,8 +1392,8 @@ int rocshmem_gin_reg_mr_vmm(rocshmem_gin_qp_set_t qp_set, void* addr, size_t siz if (!mr) return -1; *out_mr = mr; - *out_lkey = mr->lkey; - *out_rkey = mr->rkey; + *out_lkey = QueuePair::to_provider_endianness(mr->lkey); + *out_rkey = QueuePair::to_provider_endianness(mr->rkey); return 0; } @@ -1381,10 +1409,10 @@ void rocshmem_gin_dereg_mr(void* mr) { int rocshmem_gin_get_provider(rocshmem_gin_qp_set_t qp_set) { if (!qp_set) return -1; - return qp_set->provider; + return static_cast(qp_set->provider); } -static bool gin_validate_device(int provider, struct ibv_device_attr* dev_attr) { +static bool gin_validate_device(GDAProvider provider, struct ibv_device_attr* dev_attr) { #if defined(GDA_BNXT) if (provider == GDAProvider::BNXT) { const uint32_t supported_bnxt_part_ids[] = {0x1760 /* BCM57608 */}; @@ -1428,8 +1456,8 @@ int rocshmem_gin_probe_devices(void) { continue; } - int provider = gin_detect_provider(&dev_attr); - if (provider < 0 || !gin_validate_device(provider, &dev_attr)) { + GDAProvider provider = gin_detect_provider(&dev_attr); + if (provider == GDAProvider::UNSET || !gin_validate_device(provider, &dev_attr)) { ibv.close_device(ctx); continue; } diff --git a/projects/rccl/src/include/ibvcore.h b/projects/rccl/src/include/ibvcore.h index 48209a0c6426..a1f41d3e970f 100644 --- a/projects/rccl/src/include/ibvcore.h +++ b/projects/rccl/src/include/ibvcore.h @@ -63,6 +63,20 @@ union ibv_gid { } global; }; +enum ibv_gid_type { + IBV_GID_TYPE_IB, + IBV_GID_TYPE_ROCE_V1, + IBV_GID_TYPE_ROCE_V2, +}; + +struct ibv_gid_entry { + union ibv_gid gid; + uint32_t gid_index; + uint32_t port_num; + uint32_t gid_type; /* enum ibv_gid_type */ + uint32_t ndev_ifindex; +}; + #ifndef container_of /** * container_of - cast a member of a structure out to the containing structure @@ -368,6 +382,7 @@ enum ibv_access_flags { IBV_ACCESS_REMOTE_READ = (1<<2), IBV_ACCESS_REMOTE_ATOMIC = (1<<3), IBV_ACCESS_MW_BIND = (1<<4), + IBV_ACCESS_ZERO_BASED = (1<<5), IBV_ACCESS_RELAXED_ORDERING = (1<<20), }; @@ -1060,6 +1075,58 @@ struct ibv_context { void *abi_compat; }; +enum ibv_cq_init_attr_mask { + IBV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 0, + IBV_CQ_INIT_ATTR_MASK_PD = 1 << 1, +}; + +enum ibv_create_cq_attr_flags { + IBV_CREATE_CQ_ATTR_SINGLE_THREADED = 1 << 0, + IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN = 1 << 1, +}; + +struct ibv_cq_init_attr_ex { + /* Minimum number of entries required for CQ */ + uint32_t cqe; + /* Consumer-supplied context returned for completion events */ + void *cq_context; + /* Completion channel where completion events will be queued. + * May be NULL if completion events will not be used. + */ + struct ibv_comp_channel *channel; + /* Completion vector used to signal completion events. + * Must be < context->num_comp_vectors. + */ + uint32_t comp_vector; + /* Or'ed bit of enum ibv_create_cq_wc_flags. */ + uint64_t wc_flags; + /* compatibility mask (extended verb). Or'd flags of + * enum ibv_cq_init_attr_mask + */ + uint32_t comp_mask; + /* create cq attr flags - one or more flags from + * enum ibv_create_cq_attr_flags + */ + uint32_t flags; + struct ibv_pd *parent_domain; +}; + +enum ibv_parent_domain_init_attr_mask { + IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS = 1 << 0, + IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT = 1 << 1, +}; + +struct ibv_parent_domain_init_attr { + struct ibv_pd *pd; /* reference to a protection domain object, can't be NULL */ + struct ibv_td *td; /* reference to a thread domain object, or NULL */ + uint32_t comp_mask; + void *(*alloc)(struct ibv_pd *pd, void *pd_context, size_t size, + size_t alignment, uint64_t resource_type); + void (*free)(struct ibv_pd *pd, void *pd_context, void *ptr, + uint64_t resource_type); + void *pd_context; +}; + enum verbs_context_mask { VERBS_CONTEXT_XRCD = (uint64_t)1 << 0, VERBS_CONTEXT_SRQ = (uint64_t)1 << 1, diff --git a/projects/rccl/src/include/ionic/ionicdvcore.h b/projects/rccl/src/include/ionic/ionicdvcore.h index 262f2433ebf5..3c02d2a73b5d 100644 --- a/projects/rccl/src/include/ionic/ionicdvcore.h +++ b/projects/rccl/src/include/ionic/ionicdvcore.h @@ -1,6 +1,10 @@ #ifndef NCCL_IONICDV_CORE_H_ #define NCCL_IONICDV_CORE_H_ +/* Check whether the full ionic_dv.h header has already been included elsewhere. + * E.g. by rocSHMEM or GIN-GDA. */ +#ifndef IONIC_DV_H + /* Basic ionic direct verbs structs. * Needed to dynamically load ionic direct verbs functions without * explicit including of ionic direct verbs header. @@ -17,4 +21,6 @@ enum ionicdv_reg_udma_mask { IONIC_UDMA_MASK_HIGH = 2 }; +#endif // IONIC_DV_H + #endif // NCCL_IONICDV_CORE_H_ diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/README.headers.md b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/README.headers.md new file mode 100644 index 000000000000..65e6e9aa08fc --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/README.headers.md @@ -0,0 +1,61 @@ +# rocSHMEM headers + +## Copied verbatim from rocSHMEM source + +These headers are copied verbatim from the rocSHMEM sources +during RCCL configuration and installation, from CMakeLists.txt. + +### Queue Pair core +* `gda/queue_pair/queue_pair_common.hpp` +* `gda/queue_pair/queue_pair_option.hpp` +* `gda/queue_pair/queue_pair_interface.hpp` +* `gda/queue_pair/queue_pair_shmem.hpp` +* `gda/queue_pair/queue_pair_device.hpp` +* `gda/queue_pair_provider.hpp` + +### GDA\_MUX (Multiplexing provider) +* `gda/queue_pair_mux.hpp` + +### GDA\_IONIC (AMD Pensando IONIC RDMA provider) +* `gda/ionic/queue_pair_ionic.hpp` +* `gda/ionic/provider_gda_ionic.hpp` +* `gda/ionic/ionic_dv.h` +* `gda/ionic/ionic_fw.h` + +### GDA\_BNXT (Broadcom BNXT RDMA provider) +* `gda/bnxt/queue_pair_bnxt.hpp` +* `gda/bnxt/provider_gda_bnxt.hpp` +* `gda/bnxt/bnxt_re_dv.h` +* `gda/bnxt/bnxt_re_hsi.h` + +### GDA\_MLX5 (Mellanox MLX5 RDMA provider) +* `gda/mlx5/queue_pair_mlx5.hpp` +* `gda/mlx5/provider_gda_mlx5.hpp` +* `gda/mlx5/mlx5dv_core.hpp` +* `gda/mlx5/mlx5_ifc_core.hpp` + +### GDA\_QUEUEPAIR\_MOCK (QueuePair mock object) +* `gda/queue_pair/queue_pair_mock.hpp` + +### rocSHMEM internals / miscellaneous +* `gda/endian.hpp` +* `gda/gda_enums.hpp` +* `rocshmem/rocshmem_config.h` +* `bit.hpp` +* `constmem.hpp` +* `constants.hpp` + +## Replacements provided in RCCL + +Replacements for these headers are included directly within the RCCL GIN-GDA codebase. +Some of these already have RCCL analogues (e.g. IBVerbs headers) while for others +only a small portion of the original rocSHMEM header is required. + +### RCCL Analogues +* `gda/ibv_core.hpp` (includes RCCL `ibvcore.h` or ``) + +### Minimized rocSHMEM header +* `log.hpp` +* `util.hpp` +* `containers/free_list.hpp` +* `rocshmem/rocshmem_common.hpp` diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/containers/free_list.hpp b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/containers/free_list.hpp new file mode 100644 index 000000000000..5d2113f99b0f --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/containers/free_list.hpp @@ -0,0 +1,67 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_CONTAINERS_FREE_LIST_HPP_ +#define LIBRARY_SRC_CONTAINERS_FREE_LIST_HPP_ + +namespace rocshmem { + +/* + * @brief Stub class for rocSHMEM header compatability. + * + * QueuePairIONIC fetching AMOs need a real (non-stub) implementation; + * RCCL GIN GDA currently uses only non-fetching AMOs. + */ +template +class FreeList { +public: + struct PopBackResult { + TYPE value; + bool success; + }; + + /** + * @brief Constructors, assignment operators, and destructor + * for the stub FreeList are deleted. + * + * GIN-GDA only needs a stub FreeList definition. + */ + __host__ FreeList() = delete; + __host__ FreeList(const FreeList&) = delete; + __host__ FreeList(FreeList&&) = delete; + __host__ FreeList& operator=(const FreeList&) = delete; + __host__ FreeList& operator=(FreeList&&) = delete; + __host__ ~FreeList() = delete; + + /** + * @brief Inserts new element at the end of the FreeList. + * + * The element goes into the container right after its last + * element. The content of val is copied (or moved) to the inserted + * element. + * + * @note Host-side API is not thread safe. + * + * @param val The value to insert in the FreeList. + * @return @c true if the operation succeed, and @c false otherwise. + */ + __device__ bool push_back(const TYPE& val) { return false; } + + /// @copydoc bool FreeList::push_back(const TYPE&) + __device__ bool push_back(TYPE&& val) { return false; } + + /** + * @brief Removes the first element in FreeList, reducing its size by one. + * + * @return An object with two fields `value` and `success`. `success` is a + * boolean indicating if the operation succeeded, and if the operation + * succeeded, the `value` field contains the popped value. + */ + __device__ PopBackResult pop_front() { + return {{}, false}; + } +}; + +} // namespace rocshmem + +#endif // LIBRARY_SRC_CONTAINERS_FREE_LIST_HPP_ diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gda/ibv_core.hpp b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gda/ibv_core.hpp new file mode 100644 index 000000000000..5b765d73aa20 --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gda/ibv_core.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef LIBRARY_SRC_GDA_IBV_CORE_HPP_ +#define LIBRARY_SRC_GDA_IBV_CORE_HPP_ + +/* selects ibvcore.h or infiniband/verbs.h per build mode */ +#ifdef NCCL_BUILD_RDMA_CORE +#include +#else +#include +#include +#include "ibvcore.h" +#endif + +/* define ETHERNET_LL_SIZE */ +#ifndef ETHERNET_LL_SIZE +#define ETHERNET_LL_SIZE ETH_ALEN +#endif + +#endif // LIBRARY_SRC_GDA_IBV_CORE_HPP_ diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h index 3fba21dc9b0d..056baf33bdbf 100644 --- a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h @@ -9,11 +9,9 @@ #include -#define NCCL_GIN_ROCSHMEM_VERSION 100 +#include "gda/queue_pair_provider.hpp" -namespace rocshmem { -class QueuePair; -} +#define NCCL_GIN_ROCSHMEM_VERSION 100 struct ncclGinRocshmemGdaGPUContext { rocshmem::QueuePair** qps; // Array of nRanks QP pointers (GPU-accessible) diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h index 168db592cef8..bb9e3d9655a1 100644 --- a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h @@ -9,7 +9,7 @@ #include "../gin_device_common.h" #include "gin_rocshmem_device_host_common_gda.h" -#include "queue_pair_device.h" +#include "gda/queue_pair_provider.hpp" template <> struct ncclGinApi_Put { @@ -21,6 +21,7 @@ struct ncclGinApi_Put { ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault) { using nccl::utility::loadConst; + using rocshmem::PostOpt, rocshmem::RingDB; bool hasSignal = signal.type != NCCL_GIN_SIGNAL_TYPE_NONE; coop.sync(); @@ -45,7 +46,12 @@ struct ncclGinApi_Put { uint32_t dstRkey = loadConst(loadConst(&dstMh->rkeys) + peer); uint32_t srcLkey = loadConst(&srcMh->lkey); - qp->put_nbi((void*)dstAddr, dstRkey, (void*)srcAddr, srcLkey, bytes, wf_info, !hasSignal); + // GIN API design prevents us from determining at compile-time whether we have a signal + if (hasSignal) { + qp->put_nbi(dstAddr, dstRkey, srcAddr, srcLkey, bytes, wf_info, PostOpt{RingDB}); + } else { + qp->put_nbi(dstAddr, dstRkey, srcAddr, srcLkey, bytes, wf_info, PostOpt{RingDB}); + } } if (hasSignal) { @@ -53,7 +59,7 @@ struct ncclGinApi_Put { uintptr_t sigAddr = loadConst(loadConst(&rsCtx->signal_raddrs) + peer) + sizeof(uint64_t) * signal.indexedSignal.signalId; uint32_t sigRkey = loadConst(loadConst(&rsCtx->signal_rkeys) + peer); - qp->atomic_add((void*)sigAddr, sigRkey, (int64_t)signalOpArg, wf_info); + qp->atomic_add(sigAddr, sigRkey, signalOpArg, wf_info, PostOpt{RingDB}); } else if (hasCounter) { qp->quiet(wf_info); } @@ -75,6 +81,7 @@ struct ncclGinApi_PutValue { cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault) { using nccl::utility::loadConst; + using rocshmem::PostOpt, rocshmem::RingDB; bool hasSignal = signal.type != NCCL_GIN_SIGNAL_TYPE_NONE; coop.sync(); @@ -85,6 +92,7 @@ struct ncclGinApi_PutValue { ncclGinRocshmemGdaMemHandle* dstMh = (ncclGinRocshmemGdaMemHandle*)dstWin; uintptr_t dstAddr = loadConst(loadConst(&dstMh->remote_vas) + peer) + dstOff; + uintptr_t srcAddr = reinterpret_cast(&srcVal); uint32_t dstRkey = loadConst(loadConst(&dstMh->rkeys) + peer); // HIP thread_scope (hip_compat.h): system is the MAX value, so a caller that @@ -95,14 +103,21 @@ struct ncclGinApi_PutValue { // lkey=0: put_nbi copies srcVal inline into the WQE // (inline_threshold >= sizeof(T)), so no registered MR is needed. - qp->put_nbi((void*)dstAddr, dstRkey, &srcVal, 0, sizeof(T), wf_info, !hasSignal); + static_assert(rocshmem::QueuePair::can_inline(sizeof(T)), + "ncclGin::putValue must inline srcVal into WQE"); + // GIN API design prevents us from determining at compile-time whether we have a signal + if (hasSignal) { + qp->put_nbi(dstAddr, dstRkey, srcAddr, 0, sizeof(T), wf_info, PostOpt{RingDB}); + } else { + qp->put_nbi(dstAddr, dstRkey, srcAddr, 0, sizeof(T), wf_info, PostOpt{RingDB}); + } if (hasSignal) { if (signalOp == ncclGinSignalInc) signalOpArg = 1; uintptr_t sigAddr = loadConst(loadConst(&rsCtx->signal_raddrs) + peer) + sizeof(uint64_t) * signal.indexedSignal.signalId; uint32_t sigRkey = loadConst(loadConst(&rsCtx->signal_rkeys) + peer); - qp->atomic_add((void*)sigAddr, sigRkey, (int64_t)signalOpArg, wf_info); + qp->atomic_add(sigAddr, sigRkey, signalOpArg, wf_info, PostOpt{RingDB}); } } coop.sync(); diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/log.hpp b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/log.hpp new file mode 100644 index 000000000000..a0dad0cba497 --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/log.hpp @@ -0,0 +1,117 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_LOG_HPP_ +#define LIBRARY_SRC_LOG_HPP_ + +#include + +#include + +namespace rocshmem { + // __attribute__((error(...))) causes a compile error at the call site + // (not at the definition), so it is safe to declare in a header. + // __HIP_DEVICE_COMPILE__ is defined only during the device compilation + // pass, giving true host-vs-device enforcement regardless of whether the + // call site is __host__, __device__, or __host__ __device__. +#ifdef __HIP_DEVICE_COMPILE__ + __attribute__((error("host-only macro used in device code"))) + void static_assert_host_only(); + __device__ inline void static_assert_device_only() {} +#else + __host__ inline void static_assert_host_only() {} + __attribute__((error("device-only macro used in host code"))) + void static_assert_device_only(); +#endif + + template + __host__ __device__ constexpr void unused_args([[maybe_unused]] T&& ...) { } +} // namespace rocshmem + + + +/***************************************************************************** + * Host-side logging macros + * + * TODO: implement using RCCL logging + *****************************************************************************/ + +#define LOG_ERROR(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOG_ERROR_EXIT(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ + exit(EXIT_FAILURE); \ +} while (0) + +#define LOG_ERROR_ABORT(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ + abort(); \ +} while (0) + +#define LOG_WARN(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOG_INFO(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOG_API(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOG_TRACE(fmt, ...) do { \ + rocshmem::static_assert_host_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + + + +/***************************************************************************** + * Device-side logging macros + * + * TODO: implement using RCCL logging + *****************************************************************************/ + +#define LOGD_ERROR(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOGD_ERROR_ABORT(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ + abort(); \ +} while (0) + +#define LOGD_WARN(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOGD_INFO(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOGD_API(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + +#define LOGD_TRACE(fmt, ...) do { \ + rocshmem::static_assert_device_only(); \ + rocshmem::unused_args(fmt __VA_OPT__(,) __VA_ARGS__); \ +} while (0) + + + +#endif // LIBRARY_SRC_LOG_HPP_ diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/queue_pair_device.h b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/queue_pair_device.h deleted file mode 100644 index 85258bf3c84e..000000000000 --- a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/queue_pair_device.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * SPDX-License-Identifier: MIT - * - * Lightweight device-only header for QueuePair. - * Provides only the device-side declarations needed by external consumers, - * without pulling in IB verbs types or host-only dependencies. - * Link against rocshmem's device bitcode for the definitions. - */ - -#ifndef NCCL_DEVICE_GIN_ROCSHMEM_QUEUE_PAIR_DEVICE_H_ -#define NCCL_DEVICE_GIN_ROCSHMEM_QUEUE_PAIR_DEVICE_H_ - -#include -#include - -namespace rocshmem { - -static __device__ __forceinline__ uint64_t get_active_lane_mask() { - return __ballot(true); -} - -static __device__ __forceinline__ int popcount(uint64_t val) { - return __builtin_popcountll(val); -} - -static __device__ __forceinline__ uint64_t lanemask_eq() { - return 1ULL << (__builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0))); -} - -static __device__ __forceinline__ int get_active_lane_count(uint64_t mask) { - return popcount(mask); -} - -static __device__ __forceinline__ int get_active_lane_num(uint64_t mask) { - return popcount(mask & (lanemask_eq() - 1)); -} - -static __device__ __forceinline__ int get_first_active_lane_id(uint64_t mask) { - return __builtin_ctzll(mask); -} - -static __device__ __forceinline__ int get_last_active_lane_id(uint64_t mask) { - return 63 - __builtin_clzll(mask); -} - -enum class ThreadScope : int { - thread, - wave, - wg, - exclusive -}; - -class ActiveWFInfo { -public: - uint64_t activemask{0}; - uint64_t pe_group_mask{0}; - int pe{-1}; - int num_pe_group_lanes{0}; - int pe_group_logical_lane_id{0}; - int pe_group_first_phys_lane_id{0}; - int pe_group_last_phys_lane_id{0}; - ThreadScope scope{ThreadScope::thread}; - bool is_pe_group_first{false}; - bool is_pe_group_last{false}; - - __device__ explicit ActiveWFInfo(int pe, ThreadScope scope = ThreadScope::thread) : pe(pe), scope(scope) { - activemask = get_active_lane_mask(); - switch (scope) { - case ThreadScope::thread: - { - pe_group_mask = __match_any_sync(activemask, pe); - num_pe_group_lanes = get_active_lane_count(pe_group_mask); - pe_group_logical_lane_id = get_active_lane_num(pe_group_mask); - pe_group_first_phys_lane_id = get_first_active_lane_id(pe_group_mask); - pe_group_last_phys_lane_id = get_last_active_lane_id(pe_group_mask); - break; - } - case ThreadScope::exclusive: - { - pe_group_mask = 1; - num_pe_group_lanes = 1; - pe_group_logical_lane_id = 0; - pe_group_first_phys_lane_id = 0; - pe_group_last_phys_lane_id = 0; - break; - } - case ThreadScope::wave: - case ThreadScope::wg: - { - pe_group_mask = 1; - num_pe_group_lanes = 1; - pe_group_logical_lane_id = get_active_lane_num(activemask); - pe_group_first_phys_lane_id = 0; - pe_group_last_phys_lane_id = 0; - } - } - is_pe_group_first = (pe_group_logical_lane_id == 0); - is_pe_group_last = (pe_group_logical_lane_id == num_pe_group_lanes - 1); - } -}; - -class QueuePair { -public: - __device__ void put_nbi(void* raddr, uint32_t rkey, const void* laddr, uint32_t lkey, size_t length, - ActiveWFInfo& wf_info, bool ring_db = true); - - __device__ void put_nbi_single(void* raddr, uint32_t rkey, const void* laddr, uint32_t lkey, size_t length, - bool ring_db = true); - - __device__ void atomic_add(void* raddr, uint32_t rkey, int64_t value, ActiveWFInfo& wf_info, bool fence = false); - - __device__ void atomic_add_single(void* raddr, uint32_t rkey, int64_t value, bool fence = false); - - __device__ void quiet(ActiveWFInfo& wf_info); - - __device__ void quiet_single(); -}; - -} // namespace rocshmem - -#endif // NCCL_DEVICE_GIN_ROCSHMEM_QUEUE_PAIR_DEVICE_H_ diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/rocshmem/rocshmem_common.hpp b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/rocshmem/rocshmem_common.hpp new file mode 100644 index 000000000000..c888d2a0ec77 --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/rocshmem/rocshmem_common.hpp @@ -0,0 +1,18 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_INCLUDE_ROCSHMEM_COMMON_HPP +#define LIBRARY_INCLUDE_ROCSHMEM_COMMON_HPP + +namespace rocshmem { + +enum ROCSHMEM_STATUS { + ROCSHMEM_SUCCESS = 0, + ROCSHMEM_ERROR = 1, +}; + +enum class BackendType { GDA_BACKEND, }; + +} // namespace rocshmem + +#endif // LIBRARY_INCLUDE_ROCSHMEM_COMMON_HPP diff --git a/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/util.hpp b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/util.hpp new file mode 100644 index 000000000000..db1bf648f9f9 --- /dev/null +++ b/projects/rccl/src/include/nccl_device/gin/rocshmem_gda/util.hpp @@ -0,0 +1,187 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_UTIL_HPP_ +#define LIBRARY_SRC_UTIL_HPP_ + +#include + +#include "bit.hpp" + +namespace rocshmem { + +/* Helper Macros for handling dynamic libraries */ +#define PPCAT_NX(prefix, func_name) prefix##func_name +#define PPCAT(prefix, func_name) PPCAT_NX(prefix, func_name) + +#define STRINGIFY_NX(name) #name +#define STRINGIFY(name) STRINGIFY_NX(name) + +#define DLSYM_OPT_HELPER(func_struct, prefix, handle, func_name) \ +do { \ + *(void **) (&func_struct.func_name) = dlsym(handle, STRINGIFY(PPCAT(prefix, func_name))); \ +} while (0) + +#define DLSYM_HELPER(func_struct, prefix, handle, func_name) \ +do { \ + *(void **) (&func_struct.func_name) = dlsym(handle, STRINGIFY(PPCAT(prefix, func_name))); \ + if (!func_struct.func_name) { \ + LOG_WARN("Failed to find function %s", STRINGIFY(PPCAT(prefix, func_name))); \ + dlclose(handle); \ + handle = nullptr; \ + return ROCSHMEM_ERROR; \ + } \ +} while (0) + +#define DLSYM_VAR_HELPER(func_struct, handle, var_name) \ +do { \ + *(void **) (&func_struct.var_name) = dlsym(handle, STRINGIFY(var_name)); \ + if (!func_struct.var_name) { \ + LOG_WARN("Failed to find function %s", STRINGIFY(var_name)); \ + dlclose(handle); \ + handle = nullptr; \ + return ROCSHMEM_ERROR; \ + } \ +} while (0) + + + +/* Device-side internal functions */ +[[maybe_unused]] __device__ __forceinline__ uint64_t get_active_lane_mask() { + return __ballot(true); +} + +[[maybe_unused]] __device__ __forceinline__ int get_active_lane_count(uint64_t active_lane_mask) { + return popcount(active_lane_mask); +} + +[[maybe_unused]] __device__ __forceinline__ int get_active_lane_count() { + return get_active_lane_count(get_active_lane_mask()); +} + +[[maybe_unused]] __device__ __forceinline__ int get_active_lane_num(uint64_t active_lane_mask) { + return popcount(active_lane_mask & (__lanemask_eq() - 1)); +} + +[[maybe_unused]] __device__ __forceinline__ int get_active_lane_num() { + return get_active_lane_num(get_active_lane_mask()); +} + +[[maybe_unused]] __device__ __forceinline__ int get_first_active_lane_id(uint64_t active_lane_mask) { + return countr_zero(active_lane_mask); +} + +[[maybe_unused]] __device__ __forceinline__ int get_first_active_lane_id() { + return get_first_active_lane_id(get_active_lane_mask()); +} + +[[maybe_unused]] __device__ __forceinline__ bool is_first_active_lane(uint64_t active_lane_mask) { + return get_active_lane_num(active_lane_mask) == 0; +} + +[[maybe_unused]] __device__ __forceinline__ bool is_first_active_lane() { + return is_first_active_lane(get_active_lane_mask()); +} + +[[maybe_unused]] __device__ __forceinline__ int get_last_active_lane_id(uint64_t active_lane_mask) { + return bit_log2(active_lane_mask); +} + +[[maybe_unused]] __device__ __forceinline__ int get_last_active_lane_id() { + return get_last_active_lane_id(get_active_lane_mask()); +} + +[[maybe_unused]] __device__ __forceinline__ bool is_last_active_lane(uint64_t active_lane_mask) { + return get_active_lane_num(active_lane_mask) == get_active_lane_count(active_lane_mask) - 1; +} + +__device__ __forceinline__ bool is_last_active_lane() { + return is_last_active_lane(get_active_lane_mask()); +} + +#define SPIN_LOCK_INVALID 0xdead +#define SPIN_LOCK_UNLOCKED 0x1234 +#define SPIN_LOCK_LOCKED 0xabcd + +/* + * Each thread in wave tries to acquire a different lock. + */ +[[maybe_unused]] __device__ __forceinline__ bool spin_lock_try_acquire_unique(uint32_t *lock) { + uint32_t lock_val = SPIN_LOCK_UNLOCKED; + + __hip_atomic_compare_exchange_strong(lock, &lock_val, SPIN_LOCK_LOCKED, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __HIP_MEMORY_SCOPE_AGENT); + + return lock_val == SPIN_LOCK_UNLOCKED; +} + +/* + * Each thread in wave acquires a different lock. + * (deadlock if locks are not different) + */ +[[maybe_unused]] __device__ __forceinline__ void spin_lock_acquire_unique(uint32_t *lock) { + while (!spin_lock_try_acquire_unique(lock)) { + // spin + } +} + +/* + * Each thread in wave releases a different lock. + */ +[[maybe_unused]] __device__ __forceinline__ void spin_lock_release_unique(uint32_t *lock) { + __hip_atomic_store(lock, SPIN_LOCK_UNLOCKED, __ATOMIC_RELEASE, + __HIP_MEMORY_SCOPE_AGENT); +} + +/* + * Threads in activemask together try to acquire the same lock. + */ +[[maybe_unused]] __device__ __forceinline__ bool spin_lock_try_acquire_shared(uint32_t *lock, uint64_t activemask) { + uint32_t lock_val = SPIN_LOCK_INVALID; + + if (is_first_active_lane(activemask)) { + lock_val = SPIN_LOCK_UNLOCKED; + __hip_atomic_compare_exchange_strong(lock, &lock_val, SPIN_LOCK_LOCKED, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __HIP_MEMORY_SCOPE_AGENT); + } + lock_val = __shfl(lock_val, get_first_active_lane_id(activemask)); + + return lock_val == SPIN_LOCK_UNLOCKED; +} + +/* + * Threads in activemask together acquire the same lock. + */ +[[maybe_unused]] __device__ __forceinline__ void spin_lock_acquire_shared(uint32_t *lock, uint64_t activemask) { + while (!spin_lock_try_acquire_shared(lock, activemask)) { + // spin + } +} + +/* + * Threads in activemask together release the same lock. + */ +[[maybe_unused]] __device__ __forceinline__ void spin_lock_release_shared(uint32_t *lock, uint64_t activemask) { + if (is_first_active_lane(activemask)) { + __hip_atomic_store(lock, SPIN_LOCK_UNLOCKED, __ATOMIC_RELEASE, + __HIP_MEMORY_SCOPE_AGENT); + } +} + +/* Is ptr_b in range [ptr_a, ptr_a + len_a) */ +[[maybe_unused]] +__host__ __device__ static bool +is_ptr_in_range(uintptr_t ptr_a, size_t len_a, uintptr_t ptr_b) { + + if ((len_a == 0) || (ptr_b < ptr_a)) { + return false; + } + + return static_cast(ptr_b - ptr_a) < len_a; +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_UTIL_HPP_ diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt index 5452499982aa..ebc99c746f9e 100644 --- a/projects/rccl/test/CMakeLists.txt +++ b/projects/rccl/test/CMakeLists.txt @@ -490,7 +490,10 @@ if(BUILD_TESTS) target_compile_definitions(rccl-UnitTestsGinAnvilPlugin PRIVATE ENABLE_ROCSHMEM_GIN) # No rccl link below, so the implicit ordering after hipify has to be explicit. - add_dependencies(rccl-UnitTestsGinAnvilPlugin hipify_all) + add_dependencies(rccl-UnitTestsGinAnvilPlugin hipify_all copy_nccl_device_headers) + if(TARGET copy_rocshmem_headers) + add_dependencies(rccl-UnitTestsGinAnvilPlugin copy_rocshmem_headers) + endif() endif() endif() @@ -1098,6 +1101,8 @@ if(BUILD_TESTS) rocm_install(TARGETS ${test_executable} COMPONENT tests) endforeach() + # Suite G: prefer RCCL's GDA headers over rocSHMEM's (higher-priority include path). + # GDA test stubs are used by defining GDA_QUEUEPAIR_MOCK. # Suite H: shadow rocSHMEM sdma headers with no-op test stubs (higher-priority include path). # Gated on ENABLE_ROCSHMEM_GIN because that is what now turns on NCCL_GIN_ANVIL_SDMA_ENABLE, # and these are the only two test targets that define it. @@ -1108,7 +1113,10 @@ if(BUILD_TESTS) target_include_directories(rccl-UnitTestsFixtures PRIVATE ${CMAKE_SOURCE_DIR}/../rocshmem/src) endif() target_include_directories(rccl-UnitTestsFixtures BEFORE PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/device) + ${CMAKE_CURRENT_SOURCE_DIR}/device + ${PROJECT_BINARY_DIR}/include/nccl_device/gin/rocshmem_gda + ) + target_compile_definitions(rccl-UnitTestsFixtures PRIVATE GDA_QUEUEPAIR_MOCK) set_source_files_properties( ${CMAKE_CURRENT_SOURCE_DIR}/device/GinAnvilIpcDevice_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/device/GinAnvilSdmaTemplate_test.cpp @@ -1129,7 +1137,10 @@ if(BUILD_TESTS) -fgpu-rdc --hip-link ${_ROCSHMEM_FIXTURES_RDC_OFFLOAD_FLAGS} -rdynamic) if(TARGET rccl-UnitTestsGinAnvilPlugin) target_include_directories(rccl-UnitTestsGinAnvilPlugin BEFORE PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/device) + ${CMAKE_CURRENT_SOURCE_DIR}/device + ${PROJECT_BINARY_DIR}/include/nccl_device/gin/rocshmem_gda + ) + target_compile_definitions(rccl-UnitTestsGinAnvilPlugin PRIVATE GDA_QUEUEPAIR_MOCK) endif() endif() diff --git a/projects/rccl/test/device/GinRocshmemGdaTemplate_test.cpp b/projects/rccl/test/device/GinRocshmemGdaTemplate_test.cpp index 93b57a4bc5e6..a964e40cf701 100644 --- a/projects/rccl/test/device/GinRocshmemGdaTemplate_test.cpp +++ b/projects/rccl/test/device/GinRocshmemGdaTemplate_test.cpp @@ -10,11 +10,15 @@ // specializations, so this suite unit-tests the GDA AllToAll device path // without a live network. // -// The real rocshmem::QueuePair device methods (put_nbi/atomic_add/quiet) are -// only declared in queue_pair_device.h (defined in rocSHMEM device bitcode). -// This translation unit supplies inline stub definitions so the suite links -// against no librocshmem device code, mirroring how Suite H shadows the SDMA -// engine with test/device/sdma/anvil_device.hpp. +// rocshmem::QueuePair is declared as a type alias for some QueuePair type by +// the rocSHMEM-GDA header gda/queue_pair_provider.hpp. rocshmem::QueuePairMock +// provides a mock QueuePair type that conforms to rocshmem::QueuePairInterface. +// By defining GDA_QUEUEPAIR_MOCK prior to including queue_pair_provider.hpp, +// rocshmem::QueuePair is aliased to rocshmem::QueuePairMock. The definition of +// the rocshmem::QueuePairMock mock QueuePair object type mirrors how Suite H +// shadows the SDMA engine with test/device/sdma/anvil_device.hpp. +// projects/rccl/test/CMakeLists.txt defines GDA_QUEUEPAIR_MOCK for targets +// rccl-UnitTestsFixtures and rccl-UnitTestsGinAnvilPlugin #include "DeviceTestBase.hpp" @@ -22,6 +26,7 @@ #include "nccl_device/gin/gin_device_host_common.h" #include "nccl_device/gin/gin_device_common.h" #include "nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h" +#include "nccl_device/gin/rocshmem_gda/gda/queue_pair_provider.hpp" #if NCCL_GIN_ROCSHMEM_GDA_ENABLE // Count invocations of the Put/PutValue system-scope fence seam (gin_device_common.h). @@ -38,43 +43,6 @@ __device__ unsigned long long g_gdaStubThreadfenceCount = 0; #if NCCL_GIN_ROCSHMEM_GDA_ENABLE -// -------------------------------------------------------------------------- -// Stub definitions for rocshmem::QueuePair device methods. -// put_nbi : byte-copy laddr -> raddr (observe data landing at remote VA) -// atomic_add : atomic add into the remote signal word (observe signal deliver) -// quiet : bump a device-global call counter (observe completion sync) -// Non-RDC build: these must be defined in the same TU as the kernels that -// (via the inlined template) call them. -// -------------------------------------------------------------------------- -__device__ unsigned long long g_gdaStubQuietCount = 0; -__device__ unsigned long long g_gdaStubPutNbiCount = 0; - -namespace rocshmem { - -__device__ void QueuePair::put_nbi(void* raddr, uint32_t /*rkey*/, const void* laddr, uint32_t /*lkey*/, - size_t length, ActiveWFInfo& /*wf_info*/, bool /*ring_db*/) { - if (raddr == nullptr || laddr == nullptr) return; - atomicAdd(&g_gdaStubPutNbiCount, 1ULL); - auto* d = static_cast(raddr); - const auto* s = static_cast(laddr); - for (size_t i = 0; i < length; ++i) d[i] = s[i]; -} - -// Matches queue_pair_device.h: atomic_add(void* raddr, uint32_t rkey, int64_t -// value, ActiveWFInfo&, bool fence). The template's signal path calls this -// (gin_rocshmem_gda.h:54,101); rkey/fence are unused by the byte-accurate stub. -__device__ void QueuePair::atomic_add(void* raddr, uint32_t /*rkey*/, int64_t value, ActiveWFInfo& /*wf_info*/, - bool /*fence*/) { - if (raddr == nullptr) return; - atomicAdd(reinterpret_cast(raddr), static_cast(value)); -} - -__device__ void QueuePair::quiet(ActiveWFInfo& /*wf_info*/) { - atomicAdd(&g_gdaStubQuietCount, 1ULL); -} - -} // namespace rocshmem - namespace RcclUnitTesting { @@ -156,34 +124,56 @@ class GdaEnv { }; static void resetQuietCount() { - unsigned long long z = 0; - HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(g_gdaStubQuietCount), &z, sizeof(z))); + size_t z = 0; + HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(rocshmem::QueuePairMock::quiet_count), &z, sizeof(z))); } -static unsigned long long readQuietCount() { - unsigned long long q = 0; - HIP_EXPECT(hipMemcpyFromSymbol(&q, HIP_SYMBOL(g_gdaStubQuietCount), sizeof(q))); +static size_t readQuietCount() { + size_t q = 0; + HIP_EXPECT(hipMemcpyFromSymbol(&q, HIP_SYMBOL(rocshmem::QueuePairMock::quiet_count), sizeof(q))); return q; } static void resetPutNbiCount() { - unsigned long long z = 0; - HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(g_gdaStubPutNbiCount), &z, sizeof(z))); + size_t z = 0; + HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(rocshmem::QueuePairMock::rma_count), &z, sizeof(z))); } -static unsigned long long readPutNbiCount() { - unsigned long long n = 0; - HIP_EXPECT(hipMemcpyFromSymbol(&n, HIP_SYMBOL(g_gdaStubPutNbiCount), sizeof(n))); +static size_t readPutNbiCount() { + size_t n = 0; + HIP_EXPECT(hipMemcpyFromSymbol(&n, HIP_SYMBOL(rocshmem::QueuePairMock::rma_count), sizeof(n))); return n; } +static void resetPutValCount() { + size_t z = 0; + HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(rocshmem::QueuePairMock::rma_inline_count), &z, sizeof(z))); +} + +static size_t readPutValCount() { + size_t v = 0; + HIP_EXPECT(hipMemcpyFromSymbol(&v, HIP_SYMBOL(rocshmem::QueuePairMock::rma_inline_count), sizeof(v))); + return v; +} + +static void resetSignalCount() { + size_t z = 0; + HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(rocshmem::QueuePairMock::amo_count), &z, sizeof(z))); +} + +static size_t readSignalCount() { + size_t s = 0; + HIP_EXPECT(hipMemcpyFromSymbol(&s, HIP_SYMBOL(rocshmem::QueuePairMock::amo_count), sizeof(s))); + return s; +} + static void resetThreadfenceCount() { - unsigned long long z = 0; + size_t z = 0; HIP_CHECK(hipMemcpyToSymbol(HIP_SYMBOL(g_gdaStubThreadfenceCount), &z, sizeof(z))); } -static unsigned long long readThreadfenceCount() { - unsigned long long c = 0; +static size_t readThreadfenceCount() { + size_t c = 0; HIP_EXPECT(hipMemcpyFromSymbol(&c, HIP_SYMBOL(g_gdaStubThreadfenceCount), sizeof(c))); return c; } @@ -237,9 +227,11 @@ TEST_F(GinRocshmemGdaTemplateTest, Put_ZeroByteSkipsDataStillSignals) { env.dst.zero(); env.build(); resetPutNbiCount(); + resetSignalCount(); kernelPutZeroBytes<<<1, 1>>>(env.dHarness.ptr); syncAndCheck(); EXPECT_EQ(readPutNbiCount(), 0ULL); // production skips put_nbi when bytes==0 + EXPECT_EQ(readSignalCount(), 1ULL); // signal still dispatched auto got = env.dst.copyTo(); for (int i = 0; i < kN; ++i) EXPECT_EQ(got[static_cast(i)], 0u); // data write skipped auto sigs = env.signals.copyTo(); @@ -264,8 +256,10 @@ TEST_F(GinRocshmemGdaTemplateTest, Put_SignalAddDeliversArg) { GdaEnv env(32); env.src.zero(); env.build(); + resetSignalCount(); kernelPutSignalAdd<<<1, 1>>>(env.dHarness.ptr); syncAndCheck(); + EXPECT_EQ(readSignalCount(), 1ULL); // signal dispatched auto sigs = env.signals.copyTo(); EXPECT_EQ(sigs[1], 7ULL); EXPECT_EQ(sigs[0], 0ULL); @@ -414,9 +408,11 @@ TEST_F(GinRocshmemGdaTemplateTest, PutValue_InlineScalar) { GdaEnv env(sizeof(uint64_t)); env.dst.zero(); env.build(); + resetPutValCount(); const uint64_t kVal = 0xAABBCCDDEEFF0011ULL; kernelPutValueScalar<<<1, 1>>>(env.dHarness.ptr, kVal); syncAndCheck(); + EXPECT_EQ(readPutValCount(), 1ULL); // put inlined auto got = env.dst.copyTo(); uint64_t observed = 0; std::memcpy(&observed, got.data(), sizeof(observed)); @@ -440,9 +436,13 @@ TEST_F(GinRocshmemGdaTemplateTest, PutValue_WithSignal) { GdaEnv env(sizeof(uint32_t)); env.dst.zero(); env.build(); + resetPutValCount(); + resetSignalCount(); const uint32_t kVal = 0x12345678u; kernelPutValueSignal<<<1, 1>>>(env.dHarness.ptr, kVal); syncAndCheck(); + EXPECT_EQ(readPutValCount(), 1ULL); // put inlined + EXPECT_EQ(readSignalCount(), 1ULL); // signal dispatched auto got = env.dst.copyTo(); uint32_t observed = 0; std::memcpy(&observed, got.data(), sizeof(observed)); @@ -466,7 +466,7 @@ TEST_F(GinRocshmemGdaTemplateTest, Flush_QuietsAllPeers) { resetQuietCount(); kernelFlush<<<1, 1>>>(env.dHarness.ptr); syncAndCheck(); - EXPECT_EQ(readQuietCount(), static_cast(GdaEnv::kNRanks)); + EXPECT_EQ(readQuietCount(), static_cast(GdaEnv::kNRanks)); } // G11: GetSignalPtr/ResetSignal and GetCounterPtr/ResetCounter round-trip. diff --git a/projects/rocshmem/CHANGELOG.md b/projects/rocshmem/CHANGELOG.md index 6a4033b0851c..c0089835c31e 100644 --- a/projects/rocshmem/CHANGELOG.md +++ b/projects/rocshmem/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog for rocSHMEM ## Unreleased - rocSHMEM 3.8.0 for ROCm 10.x +### Added +* GDA device code refactor (#7217) + * Core GDA device code is now header-only. + * Removed indirection and branching when compiled for a single GDA provider. + * Significantly reduced size of device code GDA Queue Pair management objects. + * Reduced latency of `rocshmem_quiet` for AMD Pensando IONIC provider. + This also reduces the latency of blocking communication calls. + * Communications using the GDA backend can now selectively choose at compilation time + to reduce the internal GDA Queue Pair object correctness and safety constraints, + improving application performance when such relaxations are known to be safe by the caller. + These relaxations can be applied independently at each call site. + This is currently an internal mechanism and is not exposed by the public rocSHMEM APIs. + ## rocSHMEM 3.7.0 for ROCm 10.1 ### Added diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 5988834664d1..140ccaeb8577 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -71,10 +71,39 @@ option(BUILD_DEBUG_TRACE_DEVICE "Compile in device-side trace logging (LOGD_API/ option(BUILD_DEBUG_DEVICE "Compile in device-side logging (LOGD_WARN/INFO and GDA error checking)" OFF) option(GDA_IONIC "Build for AMD Pensando IONIC RDMA provider" OFF) -option(GDA_BNXT "Build for Broadcom RDMA provider" OFF) +option(GDA_BNXT "Build for Broadcom BNXT RDMA provider" OFF) option(GDA_MLX5 "Build for Mellanox MLX5 RDMA provider" OFF) option(ASAN "Enable AddressSanitizer" ${ASAN}) +set(GDA_PROVIDERS "") +foreach(gda_provider IN ITEMS IONIC BNXT MLX5) + if(GDA_${gda_provider}) + list(APPEND GDA_PROVIDERS "${gda_provider}") + endif() +endforeach() +list(LENGTH GDA_PROVIDERS GDA_PROVIDER_COUNT) + +# default to provider multiplexer only when multiple providers enabled +if(GDA_PROVIDER_COUNT GREATER 1) + set(GDA_MUX_DEFAULT ON) +else() + set(GDA_MUX_DEFAULT OFF) +endif() +option(GDA_MUX "Build provider multiplexer with runtime dispatch to hardware-specific provider" + ${GDA_MUX_DEFAULT}) + +if(USE_GDA) + if(GDA_PROVIDER_COUNT EQUAL 0) + message(WARNING "GDA support requested, but no GDA provider enabled - disabling GDA") + set(USE_GDA OFF) + set(GDA_MUX OFF) + elseif(GDA_PROVIDER_COUNT EQUAL 1 AND GDA_MUX) + message(WARNING "GDA provider multiplexer enabled, but only ${GDA_PROVIDERS} provider enabled") + elseif(GDA_PROVIDER_COUNT GREATER 1 AND NOT GDA_MUX) + message(FATAL_ERROR "Multiple GDA providers enabled, but provider multiplexer support disabled") + endif() +endif() + set(USE_EXTERNAL_MPI AUTO CACHE STRING "Link with an external MPI (required if used MPI is ABI incompatible with Open MPI v5)") set_property(CACHE USE_EXTERNAL_MPI PROPERTY STRINGS AUTO ON OFF) @@ -283,6 +312,9 @@ if (NOT BUILD_TESTS_ONLY) set(GDA_IONIC OFF) set(GDA_BNXT OFF) set(GDA_MLX5 OFF) + set(GDA_MUX OFF) + set(GDA_PROVIDERS "") + set(GDA_PROVIDER_COUNT 0) endif() endif() @@ -428,26 +460,23 @@ if (NOT BUILD_TESTS_ONLY) string(REPLACE ";" " " ROCSHMEM_OFFLOAD_TARGETS "${GPU_TARGETS}") - set(_backends "") + set(backends "") if(USE_GDA) - set(_gda_nics "") - foreach(_nic IN ITEMS IONIC BNXT MLX5) - if(GDA_${_nic}) - list(APPEND _gda_nics "${_nic}") - endif() - endforeach() - list(JOIN _gda_nics "/" _gda_providers) - list(APPEND _backends "GDA(${_gda_providers})") + list(JOIN GDA_PROVIDERS "/" gda_provider_string) + if(GDA_MUX) + string(PREPEND gda_provider_string "MUX:") + endif() + list(APPEND backends "GDA(${gda_provider_string})") endif() if(USE_RO) - list(APPEND _backends "RO") + list(APPEND backends "RO") endif() if(USE_IPC) - list(APPEND _backends "IPC") + list(APPEND backends "IPC") endif() - if(_backends) - list(JOIN _backends "+" ROCSHMEM_BACKEND) + if(backends) + list(JOIN backends "+" ROCSHMEM_BACKEND) else() set(ROCSHMEM_BACKEND "NONE") endif() diff --git a/projects/rocshmem/cmake/DeviceBitcode.cmake b/projects/rocshmem/cmake/DeviceBitcode.cmake index 95fb71f528c1..49c73dddc6c6 100644 --- a/projects/rocshmem/cmake/DeviceBitcode.cmake +++ b/projects/rocshmem/cmake/DeviceBitcode.cmake @@ -154,31 +154,15 @@ if(USE_IPC) ) endif() -# GDA queue_pair implementations are guarded by GDA_MLX5/GDA_IONIC/GDA_BNXT in -# queue_pair.hpp. Only compile the backend(s) enabled for this build so that -# declarations and definitions match. +# GDA QueuePair implementations are guarded by GDA_MUX/GDA_IONIC/GDA_BNXT/GDA_MLX5 +# in queue_pair_provider.hpp and (if GDA_MUX is enabled) queue_pair_mux.hpp. +# Only compile the backend(s) enabled for this build so that declarations and definitions match. if(USE_GDA) list(APPEND BITCODE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/context_gda_device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/context_gda_device_coll.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/backend_gda.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/queue_pair.cpp ) - if(GDA_MLX5) - list(APPEND BITCODE_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/mlx5/queue_pair_mlx5.cpp - ) - endif() - if(GDA_IONIC) - list(APPEND BITCODE_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/ionic/queue_pair_ionic.cpp - ) - endif() - if(GDA_BNXT) - list(APPEND BITCODE_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/src/gda/bnxt/queue_pair_bnxt.cpp - ) - endif() endif() # Build bitcode for each GPU architecture diff --git a/projects/rocshmem/cmake/rocshmem_config.h.in b/projects/rocshmem/cmake/rocshmem_config.h.in index 9c0f9342e05b..10ec550c7458 100644 --- a/projects/rocshmem/cmake/rocshmem_config.h.in +++ b/projects/rocshmem/cmake/rocshmem_config.h.in @@ -41,6 +41,7 @@ #cmakedefine GDA_IONIC #cmakedefine GDA_BNXT #cmakedefine GDA_MLX5 +#cmakedefine GDA_MUX #cmakedefine HAVE_EXTERNAL_MPI #cmakedefine HAVE_DEVICE_MALLOC_UNCACHED diff --git a/projects/rocshmem/src/constmem.cpp b/projects/rocshmem/src/constmem.cpp index 60fa9a0ff1c7..d7c0f2fd35b3 100644 --- a/projects/rocshmem/src/constmem.cpp +++ b/projects/rocshmem/src/constmem.cpp @@ -5,6 +5,10 @@ #include "gda/backend_gda.hpp" #endif +/** + * @file constmem.cpp + */ + namespace rocshmem { extern Backend *backend; @@ -46,4 +50,40 @@ void init_constant_memory(void) { CHECK_HIP(hipMemcpyToSymbol(HIP_SYMBOL(constmem), &constmem_values, sizeof(constmem_t))); } +} // namespace rocshmem + +/** + * @brief Exported C function for GIN QP factory to initialize __constant__ constmem. + * + * Lives in librocshmem.a so HIP_SYMBOL(constmem) resolves via device linking. + * Callable from librccl.so via -rdynamic symbol export. + * + * @param[in] provider GDA provider enumerator from rocshmem::gda::provider / rocshmem::GDAProvider. + * @param[in] rank Rank of this PE. + */ +extern "C" void rocshmem_gin_init_constmem(int provider, int rank) { + using namespace rocshmem; + + // Initialize constmem.gda_provider for QP device dispatch + GDAProvider gda_prov = static_cast(provider); + constmem_t* cm_addr{nullptr}; + if (hipGetSymbolAddress(reinterpret_cast(&cm_addr), + HIP_SYMBOL(constmem)) == hipSuccess) { + CHECK_HIP(hipMemcpy(&cm_addr->gda_provider, &gda_prov, sizeof(gda_prov), hipMemcpyDefault)); + } + + // Initialize logd_constants for device-side error reporting + log_pe_number = rank; + uint32_t log_flags = 0; + if (envvar::log_flags.show_error) log_flags |= logd_constants::SHOW_ERROR; + if (envvar::log_flags.show_warn) log_flags |= logd_constants::SHOW_WARN; + if (envvar::log_flags.show_info) log_flags |= logd_constants::SHOW_INFO; + if (envvar::log_flags.show_trace) log_flags |= logd_constants::SHOW_TRACE; + if (envvar::log_flags.show_color) log_flags |= logd_constants::SHOW_COLOR; + struct logd_constants host_logd{rank, log_flags}; + struct logd_constants* logd_addr{nullptr}; + if (hipGetSymbolAddress(reinterpret_cast(&logd_addr), + HIP_SYMBOL(logd_constants)) == hipSuccess) { + CHECK_HIP(hipMemcpy(logd_addr, &host_logd, sizeof(host_logd), hipMemcpyDefault)); + } } diff --git a/projects/rocshmem/src/gda/CMakeLists.txt b/projects/rocshmem/src/gda/CMakeLists.txt index 841e714ebf4e..1a5388c7b087 100644 --- a/projects/rocshmem/src/gda/CMakeLists.txt +++ b/projects/rocshmem/src/gda/CMakeLists.txt @@ -34,10 +34,11 @@ target_sources( backend_gda.cpp ibv_wrapper.cpp gda_team.cpp - queue_pair.cpp topology.cpp ) -add_subdirectory(mlx5) -add_subdirectory(bnxt) +add_subdirectory(queue_pair) + add_subdirectory(ionic) +add_subdirectory(bnxt) +add_subdirectory(mlx5) diff --git a/projects/rocshmem/src/gda/backend_gda.cpp b/projects/rocshmem/src/gda/backend_gda.cpp index 36c165eab2b2..fd597dc518a1 100644 --- a/projects/rocshmem/src/gda/backend_gda.cpp +++ b/projects/rocshmem/src/gda/backend_gda.cpp @@ -36,6 +36,7 @@ #include "debug_gda.hpp" #include "ibv_wrapper.hpp" #include "envvar.hpp" +#include "gda_enums.hpp" #include "gda_team.hpp" #include "log.hpp" #include "mpi_instance.hpp" @@ -107,7 +108,7 @@ void GDABackend::init() { configure_nic_policy(); LOG_TRACE("PE %d QP config: num_nics=%d, qps_per_pe_default_ctx=%zu, " - "qps_per_pe_usr_ctx=%zu, num_qps_per_pe=%zu, num_qps=%u, " + "qps_per_pe_usr_ctx=%zu, num_qps_per_pe=%zu, num_qps=%zu, " "nic_policy=%s", my_pe, num_nics_, qps_per_pe_default_ctx_, qps_per_pe_usr_ctx_, num_qps_per_pe, num_qps, @@ -616,7 +617,7 @@ int GDABackend::buffer_register(void *addr, size_t length) { /* Register with QPs */ for (size_t i = 0; i < num_qps; i++) { - err = host_qps[i].buffer_register((uintptr_t)addr, length); + err = host_qps[i].buffer_register(addr, length); if (ROCSHMEM_SUCCESS != err) { qp_registration_failed = true; } @@ -641,7 +642,7 @@ int GDABackend::buffer_unregister(void *addr) { /* Deregister with QPs */ for (size_t i = 0; i < num_qps; i++) { - err = host_qps[i].buffer_unregister((uintptr_t)addr); + err = host_qps[i].buffer_unregister(addr); if (ROCSHMEM_SUCCESS != err) { return ROCSHMEM_ERROR; } @@ -752,7 +753,7 @@ int GDABackend::buffer_register_symmetric([[maybe_unused]] void *addr, [[maybe_unused]] size_t length, [[maybe_unused]] void **registered_addr) { #if HIP_VERSION >= 70200000 - if (registered_addr == nullptr || symm_entries_ == nullptr) { + if (registered_addr == nullptr || symm_buffers_ == nullptr) { return ROCSHMEM_ERROR; } @@ -842,7 +843,7 @@ int GDABackend::buffer_register_symmetric([[maybe_unused]] void *addr, std::vector rkeys(static_cast(num_pes) * num_nics_, 0); for (int n = 0; n < num_nics_; n++) { - rkeys[my_pe * num_nics_ + n] = mrs[n]->rkey; + rkeys[flat_pe_nic_idx(my_pe, n)] = mrs[n]->rkey; } symm_allgather(rkeys.data(), sizeof(uint32_t) * num_nics_); @@ -876,14 +877,12 @@ int GDABackend::buffer_register_symmetric([[maybe_unused]] void *addr, for (int pe = 0; pe < num_pes; pe++) { for (int n = 0; n < num_nics_; n++) { - QpSymmEntry &e = - host_symm_entries_[(static_cast(pe) * num_nics_ + n) * - symm_capacity_ + slot]; + SymmBufferInfo &e = host_symm_buffers_[flat_pe_nic_idx(pe, n) * symm_capacity_ + slot]; e.local_base = key; e.remote_base = bases[pe]; e.length = length; - e.rkey = rkeys[static_cast(pe) * num_nics_ + n]; - e.lkey = lkeys[n]; + e.lkey = QueuePair::to_provider_endianness(lkeys[n]); + e.rkey = QueuePair::to_provider_endianness(rkeys[flat_pe_nic_idx(pe, n)]); } } @@ -892,8 +891,8 @@ int GDABackend::buffer_register_symmetric([[maybe_unused]] void *addr, * for simplicity, then publish by bumping the shared count last so a device * reader never observes count > populated entries. */ - CHECK_HIP(hipMemcpy(symm_entries_, host_symm_entries_.data(), - host_symm_entries_.size() * sizeof(QpSymmEntry), + CHECK_HIP(hipMemcpy(symm_buffers_, host_symm_buffers_.data(), + host_symm_buffers_.size() * sizeof(SymmBufferInfo), hipMemcpyHostToDevice)); symm_count_host_ = slot + 1; CHECK_HIP(hipMemcpy(symm_count_, &symm_count_host_, sizeof(int), @@ -968,9 +967,8 @@ int GDABackend::gda_nic_unregister([[maybe_unused]] uintptr_t key) { if (slot != last && last >= 0) { for (int pe = 0; pe < num_pes; pe++) { for (int n = 0; n < num_nics_; n++) { - size_t base = - (static_cast(pe) * num_nics_ + n) * symm_capacity_; - host_symm_entries_[base + slot] = host_symm_entries_[base + last]; + size_t base = flat_pe_nic_idx(pe, n) * symm_capacity_; + host_symm_buffers_[base + slot] = host_symm_buffers_[base + last]; } } for (auto &kv : gda_symm_records_) { @@ -980,8 +978,8 @@ int GDABackend::gda_nic_unregister([[maybe_unused]] uintptr_t key) { } } } - CHECK_HIP(hipMemcpy(symm_entries_, host_symm_entries_.data(), - host_symm_entries_.size() * sizeof(QpSymmEntry), + CHECK_HIP(hipMemcpy(symm_buffers_, host_symm_buffers_.data(), + host_symm_buffers_.size() * sizeof(SymmBufferInfo), hipMemcpyHostToDevice)); symm_count_host_ = (last >= 0) ? last : 0; CHECK_HIP(hipMemcpy(symm_count_, &symm_count_host_, sizeof(int), @@ -1064,7 +1062,7 @@ void GDABackend::symmetric_buffer_unregister_all() { int GDABackend::buffer_unregister_symmetric([[maybe_unused]] void *addr) { #if HIP_VERSION >= 70200000 - if (addr == nullptr || symm_entries_ == nullptr) { + if (addr == nullptr || symm_buffers_ == nullptr) { return ROCSHMEM_ERROR; } @@ -1327,15 +1325,15 @@ bool GDABackend::device_matches_provider_vendor(GDAProvider provider, switch (provider) { case GDAProvider::BNXT: - expected_vendor_id = GDA_BNXT_VENDOR_ID; + expected_vendor_id = static_cast(gda::vendor_id::BNXT); vendor_name = "BNXT/Broadcom"; break; case GDAProvider::IONIC: - expected_vendor_id = GDA_IONIC_VENDOR_ID; + expected_vendor_id = static_cast(gda::vendor_id::IONIC); vendor_name = "IONIC/Pensando"; break; case GDAProvider::MLX5: - expected_vendor_id = GDA_MLX5_VENDOR_ID; + expected_vendor_id = static_cast(gda::vendor_id::MLX5); vendor_name = "MLX5/Mellanox"; break; case GDAProvider::UNSET: @@ -1674,7 +1672,7 @@ void GDABackend::setup_heap_memory_rkey() { CHECK_HIP(hipHostMalloc(&heap_rkey, rkeys_size)); for (int n = 0; n < num_nics_; n++) { - heap_rkey[my_pe * num_nics_ + n] = nic_devices_[n].heap_mr->rkey; + heap_rkey[flat_pe_nic_idx(my_pe, n)] = nic_devices_[n].heap_mr->rkey; } hipStream_t stream; @@ -1711,16 +1709,15 @@ void GDABackend::cleanup_heap_memory_rkey() { } void GDABackend::setup_gpu_qps() { - size_t qp_objs_count; - size_t qp_objs_mem_size; - - qp_objs_count = num_qps; - qp_objs_mem_size = sizeof(QueuePair) * qp_objs_count; + size_t qp_objs_mem_size = sizeof(QueuePair) * num_qps; CHECK_HIP(hipMalloc(&gpu_qps, qp_objs_mem_size)); - host_qps = (QueuePair*) malloc(qp_objs_mem_size); - CHECK_NNULL(host_qps, "malloc (host_qps)"); + QueuePair *host_gpu_qps = + static_cast(aligned_alloc(alignof(QueuePair), qp_objs_mem_size)); + CHECK_NNULL(host_gpu_qps, "aligned_alloc (host_gpu_qps)"); + + host_qps.reserve(num_qps); #if HIP_VERSION >= 70200000 /* @@ -1736,62 +1733,42 @@ void GDABackend::setup_gpu_qps() { symm_capacity = 1; } symm_capacity_ = symm_capacity; - size_t num_entries = - static_cast(num_pes) * num_nics_ * symm_capacity_; - CHECK_HIP(hipMalloc(reinterpret_cast(&symm_entries_), - num_entries * sizeof(QpSymmEntry))); - CHECK_HIP(hipMemset(symm_entries_, 0, num_entries * sizeof(QpSymmEntry))); + size_t num_symm_buffers = static_cast(num_pes) * num_nics_ * symm_capacity_; + CHECK_HIP(hipMalloc(reinterpret_cast(&symm_buffers_), + num_symm_buffers * sizeof(SymmBufferInfo))); + CHECK_HIP(hipMemset(symm_buffers_, 0, num_symm_buffers * sizeof(SymmBufferInfo))); CHECK_HIP(hipMalloc(reinterpret_cast(&symm_count_), sizeof(int))); CHECK_HIP(hipMemset(symm_count_, 0, sizeof(int))); - host_symm_entries_.assign(num_entries, QpSymmEntry{}); + host_symm_buffers_.assign(num_symm_buffers, SymmBufferInfo{}); symm_count_host_ = 0; #endif - const auto &heap_bases = heap.get_heap_bases(); - for (size_t i = 0; i < qp_objs_count; i++) { - new (&host_qps[i]) QueuePair(nic_for_qp(i).pd_orig, gda_provider); - int qp_dest_pe = static_cast(i % num_pes); - /* Cache the peer heap base so the heap translation is load-free. */ - host_qps[i].remote_heap_base = - reinterpret_cast(heap_bases[qp_dest_pe]); -#if HIP_VERSION >= 70200000 - /* Point the QP at the registration slice for its (dest_pe, nic_idx). */ - int qp_nic_idx = nic_idx_for_qp(static_cast(i)); - host_qps[i].symm_count = symm_count_; - host_qps[i].symm_entries = - symm_entries_ + (static_cast(qp_dest_pe) * num_nics_ + - qp_nic_idx) * symm_capacity_; -#endif - CHECK_HIP(hipMemcpy(&gpu_qps[i], &host_qps[i], sizeof(QueuePair), hipMemcpyDefault)); - - initialize_gpu_qp(&gpu_qps[i], i); + for (size_t i = 0; i < num_qps; i++) { + initialize_gpu_qp(&host_gpu_qps[i], i); } + + CHECK_HIP(hipMemcpy(gpu_qps, host_gpu_qps, qp_objs_mem_size, hipMemcpyDefault)); + free(host_gpu_qps); } void GDABackend::cleanup_gpu_qps() { - size_t qp_objs_count; - - qp_objs_count = num_qps; - - for (size_t i = 0; i < qp_objs_count; i++) { - host_qps[i].~QueuePair(); - } - - free(host_qps); + /* Calls QueuePairHost::~QueuePairHost on all elements in host_qps */ + host_qps.clear(); + /* QueuePair is trivially destructible, can just free it */ CHECK_HIP(hipFree(gpu_qps)); gpu_qps = nullptr; #if HIP_VERSION >= 70200000 - if (symm_entries_ != nullptr) { - CHECK_HIP(hipFree(symm_entries_)); - symm_entries_ = nullptr; + if (symm_buffers_ != nullptr) { + CHECK_HIP(hipFree(symm_buffers_)); + symm_buffers_ = nullptr; } if (symm_count_ != nullptr) { CHECK_HIP(hipFree(symm_count_)); symm_count_ = nullptr; } - host_symm_entries_.clear(); + host_symm_buffers_.clear(); symm_capacity_ = 0; symm_count_host_ = 0; #endif @@ -1881,7 +1858,7 @@ void GDABackend::validate_ib_device(NicDevice &nic) { const std::set supported_bnxt_part_ids = { 0x1760 /* BCM57608 */}; const char min_supported_bnxt_fw_ver[12] = "233.2.104.0"; - if (nic.device_attr.vendor_id != GDA_BNXT_VENDOR_ID) { + if (nic.device_attr.vendor_id != static_cast(gda::vendor_id::BNXT)) { LOG_ERROR_EXIT("%s GDAProvider::BNXT requested but an invalid device is selected", debug_str.c_str()); } @@ -2068,15 +2045,29 @@ void GDABackend::create_queues() { mlx5_qps.resize(num_qps); - if (gda_provider == GDAProvider::BNXT) { - bnxt_create_cqs(ncqes); - bnxt_create_qps(sq_size); - } else if (gda_provider == GDAProvider::IONIC) { + switch (gda_provider) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: ionic_create_cqs(ncqes); create_qps(sq_size); - } else if (gda_provider == GDAProvider::MLX5) { + break; +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + bnxt_create_cqs(ncqes); + bnxt_create_qps(sq_size); + break; +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: // mlx5_create_qps also creates the associated CQs mlx5_create_qps(sq_size); + break; +#endif + default: + create_cqs(ncqes); + create_qps(sq_size); + break; } alternate_qp_ports(); @@ -2172,9 +2163,11 @@ void GDABackend::create_parent_domain(NicDevice &nic) { CHECK_NNULL(nic.pd_parent, "ibv_alloc_parent_domain"); dump_ibv_pd(nic.pd_parent); +#if defined(GDA_IONIC) if (gda_provider == GDAProvider::IONIC) { ionic_setup_parent_domain(nic, &pattr); } +#endif // defined(GDA_IONIC) } void GDABackend::create_cqs(int cqe) { @@ -2205,35 +2198,85 @@ void GDABackend::create_cqs(int cqe) { void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { switch (gda_provider) { +#if defined(GDA_IONIC) case GDAProvider::IONIC: ionic_initialize_gpu_qp(gpu_qp, conn_num); dump_ibv_qp(qps[conn_num], conn_num); break; +#endif +#if defined(GDA_BNXT) case GDAProvider::BNXT: bnxt_initialize_gpu_qp(gpu_qp, conn_num); dump_ibv_qp(qps[conn_num], conn_num); break; +#endif +#if defined(GDA_MLX5) case GDAProvider::MLX5: mlx5_initialize_gpu_qp(gpu_qp, conn_num); + mlx5_qps[conn_num].dump(conn_num); break; +#endif default: assert(false /* GDAProvider initialize_gpu_qp */); } } +QueuePairInitInfo GDABackend::gpu_qp_init_info(int conn_num) { + int pe = conn_num % num_pes; + int nic_idx = nic_idx_for_qp(conn_num); + const NicDevice& nic = nic_for_qp(conn_num); + + host_qps.emplace_back(nic.pd_orig); + const QueuePairHost &host_qp = host_qps.back(); + + return {.heap_laddr = reinterpret_cast(heap.get_local_heap_base()), + .heap_raddr = reinterpret_cast(heap.get_heap_bases()[pe]), + .heap_size = heap.get_size(), + .heap_lkey = nic.heap_mr->lkey, + .heap_rkey = heap_rkey[flat_pe_nic_idx(pe, nic_idx)], + .fetching_atomic = host_qp.fetching_atomic, + .nonfetching_atomic = host_qp.nonfetching_atomic, + .fetching_atomic_lkey = host_qp.fetching_atomic_mr->lkey, + .nonfetching_atomic_lkey = host_qp.nonfetching_atomic_mr->lkey, + .fetching_atomic_freelist = host_qp.fetching_atomic_freelist, + .local_buffers = host_qp.buffer_info, + .num_user_buffers = host_qp.num_user_buffers, + .symm_buffers = get_symm_buffers_slice(pe, nic_idx), + .symm_count = symm_count_}; +} + void GDABackend::create_qps(int sq_length) { struct ibv_qp_init_attr_ex attr; memset(&attr, 0, sizeof(struct ibv_qp_init_attr_ex)); attr.cap.max_send_wr = sq_length; attr.cap.max_send_sge = 1; - attr.cap.max_inline_data = inline_threshold; attr.sq_sig_all = 0; attr.qp_type = IBV_QPT_RC; attr.comp_mask = IBV_QP_INIT_ATTR_PD; - if (gda_provider == GDAProvider::IONIC) { + /* Set provider-specific QP creation attibutes + * Note that only ionic actually uses this code path currently */ + switch (gda_provider) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; attr.cap.max_recv_sge = 1; // TODO allow zero sges in the driver + break; +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; + break; +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; + break; +#endif + default: + assert(false /* invalid GDAProvider */); + break; } for (size_t i = 0; i < qps.size(); i++) { diff --git a/projects/rocshmem/src/gda/backend_gda.hpp b/projects/rocshmem/src/gda/backend_gda.hpp index 9a474dc1f55b..dc325b51ae42 100644 --- a/projects/rocshmem/src/gda/backend_gda.hpp +++ b/projects/rocshmem/src/gda/backend_gda.hpp @@ -33,14 +33,16 @@ #include "backend_bc.hpp" #include "gda_enums.hpp" -#include "gda/gda_symm_table.hpp" #include "containers/free_list_impl.hpp" #include "hdp_proxy.hpp" //TODO useless? #include "memory/hip_allocator.hpp" #include "context_incl.hpp" #include "gda_context_proxy.hpp" -#include "queue_pair.hpp" +#include "queue_pair_provider.hpp" #include "bootstrap/bootstrap.hpp" +#include "gda/queue_pair/queue_pair_common.hpp" +#include "gda/queue_pair/queue_pair_device.hpp" +#include "gda/queue_pair/queue_pair_host.hpp" #include "gda/ionic/provider_gda_ionic.hpp" #include "gda/bnxt/provider_gda_bnxt.hpp" #include "gda/mlx5/provider_gda_mlx5.hpp" @@ -49,13 +51,8 @@ namespace rocshmem { class GDAContext; class GDAHostContext; -class QueuePair; class HostInterface; -inline constexpr uint32_t GDA_IONIC_VENDOR_ID = 0x1DD8; -inline constexpr uint32_t GDA_MLX5_VENDOR_ID = 0x02c9; //PCI-ID is 15b3 -inline constexpr uint32_t GDA_BNXT_VENDOR_ID = 0x14E4; - struct NicDevice { std::string nic_name; struct ibv_device *device = nullptr; @@ -92,13 +89,13 @@ class GDABackend : public Backend { /** * @brief Device-visible flat table of symmetric user-buffer registrations. * - * Sized num_pes * num_nics_ * symm_capacity_ QpSymmEntry records, allocated + * Sized num_pes * num_nics_ * symm_capacity_ SymmBufferInfo records, allocated * in setup_gpu_qps() so QPs capture stable slice pointers. Layout: the slice * for (pe, nic) starts at (pe * num_nics_ + nic) * symm_capacity_ and holds * one entry per registration slot, pre-specialized to that (pe, nic). Each QP * points at the slice for its own (dest_pe, nic_idx). Null pre-ROCm-7.0. */ - QpSymmEntry *symm_entries_{nullptr}; + SymmBufferInfo *symm_buffers_{nullptr}; /** * @brief Device-resident shared registration count (live slots per slice). @@ -110,12 +107,12 @@ class GDABackend : public Backend { int *symm_count_{nullptr}; /** - * @brief Host mirror of symm_entries_ and its capacity/count. + * @brief Host mirror of symm_buffers_ and its capacity/count. * * The mirror is updated on the host at register/unregister and (re-)uploaded - * to symm_entries_; symm_count_host_ mirrors *symm_count_. + * to symm_buffers_; symm_count_host_ mirrors *symm_count_. */ - std::vector host_symm_entries_{}; + std::vector host_symm_buffers_{}; int symm_capacity_{0}; int symm_count_host_{0}; @@ -142,8 +139,7 @@ class GDABackend : public Backend { std::vector nic_devices_; int num_nics_{0}; - uint32_t inline_threshold = 8; - QueuePair *host_qps = nullptr; + std::vector host_qps; QueuePair *gpu_qps = nullptr; std::vector qps; std::vector cqs; @@ -186,7 +182,7 @@ class GDABackend : public Backend { * Total number of QPs created = * num_qps_per_pe * num_pes; */ - uint32_t num_qps {1}; + size_t num_qps {1}; /** * @brief Select one or more NICs based on topology/env vars. @@ -209,6 +205,17 @@ class GDABackend : public Backend { return nic_devices_[nic_idx_for_qp(qp_idx)]; } + size_t flat_pe_nic_idx(int pe, int nic_idx) const { + return static_cast(pe) * static_cast(num_nics_) + static_cast(nic_idx); + } + + const SymmBufferInfo * get_symm_buffers_slice(int pe, int nic_idx) const { + if (!symm_buffers_) { + return nullptr; + } + return &symm_buffers_[flat_pe_nic_idx(pe, nic_idx) * symm_capacity_]; + } + /** * @brief return user-preferred GDA provider (or NONE if not specified) */ @@ -551,9 +558,24 @@ class GDABackend : public Backend { void cleanup_heap_memory_rkey(); void initialize_gpu_qp(QueuePair* qp, int conn_num); - void bnxt_initialize_gpu_qp(QueuePair* qp, int conn_num); + QueuePairInitInfo gpu_qp_init_info(int conn_num); + +#if defined(GDA_IONIC) void ionic_initialize_gpu_qp(QueuePair* qp, int conn_num); + void ionic_create_cqs(int ncqes); + void ionic_setup_parent_domain(NicDevice &nic, struct ibv_parent_domain_init_attr* pattr); +#endif // defined(GDA_IONIC) + +#if defined(GDA_BNXT) + void bnxt_initialize_gpu_qp(QueuePair* qp, int conn_num); + void bnxt_create_cqs(int ncqes); + void bnxt_create_qps(int sq_length); +#endif // defined(GDA_BNXT) + +#if defined(GDA_MLX5) void mlx5_initialize_gpu_qp(QueuePair* qp, int conn_num); + void mlx5_create_qps(int sq_length); +#endif // defined(GDA_MLX5) /** * @brief Setup InfiniBand Resources @@ -599,15 +621,11 @@ class GDABackend : public Backend { * @brief Create all CQs with a of length ncqes */ void create_cqs(int ncqes); - void bnxt_create_cqs(int ncqes); - void ionic_create_cqs(int ncqes); /** * @brief Create all QPs with a SQ of length sq_length */ void create_qps(int sq_length); - void bnxt_create_qps(int sq_length); - void mlx5_create_qps(int sq_length); /** * @brief Reorders QPs to that we map rocSHMEM contexts to the correct QPs @@ -645,7 +663,6 @@ class GDABackend : public Backend { static void pd_release(ibv_pd* pd, void* pd_context, void* ptr, uint64_t resource_type); void create_parent_domain(NicDevice &nic); - void ionic_setup_parent_domain(NicDevice &nic, struct ibv_parent_domain_init_attr* pattr); void setup_gpu_qps(); void cleanup_gpu_qps(); @@ -748,22 +765,23 @@ class GDABackend : public Backend { * @brief structures holding the function pointers to the direct verbs functionality * of each network driver. */ - bnxtdv_funcs_t bnxt_re_dv; + ionicdv_funcs_t ionic_dv; /** - * @brief handle used for the dlopen of the BCOM library + * @brief handle used for the dlopen of the IONIC library */ - void *bnxtdv_handle_{nullptr}; + void *ionicdv_handle_{nullptr}; /** - * @brief initialize function table for BCOM direct verbs support + * @brief structures holding the function pointers to the direct verbs functionality + * of each network driver. */ - int bnxt_dv_dl_init(); + bnxtdv_funcs_t bnxt_re_dv; /** - * @brief open bnxt dv lib + * @brief handle used for the dlopen of the BCOM library */ - static void* bnxt_dv_dlopen(); + void *bnxtdv_handle_{nullptr}; /** * @brief structures holding the function pointers to the direct verbs functionality @@ -776,36 +794,41 @@ class GDABackend : public Backend { */ void *mlx5dv_handle_{nullptr}; +#if defined(GDA_IONIC) /** - * @brief initialize function table for MLNX direct verbs support + * @brief initialize function table for IONIC direct verbs support */ - int mlx5_dv_dl_init(); + int ionic_dv_dl_init(); /** - * @brief open mlx5 dv lib + * @brief open ionic dv lib */ - static void* mlx5_dv_dlopen(); + static void* ionic_dv_dlopen(); +#endif // defined(GDA_IONIC) +#if defined(GDA_BNXT) /** - * @brief structures holding the function pointers to the direct verbs functionality - * of each network driver. + * @brief initialize function table for BCOM direct verbs support */ - ionicdv_funcs_t ionic_dv; + int bnxt_dv_dl_init(); /** - * @brief handle used for the dlopen of the IONIC library + * @brief open bnxt dv lib */ - void *ionicdv_handle_{nullptr}; + static void* bnxt_dv_dlopen(); +#endif // defined(GDA_BNXT) +#if defined(GDA_MLX5) /** - * @brief initialize function table for IONIC direct verbs support + * @brief initialize function table for MLNX direct verbs support */ - int ionic_dv_dl_init(); + int mlx5_dv_dl_init(); /** - * @brief open ionic dv lib + * @brief open mlx5 dv lib */ - static void* ionic_dv_dlopen(); + static void* mlx5_dv_dlopen(); +#endif // defined(GDA_MLX5) }; } // namespace rocshmem diff --git a/projects/rocshmem/src/gda/bnxt/CMakeLists.txt b/projects/rocshmem/src/gda/bnxt/CMakeLists.txt index 686c83f28201..f8fc598acebb 100644 --- a/projects/rocshmem/src/gda/bnxt/CMakeLists.txt +++ b/projects/rocshmem/src/gda/bnxt/CMakeLists.txt @@ -21,16 +21,11 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. ############################################################################### -target_sources( - ${PROJECT_NAME} - PRIVATE - backend_gda_bnxt.cpp -) if(GDA_BNXT) target_sources( ${PROJECT_NAME} PRIVATE - queue_pair_bnxt.cpp + backend_gda_bnxt.cpp ) endif() diff --git a/projects/rocshmem/src/gda/bnxt/backend_gda_bnxt.cpp b/projects/rocshmem/src/gda/bnxt/backend_gda_bnxt.cpp index bb64cfb26fdf..a51294e230a7 100644 --- a/projects/rocshmem/src/gda/bnxt/backend_gda_bnxt.cpp +++ b/projects/rocshmem/src/gda/bnxt/backend_gda_bnxt.cpp @@ -22,10 +22,16 @@ * IN THE SOFTWARE. *****************************************************************************/ -#include "gda/backend_gda.hpp" +#include // getpagesize() +#include + +#include "bit.hpp" #include "log.hpp" #include "util.hpp" -#include // getpagesize() +#include "gda/backend_gda.hpp" +#include "gda/bnxt/provider_gda_bnxt.hpp" +#include "gda/bnxt/queue_pair_bnxt.hpp" +#include "gda/queue_pair_provider.hpp" namespace rocshmem { @@ -33,14 +39,10 @@ void GDABackend::bnxt_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { struct bnxt_re_dv_obj dv_obj; struct bnxt_re_dv_cq dv_cq; struct bnxt_re_dv_qp dv_qp; - struct ibv_qp *ib_qp; int err; - NicDevice &nic = nic_for_qp(conn_num); - int pe = conn_num % num_pes; - int nic_idx = nic_idx_for_qp(conn_num); - - ib_qp = qps[conn_num]; + struct ibv_qp *ib_qp = qps[conn_num]; + const NicDevice& nic = nic_for_qp(conn_num); /* Export SCQ */ memset(&dv_obj, 0, sizeof(struct bnxt_re_dv_obj)); @@ -50,11 +52,6 @@ void GDABackend::bnxt_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { err = bnxt_re_dv.init_obj(&dv_obj, BNXT_RE_DV_OBJ_CQ); CHECK_ZERO(err, "bnxt_re_dv_init_obj(CQ)"); - memset(&gpu_qp->bnxt_cq, 0, sizeof(bnxt_device_cq)); - gpu_qp->bnxt_cq.buf = bnxt_scqs[conn_num].buf; - gpu_qp->bnxt_cq.depth = bnxt_scqs[conn_num].depth; - gpu_qp->bnxt_cq.id = dv_cq.cqn; - /* Export QP */ memset(&dv_obj, 0, sizeof(struct bnxt_re_dv_obj)); dv_obj.qp.in = ib_qp; @@ -63,35 +60,35 @@ void GDABackend::bnxt_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { err = bnxt_re_dv.init_obj(&dv_obj, BNXT_RE_DV_OBJ_QP); CHECK_ZERO(err, "bnxt_re_dv_init_obj(QP)"); - memset(&gpu_qp->bnxt_sq, 0, sizeof(bnxt_device_sq)); - gpu_qp->bnxt_sq.buf = bnxt_qps[conn_num].sq_buf; - gpu_qp->bnxt_sq.depth = bnxt_qps[conn_num].mem_info.sq_slots; - - if ((gpu_qp->bnxt_sq.depth % BNXT_RE_STATIC_WQE_BB) != 0) { - LOG_WARN("SQ depth not divisible by BNXT_RE_STATIC_WQE_BB. " - "There may be runtime errors."); - } - - gpu_qp->bnxt_sq.id = ib_qp->qp_num; - gpu_qp->bnxt_sq.msntbl = bnxt_qps[conn_num].msntbl; - gpu_qp->bnxt_sq.msn_tbl_sz = bnxt_qps[conn_num].msn_tbl_sz; - gpu_qp->bnxt_sq.psn_sz_log2 = std::log2(bnxt_qps[conn_num].mem_info.sq_psn_sz); - gpu_qp->bnxt_sq.mtu = ibv_mtu_to_int(nic.portinfo.active_mtu); - /* Export DB */ + void* gpu_dbr_ptr = nullptr; CHECK_HIP(hipHostRegister(bnxt_qps[conn_num].db_region_attr->dbr, getpagesize(), hipHostRegisterDefault)); - CHECK_HIP(hipHostGetDevicePointer((void**) &gpu_qp->bnxt_dbr, bnxt_qps[conn_num].db_region_attr->dbr, 0)); + CHECK_HIP(hipHostGetDevicePointer(&gpu_dbr_ptr, bnxt_qps[conn_num].db_region_attr->dbr, 0)); + + uint64_t* dbr = reinterpret_cast(gpu_dbr_ptr); - /* Export Memory Keys */ - gpu_qp->lkey = nic.heap_mr->lkey; - gpu_qp->rkey = heap_rkey[pe * num_nics_ + nic_idx]; + void* sq_buf = bnxt_qps[conn_num].sq_buf; + uint32_t sq_depth = bnxt_qps[conn_num].mem_info.sq_slots; + void* msntbl = bnxt_qps[conn_num].msntbl; + uint32_t msn_tbl_sz = bnxt_qps[conn_num].msn_tbl_sz; + uint32_t psn_sz_log2 = bit_log2(bnxt_qps[conn_num].mem_info.sq_psn_sz); + uint32_t mtu = ibv_mtu_to_int(nic.portinfo.active_mtu); - /* Export Inline Threshold */ - gpu_qp->inline_threshold = inline_threshold; + void* cq_buf = bnxt_scqs[conn_num].buf; + uint32_t cq_depth = bnxt_scqs[conn_num].depth; + + if ((sq_depth % BNXT_RE_STATIC_WQE_BB) != 0) { + LOG_WARN("SQ depth not divisible by BNXT_RE_STATIC_WQE_BB. " + "There may be runtime errors."); + } - /* Base Heap information */ - gpu_qp->base_heap = (uintptr_t) heap.get_local_heap_base(); - gpu_qp->base_heap_size = heap.get_size(); + /* QueuePair is either QueuePairBNXT or QueuePairMux + * both have a constructor that accepts rvalue reference QueuePairBNXT&&, + * so just use that instead of trying to figure out which one we're using */ + new (gpu_qp) QueuePair{QueuePairBNXT{ib_qp->qp_num, gpu_qp_init_info(conn_num), dbr, + bnxt_device_sq{sq_buf, sq_depth, msntbl, msn_tbl_sz, + psn_sz_log2, mtu}, + bnxt_device_cq{cq_buf, cq_depth}}}; } void GDABackend::bnxt_create_cqs(int cqe) { @@ -200,7 +197,7 @@ void GDABackend::bnxt_create_qps(int sq_length) { ib_qp_attr.cap.max_recv_wr = 0; ib_qp_attr.cap.max_send_sge = 1; ib_qp_attr.cap.max_recv_sge = 0; - ib_qp_attr.cap.max_inline_data = inline_threshold; + ib_qp_attr.cap.max_inline_data = QueuePairTraits::InlineThreshold; ib_qp_attr.qp_type = IBV_QPT_RC; ib_qp_attr.sq_sig_all = 0; diff --git a/projects/rocshmem/src/gda/bnxt/bnxt_re_dv.h b/projects/rocshmem/src/gda/bnxt/bnxt_re_dv.h index 5e4143478f44..62713715b8d9 100644 --- a/projects/rocshmem/src/gda/bnxt/bnxt_re_dv.h +++ b/projects/rocshmem/src/gda/bnxt/bnxt_re_dv.h @@ -1,3 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause + +/* + * This file is derived from the rdma-core source code, + * the userspace components for the Linux kernel drivers/infiniband subsystem, + * and is distributed under the license below. + * + * The file COPYING is in the main directory of the origin source tree. + */ + /* * Copyright (c) 2025, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. diff --git a/projects/rocshmem/src/gda/bnxt/bnxt_re_hsi.h b/projects/rocshmem/src/gda/bnxt/bnxt_re_hsi.h index 24f32a4aab30..a6abf37baa81 100644 --- a/projects/rocshmem/src/gda/bnxt/bnxt_re_hsi.h +++ b/projects/rocshmem/src/gda/bnxt/bnxt_re_hsi.h @@ -1,3 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause + +/* + * This file is derived from the rdma-core source code, + * the userspace components for the Linux kernel drivers/infiniband subsystem, + * and is distributed under the license below. + * + * The file COPYING is in the main directory of the origin source tree. + */ + /* * Copyright (c) 2025, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. diff --git a/projects/rocshmem/src/gda/bnxt/provider_gda_bnxt.hpp b/projects/rocshmem/src/gda/bnxt/provider_gda_bnxt.hpp index 883a341130d6..a600eaeb050e 100644 --- a/projects/rocshmem/src/gda/bnxt/provider_gda_bnxt.hpp +++ b/projects/rocshmem/src/gda/bnxt/provider_gda_bnxt.hpp @@ -25,11 +25,17 @@ #ifndef LIBRARY_SRC_GDA_BNXT_GDA_PROVIDER_HPP_ #define LIBRARY_SRC_GDA_BNXT_GDA_PROVIDER_HPP_ +#include /* For __u32 and __u64 types needed by bnxt_re_hsi.h */ + +#include "gda/ibv_core.hpp" /* For ibv types needed by bnxt_re_dv.h */ + extern "C" { #include "gda/bnxt/bnxt_re_dv.h" #include "gda/bnxt/bnxt_re_hsi.h" } +namespace rocshmem { + #define GDA_BNXT_WQE_SLOT_COUNT 3 struct bnxt_device_wq { @@ -38,22 +44,30 @@ struct bnxt_device_wq { uint32_t head; uint32_t tail; uint32_t flags; - uint32_t id; -} __attribute__((packed)); + + __host__ inline bnxt_device_wq(void *buf, uint32_t depth) + : buf{buf}, depth{depth}, head{0}, tail{0}, flags{0} { } +}; struct bnxt_device_cq : public bnxt_device_wq { -} __attribute__((packed)); + // inherit constructors + using bnxt_device_wq::bnxt_device_wq; +}; struct bnxt_device_sq : public bnxt_device_wq { - uint32_t psn; - void *msntbl; + uint32_t psn; uint32_t msn; uint32_t msn_tbl_sz; uint32_t psn_sz_log2; - uint64_t mtu; + uint32_t mtu; uint32_t lock; -} __attribute__((packed)); + + __host__ inline bnxt_device_sq(void *buf, uint32_t depth, void *msntbl, uint32_t msn_tbl_sz, + uint32_t psn_sz_log2, uint32_t mtu) + : bnxt_device_wq{buf, depth}, msntbl{msntbl}, psn{0}, msn{0}, msn_tbl_sz{msn_tbl_sz}, + psn_sz_log2{psn_sz_log2}, mtu{mtu}, lock{0} { } +}; struct bnxt_host_cq { void *buf; @@ -104,4 +118,6 @@ struct bnxtdv_funcs_t { int (*free_db_region)(struct ibv_context *ctx, struct bnxt_re_dv_db_region_attr *attr); }; +} // namespace rocshmem + #endif //LIBRARY_SRC_GDA_BNXT_GDA_PROVIDER_HPP_ diff --git a/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.cpp b/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.cpp deleted file mode 100644 index 1e6132bd54c6..000000000000 --- a/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.cpp +++ /dev/null @@ -1,444 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#include "gda/queue_pair.hpp" -#include "log.hpp" - -namespace rocshmem { - -__device__ static inline struct bnxt_re_msns* bnxt_re_pull_psn_buff(struct bnxt_device_sq *sq) { - return (struct bnxt_re_msns*)(((char *) sq->msntbl) + ((sq->msn) << sq->psn_sz_log2)); -} - -__device__ static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, - uint32_t start_psn) { - return ((((uint64_t)(st_idx) << BNXT_RE_SQ_MSN_SEARCH_START_IDX_SHIFT) & - BNXT_RE_SQ_MSN_SEARCH_START_IDX_MASK) | - (((uint64_t)(npsn) << BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_SHIFT) & - BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_MASK) | - (((start_psn) << BNXT_RE_SQ_MSN_SEARCH_START_PSN_SHIFT) & - BNXT_RE_SQ_MSN_SEARCH_START_PSN_MASK)); -} - -__device__ static inline void bnxt_re_fill_psns_for_msntbl(struct bnxt_device_sq *sq, - uint32_t msg_len) { - uint32_t npsn = 0, start_psn = 0, next_psn = 0; - struct bnxt_re_msns msns; - uint64_t *msns_ptr; - uint32_t pkt_cnt = 0; - /* Start slot index of the WQE */ - uint32_t st_idx = sq->tail; // * BNXT_RE_STATIC_WQE_SIZE_SLOTS; Do we need this? - // Get the MSN table address - msns_ptr = (uint64_t *)bnxt_re_pull_psn_buff(sq); - // Start PSN is the last recorded PSN - // Calculate the packet count based on the len of the WQE/MTU - msns.start_idx_next_psn_start_psn = 0; - start_psn = sq->psn; - pkt_cnt = (msg_len / sq->mtu); - - if (msg_len % sq->mtu) - pkt_cnt++; - - /* Increment the psn even for 0 len packets - * e.g. for opcode rdma-write-with-imm-data - * with length field = 0 - */ - if (msg_len == 0) - pkt_cnt = 1; - - /* make it 24 bit */ - next_psn = sq->psn + pkt_cnt; - npsn = next_psn; - sq->psn = next_psn; - msns.start_idx_next_psn_start_psn |= bnxt_re_update_msn_tbl(st_idx, npsn, start_psn); - sq->msn++; - sq->msn %= sq->msn_tbl_sz; - - memcpy(msns_ptr, &msns, sizeof(uint64_t)); -} - -__device__ static inline void bnxt_re_incr_tail(struct bnxt_device_sq *sq, uint8_t cnt) -{ - sq->tail += cnt; - if (sq->tail >= sq->depth) { - sq->tail %= sq->depth; - /* Rolled over, Toggle Tail bit in epoch flags */ - sq->flags ^= 1UL << BNXT_RE_FLAG_EPOCH_TAIL_SHIFT; - } -} - -__device__ static inline void* bnxt_re_get_hwqe(struct bnxt_device_sq *sq, uint32_t idx) -{ - idx += sq->tail; - if (idx >= sq->depth) - idx -= sq->depth; - return (void *)((char*)sq->buf + (idx << 4)); -} - -__device__ static inline void lock(uint32_t *lock) { - uint32_t expected; - - do { - expected = 0; - } while (0 == __hip_atomic_compare_exchange_strong(lock, &expected, 1, - __ATOMIC_ACQUIRE, - __ATOMIC_ACQUIRE, - __HIP_MEMORY_SCOPE_SYSTEM)); -} - -__device__ static inline void unlock(uint32_t *lock) { - __hip_atomic_store(lock, 0, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ void QueuePair::bnxt_ring_doorbell(uint32_t slot_idx) { - struct bnxt_re_db_hdr hdr; - uint32_t epoch; - uint64_t key_lo; - uint64_t key_hi; - - epoch = (bnxt_sq.flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT; - - key_lo = (slot_idx | epoch); - - key_hi = (bnxt_sq.id & BNXT_RE_DB_QID_MASK) - | (((uint64_t) BNXT_RE_QUE_TYPE_SQ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) - | (0x1UL << BNXT_RE_DB_VALID_SHIFT); - - hdr.typ_qid_indx = (key_lo | (key_hi << 32)); - - __threadfence_system(); - __hip_atomic_store(bnxt_dbr, hdr.typ_qid_indx, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SYSTEM); -} - -[[maybe_unused]] __attribute__((noinline)) -__device__ void QueuePair::bnxt_print_cqe_error(uint8_t status) { - switch (status) { - case BNXT_RE_REQ_ST_BAD_RESP: - LOGD_ERROR_ABORT("CQ error BAD_RESP (%x)", status); - break; - case BNXT_RE_REQ_ST_LOC_LEN: - LOGD_ERROR_ABORT("CQ error LOC_LEN (%x)", status); - break; - case BNXT_RE_REQ_ST_LOC_QP_OP: - LOGD_ERROR_ABORT("CQ error LOC_QP_OP (%x)", status); - break; - case BNXT_RE_REQ_ST_PROT: - LOGD_ERROR_ABORT("CQ error PROT (%x)", status); - break; - case BNXT_RE_REQ_ST_MEM_OP: - LOGD_ERROR_ABORT("CQ error MEM_OP (%x)", status); - break; - case BNXT_RE_REQ_ST_REM_INVAL: - LOGD_ERROR_ABORT("CQ error REM_INVAL (%x)", status); - break; - case BNXT_RE_REQ_ST_REM_ACC: - LOGD_ERROR_ABORT("CQ error REM_ACC (%x)", status); - break; - case BNXT_RE_REQ_ST_REM_OP: - LOGD_ERROR_ABORT("CQ error REM_OP (%x)", status); - break; - case BNXT_RE_REQ_ST_RNR_NAK_XCED: - LOGD_ERROR_ABORT("CQ error RNR_NAK_XCED (%x)", status); - break; - case BNXT_RE_REQ_ST_TRNSP_XCED: - LOGD_ERROR_ABORT("CQ error TRNSP_XCED (%x)", status); - break; - case BNXT_RE_REQ_ST_WR_FLUSH: - LOGD_ERROR_ABORT("CQ error WR_FLUSH (%x)", status); - break; - default: - LOGD_ERROR_ABORT("CQ error unknown status (%x)", status); - break; - } -} - -__device__ void QueuePair::bnxt_poll_cq_until(uint32_t requested_available_slots) { - struct bnxt_re_req_cqe *cqe; - uint32_t sq_tail; - uint32_t sq_head; - uint32_t sq_depth; - uint32_t consumed_slots; - uint32_t available_slots; - - sq_depth = bnxt_sq.depth; - - do { - cqe = (struct bnxt_re_req_cqe *) bnxt_cq.buf; - -#ifdef BUILD_DEBUG_DEVICE - { - uint32_t flg_val = __hip_atomic_load( - static_cast(__builtin_assume_aligned( - (char*)cqe + sizeof(struct bnxt_re_req_cqe) + offsetof(struct bnxt_re_bcqe, flg_st_typ_ph), 4)), - __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); - uint8_t status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; - if (status != BNXT_RE_REQ_ST_OK) - bnxt_print_cqe_error(status); - } -#endif - - /* Update the SQ head - * This param provides us the wqe_idx but we need to convert to the slot idx. - * We assume a static slots size of GDA_BNXT_WQE_SLOT_COUNT thus can multiply by this value */ - sq_head = (((cqe->con_indx & 0xFFFF) * GDA_BNXT_WQE_SLOT_COUNT) % sq_depth); - bnxt_sq.head = sq_head; - - sq_tail = __hip_atomic_load(&bnxt_sq.tail, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_AGENT); - - consumed_slots = (sq_tail - sq_head + sq_depth) % sq_depth; - available_slots = sq_depth - consumed_slots; - } while (available_slots < requested_available_slots); -} - -__device__ void QueuePair::bnxt_quiet() { - bnxt_poll_cq_until(bnxt_sq.depth); -} - -/** - * TODO: This function is redundant but kept because ionic has a different - * quiet_single implementation. Remove once ionic's quiet is unified. - */ -__device__ void QueuePair::bnxt_quiet_single() { - bnxt_poll_cq_until(bnxt_sq.depth); -} - -__device__ void QueuePair::bnxt_write_rma_wqe(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, uint8_t opcode) { - struct bnxt_re_bsqe hdr; - struct bnxt_re_rdma rdma; - struct bnxt_re_sge sge; - struct bnxt_re_bsqe *hdr_ptr; - struct bnxt_re_rdma *rdma_ptr; - struct bnxt_re_sge *sge_ptr; - uint32_t wqe_size; - uint32_t wqe_type; - uint32_t hdr_flags; - uint32_t inline_msg; - - inline_msg = static_cast(length) <= static_cast(inline_threshold) && - opcode == gda_op_rdma_write; - - bnxt_poll_cq_until(GDA_BNXT_WQE_SLOT_COUNT); - - hdr_ptr = (struct bnxt_re_bsqe*) bnxt_re_get_hwqe(&bnxt_sq, 0); - rdma_ptr = (struct bnxt_re_rdma*) bnxt_re_get_hwqe(&bnxt_sq, 1); - sge_ptr = (struct bnxt_re_sge*) bnxt_re_get_hwqe(&bnxt_sq, 2); - - /* Populate Header Segment */ - wqe_type = BNXT_RE_HDR_WT_MASK & opcode; - wqe_size = BNXT_RE_HDR_WS_MASK & GDA_BNXT_WQE_SLOT_COUNT; - hdr_flags = ((uint32_t) BNXT_RE_HDR_FLAGS_MASK) - & ((uint32_t) BNXT_RE_WR_FLAGS_SIGNALED); - - if (inline_msg) { - hdr_flags |= ((uint32_t) BNXT_RE_WR_FLAGS_INLINE); - } - - hdr.rsv_ws_fl_wt = (wqe_size << BNXT_RE_HDR_WS_SHIFT) - | (hdr_flags << BNXT_RE_HDR_FLAGS_SHIFT) - | wqe_type; - hdr.key_immd = 0; - hdr.lhdr.qkey_len = length; - - /* Populate RDMA Segment */ - rdma.rva = raddr; - rdma.rkey = rkey; - - if (!inline_msg) { - /* Populate SG Segment */ - sge.pa = laddr; - sge.lkey = lkey; - sge.length = length; - } - - /* Write WQE to SQ */ - memcpy(hdr_ptr, &hdr, sizeof(struct bnxt_re_bsqe)); - memcpy(rdma_ptr, &rdma, sizeof(struct bnxt_re_rdma)); - - if (inline_msg) { - memcpy(sge_ptr, reinterpret_cast(laddr), length); - } else { - memcpy(sge_ptr, &sge, sizeof(struct bnxt_re_sge)); - } - - /* Populate MSN Table */ - bnxt_re_fill_psns_for_msntbl(&bnxt_sq, length); - - /* Update SQ Pointer */ - bnxt_re_incr_tail(&bnxt_sq, GDA_BNXT_WQE_SLOT_COUNT); -} - -__device__ void QueuePair::bnxt_post_wqe_rma(int32_t length, - uintptr_t raddr, uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db) { - if (wf_info.is_pe_group_first) { - lock(&bnxt_sq.lock); - } - - for (int i = 0; i < wf_info.num_pe_group_lanes; i++) { - if (i == wf_info.pe_group_logical_lane_id) { - bnxt_write_rma_wqe(length, raddr, rkey, laddr, lkey, opcode); - if (ring_db) { - bnxt_ring_doorbell(bnxt_sq.tail); - } - } - } - - if (wf_info.is_pe_group_first) { - unlock(&bnxt_sq.lock); - } -} - -__device__ void QueuePair::bnxt_post_wqe_rma_single(int32_t length, - uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, - uint8_t opcode, bool ring_db) { - lock(&bnxt_sq.lock); - - bnxt_write_rma_wqe(length, raddr, rkey, laddr, lkey, opcode); - - if (ring_db) { - bnxt_ring_doorbell(bnxt_sq.tail); - } - - unlock(&bnxt_sq.lock); -} - -__device__ uint32_t QueuePair::bnxt_write_amo_wqe(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence) { - struct bnxt_re_bsqe hdr; - struct bnxt_re_atomic amo; - struct bnxt_re_sge sge; - struct bnxt_re_bsqe *hdr_ptr; - struct bnxt_re_atomic *amo_ptr; - struct bnxt_re_sge *sge_ptr; - uint32_t wqe_size; - uint32_t wqe_type; - uint32_t hdr_flags; - - uint32_t atomic_idx = 0; - uint32_t length = sizeof(uint64_t); - - bnxt_poll_cq_until(GDA_BNXT_WQE_SLOT_COUNT); - - hdr_ptr = (struct bnxt_re_bsqe*) bnxt_re_get_hwqe(&bnxt_sq, 0); - amo_ptr = (struct bnxt_re_atomic*) bnxt_re_get_hwqe(&bnxt_sq, 1); - sge_ptr = (struct bnxt_re_sge*) bnxt_re_get_hwqe(&bnxt_sq, 2); - - /* Populate Header Segment */ - wqe_size = BNXT_RE_HDR_WS_MASK & GDA_BNXT_WQE_SLOT_COUNT; - hdr_flags = ((uint32_t) BNXT_RE_HDR_FLAGS_MASK) - & ((uint32_t) BNXT_RE_WR_FLAGS_SIGNALED); - if (fence) { - hdr_flags |= ((uint32_t) BNXT_RE_WR_FLAGS_UC_FENCE); - } - wqe_type = BNXT_RE_HDR_WT_MASK & opcode; - - hdr.rsv_ws_fl_wt = (wqe_size << BNXT_RE_HDR_WS_SHIFT) - | (hdr_flags << BNXT_RE_HDR_FLAGS_SHIFT) - | wqe_type; - hdr.key_immd = rkey; - hdr.lhdr.rva = raddr; - - /* Populate AMO Segment */ - amo.swp_dt = atomic_data; - amo.cmp_dt = atomic_cmp; - - /* Populate SG Segment - (Return address of atomic) */ - if (fetching) { - atomic_idx = fetching_atomic_idx++ % FETCHING_ATOMIC_CNT; - sge.pa = (uint64_t) &fetching_atomic[atomic_idx]; - sge.lkey = fetching_atomic_lkey; - } else { - sge.pa = (uint64_t) nonfetching_atomic; - sge.lkey = nonfetching_atomic_lkey; - } - sge.length = length; - - /* Write WQE to SQ */ - memcpy(hdr_ptr, &hdr, sizeof(struct bnxt_re_bsqe)); - memcpy(amo_ptr, &amo, sizeof(struct bnxt_re_atomic)); - memcpy(sge_ptr, &sge, sizeof(struct bnxt_re_sge)); - - /* Populate MSN Table */ - bnxt_re_fill_psns_for_msntbl(&bnxt_sq, length); - - /* Update SQ Pointer */ - bnxt_re_incr_tail(&bnxt_sq, GDA_BNXT_WQE_SLOT_COUNT); - - return atomic_idx; -} - -__device__ uint64_t QueuePair::bnxt_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching, bool fence) { - uint32_t atomic_idx = 0; - - if (wf_info.is_pe_group_first) { - lock(&bnxt_sq.lock); - } - - for (int i = 0; i < wf_info.num_pe_group_lanes; i++) { - if (i == wf_info.pe_group_logical_lane_id) { - atomic_idx = bnxt_write_amo_wqe(raddr, rkey, opcode, atomic_data, atomic_cmp, fetching, fence); - - /* Ring Doorbell */ - bnxt_ring_doorbell(bnxt_sq.tail); - } - } - - if (wf_info.is_pe_group_first) { - unlock(&bnxt_sq.lock); - } - - if (fetching) { - bnxt_quiet(); - return fetching_atomic[atomic_idx]; - } - - return 0; -} - -__device__ uint64_t QueuePair::bnxt_post_wqe_amo_single(uintptr_t raddr, - uint32_t rkey, uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence) { - uint32_t atomic_idx = 0; - - lock(&bnxt_sq.lock); - - atomic_idx = bnxt_write_amo_wqe(raddr, rkey, opcode, atomic_data, atomic_cmp, fetching, fence); - - bnxt_ring_doorbell(bnxt_sq.tail); - - unlock(&bnxt_sq.lock); - - if (fetching) { - bnxt_quiet_single(); - return fetching_atomic[atomic_idx]; - } - - return 0; -} - -} // namespace rocshmem diff --git a/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.hpp b/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.hpp new file mode 100644 index 000000000000..a43ca7d25128 --- /dev/null +++ b/projects/rocshmem/src/gda/bnxt/queue_pair_bnxt.hpp @@ -0,0 +1,599 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_BNXT_QUEUE_PAIR_HPP_ +#define LIBRARY_SRC_GDA_BNXT_QUEUE_PAIR_HPP_ + +#include + +#include "log.hpp" +#include "gda/endian.hpp" +#include "gda/bnxt/provider_gda_bnxt.hpp" +#include "gda/queue_pair/queue_pair_device.hpp" + +namespace rocshmem { + +class QueuePairBNXT; + +template <> struct QueuePairTraits { + enum class OpCode : uint8_t { + RDMA_WRITE = BNXT_RE_WR_OPCD_RDMA_WRITE, + RDMA_READ = BNXT_RE_WR_OPCD_RDMA_READ, + ATOMIC_CS = BNXT_RE_WR_OPCD_ATOMIC_CS, + ATOMIC_FA = BNXT_RE_WR_OPCD_ATOMIC_FA, + }; + + /** + * @brief bnxt uses little-endian ordering + */ + static constexpr endian::Order Endianness = endian::Order::Little; + + /** + * @brief bnxt inlining maximum is one WQE segment: sizeof(struct bnxt_re_sge) = 16 bytes + */ + static constexpr size_t InlineMax = sizeof(struct bnxt_re_sge); + + /** + * @brief bnxt preferred inlining threshold is 8 bytes, for performance reasons + */ + static constexpr size_t InlineThreshold = sizeof(uint64_t); +}; + +class QueuePairBNXT : public QueuePairDevice { +private: + uint64_t* dbr; + bnxt_device_sq sq; + bnxt_device_cq cq; + +public: + __host__ explicit QueuePairBNXT(uint32_t qpn, QueuePairInitInfo&& init_info, + uint64_t* dbr, bnxt_device_sq&& sq, bnxt_device_cq&& cq) + : QueuePairDevice{qpn, std::move(init_info)}, + dbr{dbr}, sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ explicit QueuePairBNXT(uint32_t qpn, + uint64_t *nonfetching_atomic, uint32_t nonfetching_atomic_lkey, + uint64_t* dbr, bnxt_device_sq&& sq, bnxt_device_cq&& cq) + : QueuePairDevice{qpn, nonfetching_atomic, nonfetching_atomic_lkey}, + dbr{dbr}, sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ QueuePairBNXT(const QueuePairBNXT& other) = delete; + __host__ QueuePairBNXT& operator=(const QueuePairBNXT& other) = delete; + __host__ QueuePairBNXT(QueuePairBNXT&& other) noexcept = default; + __host__ QueuePairBNXT& operator=(QueuePairBNXT&& other) = default; + __host__ ~QueuePairBNXT() = default; + +public: + template + __device__ __noinline__ + void post_wqe_rma(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + void post_wqe_rma_single(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + PostOpt = {}); + + __device__ __noinline__ void quiet_single(); + +private: + __device__ void ring_doorbell(uint32_t slot_idx); + __device__ void poll_cq_until(uint32_t requested_available_slots); + + static __device__ void acquire_lock(uint32_t* lock); + static __device__ void release_lock(uint32_t* lock); + + template + __device__ void write_rma_wqe(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, bool signaled); + + template + __device__ uint64_t* write_amo_wqe(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, bool signaled); + + static __device__ void* get_hwqe(const bnxt_device_sq& sq, uint32_t idx); + static __device__ void fill_psns_for_msntbl(bnxt_device_sq& sq, uint32_t msg_len); + static __device__ void incr_tail(bnxt_device_sq& sq, uint8_t cnt); + + static __device__ struct bnxt_re_msns* pull_psn_buff(const bnxt_device_sq& sq); + static __device__ uint64_t update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn); + +#if defined(BUILD_DEBUG_DEVICE) + __device__ __noinline__ void print_cqe_error(uint8_t status); +#endif +}; + +template +__device__ void QueuePairBNXT::write_rma_wqe( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, bool signaled) { + struct bnxt_re_bsqe hdr; + struct bnxt_re_rdma rdma; + struct bnxt_re_sge sge; + struct bnxt_re_bsqe *hdr_ptr; + struct bnxt_re_rdma *rdma_ptr; + struct bnxt_re_sge *sge_ptr; + uint32_t wqe_size; + uint32_t wqe_type; + uint32_t hdr_flags; + + bool inline_msg = can_inline(size); + + if constexpr (CheckSQ) { + poll_cq_until(GDA_BNXT_WQE_SLOT_COUNT); + } + + hdr_ptr = (struct bnxt_re_bsqe*) get_hwqe(sq, 0); + rdma_ptr = (struct bnxt_re_rdma*) get_hwqe(sq, 1); + sge_ptr = (struct bnxt_re_sge*) get_hwqe(sq, 2); + + /* Populate Header Segment */ + wqe_type = BNXT_RE_HDR_WT_MASK & static_cast(Op); + wqe_size = BNXT_RE_HDR_WS_MASK & GDA_BNXT_WQE_SLOT_COUNT; + hdr_flags = signaled ? (BNXT_RE_HDR_FLAGS_MASK & BNXT_RE_WR_FLAGS_SIGNALED) : 0; + + if (inline_msg) { + hdr_flags |= ((uint32_t) BNXT_RE_WR_FLAGS_INLINE); + } + + hdr.rsv_ws_fl_wt = (wqe_size << BNXT_RE_HDR_WS_SHIFT) + | (hdr_flags << BNXT_RE_HDR_FLAGS_SHIFT) + | wqe_type; + hdr.key_immd = 0; + hdr.lhdr.qkey_len = size; + + /* Populate RDMA Segment */ + rdma.rva = raddr; + rdma.rkey = rkey; + + if (!inline_msg) { + /* Populate SG Segment */ + sge.pa = laddr; + sge.lkey = lkey; + sge.length = size; + } + + /* Write WQE to SQ */ + memcpy(hdr_ptr, &hdr, sizeof(struct bnxt_re_bsqe)); + memcpy(rdma_ptr, &rdma, sizeof(struct bnxt_re_rdma)); + + if (inline_msg) { + memcpy(sge_ptr, reinterpret_cast(laddr), size); + } else { + memcpy(sge_ptr, &sge, sizeof(struct bnxt_re_sge)); + } + + /* Populate MSN Table */ + fill_psns_for_msntbl(sq, size); + + /* Update SQ Pointer */ + incr_tail(sq, GDA_BNXT_WQE_SLOT_COUNT); +} + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ void QueuePairBNXT::post_wqe_rma( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt) { + using PostOptions = PostOpt; + if constexpr (PostOptions::ThreadSafe) { + if (wf_info.is_pe_group_first) { + acquire_lock(&sq.lock); + } + } else if constexpr (!PostOptions::CheckSQ) { + // need to at least acquire so that tail is visible + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "agent"); + } + + bool signaled = PostOptions::signal_completion(wf_info); + + for (int i = 0; i < wf_info.num_pe_group_lanes; i++) { + if (i == wf_info.pe_group_logical_lane_id) { + /* Write WQE to SQ */ + write_rma_wqe(laddr, lkey, raddr, rkey, size, signaled); + + /* Ring Doorbell */ + if constexpr (PostOptions::RingDB) { + ring_doorbell(sq.tail); + } + } + } + + if constexpr (PostOptions::ThreadSafe) { + if (wf_info.is_pe_group_first) { + release_lock(&sq.lock); + } + } else if constexpr (!PostOptions::RingDB) { + // need to at least release so that tail is available + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "agent"); + } +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ void QueuePairBNXT::post_wqe_rma_single( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, PostOpt) { + using PostOptions = PostOpt; + if constexpr (PostOptions::ThreadSafe) { + acquire_lock(&sq.lock); + } else if constexpr (!PostOptions::CheckSQ) { + // need to at least acquire so that tail is visible + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "agent"); + } + + bool signaled = PostOptions::signal_completion_single(); + + /* Write WQE to SQ */ + write_rma_wqe(laddr, lkey, raddr, rkey, size, signaled); + + /* Ring Doorbell */ + if constexpr (PostOptions::RingDB) { + ring_doorbell(sq.tail); + } + + if constexpr (PostOptions::ThreadSafe) { + release_lock(&sq.lock); + } else if constexpr (!PostOptions::RingDB) { + // need to at least release so that tail is available + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "agent"); + } +} + +template +__device__ uint64_t* QueuePairBNXT::write_amo_wqe( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, bool signaled) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + static constexpr size_t size = sizeof(uint64_t); + + struct bnxt_re_bsqe hdr; + struct bnxt_re_atomic amo; + struct bnxt_re_sge sge; + struct bnxt_re_bsqe *hdr_ptr; + struct bnxt_re_atomic *amo_ptr; + struct bnxt_re_sge *sge_ptr; + uint32_t wqe_size; + uint32_t wqe_type; + uint32_t hdr_flags; + uint64_t* atomic_laddr; + + if constexpr (CheckSQ) { + poll_cq_until(GDA_BNXT_WQE_SLOT_COUNT); + } + + hdr_ptr = (struct bnxt_re_bsqe*) get_hwqe(sq, 0); + amo_ptr = (struct bnxt_re_atomic*) get_hwqe(sq, 1); + sge_ptr = (struct bnxt_re_sge*) get_hwqe(sq, 2); + + /* Populate Header Segment */ + wqe_type = BNXT_RE_HDR_WT_MASK & static_cast(Op); + wqe_size = BNXT_RE_HDR_WS_MASK & GDA_BNXT_WQE_SLOT_COUNT; + hdr_flags = signaled ? (BNXT_RE_HDR_FLAGS_MASK & BNXT_RE_WR_FLAGS_SIGNALED) : 0; + + hdr.rsv_ws_fl_wt = (wqe_size << BNXT_RE_HDR_WS_SHIFT) + | (hdr_flags << BNXT_RE_HDR_FLAGS_SHIFT) + | wqe_type; + hdr.key_immd = rkey; + hdr.lhdr.rva = raddr; + + /* Populate AMO Segment */ + amo.swp_dt = swap_add; + amo.cmp_dt = compare; + + /* Populate SG Segment - (Return address of atomic) */ + atomic_laddr = get_atomic_addr(); + if constexpr (Fetch == AMOFetchType::Blocking) { + atomic_laddr += (fetching_atomic_idx++ % FETCHING_ATOMIC_CNT); + } + sge.pa = reinterpret_cast(atomic_laddr); + sge.lkey = get_atomic_lkey(); + sge.length = size; + + /* Write WQE to SQ */ + memcpy(hdr_ptr, &hdr, sizeof(struct bnxt_re_bsqe)); + memcpy(amo_ptr, &amo, sizeof(struct bnxt_re_atomic)); + memcpy(sge_ptr, &sge, sizeof(struct bnxt_re_sge)); + + /* Populate MSN Table */ + fill_psns_for_msntbl(sq, size); + + /* Update SQ Pointer */ + incr_tail(sq, GDA_BNXT_WQE_SLOT_COUNT); + + return atomic_laddr; +} + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ QueuePairBNXT::amo_ret_t QueuePairBNXT::post_wqe_amo( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + using PostOptions = PostOpt; + uint64_t* atomic_laddr = nullptr; + + if constexpr (PostOptions::ThreadSafe) { + if (wf_info.is_pe_group_first) { + acquire_lock(&sq.lock); + } + } else if constexpr (!PostOptions::CheckSQ) { + // need to at least acquire so that tail is visible + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "agent"); + } + + bool signaled = PostOptions::signal_completion(wf_info); + + for (int i = 0; i < wf_info.num_pe_group_lanes; i++) { + if (i == wf_info.pe_group_logical_lane_id) { + /* Write WQE to SQ */ + atomic_laddr = write_amo_wqe(raddr, rkey, swap_add, compare, signaled); + + /* Ring Doorbell */ + if constexpr (PostOptions::RingDB) { + ring_doorbell(sq.tail); + } + } + } + + if constexpr (PostOptions::ThreadSafe) { + if (wf_info.is_pe_group_first) { + release_lock(&sq.lock); + } + } else if constexpr (!PostOptions::RingDB) { + // need to at least release so that tail is available + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "agent"); + } + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet(wf_info); + return *atomic_laddr; + } +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ QueuePairBNXT::amo_ret_t QueuePairBNXT::post_wqe_amo_single( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + using PostOptions = PostOpt; + uint64_t* atomic_laddr = nullptr; + + if constexpr (PostOptions::ThreadSafe) { + acquire_lock(&sq.lock); + } else if constexpr (!PostOptions::CheckSQ) { + // need to at least acquire so that tail is visible + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "agent"); + } + + bool signaled = PostOptions::signal_completion_single(); + + /* Write WQE to SQ */ + atomic_laddr = write_amo_wqe(raddr, rkey, swap_add, compare, signaled); + + /* Ring Doorbell */ + if constexpr (PostOptions::RingDB) { + ring_doorbell(sq.tail); + } + + if constexpr (PostOptions::ThreadSafe) { + release_lock(&sq.lock); + } else if constexpr (!PostOptions::RingDB) { + // need to at least release so that tail is available + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "agent"); + } + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_single(); + return *atomic_laddr; + } +} + +// precondition: called with all active lanes using different QPs +__device__ inline __noinline__ void QueuePairBNXT::quiet_single() { + poll_cq_until(sq.depth); +} + +// precondition: called with all active lanes using different QPs +__device__ inline void QueuePairBNXT::poll_cq_until(uint32_t requested_available_slots) { + struct bnxt_re_req_cqe *cqe; + uint32_t sq_tail; + uint32_t sq_head; + uint32_t sq_depth; + uint32_t consumed_slots; + uint32_t available_slots; + + sq_depth = sq.depth; + + do { + cqe = (struct bnxt_re_req_cqe *) cq.buf; + +#ifdef BUILD_DEBUG_DEVICE + uint32_t flg_val = + __hip_atomic_load(static_cast( + __builtin_assume_aligned((char*)cqe + sizeof(struct bnxt_re_req_cqe) + + offsetof(struct bnxt_re_bcqe, flg_st_typ_ph), + 4)), + __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + uint8_t status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; + if (status != BNXT_RE_REQ_ST_OK) { + print_cqe_error(status); + } +#endif + + /* Update the SQ head + * This param provides us the wqe_idx but we need to convert to the slot idx. + * We assume a static slots size of GDA_BNXT_WQE_SLOT_COUNT thus can multiply by this value */ + sq_head = (((cqe->con_indx & 0xFFFF) * GDA_BNXT_WQE_SLOT_COUNT) % sq_depth); + sq.head = sq_head; + + sq_tail = __hip_atomic_load(&sq.tail, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_AGENT); + + consumed_slots = (sq_tail - sq_head + sq_depth) % sq_depth; + available_slots = sq_depth - consumed_slots; + } while (available_slots < requested_available_slots); +} + +__device__ __forceinline__ void QueuePairBNXT::ring_doorbell(uint32_t slot_idx) { + struct bnxt_re_db_hdr hdr; + uint32_t epoch; + uint64_t key_lo; + uint64_t key_hi; + + epoch = (sq.flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT; + + key_lo = (slot_idx | epoch); + + key_hi = (qp_num & BNXT_RE_DB_QID_MASK) + | (((uint64_t) BNXT_RE_QUE_TYPE_SQ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) + | (0x1UL << BNXT_RE_DB_VALID_SHIFT); + + hdr.typ_qid_indx = (key_lo | (key_hi << 32)); + + __threadfence_system(); + __hip_atomic_store(dbr, hdr.typ_qid_indx, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SYSTEM); +} + +__device__ __forceinline__ void* QueuePairBNXT::get_hwqe( + const bnxt_device_sq& sq, uint32_t idx) { + idx += sq.tail; + if (idx >= sq.depth) + idx -= sq.depth; + return (void *)((char*)sq.buf + (idx << 4)); +} + +__device__ __forceinline__ void QueuePairBNXT::fill_psns_for_msntbl( + bnxt_device_sq& sq, uint32_t msg_len) { + uint32_t npsn = 0, start_psn = 0, next_psn = 0; + struct bnxt_re_msns msns; + uint64_t *msns_ptr; + uint32_t pkt_cnt = 0; + /* Start slot index of the WQE */ + uint32_t st_idx = sq.tail; // * BNXT_RE_STATIC_WQE_SIZE_SLOTS; Do we need this? + // Get the MSN table address + msns_ptr = (uint64_t *)pull_psn_buff(sq); + // Start PSN is the last recorded PSN + // Calculate the packet count based on the len of the WQE/MTU + msns.start_idx_next_psn_start_psn = 0; + start_psn = sq.psn; + pkt_cnt = (msg_len / sq.mtu); + + if (msg_len % sq.mtu) + pkt_cnt++; + + /* Increment the psn even for 0 len packets + * e.g. for opcode rdma-write-with-imm-data + * with length field = 0 + */ + if (msg_len == 0) + pkt_cnt = 1; + + /* make it 24 bit */ + next_psn = sq.psn + pkt_cnt; + npsn = next_psn; + sq.psn = next_psn; + msns.start_idx_next_psn_start_psn |= update_msn_tbl(st_idx, npsn, start_psn); + sq.msn++; + sq.msn %= sq.msn_tbl_sz; + + memcpy(msns_ptr, &msns, sizeof(uint64_t)); +} + +__device__ __forceinline__ void QueuePairBNXT::incr_tail( + bnxt_device_sq& sq, uint8_t cnt) { + sq.tail += cnt; + if (sq.tail >= sq.depth) { + sq.tail %= sq.depth; + /* Rolled over, Toggle Tail bit in epoch flags */ + sq.flags ^= 1UL << BNXT_RE_FLAG_EPOCH_TAIL_SHIFT; + } +} + +__device__ __forceinline__ struct bnxt_re_msns* QueuePairBNXT::pull_psn_buff( + const bnxt_device_sq& sq) { + return (struct bnxt_re_msns*)(((char *) sq.msntbl) + ((sq.msn) << sq.psn_sz_log2)); +} + +__device__ __forceinline__ uint64_t QueuePairBNXT::update_msn_tbl( + uint32_t st_idx, uint32_t npsn, uint32_t start_psn) { + return ((((uint64_t)(st_idx) << BNXT_RE_SQ_MSN_SEARCH_START_IDX_SHIFT) + & BNXT_RE_SQ_MSN_SEARCH_START_IDX_MASK) | + (((uint64_t)(npsn) << BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_SHIFT) + & BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_MASK) | + (((uint64_t)(start_psn) << BNXT_RE_SQ_MSN_SEARCH_START_PSN_SHIFT) + & BNXT_RE_SQ_MSN_SEARCH_START_PSN_MASK)); +} + +__device__ __forceinline__ void QueuePairBNXT::acquire_lock(uint32_t* lock) { + uint32_t expected; + + do { + expected = 0; + } while (0 == __hip_atomic_compare_exchange_strong(lock, &expected, 1, + __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __HIP_MEMORY_SCOPE_SYSTEM)); +} + +__device__ __forceinline__ void QueuePairBNXT::release_lock(uint32_t* lock) { + __hip_atomic_store(lock, 0, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); +} + +#if defined(BUILD_DEBUG_DEVICE) +__device__ inline __noinline__ void QueuePairBNXT::print_cqe_error(uint8_t status) { + switch (status) { + case BNXT_RE_REQ_ST_BAD_RESP: + LOGD_ERROR_ABORT("CQ error BAD_RESP (%x)", status); + break; + case BNXT_RE_REQ_ST_LOC_LEN: + LOGD_ERROR_ABORT("CQ error LOC_LEN (%x)", status); + break; + case BNXT_RE_REQ_ST_LOC_QP_OP: + LOGD_ERROR_ABORT("CQ error LOC_QP_OP (%x)", status); + break; + case BNXT_RE_REQ_ST_PROT: + LOGD_ERROR_ABORT("CQ error PROT (%x)", status); + break; + case BNXT_RE_REQ_ST_MEM_OP: + LOGD_ERROR_ABORT("CQ error MEM_OP (%x)", status); + break; + case BNXT_RE_REQ_ST_REM_INVAL: + LOGD_ERROR_ABORT("CQ error REM_INVAL (%x)", status); + break; + case BNXT_RE_REQ_ST_REM_ACC: + LOGD_ERROR_ABORT("CQ error REM_ACC (%x)", status); + break; + case BNXT_RE_REQ_ST_REM_OP: + LOGD_ERROR_ABORT("CQ error REM_OP (%x)", status); + break; + case BNXT_RE_REQ_ST_RNR_NAK_XCED: + LOGD_ERROR_ABORT("CQ error RNR_NAK_XCED (%x)", status); + break; + case BNXT_RE_REQ_ST_TRNSP_XCED: + LOGD_ERROR_ABORT("CQ error TRNSP_XCED (%x)", status); + break; + case BNXT_RE_REQ_ST_WR_FLUSH: + LOGD_ERROR_ABORT("CQ error WR_FLUSH (%x)", status); + break; + default: + LOGD_ERROR_ABORT("CQ error unknown status (%x)", status); + break; + } +} +#endif // BUILD_DEBUG_DEVICE + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_BNXT_QUEUE_PAIR_HPP_ + diff --git a/projects/rocshmem/src/gda/context_gda_device.hpp b/projects/rocshmem/src/gda/context_gda_device.hpp index f1515647af3b..aa151d98701d 100644 --- a/projects/rocshmem/src/gda/context_gda_device.hpp +++ b/projects/rocshmem/src/gda/context_gda_device.hpp @@ -27,14 +27,11 @@ #include "context.hpp" #include "team.hpp" -#include "queue_pair.hpp" +#include "queue_pair_provider.hpp" #include "constmem.hpp" -#include "gda/gda_symm_table.hpp" namespace rocshmem { -class QueuePair; - class GDAContext : public Context { public: __host__ GDAContext(Backend *b, unsigned int ctx_id); diff --git a/projects/rocshmem/src/gda/context_gda_device_coll.cpp b/projects/rocshmem/src/gda/context_gda_device_coll.cpp index cd09e9b2278f..1270ccc07c26 100644 --- a/projects/rocshmem/src/gda/context_gda_device_coll.cpp +++ b/projects/rocshmem/src/gda/context_gda_device_coll.cpp @@ -429,26 +429,10 @@ __device__ void GDAContext::alltoallmem_linear_thread_puts_wg(rocshmem_team_t te // Have each PE put their designated data to the other PEs for (int j = tid; j < pe_size; j += step_size) { int dest_pe = team_obj->get_pe_in_world(j); - uint64_t base_heap_offset = base_heap[dest_pe] - base_heap[constmem.my_pe]; - - /* - * dst may be a symmetric-heap object or a registered user buffer, so - * resolve its remote address and key through the QP. src and pSync are - * always heap; for a heap dst this reproduces the base_heap_offset math - * and default keys exactly. - */ char *dst_local = reinterpret_cast(dst) + my_pe_in_team * nelems; const char *src_local = reinterpret_cast(src) + j * nelems; - auto [dst_raddr, dst_rkey] = qps[dest_pe].get_raddr_info(dst_local); - uint32_t src_lkey = - (static_cast(nelems) <= - static_cast(qps[dest_pe].inline_threshold)) - ? 0 : qps[dest_pe].get_lkey(reinterpret_cast(src_local)); - qps[dest_pe].put_nbi_single(reinterpret_cast(dst_raddr), dst_rkey, - src_local, src_lkey, nelems, false); - qps[dest_pe].atomic_add_single( - reinterpret_cast(&pSync[alltoall_pSync_offset + my_pe_in_team]) + - base_heap_offset, 1); + qps[dest_pe].put_nbi_single(dst_local, src_local, nelems, PostOpt{RingDB}); + qps[dest_pe].atomic_add_single(&pSync[alltoall_pSync_offset + my_pe_in_team], 1); } // wait until everyone has obtained their designated data @@ -528,26 +512,10 @@ __device__ void GDAContext::alltoallmem_linear_thread_puts_wave(rocshmem_team_t // Have each PE put their designated data to the other PEs for (int j = tid; j < pe_size; j += step_size) { int dest_pe = team_obj->get_pe_in_world(j); - uint64_t base_heap_offset = base_heap[dest_pe] - base_heap[constmem.my_pe]; - - /* - * dst may be a symmetric-heap object or a registered user buffer, so - * resolve its remote address and key through the QP. src and pSync are - * always heap; for a heap dst this reproduces the base_heap_offset math - * and default keys exactly. - */ char *dst_local = reinterpret_cast(dst) + my_pe_in_team * nelems; const char *src_local = reinterpret_cast(src) + j * nelems; - auto [dst_raddr, dst_rkey] = qps[dest_pe].get_raddr_info(dst_local); - uint32_t src_lkey = - (static_cast(nelems) <= - static_cast(qps[dest_pe].inline_threshold)) - ? 0 : qps[dest_pe].get_lkey(reinterpret_cast(src_local)); - qps[dest_pe].put_nbi_single(reinterpret_cast(dst_raddr), dst_rkey, - src_local, src_lkey, nelems, false); - qps[dest_pe].atomic_add_single( - reinterpret_cast(&pSync[alltoall_pSync_offset + my_pe_in_team]) + - base_heap_offset, 1); + qps[dest_pe].put_nbi_single(dst_local, src_local, nelems, PostOpt{RingDB}); + qps[dest_pe].atomic_add_single(&pSync[alltoall_pSync_offset + my_pe_in_team], 1); } // wait until everyone has obtained their designated data diff --git a/projects/rocshmem/src/gda/context_gda_tmpl_device.hpp b/projects/rocshmem/src/gda/context_gda_tmpl_device.hpp index 2a7c5e83bdc3..c5ae3eba7c8a 100644 --- a/projects/rocshmem/src/gda/context_gda_tmpl_device.hpp +++ b/projects/rocshmem/src/gda/context_gda_tmpl_device.hpp @@ -34,7 +34,7 @@ #include "tile_memcpy_device.hpp" #include "context_gda_device.hpp" #include "gda_team.hpp" -#include "queue_pair.hpp" +#include "queue_pair_provider.hpp" #include "rocshmem_calc.hpp" #include "backend_gda.hpp" @@ -99,14 +99,13 @@ __device__ void GDAContext::amo_add(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_add not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); while (turns) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - qps[qp_index].atomic_add(reinterpret_cast(raddr), rkey, value, wf_info); + qps[qp_index].atomic_add(dst, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -123,7 +122,6 @@ __device__ T GDAContext::amo_swap(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_set not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -137,8 +135,8 @@ __device__ T GDAContext::amo_swap(void *dst, T value, int pe) { * The compare-and-swap loop will execute at least twice if wrong. * It may run additional times if contention on memory location. */ - while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_cas( - reinterpret_cast(raddr), rkey, value, cond, wf_info)) != cond) { + while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_compare_swap( + dst, cond, value, wf_info)) != cond) { cond = ret_val; } need_turn = false; @@ -153,7 +151,6 @@ __device__ T GDAContext::amo_fetch_and(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fetch_and not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -163,8 +160,8 @@ __device__ T GDAContext::amo_fetch_and(void *dst, T value, int pe) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_cas( - reinterpret_cast(raddr), rkey, desired_val, cond, wf_info)) != cond) { + while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_compare_swap( + dst, cond, desired_val, wf_info)) != cond) { cond = ret_val; desired_val = ret_val & value; } @@ -185,7 +182,6 @@ __device__ T GDAContext::amo_fetch_or(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fetch_or not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -195,8 +191,8 @@ __device__ T GDAContext::amo_fetch_or(void *dst, T value, int pe) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_cas( - reinterpret_cast(raddr), rkey, desired_val, cond, wf_info)) != cond) { + while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_compare_swap( + dst, cond, desired_val, wf_info)) != cond) { cond = ret_val; desired_val = ret_val | value; } @@ -217,7 +213,6 @@ __device__ T GDAContext::amo_fetch_xor(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fetch_xor not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -227,8 +222,8 @@ __device__ T GDAContext::amo_fetch_xor(void *dst, T value, int pe) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_cas( - reinterpret_cast(raddr), rkey, desired_val, cond, wf_info)) != cond) { + while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_compare_swap( + dst, cond, desired_val, wf_info)) != cond) { cond = ret_val; desired_val = ret_val ^ value; } @@ -249,14 +244,13 @@ __device__ void GDAContext::amo_cas(void *dst, T value, T cond, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_cas not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); while (turns) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - qps[qp_index].atomic_cas_nofetch(reinterpret_cast(raddr), rkey, value, cond, wf_info); + qps[qp_index].atomic_compare_swap_nofetch(dst, cond, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -268,7 +262,6 @@ __device__ T GDAContext::amo_fetch_add(void *dst, T value, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fadd not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); T ret_val = 0; bool need_turn {true}; uint64_t turns = __ballot(need_turn); @@ -276,7 +269,7 @@ __device__ T GDAContext::amo_fetch_add(void *dst, T value, int pe) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - ret_val = qps[qp_index].atomic_fetch_add(reinterpret_cast(raddr), rkey, value, wf_info); + ret_val = qps[qp_index].atomic_fetch_add(dst, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -289,7 +282,6 @@ __device__ T GDAContext::amo_fetch_cas(void *dst, T value, T cond, int pe) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fcas not implemented for non-64bit types"); }//TODO:support for non-uint64t ActiveWFInfo wf_info(pe); int qp_index = get_qp_index(pe, wf_info); - auto [raddr, rkey] = qps[qp_index].get_raddr_info(dst); bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -297,7 +289,7 @@ __device__ T GDAContext::amo_fetch_cas(void *dst, T value, T cond, int pe) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - ret_val = qps[qp_index].atomic_cas(reinterpret_cast(raddr), rkey, value, cond, wf_info); + ret_val = qps[qp_index].atomic_compare_swap(dst, cond, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -1058,14 +1050,13 @@ __device__ void GDAContext::alltoallv_copy(rocshmem_team_t team, T *dest, // Have each PE put their designated data to the other PEs for (int j = tid; j < pe_size; j+= step_size) { int dest_pe = team_obj->get_pe_in_world(j); - uint64_t base_heap_offset = base_heap[dest_pe] - base_heap[constmem.my_pe]; size_t nelems = source_nelems[j] * sizeof(T); - char* amo_dst = ((char*)&pSync[alltoall_pSync_offset + my_pe_in_team] + base_heap_offset); + long *amo_dst = &pSync[alltoall_pSync_offset + my_pe_in_team]; if (nelems != 0) { - T* src = (T*)((char*)source + (source_displs[j] * sizeof(T))); - T* dst = (T*)((char*)&tmp_buf[my_pe_in_team * tmp_buf_off] + base_heap_offset); - qps[dest_pe].put_nbi_single(dst, src, nelems, false); + T *src = &source[source_displs[j]]; + T *dst = &tmp_buf[my_pe_in_team * tmp_buf_off]; + qps[dest_pe].put_nbi_single(dst, src, nelems, PostOpt{RingDB}); } qps[dest_pe].atomic_add_single(amo_dst, 1); @@ -1090,8 +1081,8 @@ __device__ void GDAContext::alltoallv_copy(rocshmem_team_t team, T *dest, size_t nelems = dest_nelems[j] * sizeof(T); if (nelems != 0) { - T* dst = (T*)((char*) dest + dest_displs[j] * sizeof(T)); - T* src = (T*)((char*) &tmp_buf[j * tmp_buf_off]); + T *dst = &dest[dest_displs[j]]; + T *src = &tmp_buf[j * tmp_buf_off]; memcpy_wg(dst, src, nelems); } } @@ -1124,24 +1115,20 @@ __device__ void GDAContext::alltoallv_get(rocshmem_team_t team, T *dest, /* Put Ctrl Message */ for (int j = tid; j < pe_size; j+= step_size) { - uint64_t *src; - uint64_t *dst; - uint64_t seq_bits; - uint64_t displ_bits; - int dest_pe = team_obj->get_pe_in_world(j); - uint64_t base_heap_offset = base_heap[dest_pe] - base_heap[constmem.my_pe]; /* Pack Ctrl Message * 16 bits seq | 48bit displ */ - seq_bits = (seq_mask & (a2a_sn + 1)) << seq_shift; - displ_bits = (displs_mask & source_displs[j]); + uint64_t seq_bits = (seq_mask & (a2a_sn + 1)) << seq_shift; + uint64_t displ_bits = (displs_mask & source_displs[j]); uint64_t ctrl_msg = seq_bits | displ_bits; /* Prepare Ctrl Message */ - src = (uint64_t*)&ctrl_msg; - dst = (uint64_t*)((char*)&tmp_buf[my_pe_in_team] + base_heap_offset); + uint64_t *ctrl_src = &ctrl_msg; + uint64_t *ctrl_dst = &tmp_buf[my_pe_in_team]; - qps[dest_pe].put_nbi_single(dst, src, sizeof(uint64_t), true); + static_assert(QueuePair::can_inline(sizeof(ctrl_msg)), + "alltoallv_get control message must be posted inline"); + qps[dest_pe].put_nbi_single(ctrl_dst, ctrl_src, sizeof(ctrl_msg), PostOpt{RingDB}); /* Wait for Ctrl Message */ uint64_t ctrl_value; @@ -1155,13 +1142,13 @@ __device__ void GDAContext::alltoallv_get(rocshmem_team_t team, T *dest, /* Get data */ size_t nelems = dest_nelems[j] * sizeof(T); - src = (uint64_t*)((char*)source + (displ_bits * sizeof(T))); - dst = (uint64_t*)((char*)dest + (dest_displs[j] * sizeof(T))); + T *src = &source[displ_bits]; + T *dst = &dest[dest_displs[j]]; - qps[dest_pe].get_nbi_single(dst, src, nelems, true); + qps[dest_pe].get_nbi_single(dst, src, nelems, PostOpt{RingDB}); /* Put Completion */ - char* amo_dst = ((char*)&pSync[alltoall_pSync_offset + my_pe_in_team] + base_heap_offset); + long *amo_dst = &pSync[alltoall_pSync_offset + my_pe_in_team]; qps[dest_pe].atomic_add_single(amo_dst, 1); long *sync_flags = &pSync[alltoall_pSync_offset + j]; @@ -1303,14 +1290,13 @@ template __device__ void GDAContext::internal_amo_add(void *dst, T value, int pe, int qp_index, ActiveWFInfo &wf_info) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_add not implemented for non-64bit types"); }//TODO:support for non-uint64t - uint64_t L_offset = reinterpret_cast(dst) - base_heap[constmem.my_pe]; bool need_turn {true}; uint64_t turns = __ballot(need_turn); while (turns) { uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - qps[qp_index].atomic_add(base_heap[pe] + L_offset, value, wf_info); + qps[qp_index].atomic_add(dst, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -1321,7 +1307,6 @@ template __device__ T GDAContext::internal_amo_fetch_add(void *dst, T value, int pe, int qp_index, ActiveWFInfo &wf_info) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_fadd not implemented for non-64bit types"); }//TODO:support for non-uint64t - uint64_t L_offset = reinterpret_cast(dst) - base_heap[constmem.my_pe]; T ret_val = 0; bool need_turn {true}; uint64_t turns = __ballot(need_turn); @@ -1329,7 +1314,7 @@ __device__ T GDAContext::internal_amo_fetch_add(void *dst, T value, int pe, uint8_t lane = __ffsll((unsigned long long)turns) - 1; int pe_turn = __shfl(pe, lane); if (pe_turn == pe) { - ret_val = qps[qp_index].atomic_fetch_add(base_heap[pe] + L_offset, value, wf_info); + ret_val = qps[qp_index].atomic_fetch_add(dst, value, wf_info); need_turn = false; } turns = __ballot(need_turn); @@ -1341,7 +1326,6 @@ template __device__ T GDAContext::internal_amo_swap(void *dst, T value, int pe, int qp_index, ActiveWFInfo &wf_info) { if constexpr (sizeof(T) != 8) { LOGD_ERROR_ABORT("gda::amo_set not implemented for non-64bit types"); }//TODO:support for non-uint64t - uint64_t L_offset = reinterpret_cast(dst) - base_heap[constmem.my_pe]; bool need_turn {true}; uint64_t turns = __ballot(need_turn); T ret_val; @@ -1355,8 +1339,8 @@ __device__ T GDAContext::internal_amo_swap(void *dst, T value, int pe, * The compare-and-swap loop will execute at least twice if wrong. * It may run additional times if contention on memory location. */ - while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_cas( - base_heap[pe] + L_offset, value, cond, wf_info)) != cond) { + while (wf_info.update(pe), (ret_val = qps[qp_index].atomic_compare_swap( + dst, cond, value, wf_info)) != cond) { cond = ret_val; } need_turn = false; @@ -1458,14 +1442,7 @@ __device__ inline void GDAContext::tile_put_chunk_nbi(char *dst, const char *src * execution mask at the point the WQEs are written. */ ActiveWFInfo wf_info(pe); - auto [dst_raddr, dst_rkey] = qps[qp_index].get_raddr_info(dst); - uint32_t src_lkey = - (static_cast(bytes) <= - static_cast(qps[qp_index].inline_threshold)) - ? 0 - : qps[qp_index].get_lkey(reinterpret_cast(src)); - qps[qp_index].put_nbi(reinterpret_cast(dst_raddr), dst_rkey, - src, src_lkey, bytes, wf_info, true); + qps[qp_index].put_nbi(dst, src, bytes, wf_info); } __device__ inline void GDAContext::tile_get_chunk_nbi(char *dst, const char *src, diff --git a/projects/rocshmem/src/gda/debug_gda.hpp b/projects/rocshmem/src/gda/debug_gda.hpp index b21200735b91..673f3734cb06 100644 --- a/projects/rocshmem/src/gda/debug_gda.hpp +++ b/projects/rocshmem/src/gda/debug_gda.hpp @@ -90,6 +90,7 @@ static void dump_ibv_port_attr([[maybe_unused]] struct ibv_port_attr* x) { x->link_layer, x->flags, x->port_cap_flags2); } +[[maybe_unused]] static void dump_ibv_qp([[maybe_unused]] struct ibv_qp *qp, [[maybe_unused]] int conn_num) { LOG_TRACE("== IBV_QP CONNECTION#%d ======================================\n" " (ibv_context*) context = %p\n" diff --git a/projects/rocshmem/src/gda/gda_enums.hpp b/projects/rocshmem/src/gda/gda_enums.hpp index c99f6e8b460b..61a83cc6f3a7 100644 --- a/projects/rocshmem/src/gda/gda_enums.hpp +++ b/projects/rocshmem/src/gda/gda_enums.hpp @@ -25,15 +25,23 @@ #ifndef LIBRARY_SRC_GDA_ENUMS_HPP_ #define LIBRARY_SRC_GDA_ENUMS_HPP_ +#include + namespace rocshmem { namespace gda { - enum provider { + enum class provider : int { UNSET, IONIC, BNXT, MLX5 }; + enum class vendor_id : uint32_t { + IONIC = 0x1dd8, + BNXT = 0x14e4, + MLX5 = 0x02c9, // PCI-ID is 15b3 + }; + enum alltoallv_wg_algos { ALLTOALLV_WG_ALGO_GET = 0, ALLTOALLV_WG_ALGO_COPY, diff --git a/projects/rocshmem/src/gda/gda_symm_table.hpp b/projects/rocshmem/src/gda/gda_symm_table.hpp deleted file mode 100644 index 7474e051de3f..000000000000 --- a/projects/rocshmem/src/gda/gda_symm_table.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef LIBRARY_SRC_GDA_GDA_SYMM_TABLE_HPP_ -#define LIBRARY_SRC_GDA_GDA_SYMM_TABLE_HPP_ - -#include -#include - -namespace rocshmem { - -/** - * @brief One symmetric user-buffer registration, pre-specialized to a single - * QueuePair's fixed (dest_pe, nic_idx). - * - * Because every QP talks to exactly one peer over exactly one NIC, all the - * per-PE / per-NIC indexing can be resolved on the host at registration time, - * leaving the device with a self-contained 32-byte record: matching an address - * and producing the remote address + keys needs no pointer chasing. The host - * fills one entry per (pe, nic) per registration (see GDABackend's flat entry - * table), and each QP is handed the contiguous slice for its own (dest_pe, - * nic_idx), scanned by registration slot. - */ -struct QpSymmEntry { - /** - * @brief This PE's registered alias base (identical across all slices). - * - * Used to recognize whether a symmetric address falls in this registration - * and to compute the intra-registration offset. - */ - uintptr_t local_base; - - /** - * @brief The peer's alias base for this slice's dest_pe. - * - * Remote address for a transfer is remote_base + (sym_addr - local_base). - */ - uintptr_t remote_base; - - /** - * @brief Registered length in bytes. - */ - uint64_t length; - - /** - * @brief Remote key for this slice's (dest_pe, nic_idx). - */ - uint32_t rkey; - - /** - * @brief Local key for this slice's nic_idx (locally-sourced buffers). - */ - uint32_t lkey; -}; - -} // namespace rocshmem - -#endif // LIBRARY_SRC_GDA_GDA_SYMM_TABLE_HPP_ diff --git a/projects/rocshmem/src/gda/ionic/CMakeLists.txt b/projects/rocshmem/src/gda/ionic/CMakeLists.txt index 896fb2c4dd9c..087dbd343e58 100644 --- a/projects/rocshmem/src/gda/ionic/CMakeLists.txt +++ b/projects/rocshmem/src/gda/ionic/CMakeLists.txt @@ -21,16 +21,11 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. ############################################################################### -target_sources( - ${PROJECT_NAME} - PRIVATE - backend_gda_ionic.cpp -) if(GDA_IONIC) target_sources( ${PROJECT_NAME} PRIVATE - queue_pair_ionic.cpp + backend_gda_ionic.cpp ) endif() diff --git a/projects/rocshmem/src/gda/ionic/backend_gda_ionic.cpp b/projects/rocshmem/src/gda/ionic/backend_gda_ionic.cpp index 4fd8adf438bc..5d68f60a164a 100644 --- a/projects/rocshmem/src/gda/ionic/backend_gda_ionic.cpp +++ b/projects/rocshmem/src/gda/ionic/backend_gda_ionic.cpp @@ -22,9 +22,14 @@ * IN THE SOFTWARE. *****************************************************************************/ -#include "gda/backend_gda.hpp" +#include + #include "log.hpp" #include "util.hpp" +#include "gda/backend_gda.hpp" +#include "gda/ionic/provider_gda_ionic.hpp" +#include "gda/ionic/queue_pair_ionic.hpp" +#include "gda/queue_pair_provider.hpp" namespace rocshmem { @@ -68,7 +73,9 @@ void GDABackend::ionic_create_cqs(int ncqes) { } void GDABackend::ionic_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { - NicDevice &nic = nic_for_qp(conn_num); + ibv_qp* qp = qps[conn_num]; + const NicDevice& nic = nic_for_qp(conn_num); + ionic_dv_ctx dvctx; ionic_dv.get_ctx(&dvctx, nic.context); @@ -86,40 +93,30 @@ void GDABackend::ionic_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { gpu_db_cq = &gpu_db_ptr[dvctx.cq_qtype]; gpu_db_sq = &gpu_db_ptr[dvctx.sq_qtype]; - uint8_t udma_idx = ionic_dv.qp_get_udma_idx(qps[conn_num]); + uint8_t udma_idx = ionic_dv.qp_get_udma_idx(qp); ionic_dv_cq dvcq; ionic_dv.get_cq(&dvcq, cqs[conn_num], udma_idx); - gpu_qp->cq_dbreg = gpu_db_cq; - gpu_qp->cq_dbval = dvcq.q.db_val; - gpu_qp->cq_mask = dvcq.q.mask; - - gpu_qp->ionic_cq_buf = reinterpret_cast(dvcq.q.ptr); - ionic_dv_qp dvqp; - ionic_dv.get_qp(&dvqp, qps[conn_num]); - - gpu_qp->sq_dbreg = gpu_db_sq; - gpu_qp->sq_dbval = dvqp.sq.db_val; - gpu_qp->sq_mask = dvqp.sq.mask; - gpu_qp->ionic_sq_buf = reinterpret_cast(dvqp.sq.ptr); - - strncpy(gpu_qp->dev_name, - qps[conn_num]->context->device->name, - sizeof(gpu_qp->dev_name)); - gpu_qp->dev_name[sizeof(gpu_qp->dev_name) - 1] = 0; - - gpu_qp->qp_num = qps[conn_num]->qp_num; - int pe = conn_num % num_pes; - int nic_idx = nic_idx_for_qp(conn_num); - gpu_qp->lkey = nic.heap_mr->lkey; - gpu_qp->rkey = heap_rkey[pe * num_nics_ + nic_idx]; - gpu_qp->inline_threshold = 32; - - /* Base Heap information */ - gpu_qp->base_heap = (uintptr_t) heap.get_local_heap_base(); - gpu_qp->base_heap_size = heap.get_size(); + ionic_dv.get_qp(&dvqp, qp); + + ionic_v1_wqe* sq_buf = reinterpret_cast(dvqp.sq.ptr); + uint64_t* sq_dbreg = gpu_db_sq; + uint64_t sq_dbval = dvqp.sq.db_val; + uint16_t sq_mask = dvqp.sq.mask; + + ionic_v1_cqe* cq_buf = reinterpret_cast(dvcq.q.ptr); + uint64_t* cq_dbreg = gpu_db_cq; + uint64_t cq_dbval = dvcq.q.db_val; + uint16_t cq_mask = dvcq.q.mask; + + /* QueuePair is either QueuePairIONIC or QueuePairMux + * both have a constructor that accepts rvalue reference QueuePairIONIC&&, + * so just use that instead of trying to figure out which one we're using */ + new (gpu_qp) QueuePair{QueuePairIONIC{qp->qp_num, gpu_qp_init_info(conn_num), + ionic_device_sq{sq_buf, sq_dbreg, sq_dbval, sq_mask}, + ionic_device_cq{cq_buf, cq_dbreg, cq_dbval, cq_mask}}}; } void GDABackend::ionic_setup_parent_domain(NicDevice &nic, struct ibv_parent_domain_init_attr* pattr) { diff --git a/projects/rocshmem/src/gda/ionic/ionic_fw.h b/projects/rocshmem/src/gda/ionic/ionic_fw.h index 4f00043b273b..2357feae5b76 100644 --- a/projects/rocshmem/src/gda/ionic/ionic_fw.h +++ b/projects/rocshmem/src/gda/ionic/ionic_fw.h @@ -7,13 +7,7 @@ #ifndef IONIC_FW_H #define IONIC_FW_H -#if !defined(__cplusplus) -#include -#else -#ifndef BIT -#define BIT(n) (1u << (n)) -#endif -#endif +#define IONIC_BIT(n) (1u << (n)) #define IONIC_EXP_DBELL_SZ 8 @@ -29,19 +23,19 @@ struct ionic_sge { /* admin queue mr type */ enum ionic_mr_flags { /* bits that determine mr access */ - IONIC_MRF_LOCAL_WRITE = BIT(0), - IONIC_MRF_REMOTE_WRITE = BIT(1), - IONIC_MRF_REMOTE_READ = BIT(2), - IONIC_MRF_REMOTE_ATOMIC = BIT(3), - IONIC_MRF_MW_BIND = BIT(4), - IONIC_MRF_ZERO_BASED = BIT(5), - IONIC_MRF_ON_DEMAND = BIT(6), - IONIC_MRF_PB = BIT(7), - IONIC_MRF_ACCESS_MASK = BIT(12) - 1, + IONIC_MRF_LOCAL_WRITE = IONIC_BIT(0), + IONIC_MRF_REMOTE_WRITE = IONIC_BIT(1), + IONIC_MRF_REMOTE_READ = IONIC_BIT(2), + IONIC_MRF_REMOTE_ATOMIC = IONIC_BIT(3), + IONIC_MRF_MW_BIND = IONIC_BIT(4), + IONIC_MRF_ZERO_BASED = IONIC_BIT(5), + IONIC_MRF_ON_DEMAND = IONIC_BIT(6), + IONIC_MRF_PB = IONIC_BIT(7), + IONIC_MRF_ACCESS_MASK = IONIC_BIT(12) - 1, /* bits that determine mr type */ - IONIC_MRF_IS_MW = BIT(14), - IONIC_MRF_INV_EN = BIT(15), + IONIC_MRF_IS_MW = IONIC_BIT(14), + IONIC_MRF_INV_EN = IONIC_BIT(15), /* base flags combinations for mr types */ IONIC_MRF_USER_MR = 0, @@ -180,8 +174,8 @@ enum ionic_v1_cqe_wqe_idx_timestamp_bits { /* bits for rcqe seq_op_flags */ enum ionic_v1_cqe_rcqe_op_flag_bits { IONIC_V1_CQE_RCQE_SEQ_MASK = 0xffffff, - IONIC_V1_CQE_RCQE_FLAG_V = BIT(24), - IONIC_V1_CQE_RCQE_FLAG_I = BIT(25), + IONIC_V1_CQE_RCQE_FLAG_V = IONIC_BIT(24), + IONIC_V1_CQE_RCQE_FLAG_I = IONIC_BIT(25), IONIC_V1_CQE_RCQE_OP_SHIFT = 28, }; @@ -219,14 +213,14 @@ enum ionic_v1_cqe_src_qpn_bits { IONIC_V1_CQE_RECV_OP_SEND_IMM = 2, IONIC_V1_CQE_RECV_OP_RDMA_IMM = 3, - IONIC_V1_CQE_RECV_IS_IPV4 = BIT(7 + IONIC_V1_CQE_RECV_OP_SHIFT), - IONIC_V1_CQE_RECV_IS_VLAN = BIT(6 + IONIC_V1_CQE_RECV_OP_SHIFT), + IONIC_V1_CQE_RECV_IS_IPV4 = IONIC_BIT(7 + IONIC_V1_CQE_RECV_OP_SHIFT), + IONIC_V1_CQE_RECV_IS_VLAN = IONIC_BIT(6 + IONIC_V1_CQE_RECV_OP_SHIFT), }; /* bits for cqe qid_type_flags */ enum ionic_v1_cqe_qtf_bits { - IONIC_V1_CQE_COLOR = BIT(0), - IONIC_V1_CQE_ERROR = BIT(1), + IONIC_V1_CQE_COLOR = IONIC_BIT(0), + IONIC_V1_CQE_ERROR = IONIC_BIT(1), IONIC_V1_CQE_TYPE_SHIFT = 5, IONIC_V1_CQE_TYPE_MASK = 0x7, IONIC_V1_CQE_QID_SHIFT = 8, @@ -379,11 +373,11 @@ enum ionic_v1_op { IONIC_V1_OP_BIND_MW, /* flags */ - IONIC_V1_FLAG_FENCE = BIT(0), - IONIC_V1_FLAG_SOL = BIT(1), - IONIC_V1_FLAG_INL = BIT(2), - IONIC_V1_FLAG_SIG = BIT(3), - IONIC_V1_FLAG_COLOR = BIT(4), + IONIC_V1_FLAG_FENCE = IONIC_BIT(0), + IONIC_V1_FLAG_SOL = IONIC_BIT(1), + IONIC_V1_FLAG_INL = IONIC_BIT(2), + IONIC_V1_FLAG_SIG = IONIC_BIT(3), + IONIC_V1_FLAG_COLOR = IONIC_BIT(4), /* flags last four bits for sgl spec format */ IONIC_V1_FLAG_SPEC32 = (1u << 12), @@ -548,4 +542,6 @@ static inline void ionic_rcq_ack(struct ionic_rcq *rcq, uint32_t ack) #endif // !defined(__cplusplus) +#undef IONIC_BIT + #endif /* IONIC_FW_H */ diff --git a/projects/rocshmem/src/gda/ionic/provider_gda_ionic.hpp b/projects/rocshmem/src/gda/ionic/provider_gda_ionic.hpp index 17aff152880f..04fb96712978 100644 --- a/projects/rocshmem/src/gda/ionic/provider_gda_ionic.hpp +++ b/projects/rocshmem/src/gda/ionic/provider_gda_ionic.hpp @@ -25,11 +25,46 @@ #ifndef LIBRARY_SRC_GDA_IONIC_GDA_PROVIDER_HPP_ #define LIBRARY_SRC_GDA_IONIC_GDA_PROVIDER_HPP_ +#include "util.hpp" + extern "C" { #include "gda/ionic/ionic_dv.h" #include "gda/ionic/ionic_fw.h" } +namespace rocshmem { + +template +struct ionic_device_queue { + T* buf; + uint64_t* dbreg; + uint64_t dbval; + uint64_t mask; + uint32_t pos; + uint32_t dbpos; + uint32_t lock; + + __host__ inline ionic_device_queue(T* buf, uint64_t* dbreg, uint64_t dbval, uint16_t mask) + : buf{buf}, dbreg{dbreg}, dbval{dbval}, mask{static_cast(mask)}, + pos{0}, dbpos{0}, lock{SPIN_LOCK_UNLOCKED} { } +}; + +struct ionic_device_cq : public ionic_device_queue { + // inherit constructors + using ionic_device_queue::ionic_device_queue; +}; +static_assert(sizeof(ionic_device_cq) == 48); + +struct ionic_device_sq : public ionic_device_queue { + uint32_t msn; + + __host__ inline ionic_device_sq(ionic_v1_wqe* sq_buf, uint64_t* sq_dbreg, uint64_t sq_dbval, + uint16_t sq_mask) + : ionic_device_queue{sq_buf, sq_dbreg, sq_dbval, sq_mask}, msn{0} { } +}; +// check if ionic_device_sq::msn is packed into tail padding of ionic_device_queue +static_assert(sizeof(ionic_device_sq) == 48); + struct ionicdv_funcs_t { int (*get_ctx)(struct ionic_dv_ctx *dvctx, struct ibv_context *ibctx); uint8_t (*qp_get_udma_idx)(struct ibv_qp *ibqp); @@ -43,4 +78,6 @@ struct ionicdv_funcs_t { struct ionic_cq_init_attr_ex *ionic_ex); }; +} // namespace rocshmem + #endif //LIBRARY_SRC_GDA_IONIC_GDA_PROVIDER_HPP_ diff --git a/projects/rocshmem/src/gda/ionic/queue_pair_ionic.cpp b/projects/rocshmem/src/gda/ionic/queue_pair_ionic.cpp deleted file mode 100644 index 13268debfbba..000000000000 --- a/projects/rocshmem/src/gda/ionic/queue_pair_ionic.cpp +++ /dev/null @@ -1,545 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#include "gda/queue_pair.hpp" -#include "gda/endian.hpp" -#include "util.hpp" -#include "log.hpp" -#include "containers/free_list_impl.hpp" - -namespace rocshmem { - -__device__ uint32_t QueuePair::reserve_sq(ActiveWFInfo &wf_info, - uint32_t num_wqes) { - uint32_t my_sq_prod = 0; - - // reserve space for wqes in sq - if (wf_info.is_pe_group_first) { - my_sq_prod = __hip_atomic_fetch_add(&sq_prod, num_wqes, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); - } - my_sq_prod = __shfl(my_sq_prod, wf_info.pe_group_first_phys_lane_id); - - // wait for that space to be available - ionic_quiet_internal(wf_info, my_sq_prod + num_wqes - sq_mask); - - return my_sq_prod; -} - -__device__ uint32_t QueuePair::reserve_sq_single(uint32_t num_wqes) { - uint32_t my_sq_prod = 0; - - // reserve space for wqes in sq - my_sq_prod = __hip_atomic_fetch_add(&sq_prod, num_wqes, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - - // wait for that space to be available - ionic_quiet_internal_ccqe_single(my_sq_prod + num_wqes - sq_mask); - - return my_sq_prod; -} - -__device__ uint32_t QueuePair::commit_sq_single(uint32_t my_sq_prod, [[maybe_unused]] uint32_t my_sq_pos, uint32_t num_wqes) { - uint32_t dbprod = my_sq_prod + num_wqes; - - spin_lock_acquire_unique(&sq_lock); - - if ((sq_dbprod - dbprod) & (1u << 31)) { - sq_dbprod = dbprod; - - ionic_ring_doorbell_single(dbprod); - } - - spin_lock_release_unique(&sq_lock); - - return dbprod; -} - -__device__ uint32_t QueuePair::commit_sq(ActiveWFInfo &wf_info, - uint32_t my_sq_prod, [[maybe_unused]] uint32_t my_sq_pos, - uint32_t num_wqes) { - uint32_t dbprod = my_sq_prod + num_wqes; - - spin_lock_acquire_shared(&sq_lock, wf_info.pe_group_mask); - - if (wf_info.is_pe_group_first && ((sq_dbprod - dbprod) & (1u << 31))) { - sq_dbprod = dbprod; - - ionic_ring_doorbell(dbprod); - } - - spin_lock_release_shared(&sq_lock, wf_info.pe_group_mask); - - return dbprod; -} - -__device__ void QueuePair::poll_wave_cqes(uint64_t activemask) { - int my_logical_lane_id = get_active_lane_num(activemask); - uint32_t my_cq_pos = cq_pos + my_logical_lane_id; - - /* Look at the cqe at the current position in the cq buffer */ - struct ionic_v1_cqe *cqe = &ionic_cq_buf[my_cq_pos & cq_mask]; - - /* Determine expected color based on cq wrap count */ - uint32_t qtf_color_bit = byteswap(IONIC_V1_CQE_COLOR); - uint32_t qtf_color_exp = qtf_color_bit; - if (my_cq_pos & (cq_mask + 1)) { - qtf_color_exp = 0; - } - - /* Check if my cqe color == expected color */ - uint32_t qtf_be = *(volatile uint32_t *)(&cqe->qid_type_flags); - if ((qtf_be & qtf_color_bit) != qtf_color_exp) { - return; - } - - uint32_t msn = byteswap(cqe->send.msg_msn); - - /* Report if the completion indicates an error. */ - if (!!(qtf_be & byteswap(IONIC_V1_CQE_ERROR))) { -#if defined(BUILD_DEBUG_DEVICE) - uint32_t qtf = byteswap(qtf_be); - uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; - uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; - uint32_t flag = qtf & 0xf; - uint32_t status = byteswap(cqe->status_length); - uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; - - LOGD_ERROR("QUIET ERROR: qid %u type %u flag %#x status %u msn %u npg %lu", - qid, type, flag, status, msn, npg); -#endif - /* No other way to signal an error, so just crash. */ - abort(); - } - - /* Only proceed with the furthest ahead cqe to update the sq state */ - uint64_t my_lane_mask = 1ull << __lane_id(); - uint64_t lesser_lane_mask = my_lane_mask - 1; - if (my_lane_mask != (__ballot(true) & activemask & ~lesser_lane_mask)) { - return; - } - - /* update position in the cq */ - cq_pos = my_cq_pos + 1; - - /* - * Ring cq doorbell frequently enough to avoid cq full. - * - * NB: IONIC_CQ_GRACE is 100 - */ - if (((cq_pos - cq_dbpos) & cq_mask) >= 100) { - cq_dbpos = cq_pos; - __atomic_store_n(cq_dbreg, cq_dbval | (cq_mask & cq_dbpos), __ATOMIC_SEQ_CST); //TODO:maybe relaxed? - } - - sq_msn = msn; -} - -__device__ void QueuePair::ionic_quiet_internal_ccqe(ActiveWFInfo &wf_info, - uint32_t cons) { - if (!wf_info.is_pe_group_first) { - return; - } - - volatile struct ionic_v1_cqe *cqe = &ionic_cq_buf[0]; - uint32_t qtf_be = cqe->qid_type_flags; - uint32_t msn = byteswap(cqe->send.msg_msn); - while ((msn - cons) & 0x800000) { - if (!!(qtf_be & byteswap(IONIC_V1_CQE_ERROR))) { - break; - } - - qtf_be = cqe->qid_type_flags; - msn = byteswap(cqe->send.msg_msn); - } - - if (!!(qtf_be & byteswap(IONIC_V1_CQE_ERROR))) { -#if defined(BUILD_DEBUG_DEVICE) - uint32_t qtf = byteswap(qtf_be); - uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; - uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; - uint32_t flag = qtf & 0xf; - uint32_t status = byteswap(cqe->status_length); - uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; - - LOGD_ERROR("QUIET ERROR (CCQE): qid %u type %u flag %#x status %u msn %u npg %lu", - qid, type, flag, status, msn, npg); -#endif - /* No other way to signal an error, so just crash. */ - abort(); - } -} - -__device__ void QueuePair::ionic_quiet_internal_ccqe_single(uint32_t cons) { - volatile struct ionic_v1_cqe *cqe = &ionic_cq_buf[0]; - uint32_t qtf_be = cqe->qid_type_flags; - uint32_t msn = byteswap(cqe->send.msg_msn); - while ((msn - cons) & 0x800000) { - if (!!(qtf_be & byteswap(IONIC_V1_CQE_ERROR))) { - break; - } - - qtf_be = cqe->qid_type_flags; - msn = byteswap(cqe->send.msg_msn); - } - - if (!!(qtf_be & byteswap(IONIC_V1_CQE_ERROR))) { -#if defined(BUILD_DEBUG_DEVICE) - uint32_t qtf = byteswap(qtf_be); - uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; - uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; - uint32_t flag = qtf & 0xf; - uint32_t status = byteswap(cqe->status_length); - uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; - - LOGD_ERROR("QUIET ERROR (CCQE): qid %u type %u flag %#x status %u msn %u npg %lu", - qid, type, flag, status, msn, npg); -#endif - /* No other way to signal an error, so just crash. */ - abort(); - } -} - -__device__ void QueuePair::ionic_quiet_internal(ActiveWFInfo &wf_info, uint32_t cons) { - uint32_t greed = 10; - - if (!cq_mask) { - ionic_quiet_internal_ccqe(wf_info, cons); - return; - } - - /* wait for sq_msn to catch up or pass cons. */ - /* 0x800000 - sign bit for 24-bit fields */ - while ((sq_msn - cons) & 0x800000) { - if (!spin_lock_try_acquire_shared(&cq_lock, wf_info.pe_group_mask)) { - continue; - } - - /* with lock acquired, this wave polls cqes until caught up */ - while ((sq_msn - cons) & 0x800000) { - uint32_t old_sq_msn = sq_msn; - - poll_wave_cqes(wf_info.pe_group_mask); - - if (!((sq_msn - cons) & 0x800000)) { - if (sq_msn == old_sq_msn) { - break; - } - if (!greed) { - break; - } - --greed; - } - } - - spin_lock_release_shared(&cq_lock, wf_info.pe_group_mask); - break; - } -} - -__device__ void QueuePair::ionic_ring_doorbell(uint32_t pos) { - // When threads write at once to the same address, not all writes reach the bus. - // Take turns and insert a thread fence between writes to the same address. - uint64_t activemask = get_active_lane_mask(); - int lane_id = get_active_lane_num(activemask); - int lane_count = get_active_lane_count(activemask); - - for (int i = 0; i < lane_count; i++) { - if (lane_id == i) { - __threadfence(); - __atomic_store_n(sq_dbreg, sq_dbval | (sq_mask & pos), __ATOMIC_SEQ_CST); - } - } - __threadfence(); -} - -__device__ void QueuePair::ionic_ring_doorbell_single(uint32_t pos) { - // When threads write at once to the same address, not all writes reach the bus. - // Take turns and insert a thread fence between writes to the same address. - __threadfence(); - __atomic_store_n(&sq_dbreg[8 * __lane_id()], sq_dbval | (sq_mask & pos), __ATOMIC_SEQ_CST); -} - -__device__ void QueuePair::ionic_quiet(ActiveWFInfo &wf_info) { - ionic_quiet_internal(wf_info, sq_prod); -} - -__device__ void QueuePair::ionic_quiet_single() { - ionic_quiet_internal_ccqe_single(sq_prod); -} - -__device__ void QueuePair::ionic_post_wqe_rma(int32_t length, - uintptr_t raddr, uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db) { - uint32_t num_wqes = 1; - if (wf_info.scope == ThreadScope::thread) { - num_wqes = wf_info.num_pe_group_lanes; - } - - uint32_t my_sq_prod = reserve_sq(wf_info, num_wqes); - - uint32_t my_sq_pos = my_sq_prod + wf_info.pe_group_logical_lane_id; - struct ionic_v1_wqe *wqe = &ionic_sq_buf[my_sq_pos & sq_mask]; - uint16_t wqe_flags = 0; - - if (!(my_sq_pos & (sq_mask + 1))) { - wqe_flags |= byteswap(IONIC_V1_FLAG_COLOR); - } - - if (wf_info.is_pe_group_last) { - wqe_flags |= byteswap(IONIC_V1_FLAG_SIG); - } - - // TODO why is this needed? - if (length && !laddr && opcode == IONIC_V2_OP_RDMA_WRITE) { - length = 1; - } - - wqe->base.wqe_idx = my_sq_pos; - wqe->base.op = opcode; - wqe->base.num_sge_key = length ? 1 : 0; - wqe->base.imm_data_key = byteswap(0); - - wqe->common.rdma.remote_va_high = byteswap(raddr >> 32); - wqe->common.rdma.remote_va_low = byteswap(raddr); - wqe->common.rdma.remote_rkey = byteswap(rkey); - wqe->common.length = byteswap(length); - - if (length) { - if (opcode == IONIC_V2_OP_RDMA_WRITE && static_cast(length) <= static_cast(inline_threshold)) { - wqe_flags |= byteswap(IONIC_V1_FLAG_INL); - wqe->base.num_sge_key = 0; - if (!laddr) { - // TODO why is this needed? - wqe->common.pld.data[0] = 1; - } else { - memcpy(wqe->common.pld.data, reinterpret_cast(laddr), length); - } - } else { - wqe->common.pld.sgl[0].va = byteswap(laddr); - wqe->common.pld.sgl[0].len = byteswap(length); - wqe->common.pld.sgl[0].lkey = byteswap(lkey); - } - } - - __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, - __HIP_MEMORY_SCOPE_AGENT); - - if (ring_db) { - commit_sq(wf_info, my_sq_prod, my_sq_pos, num_wqes); - } -} - -__device__ void QueuePair::ionic_post_wqe_rma_single(int32_t length, - uintptr_t laddr, uint32_t lkey, uintptr_t raddr, - uint32_t rkey, uint8_t opcode, bool ring_db) { - uint32_t num_wqes = 1; - uint32_t my_sq_prod = reserve_sq_single(num_wqes); - uint32_t my_sq_pos = my_sq_prod; - struct ionic_v1_wqe *wqe = &ionic_sq_buf[my_sq_pos & sq_mask]; - uint16_t wqe_flags = 0; - - if (!(my_sq_pos & (sq_mask + 1))) { - wqe_flags |= byteswap(IONIC_V1_FLAG_COLOR); - } - - wqe_flags |= byteswap(IONIC_V1_FLAG_SIG); - - // TODO why is this needed? - if (length && !laddr && opcode == IONIC_V2_OP_RDMA_WRITE) { - length = 1; - } - - wqe->base.wqe_idx = my_sq_pos; - wqe->base.op = opcode; - wqe->base.num_sge_key = length ? 1 : 0; - wqe->base.imm_data_key = byteswap(0); - - wqe->common.rdma.remote_va_high = byteswap(raddr >> 32); - wqe->common.rdma.remote_va_low = byteswap(raddr); - wqe->common.rdma.remote_rkey = byteswap(rkey); - wqe->common.length = byteswap(length); - - if (length) { - if (opcode == IONIC_V2_OP_RDMA_WRITE && static_cast(length) <= static_cast(inline_threshold)) { - wqe_flags |= byteswap(IONIC_V1_FLAG_INL); - wqe->base.num_sge_key = 0; - if (!laddr) { - // TODO why is this needed? - wqe->common.pld.data[0] = 1; - } else { - memcpy(wqe->common.pld.data, reinterpret_cast(laddr), length); - } - } else { - wqe->common.pld.sgl[0].va = byteswap(laddr); - wqe->common.pld.sgl[0].len = byteswap(length); - wqe->common.pld.sgl[0].lkey = byteswap(lkey); - } - } - - __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); - - if (ring_db) { - commit_sq_single(my_sq_prod, my_sq_pos, num_wqes); - } -} - -__device__ uint64_t QueuePair::ionic_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching, bool fence) { - uint32_t num_wqes = wf_info.num_pe_group_lanes; - uint32_t my_sq_prod = reserve_sq(wf_info, num_wqes); - uint32_t my_sq_pos = my_sq_prod + wf_info.pe_group_logical_lane_id; - struct ionic_v1_wqe *wqe = &ionic_sq_buf[my_sq_pos & sq_mask]; - uint16_t wqe_flags = 0; - uint32_t cons; - - uint64_t* wave_fetch_atomic{nullptr}; - if (fetching) { - if (wf_info.is_pe_group_first) { - auto res = fetching_atomic_freelist->pop_front(); - while (!res.success) { - res = fetching_atomic_freelist->pop_front(); - } - wave_fetch_atomic = res.value; - } - wave_fetch_atomic = (uint64_t*)__shfl((uint64_t)wave_fetch_atomic, - wf_info.pe_group_first_phys_lane_id); - } - - if (!(my_sq_pos & (sq_mask + 1))) { - wqe_flags |= byteswap(IONIC_V1_FLAG_COLOR); - } - - if (wf_info.is_pe_group_last) { - wqe_flags |= byteswap(IONIC_V1_FLAG_SIG); - } - if (fence) { - wqe_flags |= byteswap(IONIC_V1_FLAG_FENCE); - } - - wqe->base.wqe_idx = my_sq_pos; - wqe->base.op = opcode; - wqe->base.num_sge_key = 1; - wqe->base.imm_data_key = byteswap(0); - - wqe->atomic_v2.remote_va_high = byteswap(raddr >> 32); - wqe->atomic_v2.remote_va_low = byteswap(raddr); - wqe->atomic_v2.remote_rkey = byteswap(rkey); - wqe->atomic_v2.swap_add_high = byteswap(atomic_data >> 32); - wqe->atomic_v2.swap_add_low = byteswap(atomic_data); - wqe->atomic_v2.compare_high = byteswap(atomic_cmp >> 32); - wqe->atomic_v2.compare_low = byteswap(atomic_cmp); - - if (fetching) { - wqe->atomic_v2.local_va = - byteswap(reinterpret_cast( - wave_fetch_atomic + wf_info.pe_group_logical_lane_id)); - wqe->atomic_v2.lkey = byteswap(fetching_atomic_lkey); - } else { - wqe->atomic_v2.local_va = - byteswap(reinterpret_cast(nonfetching_atomic)); - wqe->atomic_v2.lkey = byteswap(nonfetching_atomic_lkey); - } - - __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, - __HIP_MEMORY_SCOPE_AGENT); - - cons = commit_sq(wf_info, my_sq_prod, my_sq_pos, num_wqes); - - uint64_t ret{0}; - if (fetching) { - ionic_quiet_internal(wf_info, cons); - ret = wave_fetch_atomic[wf_info.pe_group_logical_lane_id]; - __atomic_signal_fence(__ATOMIC_SEQ_CST); - if (wf_info.is_pe_group_first) { - fetching_atomic_freelist->push_back(wave_fetch_atomic); - } - } - return ret; -} - -__device__ uint64_t QueuePair::ionic_post_wqe_amo_single(uintptr_t raddr, - uint32_t rkey, uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence) { - uint32_t num_wqes = 1; - uint32_t my_sq_prod = reserve_sq_single(num_wqes); - uint32_t my_sq_pos = my_sq_prod; - struct ionic_v1_wqe *wqe = &ionic_sq_buf[my_sq_pos & sq_mask]; - uint16_t wqe_flags = 0; - uint32_t cons; - - uint64_t* wave_fetch_atomic{nullptr}; - if (fetching) { - auto res = fetching_atomic_freelist->pop_front(); - while (!res.success) { - res = fetching_atomic_freelist->pop_front(); - } - wave_fetch_atomic = res.value; - } - - if (!(my_sq_pos & (sq_mask + 1))) { - wqe_flags |= byteswap(IONIC_V1_FLAG_COLOR); - } - - wqe_flags |= byteswap(IONIC_V1_FLAG_SIG); - if (fence) { - wqe_flags |= byteswap(IONIC_V1_FLAG_FENCE); - } - - wqe->base.wqe_idx = my_sq_pos; - wqe->base.op = opcode; - wqe->base.num_sge_key = 1; - wqe->base.imm_data_key = byteswap(0); - - wqe->atomic_v2.remote_va_high = byteswap(raddr >> 32); - wqe->atomic_v2.remote_va_low = byteswap(raddr); - wqe->atomic_v2.remote_rkey = byteswap(rkey); - wqe->atomic_v2.swap_add_high = byteswap(atomic_data >> 32); - wqe->atomic_v2.swap_add_low = byteswap(atomic_data); - wqe->atomic_v2.compare_high = byteswap(atomic_cmp >> 32); - wqe->atomic_v2.compare_low = byteswap(atomic_cmp); - - if (fetching) { - wqe->atomic_v2.local_va = byteswap(reinterpret_cast(wave_fetch_atomic)); - wqe->atomic_v2.lkey = byteswap(fetching_atomic_lkey); - } else { - wqe->atomic_v2.local_va = byteswap(reinterpret_cast(nonfetching_atomic)); - wqe->atomic_v2.lkey = byteswap(nonfetching_atomic_lkey); - } - - __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); - - cons = commit_sq_single(my_sq_prod, my_sq_pos, num_wqes); - - uint64_t ret{0}; - if (fetching) { - ionic_quiet_internal_ccqe_single(cons); - ret = wave_fetch_atomic[0]; - __atomic_signal_fence(__ATOMIC_SEQ_CST); - fetching_atomic_freelist->push_back(wave_fetch_atomic); - } - return ret; -} - -} // namespace rocshmem diff --git a/projects/rocshmem/src/gda/ionic/queue_pair_ionic.hpp b/projects/rocshmem/src/gda/ionic/queue_pair_ionic.hpp new file mode 100644 index 000000000000..2da4e6bbbcb3 --- /dev/null +++ b/projects/rocshmem/src/gda/ionic/queue_pair_ionic.hpp @@ -0,0 +1,667 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_IONIC_QUEUE_PAIR_HPP_ +#define LIBRARY_SRC_GDA_IONIC_QUEUE_PAIR_HPP_ + +#include + +#include "log.hpp" +#include "util.hpp" + +#include "gda/endian.hpp" +#include "gda/ionic/provider_gda_ionic.hpp" +#include "gda/queue_pair/queue_pair_device.hpp" + +namespace rocshmem { + +class QueuePairIONIC; + +template <> struct QueuePairTraits { + enum class OpCode : uint8_t { + RDMA_WRITE = IONIC_V2_OP_RDMA_WRITE, + RDMA_READ = IONIC_V2_OP_RDMA_READ, + ATOMIC_CS = IONIC_V2_OP_ATOMIC_CS, + ATOMIC_FA = IONIC_V2_OP_ATOMIC_FA, + }; + + /** + * @brief ionic uses big-endian ordering + */ + static constexpr endian::Order Endianness = endian::Order::Big; + + /** + * @brief ionic inlining maximum is the WQE's payload segment, sizeof(ionic_v1_pld) = 32 bytes + */ + static constexpr size_t InlineMax = sizeof(ionic_v1_pld); + static_assert(InlineMax == 32, "ionic can send up to 32 bytes inline in a WQE"); + + /* + * @brief ionic preferred inlining threshold is the same as the maximum + */ + static constexpr size_t InlineThreshold = InlineMax; +}; + +class QueuePairIONIC : public QueuePairDevice { +private: + ionic_device_sq sq; + ionic_device_cq cq; + +public: + __host__ explicit QueuePairIONIC(uint32_t qpn, QueuePairInitInfo&& init_info, + ionic_device_sq&& sq, ionic_device_cq&& cq) + : QueuePairDevice{qpn, std::move(init_info)}, + sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ explicit QueuePairIONIC(uint32_t qpn, + uint64_t *nonfetching_atomic, uint32_t nonfetching_atomic_lkey, + ionic_device_sq&& sq, ionic_device_cq&& cq) + : QueuePairDevice{qpn, nonfetching_atomic, nonfetching_atomic_lkey}, + sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ QueuePairIONIC(const QueuePairIONIC& other) = delete; + __host__ QueuePairIONIC& operator=(const QueuePairIONIC& other) = delete; + __host__ QueuePairIONIC(QueuePairIONIC&& other) noexcept = default; + __host__ QueuePairIONIC& operator=(QueuePairIONIC&& other) = default; + __host__ ~QueuePairIONIC() = default; + +public: + template + __device__ __noinline__ + void post_wqe_rma(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + void post_wqe_rma_single(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + PostOpt = {}); + + __device__ __noinline__ void quiet_single(); + +private: + /** + * @brief Reserve space in the sq to post this many wqes. + * @param wf_info Wavefront information. + * @param num_wqes number of sq wqes to reserve for this wave. + * @return position of my_tid=0's wqe. + */ + __device__ uint32_t reserve_sq(const ActiveWFInfo& wf_info, uint32_t num_wqes); + __device__ uint32_t reserve_sq_single(uint32_t num_wqes); + + /** + * @brief Ring the sq doorbell maintaining order between waves. + * @param wf_info Wavefront information. + * @param my_sq_prod position of my_tid=0's wqe. + * @param num_wqes number of sq wqes posted in this wave. + * @param wqe this thread's wqe. + * @return doorbell producer index. + */ + __device__ uint32_t commit_sq(const ActiveWFInfo& wf_info, uint32_t my_sq_prod, + uint32_t num_wqes); + __device__ uint32_t commit_sq_single(uint32_t my_sq_prod, uint32_t num_wqes); + + __device__ void ring_doorbell(uint32_t pos); + __device__ void ring_doorbell_single(uint32_t pos); + + /** + * @brief Helper method to poll the next completion queue entry. + */ + __device__ void poll_wave_cqes(uint64_t activemask); + + /** + * @brief Helper method to drain completion queue entries. + * @param wf_info Wavefront information. + * @param cons wait for sq.msn to catch up to this position. + */ + __device__ void quiet_internal_ccqe(const ActiveWFInfo& wf_info, uint32_t cons); + __device__ void quiet_internal_ccqe_single(uint32_t cons); + + /** + * @brief Helper method to drain completion queue entries. + * @param wf_info Wavefront information. + * @param cons wait for sq.msn to catch up to this position. + */ + __device__ void quiet_internal(const ActiveWFInfo& wf_info, uint32_t cons); + + enum ionic_v1_cqe_qtf_bits_be : uint32_t { + IONIC_V1_CQE_COLOR_BE = endian::to_be(IONIC_V1_CQE_COLOR), + IONIC_V1_CQE_ERROR_BE = endian::to_be(IONIC_V1_CQE_ERROR), + }; + + enum ionic_v1_op_be : uint16_t { + IONIC_V1_FLAG_INL_BE = endian::to_be(IONIC_V1_FLAG_INL), + IONIC_V1_FLAG_SIG_BE = endian::to_be(IONIC_V1_FLAG_SIG), + IONIC_V1_FLAG_COLOR_BE = endian::to_be(IONIC_V1_FLAG_COLOR), + }; +}; + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ void QueuePairIONIC::post_wqe_rma( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt) { + //using PostOptions = PostOpt; + uint32_t num_wqes = 1; + if (wf_info.scope == ThreadScope::thread) { + num_wqes = wf_info.num_pe_group_lanes; + } + + uint32_t my_sq_prod = reserve_sq(wf_info, num_wqes); + + uint32_t my_sq_pos = my_sq_prod + wf_info.pe_group_logical_lane_id; + struct ionic_v1_wqe *wqe = &sq.buf[my_sq_pos & sq.mask]; + uint16_t wqe_flags = 0; + + if (!(my_sq_pos & (sq.mask + 1))) { + wqe_flags |= IONIC_V1_FLAG_COLOR_BE; + } + + if (wf_info.is_pe_group_last) { + wqe_flags |= IONIC_V1_FLAG_SIG_BE; + } + + // TODO why is this needed? + if constexpr (Op == OpCode::RDMA_WRITE) { + if (size && !laddr) { + size = 1; + } + } + + wqe->base.wqe_idx = my_sq_pos; + wqe->base.op = static_cast(Op); + wqe->base.num_sge_key = size ? 1 : 0; + wqe->base.imm_data_key = 0; + + wqe->common.rdma.remote_va_high = endian::to_be(raddr >> 32); + wqe->common.rdma.remote_va_low = endian::to_be(raddr); + wqe->common.rdma.remote_rkey = rkey; + wqe->common.length = endian::to_be(size); + + if (size) { + bool send_inline = can_inline(size); + if (send_inline) { + wqe_flags |= IONIC_V1_FLAG_INL_BE; + wqe->base.num_sge_key = 0; + if (!laddr) { + // TODO why is this needed? + wqe->common.pld.data[0] = 1; + } else { + memcpy(wqe->common.pld.data, reinterpret_cast(laddr), size); + } + } else { + wqe->common.pld.sgl[0].va = endian::to_be(laddr); + wqe->common.pld.sgl[0].len = endian::to_be(size); + wqe->common.pld.sgl[0].lkey = lkey; + } + } + + __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, + __HIP_MEMORY_SCOPE_AGENT); + + commit_sq(wf_info, my_sq_prod, num_wqes); +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ void QueuePairIONIC::post_wqe_rma_single( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, PostOpt) { + //using PostOptions = PostOpt; + uint32_t num_wqes = 1; + uint32_t my_sq_prod = reserve_sq_single(num_wqes); + uint32_t my_sq_pos = my_sq_prod; + struct ionic_v1_wqe *wqe = &sq.buf[my_sq_pos & sq.mask]; + uint16_t wqe_flags = 0; + + if (!(my_sq_pos & (sq.mask + 1))) { + wqe_flags |= IONIC_V1_FLAG_COLOR_BE; + } + + wqe_flags |= IONIC_V1_FLAG_SIG_BE; + + // TODO why is this needed? + if constexpr (Op == OpCode::RDMA_WRITE) { + if (size && !laddr) { + size = 1; + } + } + + wqe->base.wqe_idx = my_sq_pos; + wqe->base.op = static_cast(Op); + wqe->base.num_sge_key = size ? 1 : 0; + wqe->base.imm_data_key = 0; + + wqe->common.rdma.remote_va_high = endian::to_be(raddr >> 32); + wqe->common.rdma.remote_va_low = endian::to_be(raddr); + wqe->common.rdma.remote_rkey = rkey; + wqe->common.length = endian::to_be(size); + + if (size) { + bool send_inline = can_inline(size); + if (send_inline) { + wqe_flags |= IONIC_V1_FLAG_INL_BE; + wqe->base.num_sge_key = 0; + if (!laddr) { + // TODO why is this needed? + wqe->common.pld.data[0] = 1; + } else { + memcpy(wqe->common.pld.data, reinterpret_cast(laddr), size); + } + } else { + wqe->common.pld.sgl[0].va = endian::to_be(laddr); + wqe->common.pld.sgl[0].len = endian::to_be(size); + wqe->common.pld.sgl[0].lkey = lkey; + } + } + + __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); + + commit_sq_single(my_sq_prod, num_wqes); +} + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ QueuePairIONIC::amo_ret_t QueuePairIONIC::post_wqe_amo( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + //using PostOptions = PostOpt; + uint32_t num_wqes = wf_info.num_pe_group_lanes; + uint32_t my_sq_prod = reserve_sq(wf_info, num_wqes); + uint32_t my_sq_pos = my_sq_prod + wf_info.pe_group_logical_lane_id; + struct ionic_v1_wqe *wqe = &sq.buf[my_sq_pos & sq.mask]; + uint16_t wqe_flags = 0; + uint32_t cons; + + uint64_t* wave_fetch_atomic{nullptr}; + if constexpr (Fetch == AMOFetchType::Blocking) { + if (wf_info.is_pe_group_first) { + auto res = fetching_atomic_freelist->pop_front(); + while (!res.success) { + res = fetching_atomic_freelist->pop_front(); + } + wave_fetch_atomic = res.value; + } + wave_fetch_atomic = (uint64_t*)__shfl((uint64_t)wave_fetch_atomic, + wf_info.pe_group_first_phys_lane_id); + } + + if (!(my_sq_pos & (sq.mask + 1))) { + wqe_flags |= IONIC_V1_FLAG_COLOR_BE; + } + + if (wf_info.is_pe_group_last) { + wqe_flags |= IONIC_V1_FLAG_SIG_BE; + } + + wqe->base.wqe_idx = my_sq_pos; + wqe->base.op = static_cast(Op); + wqe->base.num_sge_key = 1; + wqe->base.imm_data_key = 0; + + wqe->atomic_v2.remote_va_high = endian::to_be(raddr >> 32); + wqe->atomic_v2.remote_va_low = endian::to_be(raddr); + wqe->atomic_v2.remote_rkey = rkey; + wqe->atomic_v2.swap_add_high = endian::to_be(swap_add >> 32); + wqe->atomic_v2.swap_add_low = endian::to_be(swap_add); + wqe->atomic_v2.compare_high = endian::to_be(compare >> 32); + wqe->atomic_v2.compare_low = endian::to_be(compare); + + uint64_t* atomic_laddr = nonfetching_atomic; + if constexpr (Fetch == AMOFetchType::Blocking) { + atomic_laddr = wave_fetch_atomic + wf_info.pe_group_logical_lane_id; + } + wqe->atomic_v2.local_va = endian::to_be(reinterpret_cast(atomic_laddr)); + wqe->atomic_v2.lkey = get_atomic_lkey(); + + __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, + __HIP_MEMORY_SCOPE_AGENT); + + cons = commit_sq(wf_info, my_sq_prod, num_wqes); + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_internal(wf_info, cons); + uint64_t ret = wave_fetch_atomic[wf_info.pe_group_logical_lane_id]; + __atomic_signal_fence(__ATOMIC_SEQ_CST); + if (wf_info.is_pe_group_first) { + fetching_atomic_freelist->push_back(wave_fetch_atomic); + } + return ret; + } +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ QueuePairIONIC::amo_ret_t QueuePairIONIC::post_wqe_amo_single( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + //using PostOptions = PostOpt; + uint32_t num_wqes = 1; + uint32_t my_sq_prod = reserve_sq_single(num_wqes); + uint32_t my_sq_pos = my_sq_prod; + struct ionic_v1_wqe *wqe = &sq.buf[my_sq_pos & sq.mask]; + uint16_t wqe_flags = 0; + uint32_t cons; + + uint64_t* wave_fetch_atomic{nullptr}; + if constexpr (Fetch == AMOFetchType::Blocking) { + auto res = fetching_atomic_freelist->pop_front(); + while (!res.success) { + res = fetching_atomic_freelist->pop_front(); + } + wave_fetch_atomic = res.value; + } + + if (!(my_sq_pos & (sq.mask + 1))) { + wqe_flags |= IONIC_V1_FLAG_COLOR_BE; + } + + wqe_flags |= IONIC_V1_FLAG_SIG_BE; + + wqe->base.wqe_idx = my_sq_pos; + wqe->base.op = static_cast(Op); + wqe->base.num_sge_key = 1; + wqe->base.imm_data_key = 0; + + wqe->atomic_v2.remote_va_high = endian::to_be(raddr >> 32); + wqe->atomic_v2.remote_va_low = endian::to_be(raddr); + wqe->atomic_v2.remote_rkey = rkey; + wqe->atomic_v2.swap_add_high = endian::to_be(swap_add >> 32); + wqe->atomic_v2.swap_add_low = endian::to_be(swap_add); + wqe->atomic_v2.compare_high = endian::to_be(compare >> 32); + wqe->atomic_v2.compare_low = endian::to_be(compare); + + uint64_t* atomic_laddr = nonfetching_atomic; + if constexpr (Fetch == AMOFetchType::Blocking) { + atomic_laddr = wave_fetch_atomic; + } + wqe->atomic_v2.local_va = endian::to_be(reinterpret_cast(atomic_laddr)); + wqe->atomic_v2.lkey = get_atomic_lkey(); + + __hip_atomic_store(&wqe->base.flags, wqe_flags, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); + + cons = commit_sq_single(my_sq_prod, num_wqes); + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_internal_ccqe_single(cons); + uint64_t ret = wave_fetch_atomic[0]; + __atomic_signal_fence(__ATOMIC_SEQ_CST); + fetching_atomic_freelist->push_back(wave_fetch_atomic); + return ret; + } +} + +// precondition: called with all active lanes using different QPs +__device__ inline __noinline__ void QueuePairIONIC::quiet_single() { + quiet_internal_ccqe_single(sq.pos); +} + +__device__ __forceinline__ uint32_t QueuePairIONIC::reserve_sq( + const ActiveWFInfo& wf_info, uint32_t num_wqes) { + uint32_t my_sq_prod = 0; + + // reserve space for wqes in sq + if (wf_info.is_pe_group_first) { + my_sq_prod = __hip_atomic_fetch_add(&sq.pos, num_wqes, __ATOMIC_RELAXED, + __HIP_MEMORY_SCOPE_AGENT); + } + my_sq_prod = __shfl(my_sq_prod, wf_info.pe_group_first_phys_lane_id); + + // wait for that space to be available + quiet_internal(wf_info, my_sq_prod + num_wqes - sq.mask); + + return my_sq_prod; +} + +__device__ __forceinline__ uint32_t QueuePairIONIC::reserve_sq_single( + uint32_t num_wqes) { + uint32_t my_sq_prod = 0; + + // reserve space for wqes in sq + my_sq_prod = __hip_atomic_fetch_add(&sq.pos, num_wqes, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + + // wait for that space to be available + quiet_internal_ccqe_single(my_sq_prod + num_wqes - sq.mask); + + return my_sq_prod; +} + +__device__ __forceinline__ uint32_t QueuePairIONIC::commit_sq( + const ActiveWFInfo& wf_info, uint32_t my_sq_prod, uint32_t num_wqes) { + uint32_t dbprod = my_sq_prod + num_wqes; + + spin_lock_acquire_shared(&sq.lock, wf_info.pe_group_mask); + + if (wf_info.is_pe_group_first && ((sq.dbpos - dbprod) & (1u << 31))) { + sq.dbpos = dbprod; + + ring_doorbell(dbprod); + } + + spin_lock_release_shared(&sq.lock, wf_info.pe_group_mask); + + return dbprod; +} + +__device__ __forceinline__ uint32_t QueuePairIONIC::commit_sq_single( + uint32_t my_sq_prod, uint32_t num_wqes) { + uint32_t dbprod = my_sq_prod + num_wqes; + + spin_lock_acquire_unique(&sq.lock); + + if ((sq.dbpos - dbprod) & (1u << 31)) { + sq.dbpos = dbprod; + + ring_doorbell_single(dbprod); + } + + spin_lock_release_unique(&sq.lock); + + return dbprod; +} + +__device__ __forceinline__ void QueuePairIONIC::ring_doorbell(uint32_t pos) { + // When threads write at once to the same address, not all writes reach the bus. + // Take turns and insert a thread fence between writes to the same address. + uint64_t activemask = get_active_lane_mask(); + int lane_id = get_active_lane_num(activemask); + int lane_count = get_active_lane_count(activemask); + + for (int i = 0; i < lane_count; i++) { + if (lane_id == i) { + __threadfence(); + __atomic_store_n(sq.dbreg, sq.dbval | (sq.mask & pos), __ATOMIC_SEQ_CST); + } + } + __threadfence(); +} + +__device__ __forceinline__ void QueuePairIONIC::ring_doorbell_single(uint32_t pos) { + // When threads write at once to the same address, not all writes reach the bus. + // Take turns and insert a thread fence between writes to the same address. + __threadfence(); + __atomic_store_n(&sq.dbreg[8 * __lane_id()], sq.dbval | (sq.mask & pos), __ATOMIC_SEQ_CST); +} + +__device__ inline void QueuePairIONIC::poll_wave_cqes(uint64_t activemask) { + int my_logical_lane_id = get_active_lane_num(activemask); + uint32_t my_cq_pos = cq.pos + my_logical_lane_id; + + /* Look at the cqe at the current position in the cq buffer */ + struct ionic_v1_cqe *cqe = &cq.buf[my_cq_pos & cq.mask]; + + /* Determine expected color based on cq wrap count */ + uint32_t qtf_color_bit = IONIC_V1_CQE_COLOR_BE; + uint32_t qtf_color_exp = qtf_color_bit; + if (my_cq_pos & (cq.mask + 1)) { + qtf_color_exp = 0; + } + + /* Check if my cqe color == expected color */ + uint32_t qtf_be = *(volatile uint32_t *)(&cqe->qid_type_flags); + if ((qtf_be & qtf_color_bit) != qtf_color_exp) { + return; + } + + uint32_t msn = endian::from_be(cqe->send.msg_msn); + + /* Report if the completion indicates an error. */ + if (!!(qtf_be & IONIC_V1_CQE_ERROR_BE)) { +#if defined(BUILD_DEBUG_DEVICE) + uint32_t qtf = endian::from_be(qtf_be); + uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; + uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; + uint32_t flag = qtf & 0xf; + uint32_t status = endian::from_be(cqe->status_length); + uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; + + LOGD_ERROR("QUIET ERROR: qid %u type %u flag %#x status %u msn %u npg %lu", + qid, type, flag, status, msn, npg); +#endif + /* No other way to signal an error, so just crash. */ + abort(); + } + + /* Only proceed with the furthest ahead cqe to update the sq state */ + uint64_t my_lane_mask = 1ull << __lane_id(); + uint64_t lesser_lane_mask = my_lane_mask - 1; + if (my_lane_mask != (__ballot(true) & activemask & ~lesser_lane_mask)) { + return; + } + + /* update position in the cq */ + cq.pos = my_cq_pos + 1; + + /* + * Ring cq doorbell frequently enough to avoid cq full. + * + * NB: IONIC_CQ_GRACE is 100 + */ + if (((cq.pos - cq.dbpos) & cq.mask) >= 100) { + cq.dbpos = cq.pos; + __atomic_store_n(cq.dbreg, cq.dbval | (cq.mask & cq.dbpos), __ATOMIC_SEQ_CST); //TODO:maybe relaxed? + } + + sq.msn = msn; +} + +__device__ inline void QueuePairIONIC::quiet_internal_ccqe( + const ActiveWFInfo& wf_info, uint32_t cons) { + if (!wf_info.is_pe_group_first) { + return; + } + + volatile struct ionic_v1_cqe *cqe = &cq.buf[0]; + uint32_t qtf_be = cqe->qid_type_flags; + uint32_t msn = endian::from_be(cqe->send.msg_msn); + while ((msn - cons) & 0x800000) { + if (!!(qtf_be & IONIC_V1_CQE_ERROR_BE)) { + break; + } + + qtf_be = cqe->qid_type_flags; + msn = endian::from_be(cqe->send.msg_msn); + } + + if (!!(qtf_be & IONIC_V1_CQE_ERROR_BE)) { +#if defined(BUILD_DEBUG_DEVICE) + uint32_t qtf = endian::from_be(qtf_be); + uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; + uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; + uint32_t flag = qtf & 0xf; + uint32_t status = endian::from_be(cqe->status_length); + uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; + + LOGD_ERROR("QUIET ERROR (CCQE): qid %u type %u flag %#x status %u msn %u npg %lu", + qid, type, flag, status, msn, npg); +#endif + /* No other way to signal an error, so just crash. */ + abort(); + } +} + +__device__ inline void QueuePairIONIC::quiet_internal_ccqe_single( + uint32_t cons) { + volatile struct ionic_v1_cqe *cqe = &cq.buf[0]; + uint32_t qtf_be = cqe->qid_type_flags; + uint32_t msn = endian::from_be(cqe->send.msg_msn); + while ((msn - cons) & 0x800000) { + if (!!(qtf_be & IONIC_V1_CQE_ERROR_BE)) { + break; + } + + qtf_be = cqe->qid_type_flags; + msn = endian::from_be(cqe->send.msg_msn); + } + + if (!!(qtf_be & IONIC_V1_CQE_ERROR_BE)) { +#if defined(BUILD_DEBUG_DEVICE) + uint32_t qtf = endian::from_be(qtf_be); + uint32_t qid = qtf >> IONIC_V1_CQE_QID_SHIFT; + uint32_t type = (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; + uint32_t flag = qtf & 0xf; + uint32_t status = endian::from_be(cqe->status_length); + uint64_t npg = cqe->send.npg_wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; + + LOGD_ERROR("QUIET ERROR (CCQE): qid %u type %u flag %#x status %u msn %u npg %lu", + qid, type, flag, status, msn, npg); +#endif + /* No other way to signal an error, so just crash. */ + abort(); + } +} + +__device__ inline void QueuePairIONIC::quiet_internal( + const ActiveWFInfo& wf_info, uint32_t cons) { + uint32_t greed = 10; + + if (!cq.mask) { + quiet_internal_ccqe(wf_info, cons); + return; + } + + /* wait for sq.msn to catch up or pass cons. */ + /* 0x800000 - sign bit for 24-bit fields */ + while ((sq.msn - cons) & 0x800000) { + if (!spin_lock_try_acquire_shared(&cq.lock, wf_info.pe_group_mask)) { + continue; + } + + /* with lock acquired, this wave polls cqes until caught up */ + while ((sq.msn - cons) & 0x800000) { + uint32_t old_sq_msn = sq.msn; + + poll_wave_cqes(wf_info.pe_group_mask); + + if (!((sq.msn - cons) & 0x800000)) { + if (sq.msn == old_sq_msn) { + break; + } + if (!greed) { + break; + } + --greed; + } + } + + spin_lock_release_shared(&cq.lock, wf_info.pe_group_mask); + break; + } +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_IONIC_QUEUE_PAIR_HPP_ diff --git a/projects/rocshmem/src/gda/mlx5/CMakeLists.txt b/projects/rocshmem/src/gda/mlx5/CMakeLists.txt index 6e9d01fa959e..d93a40705289 100644 --- a/projects/rocshmem/src/gda/mlx5/CMakeLists.txt +++ b/projects/rocshmem/src/gda/mlx5/CMakeLists.txt @@ -24,7 +24,6 @@ target_sources( ${PROJECT_NAME} PRIVATE - backend_gda_mlx5.cpp provider_gda_mlx5.cpp ) @@ -32,6 +31,6 @@ if(GDA_MLX5) target_sources( ${PROJECT_NAME} PRIVATE - queue_pair_mlx5.cpp + backend_gda_mlx5.cpp ) endif() diff --git a/projects/rocshmem/src/gda/mlx5/backend_gda_mlx5.cpp b/projects/rocshmem/src/gda/mlx5/backend_gda_mlx5.cpp index dbd1d9d49b4f..8c479d1679ff 100644 --- a/projects/rocshmem/src/gda/mlx5/backend_gda_mlx5.cpp +++ b/projects/rocshmem/src/gda/mlx5/backend_gda_mlx5.cpp @@ -22,11 +22,14 @@ * IN THE SOFTWARE. *****************************************************************************/ +#include + #include "log.hpp" #include "util.hpp" #include "gda/backend_gda.hpp" -#include "gda/mlx5/mlx5dv_core.hpp" -#include "gda/mlx5/mlx5_ifc_core.hpp" +#include "gda/mlx5/provider_gda_mlx5.hpp" +#include "gda/mlx5/queue_pair_mlx5.hpp" +#include "gda/queue_pair_provider.hpp" namespace rocshmem { @@ -58,8 +61,6 @@ int GDABackend::mlx5_dv_dl_init() { } void GDABackend::mlx5_create_qps(int sq_length) { - // mlx5 provider can support up to 28B of inline data in a WQE - inline_threshold = sizeof(gda_mlx5_wqe_inline_data::data); for (size_t i = 0; i < mlx5_qps.size(); i++) { NicDevice &nic = nic_for_qp(i); int err = mlx5dv.create_qp(mlx5_qps[i], nic.context, nic.pd_orig, sq_length); @@ -69,7 +70,6 @@ void GDABackend::mlx5_create_qps(int sq_length) { void GDABackend::mlx5_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { mlx5_devx_qp& qp = mlx5_qps[conn_num]; - qp.dump(conn_num); /* * struct mlx5_devx_qp { @@ -96,8 +96,6 @@ void GDABackend::mlx5_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { * }; */ - gpu_qp->mlx5_cq = gda_mlx5_device_cq(reinterpret_cast(qp.cq), qp.cq_dbrec); - int hip_dev_id{-1}; CHECK_HIP(hipGetDevice(&hip_dev_id)); void* gpu_db_ptr{nullptr}; @@ -105,23 +103,21 @@ void GDABackend::mlx5_initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) { rocm_memory_lock_to_fine_grain(qp.uar->reg_addr, MLX5_DB_BLUEFLAME_BUFFER_SIZE, &gpu_db_ptr, hip_dev_id); + gda_mlx5_wqe* sq_buf = reinterpret_cast(qp.sq); // qp.dbrec points to two __be32 values: RQ dbrec at MLX5_RCV_DBR and SQ dbrec at MLX5_SND_DBR - gpu_qp->mlx5_sq = gda_mlx5_device_sq{reinterpret_cast(qp.sq), - &qp.qp_dbrec[MLX5_SND_DBR], - reinterpret_cast(gpu_db_ptr), - static_cast(qp.sq_depth)}; + __be32* sq_dbrec = &qp.qp_dbrec[MLX5_SND_DBR]; + gda_mlx5_doorbell* sq_db = reinterpret_cast(gpu_db_ptr); + uint16_t sq_depth = static_cast(qp.sq_depth); - int pe = conn_num % num_pes; - int nic_idx = nic_idx_for_qp(conn_num); - NicDevice &nic = nic_for_qp(conn_num); - gpu_qp->rkey = heap_rkey[pe * num_nics_ + nic_idx]; - gpu_qp->lkey = nic.heap_mr->lkey; - gpu_qp->qp_num = qp.qpn; - gpu_qp->inline_threshold = inline_threshold; + mlx5_cqe64* cq_buf = reinterpret_cast(qp.cq); + __be32* cq_dbrec = qp.cq_dbrec; - /* Base Heap information */ - gpu_qp->base_heap = (uintptr_t) heap.get_local_heap_base(); - gpu_qp->base_heap_size = heap.get_size(); + /* QueuePair is either QueuePairMLX5 or QueuePairMux + * both have a constructor that accepts rvalue reference QueuePairMLX5&&, + * so just use that instead of trying to figure out which one we're using */ + new (gpu_qp) QueuePair{QueuePairMLX5{qp.qpn, gpu_qp_init_info(conn_num), + gda_mlx5_device_sq{sq_buf, sq_dbrec, sq_db, sq_depth}, + gda_mlx5_device_cq{cq_buf, cq_dbrec}}}; } } // namespace rocshmem diff --git a/projects/rocshmem/src/gda/mlx5/mlx5_ifc_core.hpp b/projects/rocshmem/src/gda/mlx5/mlx5_ifc_core.hpp index 85f81122483b..c94125e8627c 100644 --- a/projects/rocshmem/src/gda/mlx5/mlx5_ifc_core.hpp +++ b/projects/rocshmem/src/gda/mlx5/mlx5_ifc_core.hpp @@ -1,9 +1,14 @@ +// Modifications Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: GPL-2.0-only OR Linux-OpenIB + /* * This file consists of hardware interface descriptions for the mlx5 driver, * as described by the Mellanox Adapters Programmer's Reference Manual Rev 0.40 and other sources. * It has been generated by extracting select portions * of the Linux kernel source under the license below. * See include/linux/mlx5/mlx5_ifc.h for additional details. + * + * The file COPYING is in the main directory of the origin source tree. */ /* @@ -37,6 +42,7 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ + #ifndef LIBRARY_SRC_GDA_MLX5_IFC_CORE_HPP_ #define LIBRARY_SRC_GDA_MLX5_IFC_CORE_HPP_ diff --git a/projects/rocshmem/src/gda/mlx5/mlx5dv_core.hpp b/projects/rocshmem/src/gda/mlx5/mlx5dv_core.hpp index 63c844b9bd39..0d5be16d10c0 100644 --- a/projects/rocshmem/src/gda/mlx5/mlx5dv_core.hpp +++ b/projects/rocshmem/src/gda/mlx5/mlx5dv_core.hpp @@ -1,3 +1,20 @@ +// Modifications Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: GPL-2.0-only OR Linux-OpenIB + +/* + * This file consists of API declarations for mlx5 direct verbs. + * It has been generated by extracting select portions + * of the rdma-core source code, the userspace components + * for the Linux kernel drivers/infiniband subsystem, under the license below. + * For additional details, see: + * - libibverbs/tm_types.h + * - kernel-headers/rdma/mlx5_user_ioctl_verbs.h + * - providers/mlx5/mlx5_api.h + * - providers/mlx5/mlx5dv.h + * + * The file COPYING is in the main directory of the origin source tree. +*/ + /* * Copyright (c) 2017 Mellanox Technologies, Inc. All rights reserved. * diff --git a/projects/rocshmem/src/gda/mlx5/provider_gda_mlx5.hpp b/projects/rocshmem/src/gda/mlx5/provider_gda_mlx5.hpp index c19788a14b8f..e9e3550c1707 100644 --- a/projects/rocshmem/src/gda/mlx5/provider_gda_mlx5.hpp +++ b/projects/rocshmem/src/gda/mlx5/provider_gda_mlx5.hpp @@ -116,11 +116,6 @@ union gda_mlx5_wqe_rma { __device__ static constexpr inline uint8_t inline_ds(uint32_t byte_count) { return (byte_count <= 12 ? 3 : 4); } - - __device__ static constexpr inline bool can_inline( - uint8_t opcode, uint32_t byte_count, uint32_t inline_threshold) { - return (opcode == MLX5_OPCODE_RDMA_WRITE) && (byte_count <= inline_threshold); - } } __attribute__((__packed__)) __attribute__((__aligned__(16))); // AMO WQEs have a 16B atomic segment and an (optional?) indirect data segment for fetching atomics diff --git a/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.cpp b/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.cpp deleted file mode 100644 index 79daf9dd4867..000000000000 --- a/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.cpp +++ /dev/null @@ -1,441 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#include "log.hpp" -#include "bit.hpp" -#include "util.hpp" - -#include "gda/endian.hpp" -#include "gda/queue_pair.hpp" - -namespace rocshmem { - -#define MLX5_LOCK_USE_S_SLEEP 1 -#define MLX5_LOCK_USE_S_WAKEUP (0 && MLX5_LOCK_USE_S_SLEEP) -// sleep for up to 64 * MLX5_LOCK_S_SLEEP_DELAY clock cycles -static constexpr int MLX5_LOCK_S_SLEEP_DELAY = 2; - -#if MLX5_LOCK_USE_S_WAKEUP -__device__ static inline void amdgcn_s_wakeup() { - /* why doesn't __builtin_amdgcn_s_wakeup() exist? - * signals other wavefronts in the same workgroup to exit early from s_sleep */ - asm volatile("s_wakeup"); -} -#endif - -__device__ static inline void acquire_lock(uint32_t *lock) { - /* acquire lock when new value 1 (locked) is exchanged with prior value 0 (unlocked) - * - * the __ATOMIC_ACQUIRE load synchronizes with the __ATOMIC_RELEASE store in release_lock(), - * but not with the (implicit) __ATOMIC_RELAXED store part of the exchange - * this is fine, since we only need to ensure happens-before between the threads - * that released and acquired the lock, not between the different threads contending on the lock - * when they (eventually) acquire the lock, *then* they will synchronize */ - while (__hip_atomic_exchange(lock, 1, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT)) { -#if MLX5_LOCK_USE_S_SLEEP - // sleep so we don't hammer the memory - __builtin_amdgcn_s_sleep(MLX5_LOCK_S_SLEEP_DELAY); -#endif - } -} - -__device__ static inline void release_lock(uint32_t *lock) { - // release lock by storing 0 (unlocked) - __hip_atomic_store(lock, 0, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); -#if MLX5_LOCK_USE_S_WAKEUP - // wake up any other sleeping waves (in the same workgroup) - amdgcn_s_wakeup(); -#endif -} - -__device__ static inline uint16_t mlx5_wqe_idx(const gda_mlx5_device_sq& sq, uint8_t lane_id) { - return static_cast(sq.post + lane_id); -} - -__device__ static inline uint16_t mlx5_sq_idx(const gda_mlx5_device_sq& sq, uint16_t wqe_idx) { - // sq.depth is a power of 2, so just mask off everything above that - return wqe_idx & sq.depth_mask; -} - -__device__ void QueuePair::mlx5_ring_doorbell(uint64_t sq_post, const gda_mlx5_wqe& wqe) { - // sq_wqebb_counter is the least significant bits of the post counter - uint16_t sq_wqebb_counter = static_cast(sq_post); - // gda_mlx5_db_register constructor extracts first 8 bytes of WQE - gda_mlx5_db_register db_val{wqe}; - __be32 be_sq_wqebb_counter = endian::to_be(sq_wqebb_counter); - - // get BlueFlame buffer from SQ - gda_mlx5_bf_buffer* bf = mlx5_sq.bf_buffer(); - - // store sq_wqebb_counter to doorbell record - __hip_atomic_store(mlx5_sq.dbrec, be_sq_wqebb_counter, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); - // ring doorbell by storing first 8B of WQE to the doorbell register - __hip_atomic_store(&bf->db_reg, db_val, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); - - LOGD_TRACE("SQ: posted WQEs with dbrec(%p)=%x (%hu), dbreg(%p)=%lx (%x, %x)", - mlx5_sq.dbrec, be_sq_wqebb_counter, sq_wqebb_counter, - &bf->db_reg, db_val.val, db_val.wqe_header.opmod_idx_opcode, db_val.wqe_header.qpn_ds); -} - -[[maybe_unused]] __attribute__((noinline)) -__device__ void QueuePair::mlx5_print_cqe_error(const mlx5_cqe64* cqe, uint8_t opcode) { - const mlx5_err_cqe* err_cqe = reinterpret_cast(cqe); - uint8_t syndrome = 0x0; - - switch (opcode) { - case MLX5_CQE_RESP_WR_IMM: - case MLX5_CQE_RESP_SEND: - case MLX5_CQE_RESP_SEND_IMM: - case MLX5_CQE_RESP_SEND_INV: - // (valid) responder completion?! - LOGD_ERROR("CQ: unexpected responder completion (%x)", opcode); - break; - case MLX5_CQE_RESIZE_CQ: - case MLX5_CQE_NO_PACKET: - LOGD_ERROR("CQ: unexpected completion type (%x)", opcode); - break; - case MLX5_CQE_SIG_ERR: - LOGD_ERROR("CQ: unexpected signature error (%x)", opcode); - break; - case MLX5_CQE_REQ_ERR: - syndrome = __hip_atomic_load(&err_cqe->syndrome, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); - switch (syndrome) { - case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR: - LOGD_ERROR("CQ requester error LOCAL_LENGTH_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR: - LOGD_ERROR("CQ requester error LOCAL_QP_OP_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR: - LOGD_ERROR("CQ requester error LOCAL_PROT_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_WR_FLUSH_ERR: - LOGD_ERROR("CQ requester error WR_FLUSH_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_MW_BIND_ERR: - LOGD_ERROR("CQ requester error MW_BIND_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_BAD_RESP_ERR: - LOGD_ERROR("CQ requester error BAD_RESP_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR: - LOGD_ERROR("CQ requester error LOCAL_ACCESS_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: - LOGD_ERROR("CQ requester error REMOTE_INVAL_REQ_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR: - LOGD_ERROR("CQ requester error REMOTE_ACCESS_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_REMOTE_OP_ERR: - LOGD_ERROR("CQ requester error REMOTE_OP_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: - LOGD_ERROR("CQ requester error TRANSPORT_RETRY_EXC_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR: - LOGD_ERROR("CQ requester error RNR_RETRY_EXC_ERR (%x)", syndrome); - break; - case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR: - LOGD_ERROR("CQ requester error REMOTE_ABORTED_ERR (%x)", syndrome); - break; - default: - LOGD_ERROR("CQ requester error unknown syndrome type (%x)", syndrome); - break; - } - break; - case MLX5_CQE_RESP_ERR: - LOGD_ERROR("CQ: unexpected responder error (%x)", opcode); - break; - case MLX5_CQE_INVALID: { - uint8_t owner = __hip_atomic_load(&cqe->op_own, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM) - & MLX5_CQE_OWNER_MASK; - LOGD_ERROR("CQ: invalid completion (%x), check owner bit = %u?", opcode, owner); - break; - } - default: - LOGD_ERROR("CQ: unknown completion type (%x)", opcode); - break; - } - abort(); -} - -// precondition: called with all active lanes using different QPs -__device__ void QueuePair::mlx5_poll_cq_until(uint16_t requested_available_slots) { - uint16_t sq_depth = mlx5_sq.depth; - - uint64_t sq_post = __hip_atomic_load(&mlx5_sq.post, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT); - // don't need to check CQ if we haven't ever filled SQ and there's enough space left - if (sq_post + requested_available_slots <= sq_depth) { - return; - } - - while (true) { - struct mlx5_cqe64* cqe = mlx5_cq.buf; - - /* Update the SQ head - * This param provides us the sq_wqebb_counter; all our WQEs are exactly one WQEBB (64B) */ - // 32-bit load: big-endian 16-bit field, then two 8-bit fields - uint32_t wqecnt_sig_op_own = __hip_atomic_load(reinterpret_cast(&cqe->wqe_counter), - __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SYSTEM); - // GPU is little-endian, so wqe_counter is loaded into the low half of wqecnt_sig_op_own - __be16 be_wqe_counter = static_cast<__be16>(wqecnt_sig_op_own); - /* GPU is little-endian, so op_own is loaded into the top half of wqecnt_sig_op_own; - * opcode is the top 4 bits of op_own */ - uint8_t opcode = static_cast(wqecnt_sig_op_own >> 28); - uint16_t sq_head = endian::from_be(be_wqe_counter); - - // sq_tail is the least significant bits of the post counter - uint16_t sq_tail = static_cast(sq_post); - - // CQEs are initially invalid, retry until we see a valid CQE - if (opcode == MLX5_CQE_INVALID) { - LOGD_TRACE("CQ: invalid completion (%x)", opcode); - continue; - } - -#if defined(BUILD_DEBUG_DEVICE) - if (opcode != MLX5_CQE_REQ) - mlx5_print_cqe_error(cqe, opcode); -#endif - - /* sq_tail is an index to the next free WQE i.e. counts number of posted WQEs - * sq_head is an index to the *last* completed WQE - need to add one to get *count* of completed WQEs */ - uint16_t posted = sq_tail; - uint16_t completed = sq_head + 1; - - /* posted >= completed, except when posted has wrapped around 0xFFFF and completed hasn't - * but posted - completed is correct even when it wraps around - * in some marginal cases it's maybe possible to see consumed_slots > sq_depth, - * but in that case available_slots will be very large, > requested_available_slots, - * and the loop will continue for another iteration */ - uint16_t consumed_slots = posted - completed; - uint16_t available_slots = sq_depth - consumed_slots; - - /* continue until both: - * - no additional WQEs have been posted - * - the number of requested SQ slots are available */ - uint64_t prior_sq_post = sq_post; - sq_post = __hip_atomic_load(&mlx5_sq.post, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT); - if (sq_post == prior_sq_post && available_slots >= requested_available_slots) { - return; - } - } -} - -// precondition: called with all active lanes using different QPs -__device__ void QueuePair::mlx5_quiet() { - mlx5_poll_cq_until(mlx5_sq.depth); -} - -/** - * TODO: This function is redundant but kept because ionic has a different - * quiet_single implementation. Remove once ionic's quiet is unified. - */ -__device__ void QueuePair::mlx5_quiet_single() { - mlx5_poll_cq_until(mlx5_sq.depth); -} - -// can be called with all active lanes using any number of different QPs, don't assume anything -__device__ void QueuePair::mlx5_post_wqe_rma(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db) { - if (wf_info.is_pe_group_last) { - // get SQ lock - acquire_lock(&mlx5_sq.lock); - // poll until we have enough WQEBB for all lanes using this QP - mlx5_poll_cq_until(wf_info.num_pe_group_lanes); - } - - // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ - uint16_t wqe_idx = mlx5_wqe_idx(mlx5_sq, wf_info.pe_group_logical_lane_id); - uint16_t sq_idx = mlx5_sq_idx(mlx5_sq, wqe_idx); - - // can we inline the data into the WQE? - bool send_inline = gda_mlx5_wqe_rma::can_inline(opcode, length, inline_threshold); - - // construct the WQE on the stack - gda_mlx5_wqe wqe{wqe_idx, opcode, qp_num, MLX5_WQE_CTRL_CQ_UPDATE, - raddr, byteswap(rkey), laddr, byteswap(lkey), - static_cast(length), send_inline}; - - // copy to SQ - mlx5_sq.buf[sq_idx] = wqe; - - if (wf_info.is_pe_group_last) { - // increment post counter - mlx5_sq.post += wf_info.num_pe_group_lanes; - // we are the last thread in the wavefront, so we have the last WQE posted - if (ring_db) { - mlx5_ring_doorbell(mlx5_sq.post, wqe); - } - // release SQ lock - release_lock(&mlx5_sq.lock); - } -} - -// precondition: called with all active lanes using different QPs -__device__ void QueuePair::mlx5_post_wqe_rma_single(int32_t length, uintptr_t laddr, - uint32_t lkey, uintptr_t raddr, - uint32_t rkey, uint8_t opcode, - bool ring_db) { - bool send_inline = gda_mlx5_wqe_rma::can_inline(opcode, length, inline_threshold); - - // get SQ lock - acquire_lock(&mlx5_sq.lock); - // poll until we have enough space for at least one WQE - mlx5_poll_cq_until(1); - - // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ - uint16_t wqe_idx = mlx5_wqe_idx(mlx5_sq, 0); - uint16_t sq_idx = mlx5_sq_idx(mlx5_sq, wqe_idx); - - // construct the WQE on the stack - gda_mlx5_wqe wqe{wqe_idx, opcode, qp_num, MLX5_WQE_CTRL_CQ_UPDATE, - raddr, byteswap(rkey), laddr, byteswap(lkey), - static_cast(length), send_inline}; - - // copy to SQ - mlx5_sq.buf[sq_idx] = wqe; - - // increment post counter - mlx5_sq.post += 1; - - if (ring_db) { - // ring doorbell for this WQE - mlx5_ring_doorbell(mlx5_sq.post, wqe); - } - - // release SQ lock - release_lock(&mlx5_sq.lock); -} - -/* can be called with all active lanes using any number of different QPs, don't assume anything - * assumes that `fetching' is constant across all lanes using the same QP - * TODO: make `fetching' a template parameter */ -__device__ uint64_t QueuePair::mlx5_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching, bool fence) { - if (wf_info.is_pe_group_last) { - // get SQ lock - acquire_lock(&mlx5_sq.lock); - // poll until we have enough WQEBB for all lanes using this QP - mlx5_poll_cq_until(wf_info.num_pe_group_lanes); - } - - uint64_t* atomic_laddr = nonfetching_atomic; - uint32_t atomic_lkey = nonfetching_atomic_lkey; - if (fetching) { - uint32_t atomic_idx = (fetching_atomic_idx + wf_info.pe_group_logical_lane_id) % FETCHING_ATOMIC_CNT; - atomic_laddr = &fetching_atomic[atomic_idx]; - atomic_lkey = fetching_atomic_lkey; - } - - // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ - uint16_t wqe_idx = mlx5_wqe_idx(mlx5_sq, wf_info.pe_group_logical_lane_id); - uint16_t sq_idx = mlx5_sq_idx(mlx5_sq, wqe_idx); - - uint8_t fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; - if (fence) fm_ce_se |= MLX5_WQE_CTRL_FENCE; - - // construct the WQE on the stack - gda_mlx5_wqe wqe{wqe_idx, opcode, qp_num, fm_ce_se, - raddr, byteswap(rkey), - static_cast(atomic_data), static_cast(atomic_cmp), - reinterpret_cast(atomic_laddr), byteswap(atomic_lkey)}; - - // copy to SQ - mlx5_sq.buf[sq_idx] = wqe; - - if (wf_info.is_pe_group_last) { - // increment post and fetching atomic counters - mlx5_sq.post += wf_info.num_pe_group_lanes; - if (fetching) { - fetching_atomic_idx += wf_info.num_pe_group_lanes; - } - // we are the last thread in the wavefront, so we have the last WQE posted - mlx5_ring_doorbell(mlx5_sq.post, wqe); - // release SQ lock - release_lock(&mlx5_sq.lock); - // wait until fetch completes - if (fetching) { - mlx5_quiet_single(); - } - } - - return fetching ? *atomic_laddr : 0; -} - -// precondition: called with all active lanes using different QPs -__device__ uint64_t QueuePair::mlx5_post_wqe_amo_single(uintptr_t raddr, - uint32_t rkey, uint8_t opcode, - int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence) { - // get SQ lock - acquire_lock(&mlx5_sq.lock); - // poll until we have enough space for at least one WQE - mlx5_poll_cq_until(1); - - uint64_t* atomic_laddr = nonfetching_atomic; - uint32_t atomic_lkey = nonfetching_atomic_lkey; - if (fetching) { - uint32_t atomic_idx = fetching_atomic_idx % FETCHING_ATOMIC_CNT; - atomic_laddr = &fetching_atomic[atomic_idx]; - atomic_lkey = fetching_atomic_lkey; - } - - // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ - uint16_t wqe_idx = mlx5_wqe_idx(mlx5_sq, 0); - uint16_t sq_idx = mlx5_sq_idx(mlx5_sq, wqe_idx); - - uint8_t fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; - if (fence) fm_ce_se |= MLX5_WQE_CTRL_FENCE; - - // construct the WQE on the stack - gda_mlx5_wqe wqe{wqe_idx, opcode, qp_num, fm_ce_se, - raddr, byteswap(rkey), - static_cast(atomic_data), static_cast(atomic_cmp), - reinterpret_cast(atomic_laddr), byteswap(atomic_lkey)}; - - // copy to SQ - mlx5_sq.buf[sq_idx] = wqe; - - // increment post counter and fetching-atomic counters - mlx5_sq.post += 1; - if (fetching) { - fetching_atomic_idx += 1; - } - // ring doorbell for this WQE - mlx5_ring_doorbell(mlx5_sq.post, wqe); - // release SQ lock - release_lock(&mlx5_sq.lock); - // wait until fetch completes - if (fetching) { - mlx5_quiet_single(); - } - - return fetching ? *atomic_laddr : 0; -} - -} // namespace rocshmem diff --git a/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.hpp b/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.hpp new file mode 100644 index 000000000000..aef104bd34aa --- /dev/null +++ b/projects/rocshmem/src/gda/mlx5/queue_pair_mlx5.hpp @@ -0,0 +1,536 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_MLX5_QUEUE_PAIR_HPP_ +#define LIBRARY_SRC_GDA_MLX5_QUEUE_PAIR_HPP_ + +#include + +#include "log.hpp" +#include "gda/endian.hpp" +#include "gda/mlx5/provider_gda_mlx5.hpp" +#include "gda/queue_pair/queue_pair_device.hpp" + +#define GDA_MLX5_LOCK_USE_S_SLEEP 1 +#define GDA_MLX5_LOCK_USE_S_WAKEUP (0 && GDA_MLX5_LOCK_USE_S_SLEEP) + +namespace rocshmem { + +class QueuePairMLX5; + +template <> struct QueuePairTraits { + enum class OpCode : uint8_t { + RDMA_WRITE = MLX5_OPCODE_RDMA_WRITE, + RDMA_READ = MLX5_OPCODE_RDMA_READ, + ATOMIC_CS = MLX5_OPCODE_ATOMIC_CS, + ATOMIC_FA = MLX5_OPCODE_ATOMIC_FA, + }; + + /** + * @brief mlx5 uses big-endian ordering + */ + static constexpr endian::Order Endianness = endian::Order::Big; + + /** + * @brief mlx5 inlining maximum is 28 bytes + * + * mlx5 inlining can use up to 2 inline data WQE segments: + * the first segment uses 4 (out of 16) bytes for encoding the data transfer size, + * leaving 12 bytes from the first segment and the full 16 bytes of the second segment + * for the actual inlined data. + */ + static constexpr size_t InlineMax = sizeof(gda_mlx5_wqe_inline_data::data); + + /* + * @brief mlx5 preferred inlining threshold is the same as the maximum + */ + static constexpr size_t InlineThreshold = InlineMax; +}; + +class QueuePairMLX5 : public QueuePairDevice { +private: + gda_mlx5_device_sq sq; + gda_mlx5_device_cq cq; + +#if GDA_MLX5_LOCK_USE_S_SLEEP + // sleep for up to 64 * LOCK_S_SLEEP_DELAY clock cycles + static constexpr int LOCK_S_SLEEP_DELAY = 2; +#endif + +public: + __host__ explicit QueuePairMLX5(uint32_t qpn, QueuePairInitInfo&& init_info, + gda_mlx5_device_sq&& sq, gda_mlx5_device_cq&& cq) + : QueuePairDevice{qpn, std::move(init_info)}, + sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ explicit QueuePairMLX5(uint32_t qpn, + uint64_t *nonfetching_atomic, uint32_t nonfetching_atomic_lkey, + gda_mlx5_device_sq&& sq, gda_mlx5_device_cq&& cq) + : QueuePairDevice{qpn, nonfetching_atomic, nonfetching_atomic_lkey}, + sq{std::move(sq)}, cq{std::move(cq)} { } + + __host__ QueuePairMLX5(const QueuePairMLX5& other) = delete; + __host__ QueuePairMLX5& operator=(const QueuePairMLX5& other) = delete; + __host__ QueuePairMLX5(QueuePairMLX5&& other) noexcept = default; + __host__ QueuePairMLX5& operator=(QueuePairMLX5&& other) = default; + __host__ ~QueuePairMLX5() = default; + +public: + template + __device__ __noinline__ + void post_wqe_rma(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + void post_wqe_rma_single(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __noinline__ + amo_ret_t post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + PostOpt = {}); + + __device__ __noinline__ void quiet_single(); + +private: +#if GDA_MLX5_LOCK_USE_S_WAKEUP + static __device__ __forceinline__ void amdgcn_s_wakeup() { + /* why doesn't __builtin_amdgcn_s_wakeup() exist? + * signals other wavefronts in the same workgroup to exit early from s_sleep */ + asm volatile("s_wakeup"); + } +#endif + + __device__ void ring_doorbell(uint64_t sq_post, const gda_mlx5_wqe& wqe); + __device__ void poll_cq_until(uint16_t requested_available_slots); + + static __device__ void acquire_lock(uint32_t* lock); + static __device__ void release_lock(uint32_t* lock); + + __device__ __forceinline__ uint16_t get_wqe_idx(uint8_t lane_id) { + return static_cast(sq.post + lane_id); + } + + __device__ __forceinline__ uint16_t get_sq_idx(uint16_t wqe_idx) { + // sq.depth is a power of 2, so just mask off everything above that + return wqe_idx & sq.depth_mask; + } + + template + __device__ void lock_pollcq(int wqe_count); + + template + __device__ void post_ringdb_unlock(int wqe_count, const gda_mlx5_wqe& wqe); + +#if defined(BUILD_DEBUG_DEVICE) + __device__ __noinline__ void print_cqe_error(const mlx5_cqe64* cqe, + uint8_t opcode, uint8_t owner); +#endif +}; + +template +__device__ void QueuePairMLX5::lock_pollcq(int wqe_count) { + if constexpr (PostOptions::ThreadSafe || PostOptions::CheckSQ) { + if constexpr (PostOptions::ThreadSafe) { + acquire_lock(&sq.lock); + } + if constexpr (PostOptions::CheckSQ) { + poll_cq_until(wqe_count); + } + } else { + // need to at least acquire so that post counter is visible + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "agent"); + } +} + +template +__device__ void QueuePairMLX5::post_ringdb_unlock(int wqe_count, const gda_mlx5_wqe& wqe) { + sq.post += wqe_count; + if constexpr (PostOptions::RingDB || PostOptions::ThreadSafe) { + if constexpr (PostOptions::RingDB) { + ring_doorbell(sq.post, wqe); + } + if constexpr (PostOptions::ThreadSafe) { + release_lock(&sq.lock); + } + } else { + // need to at least release so that post counter is available + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "agent"); + } +} + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ void QueuePairMLX5::post_wqe_rma( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt) { + using PostOptions = PostOpt; + uint32_t byte_count = static_cast(size); + if (wf_info.is_pe_group_last) { + // acquire SQ lock and poll until we have enough WQEBB for all lanes using this QP + lock_pollcq(wf_info.num_pe_group_lanes); + } + + // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ + uint16_t wqe_idx = get_wqe_idx(wf_info.pe_group_logical_lane_id); + uint16_t sq_idx = get_sq_idx(wqe_idx); + + // can we inline the data into the WQE? + bool send_inline = can_inline(size); + + // should we update CQ for this WQE? + uint8_t fm_ce_se = PostOptions::signal_completion(wf_info) ? MLX5_WQE_CTRL_CQ_UPDATE : 0; + + // construct the WQE on the stack + gda_mlx5_wqe wqe{wqe_idx, static_cast(Op), qp_num, fm_ce_se, + raddr, rkey, laddr, lkey, byte_count, send_inline}; + + // copy to SQ + sq.buf[sq_idx] = wqe; + + if (wf_info.is_pe_group_last) { + /* increment post counter, ring doorbell, and release SQ lock + * we are the last thread in the wavefront, so we have the last WQE posted */ + post_ringdb_unlock(wf_info.num_pe_group_lanes, wqe); + } +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ void QueuePairMLX5::post_wqe_rma_single( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, PostOpt) { + using PostOptions = PostOpt; + uint32_t byte_count = static_cast(size); + // acquire SQ lock and poll until we have enough space for at least one WQEBB + lock_pollcq(1); + + // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ + uint16_t wqe_idx = get_wqe_idx(0); + uint16_t sq_idx = get_sq_idx(wqe_idx); + + // can we inline the data into the WQE? + bool send_inline = can_inline(size); + + // should we update CQ for this WQE? + uint8_t fm_ce_se = PostOptions::signal_completion_single() ? MLX5_WQE_CTRL_CQ_UPDATE : 0; + + // construct the WQE on the stack + gda_mlx5_wqe wqe{wqe_idx, static_cast(Op), qp_num, fm_ce_se, + raddr, rkey, laddr, lkey, byte_count, send_inline}; + + // copy to SQ + sq.buf[sq_idx] = wqe; + + // increment post counter, ring doorbell for this WQE, and release SQ lock + post_ringdb_unlock(1, wqe); +} + +// can be called with all active lanes using any number of different QPs, don't assume anything +template +__device__ __noinline__ QueuePairMLX5::amo_ret_t QueuePairMLX5::post_wqe_amo( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + using PostOptions = PostOpt; + if (wf_info.is_pe_group_last) { + // acquire SQ lock and poll until we have enough WQEBB for all lanes using this QP + lock_pollcq(wf_info.num_pe_group_lanes); + } + + uint64_t* atomic_laddr = get_atomic_addr(); + uint32_t atomic_lkey = get_atomic_lkey(); + if constexpr (Fetch == AMOFetchType::Blocking) { + uint32_t atomic_idx = (fetching_atomic_idx + wf_info.pe_group_logical_lane_id) % FETCHING_ATOMIC_CNT; + atomic_laddr += atomic_idx; + } + + // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ + uint16_t wqe_idx = get_wqe_idx(wf_info.pe_group_logical_lane_id); + uint16_t sq_idx = get_sq_idx(wqe_idx); + + // should we update CQ for this WQE? + uint8_t fm_ce_se = PostOptions::signal_completion(wf_info) ? MLX5_WQE_CTRL_CQ_UPDATE : 0; + + // construct the WQE on the stack + gda_mlx5_wqe wqe{wqe_idx, static_cast(Op), qp_num, fm_ce_se, + raddr, rkey, swap_add, compare, reinterpret_cast(atomic_laddr), atomic_lkey}; + + // copy to SQ + sq.buf[sq_idx] = wqe; + + if (wf_info.is_pe_group_last) { + // increment fetching-atomic counter + if constexpr (Fetch == AMOFetchType::Blocking) { + fetching_atomic_idx += wf_info.num_pe_group_lanes; + } + /* increment post counter, ring doorbell, and release SQ lock + * we are the last thread in the wavefront, so we have the last WQE posted */ + post_ringdb_unlock(wf_info.num_pe_group_lanes, wqe); + // wait until fetch completes + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_single(); + } + } + + if constexpr (Fetch == AMOFetchType::Blocking) { + return *atomic_laddr; + } +} + +// precondition: called with all active lanes using different QPs +template +__device__ __noinline__ QueuePairMLX5::amo_ret_t QueuePairMLX5::post_wqe_amo_single( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, PostOpt) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + using PostOptions = PostOpt; + // acquire SQ lock and poll until we have enough space for at least one WQEBB + lock_pollcq(1); + + uint64_t* atomic_laddr = get_atomic_addr(); + uint32_t atomic_lkey = get_atomic_lkey(); + if constexpr (Fetch == AMOFetchType::Blocking) { + uint32_t atomic_idx = fetching_atomic_idx % FETCHING_ATOMIC_CNT; + atomic_laddr += atomic_idx; + } + + // wqe_idx is the logical WQE id that wraps at 0xFFFF, sq_idx is the index into the actual SQ + uint16_t wqe_idx = get_wqe_idx(0); + uint16_t sq_idx = get_sq_idx(wqe_idx); + + // should we update CQ for this WQE? + uint8_t fm_ce_se = PostOptions::signal_completion_single() ? MLX5_WQE_CTRL_CQ_UPDATE : 0; + + // construct the WQE on the stack + gda_mlx5_wqe wqe{wqe_idx, static_cast(Op), qp_num, fm_ce_se, + raddr, rkey, swap_add, compare, reinterpret_cast(atomic_laddr), atomic_lkey}; + + // copy to SQ + sq.buf[sq_idx] = wqe; + + // increment fetching-atomic counter + if constexpr (Fetch == AMOFetchType::Blocking) { + fetching_atomic_idx += 1; + } + // increment post counter, ring doorbell for this WQE, and release SQ lock + post_ringdb_unlock(1, wqe); + // wait until fetch completes + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_single(); + return *atomic_laddr; + } +} + +// precondition: called with all active lanes using different QPs +__device__ inline __noinline__ void QueuePairMLX5::quiet_single() { + poll_cq_until(sq.depth); +} + +// precondition: called with all active lanes using different QPs +__device__ inline void QueuePairMLX5::poll_cq_until(uint16_t requested_available_slots) { + uint16_t sq_depth = sq.depth; + + uint64_t sq_post = __hip_atomic_load(&sq.post, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT); + // don't need to check CQ if we haven't ever filled SQ and there's enough space left + if (sq_post + requested_available_slots <= sq_depth) { + return; + } + + while (true) { + struct mlx5_cqe64* cqe = cq.buf; + + /* Update the SQ head + * This param provides us the sq_wqebb_counter; all our WQEs are exactly one WQEBB (64B) */ + // 32-bit load: big-endian 16-bit field, then two 8-bit fields + uint32_t wqecnt_sig_op_own = __hip_atomic_load(reinterpret_cast(&cqe->wqe_counter), + __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SYSTEM); + // GPU is little-endian, so wqe_counter is loaded into the low half of wqecnt_sig_op_own + __be16 be_wqe_counter = static_cast<__be16>(wqecnt_sig_op_own); + /* GPU is little-endian, so op_own is loaded into the top byte of wqecnt_sig_op_own; + * opcode is the top 4 bits of op_own */ + uint8_t opcode = static_cast(wqecnt_sig_op_own >> 28); + uint16_t sq_head = endian::from_be(be_wqe_counter); + + // sq_tail is the least significant bits of the post counter + uint16_t sq_tail = static_cast(sq_post); + + // CQEs are initially invalid, retry until we see a valid CQE + if (opcode == MLX5_CQE_INVALID) { + LOGD_TRACE("CQ: invalid completion (%x)", opcode); + continue; + } + +#if defined(BUILD_DEBUG_DEVICE) + if (opcode != MLX5_CQE_REQ) { + /* GPU is little-endian, so op_own is loaded into the top byte of wqecnt_sig_op_own; + * opcode is the top 4 bits of op_own */ + uint8_t owner = static_cast(wqecnt_sig_op_own >> 24) & MLX5_CQE_OWNER_MASK; + print_cqe_error(cqe, opcode, owner); + } +#endif // BUILD_DEBUG_DEVICE + + /* sq_tail is an index to the next free WQE i.e. counts number of posted WQEs + * sq_head is an index to the *last* completed WQE - need to add one to get *count* of completed WQEs */ + uint16_t posted = sq_tail; + uint16_t completed = sq_head + 1; + + /* posted >= completed, except when posted has wrapped around 0xFFFF and completed hasn't + * but posted - completed is correct even when it wraps around + * in some marginal cases it's maybe possible to see consumed_slots > sq_depth, + * but in that case available_slots will be very large, > requested_available_slots, + * and the loop will continue for another iteration */ + uint16_t consumed_slots = posted - completed; + uint16_t available_slots = sq_depth - consumed_slots; + + /* continue until both: + * - no additional WQEs have been posted + * - the number of requested SQ slots are available */ + uint64_t prior_sq_post = sq_post; + sq_post = __hip_atomic_load(&sq.post, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT); + if (sq_post == prior_sq_post && available_slots >= requested_available_slots) { + return; + } + } +} + +__device__ __forceinline__ void QueuePairMLX5::ring_doorbell( + uint64_t sq_post, const gda_mlx5_wqe& wqe) { + // sq_wqebb_counter is the least significant bits of the post counter + uint16_t sq_wqebb_counter = static_cast(sq_post); + // gda_mlx5_db_register constructor extracts first 8 bytes of WQE + gda_mlx5_db_register db_val{wqe}; + __be32 be_sq_wqebb_counter = endian::to_be(sq_wqebb_counter); + + // get BlueFlame buffer from SQ + gda_mlx5_bf_buffer* bf = sq.bf_buffer(); + + // store sq_wqebb_counter to doorbell record + __hip_atomic_store(sq.dbrec, be_sq_wqebb_counter, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + // ring doorbell by storing first 8B of WQE to the doorbell register + __hip_atomic_store(&bf->db_reg, db_val, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + + LOGD_TRACE("SQ: posted WQEs with dbrec(%p)=%x (%hu), dbreg(%p)=%lx (%x, %x)", + sq.dbrec, be_sq_wqebb_counter, sq_wqebb_counter, &bf->db_reg, db_val.val, + db_val.wqe_header.opmod_idx_opcode, db_val.wqe_header.qpn_ds); +} + +__device__ __forceinline__ void QueuePairMLX5::acquire_lock(uint32_t* lock) { + /* acquire lock when new value 1 (locked) is exchanged with prior value 0 (unlocked) + * + * the __ATOMIC_ACQUIRE load synchronizes with the __ATOMIC_RELEASE store in release_lock(), + * but not with the (implicit) __ATOMIC_RELAXED store part of the exchange + * this is fine, since we only need to ensure happens-before between the threads + * that released and acquired the lock, not between the different threads contending on the lock + * when they (eventually) acquire the lock, *then* they will synchronize */ + while (__hip_atomic_exchange(lock, 1, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT)) { +#if GDA_MLX5_LOCK_USE_S_SLEEP + // sleep so we don't hammer the memory + __builtin_amdgcn_s_sleep(LOCK_S_SLEEP_DELAY); +#endif + } +} + +__device__ __forceinline__ void QueuePairMLX5::release_lock(uint32_t* lock) { + // release lock by storing 0 (unlocked) + __hip_atomic_store(lock, 0, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); +#if GDA_MLX5_LOCK_USE_S_WAKEUP + // wake up any other sleeping waves (in the same workgroup) + amdgcn_s_wakeup(); +#endif +} + +#if defined(BUILD_DEBUG_DEVICE) +__device__ inline __noinline__ void QueuePairMLX5::print_cqe_error( + const mlx5_cqe64* cqe, uint8_t opcode, uint8_t owner) { + const mlx5_err_cqe* err_cqe = reinterpret_cast(cqe); + uint8_t syndrome = 0x0; + + switch (opcode) { + case MLX5_CQE_RESP_WR_IMM: + case MLX5_CQE_RESP_SEND: + case MLX5_CQE_RESP_SEND_IMM: + case MLX5_CQE_RESP_SEND_INV: + // (valid) responder completion?! + LOGD_ERROR("CQ: unexpected responder completion (%x)", opcode); + break; + case MLX5_CQE_RESIZE_CQ: + case MLX5_CQE_NO_PACKET: + LOGD_ERROR("CQ: unexpected completion type (%x)", opcode); + break; + case MLX5_CQE_SIG_ERR: + LOGD_ERROR("CQ: unexpected signature error (%x)", opcode); + break; + case MLX5_CQE_REQ_ERR: + syndrome = __hip_atomic_load(&err_cqe->syndrome, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + switch (syndrome) { + case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR: + LOGD_ERROR("CQ requester error LOCAL_LENGTH_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR: + LOGD_ERROR("CQ requester error LOCAL_QP_OP_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR: + LOGD_ERROR("CQ requester error LOCAL_PROT_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_WR_FLUSH_ERR: + LOGD_ERROR("CQ requester error WR_FLUSH_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_MW_BIND_ERR: + LOGD_ERROR("CQ requester error MW_BIND_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_BAD_RESP_ERR: + LOGD_ERROR("CQ requester error BAD_RESP_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR: + LOGD_ERROR("CQ requester error LOCAL_ACCESS_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: + LOGD_ERROR("CQ requester error REMOTE_INVAL_REQ_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR: + LOGD_ERROR("CQ requester error REMOTE_ACCESS_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_REMOTE_OP_ERR: + LOGD_ERROR("CQ requester error REMOTE_OP_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: + LOGD_ERROR("CQ requester error TRANSPORT_RETRY_EXC_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR: + LOGD_ERROR("CQ requester error RNR_RETRY_EXC_ERR (%x)", syndrome); + break; + case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR: + LOGD_ERROR("CQ requester error REMOTE_ABORTED_ERR (%x)", syndrome); + break; + default: + LOGD_ERROR("CQ requester error unknown syndrome type (%x)", syndrome); + break; + } + break; + case MLX5_CQE_RESP_ERR: + LOGD_ERROR("CQ: unexpected responder error (%x)", opcode); + break; + case MLX5_CQE_INVALID: { + LOGD_ERROR("CQ: invalid completion (%x), check owner bit = %u?", opcode, owner); + break; + } + default: + LOGD_ERROR("CQ: unknown completion type (%x)", opcode); + break; + } + abort(); +} +#endif // BUILD_DEBUG_DEVICE + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_MLX5_QUEUE_PAIR_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair.cpp b/projects/rocshmem/src/gda/queue_pair.cpp deleted file mode 100644 index bec138516ceb..000000000000 --- a/projects/rocshmem/src/gda/queue_pair.cpp +++ /dev/null @@ -1,523 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#include "queue_pair.hpp" - -#include - -#include "backend_gda.hpp" -#include "constmem.hpp" -#include "constants.hpp" -#include "util.hpp" - -namespace rocshmem { - -QueuePair::QueuePair(struct ibv_pd* pd, int gda_provider) { - int access = IBV_ACCESS_LOCAL_WRITE - | IBV_ACCESS_REMOTE_WRITE - | IBV_ACCESS_REMOTE_READ - | IBV_ACCESS_REMOTE_ATOMIC; - - if (envvar::gda::pcie_relaxed_ordering) { - access |= IBV_ACCESS_RELAXED_ORDERING; - } - allocator.allocate((void**)&nonfetching_atomic, 8); - allocator.allocate((void**)&fetching_atomic, 8 * FETCHING_ATOMIC_CNT); - allocator.allocate((void**)&fetching_atomic_freelist, sizeof(FreeListT*)); - new (fetching_atomic_freelist) FreeListT(HIPAllocator()); - - CHECK_HIP(hipMemset(nonfetching_atomic, 0, 8)); - CHECK_HIP(hipMemset(fetching_atomic, 0, 8 * FETCHING_ATOMIC_CNT)); - - mr_nonfetching_atomic = ibv.reg_mr(pd, nonfetching_atomic, 8, access, &allocator); - CHECK_NNULL(mr_nonfetching_atomic, "ibv_reg_mr"); - - mr_fetching_atomic = ibv.reg_mr(pd, fetching_atomic, 8 * FETCHING_ATOMIC_CNT, access, &allocator); - CHECK_NNULL(mr_fetching_atomic, "ibv_reg_mr"); - - nonfetching_atomic_lkey = mr_nonfetching_atomic->lkey; - fetching_atomic_lkey = mr_fetching_atomic->lkey; - - static int wf_size = 0; - if (wf_size == 0) { - int deviceId; - CHECK_HIP(hipGetDevice(&deviceId)); - hipDeviceProp_t prop; - CHECK_HIP(hipGetDeviceProperties(&prop, deviceId)); - wf_size = prop.warpSize; - } - for(uint32_t i{0}; i < FETCHING_ATOMIC_CNT; i+=wf_size) { - fetching_atomic_freelist->push_back(fetching_atomic + i); - } - - /* Set Correct opcodes for each NIC */ - switch (gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - gda_op_rdma_write = IONIC_V2_OP_RDMA_WRITE; - gda_op_rdma_read = IONIC_V2_OP_RDMA_READ; - gda_op_atomic_fa = IONIC_V2_OP_ATOMIC_FA; - gda_op_atomic_cs = IONIC_V2_OP_ATOMIC_CS; - break; -#endif //defined(GDA_IONIC) -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - gda_op_rdma_write = BNXT_RE_WR_OPCD_RDMA_WRITE; - gda_op_rdma_read = BNXT_RE_WR_OPCD_RDMA_READ; - gda_op_atomic_fa = BNXT_RE_WR_OPCD_ATOMIC_FA; - gda_op_atomic_cs = BNXT_RE_WR_OPCD_ATOMIC_CS; - break; -#endif //defined(GDA_BNXT) -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - gda_op_rdma_write = MLX5_OPCODE_RDMA_WRITE; - gda_op_rdma_read = MLX5_OPCODE_RDMA_READ; - gda_op_atomic_fa = MLX5_OPCODE_ATOMIC_FA; - gda_op_atomic_cs = MLX5_OPCODE_ATOMIC_CS; - break; -#endif //defined(GDA_MLX5) - default: - assert(false /* invalid nic provider */); - } - gda_provider_ = gda_provider; - /* Setup User Buffer Registration Mechanism */ - pd_ = pd; - num_user_buffers = envvar::gda::num_user_buffers; - - CHECK_HIP(hipMalloc(&user_buf_info, sizeof(struct user_buf_info_t) * num_user_buffers)); - CHECK_HIP(hipMemset(user_buf_info, 0, sizeof(struct user_buf_info_t) * num_user_buffers)); -} - -QueuePair::~QueuePair() { - int err; - - err = ibv.dereg_mr(mr_nonfetching_atomic); - CHECK_ZERO(err, "ibv_dereg_mr (nonfetching_atomic)"); - - err = ibv.dereg_mr(mr_fetching_atomic); - CHECK_ZERO(err, "ibv_dereg_mr (fetching_atomic)"); - - allocator.deallocate((void*)nonfetching_atomic); - allocator.deallocate((void*)fetching_atomic); - - fetching_atomic_freelist->~FreeListT(); - allocator.deallocate((void*)fetching_atomic_freelist); - - if (user_buf_info) { - CHECK_HIP(hipFree(user_buf_info)); - user_buf_info = nullptr; - } -} - -__device__ uint64_t QueuePair::get_same_qp_lane_mask() { - uint64_t active = get_active_lane_mask(); - uintptr_t this_qp = reinterpret_cast(this); - // Bitmask of lanes in this warp whose value == this_qp - uint64_t same_qp_mask = __match_any_sync(active, this_qp); - return same_qp_mask; -} - -/****************************************************************************** - ************************ PROVIDER-SPECIFIC HELPERS *************************** - *****************************************************************************/ -__device__ void QueuePair::post_wqe_rma( - int32_t length, uintptr_t raddr, uint32_t rkey, - uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db) { - switch (constmem.gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - ionic_post_wqe_rma(length, raddr, rkey, laddr, lkey, opcode, wf_info, ring_db); - return; -#endif -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - bnxt_post_wqe_rma(length, raddr, rkey, laddr, lkey, opcode, wf_info, ring_db); - return; -#endif -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - mlx5_post_wqe_rma(length, raddr, rkey, laddr, lkey, opcode, wf_info, ring_db); - return; -#endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } -} - -__device__ void QueuePair::post_wqe_rma_single(int32_t length, - uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, - uint8_t opcode, bool ring_db) { - switch (constmem.gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - ionic_post_wqe_rma_single(length, laddr, lkey, raddr, rkey, opcode, ring_db); - return; -#endif -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - bnxt_post_wqe_rma_single(length, laddr, lkey, raddr, rkey, opcode, ring_db); - return; -#endif -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - mlx5_post_wqe_rma_single(length, laddr, lkey, raddr, rkey, opcode, ring_db); - return; -#endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } -} - -__device__ uint64_t QueuePair::post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching, bool fence) { - switch (constmem.gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - return ionic_post_wqe_amo(raddr, rkey, opcode, atomic_data, atomic_cmp, - wf_info, fetching, fence); -#endif -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - return bnxt_post_wqe_amo(raddr, rkey, opcode, atomic_data, atomic_cmp, - wf_info, fetching, fence); -#endif -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - return mlx5_post_wqe_amo(raddr, rkey, opcode, atomic_data, atomic_cmp, - wf_info, fetching, fence); -#endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } -} - -__device__ uint64_t QueuePair::post_wqe_amo_single(uintptr_t raddr, - uint32_t rkey, uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence) { - switch (constmem.gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - return ionic_post_wqe_amo_single(raddr, rkey, opcode, atomic_data, atomic_cmp, fetching, fence); -#endif -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - return bnxt_post_wqe_amo_single(raddr, rkey, opcode, atomic_data, atomic_cmp, - fetching, fence); -#endif -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - return mlx5_post_wqe_amo_single(raddr, rkey, opcode, atomic_data, atomic_cmp, fetching, fence); -#endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } -} - -__device__ void QueuePair::quiet(ActiveWFInfo &wf_info) { - if(wf_info.is_pe_group_first) { - switch (constmem.gda_provider) { - #if defined(GDA_IONIC) - case GDAProvider::IONIC: - ionic_quiet(wf_info); - return; - #endif - #if defined(GDA_BNXT) - case GDAProvider::BNXT: - bnxt_quiet(); - return; - #endif - #if defined(GDA_MLX5) - case GDAProvider::MLX5: - mlx5_quiet(); - return; - #endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } - } -} - -__device__ void QueuePair::quiet_single() { - switch (constmem.gda_provider) { -#if defined(GDA_IONIC) - case GDAProvider::IONIC: - ionic_quiet_single(); - return; -#endif -#if defined(GDA_BNXT) - case GDAProvider::BNXT: - bnxt_quiet_single(); - return; -#endif -#if defined(GDA_MLX5) - case GDAProvider::MLX5: - mlx5_quiet_single(); - return; -#endif - default: - assert(false /* invalid nic provider */); - __builtin_unreachable(); - } -} - -/****************************************************************************** - ****************************** SHMEM INTERFACE ******************************* - *****************************************************************************/ -__device__ void QueuePair::put_nbi(void *dest, const void *source, - size_t length, ActiveWFInfo &wf_info) { - auto [raddr, dst_rkey] = get_raddr_info(dest); - uint32_t src_lkey = - (static_cast(length) <= static_cast(inline_threshold)) - ? 0 : get_lkey(reinterpret_cast(source)); - put_nbi(reinterpret_cast(raddr), dst_rkey, source, src_lkey, length, - wf_info); -} - -__device__ void QueuePair::put_nbi(void *raddr, uint32_t rkey, - const void *laddr, uint32_t lkey, - size_t length, ActiveWFInfo &wf_info, bool ring_db) { - uintptr_t l = reinterpret_cast(laddr); - uintptr_t r = reinterpret_cast(raddr); - post_wqe_rma(static_cast(length), r, rkey, l, lkey, - gda_op_rdma_write, wf_info, ring_db); -} - -// Used in all to all -__device__ void QueuePair::put_nbi_single(void *dest, const void *source, - size_t length, bool ring_db) { - uintptr_t src = reinterpret_cast(source); - uintptr_t dst = reinterpret_cast(dest); - uint32_t src_lkey = (static_cast(length) <= static_cast(inline_threshold)) - ? 0 : get_lkey(src); - post_wqe_rma_single(length, src, src_lkey, dst, rkey, - gda_op_rdma_write, ring_db); -} - -__device__ void QueuePair::put_nbi_single(void *raddr, uint32_t rkey, - const void *laddr, uint32_t lkey, - size_t length, bool ring_db) { - uintptr_t l = reinterpret_cast(laddr); - uintptr_t r = reinterpret_cast(raddr); - post_wqe_rma_single(length, l, lkey, r, rkey, - gda_op_rdma_write, ring_db); -} - -__device__ void QueuePair::get_nbi_single(void *dest, const void *source, size_t length, bool ring_db) { - auto [raddr, src_rkey] = get_raddr_info(source); - uintptr_t dst = reinterpret_cast(dest); - uint32_t dst_lkey = get_lkey(dst); - post_wqe_rma_single(length, dst, dst_lkey, raddr, src_rkey, - gda_op_rdma_read, ring_db); -} - - -__device__ void QueuePair::get_nbi(void *dest, const void *source, - size_t length, ActiveWFInfo &wf_info) { - auto [raddr, src_rkey] = get_raddr_info(source); - uint32_t dst_lkey = get_lkey(reinterpret_cast(dest)); - get_nbi(dest, dst_lkey, reinterpret_cast(raddr), src_rkey, length, - wf_info); -} - -__device__ int64_t QueuePair::atomic_cas(void *dest, int64_t atomic_data, - int64_t atomic_cmp, ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_cs, atomic_data, - atomic_cmp, wf_info, true); -} - -__device__ int64_t QueuePair::atomic_cas_nofetch(void *dest, - int64_t atomic_data, int64_t atomic_cmp, ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_cs, atomic_data, - atomic_cmp, wf_info); -} - -__device__ int64_t QueuePair::atomic_fetch_add(void *dest, int64_t value, - ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_fa, value, 0, wf_info, true); -} - -__device__ void QueuePair::atomic_add(void *dest, int64_t value, - ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - post_wqe_amo(dst, rkey, gda_op_atomic_fa, value, 0, wf_info); -} - -__device__ int64_t QueuePair::atomic_fetch_add(void *dest, uint32_t rkey, - int64_t value, ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_fa, value, 0, wf_info, true); -} - -__device__ void QueuePair::atomic_add(void *raddr, uint32_t rkey, - int64_t value, ActiveWFInfo &wf_info, bool fence) { - uintptr_t r = reinterpret_cast(raddr); - post_wqe_amo(r, rkey, gda_op_atomic_fa, value, 0, wf_info, false, fence); -} - -__device__ void QueuePair::atomic_add_single(void *raddr, uint32_t rkey, - int64_t value, bool fence) { - uintptr_t r = reinterpret_cast(raddr); - post_wqe_amo_single(r, rkey, gda_op_atomic_fa, value, 0, false, fence); -} - -__device__ int64_t QueuePair::atomic_cas(void *dest, uint32_t rkey, - int64_t atomic_data, int64_t atomic_cmp, ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_cs, atomic_data, atomic_cmp, - wf_info, true); -} - -__device__ int64_t QueuePair::atomic_cas_nofetch(void *dest, uint32_t rkey, - int64_t atomic_data, int64_t atomic_cmp, ActiveWFInfo &wf_info) { - uintptr_t dst = reinterpret_cast(dest); - return post_wqe_amo(dst, rkey, gda_op_atomic_cs, atomic_data, atomic_cmp, - wf_info); -} - -__device__ void QueuePair::get_nbi(void *dest, uint32_t lkey, - const void *source, uint32_t rkey, size_t length, ActiveWFInfo &wf_info) { - uintptr_t src = reinterpret_cast(source); - uintptr_t dst = reinterpret_cast(dest); - post_wqe_rma(static_cast(length), src, rkey, dst, lkey, - gda_op_rdma_read, wf_info, true); -} - -__device__ void QueuePair::atomic_add_single(void *dest, int64_t value) { - uintptr_t dst = reinterpret_cast(dest); - post_wqe_amo_single(dst, rkey, gda_op_atomic_fa, value, 0); -} - -int QueuePair::buffer_register(uintptr_t addr, size_t length) { - struct ibv_mr *mr = nullptr; - int access = 0; - - if (user_buffer_mrs.size() >= num_user_buffers) { - LOG_WARN("Unable to register user buffer with QP. " - "Please increase the value of ROCSHMEM_GDA_NUM_USER_BUFFERS"); - return ROCSHMEM_ERROR; - } - - access = IBV_ACCESS_LOCAL_WRITE - | IBV_ACCESS_REMOTE_WRITE - | IBV_ACCESS_REMOTE_READ - | IBV_ACCESS_REMOTE_ATOMIC; - - if (envvar::gda::pcie_relaxed_ordering) { - access |= IBV_ACCESS_RELAXED_ORDERING; - } - - mr = ibv.reg_mr(pd_, (void*)addr, length, access, &allocator); - CHECK_NNULL(mr, "ibv_reg_mr (buffer_register)"); - - user_buffer_mrs[addr] = mr; - - for (size_t i=0; ilkey; - - break; - } - } - - return ROCSHMEM_SUCCESS; -} - -int QueuePair::buffer_unregister(uintptr_t addr) { - int err; - - for (size_t i=0; i(provider); - constmem_t* cm_addr{nullptr}; - if (hipGetSymbolAddress(reinterpret_cast(&cm_addr), - HIP_SYMBOL(constmem)) == hipSuccess) { - CHECK_HIP(hipMemcpy(&cm_addr->gda_provider, &gda_prov, sizeof(gda_prov), hipMemcpyDefault)); - } - - // Initialize logd_constants for device-side error reporting - log_pe_number = rank; - uint32_t log_flags = 0; - if (envvar::log_flags.show_error) log_flags |= logd_constants::SHOW_ERROR; - if (envvar::log_flags.show_warn) log_flags |= logd_constants::SHOW_WARN; - if (envvar::log_flags.show_info) log_flags |= logd_constants::SHOW_INFO; - if (envvar::log_flags.show_trace) log_flags |= logd_constants::SHOW_TRACE; - if (envvar::log_flags.show_color) log_flags |= logd_constants::SHOW_COLOR; - struct logd_constants host_logd{rank, log_flags}; - struct logd_constants* logd_addr{nullptr}; - if (hipGetSymbolAddress(reinterpret_cast(&logd_addr), - HIP_SYMBOL(logd_constants)) == hipSuccess) { - CHECK_HIP(hipMemcpy(logd_addr, &host_logd, sizeof(host_logd), hipMemcpyDefault)); - } -} diff --git a/projects/rocshmem/src/gda/queue_pair.hpp b/projects/rocshmem/src/gda/queue_pair.hpp deleted file mode 100644 index de31832bf2b5..000000000000 --- a/projects/rocshmem/src/gda/queue_pair.hpp +++ /dev/null @@ -1,675 +0,0 @@ -/****************************************************************************** - * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_HPP_ -#define LIBRARY_SRC_GDA_QUEUE_PAIR_HPP_ - -/** - * @file queue_pair.hpp - * - * @section DESCRIPTION - * An IB QueuePair (SQ and CQ) that the device can use to perform network - * operations. Most important rocSHMEM operations are performed by this - * class. - */ - -#include "rocshmem_config.h" -#include "endian.h" -#include "constants.hpp" -#include "util.hpp" - -#include "ibv_wrapper.hpp" - -#include "gda/ionic/provider_gda_ionic.hpp" -#include "gda/mlx5/provider_gda_mlx5.hpp" -#include "gda/bnxt/provider_gda_bnxt.hpp" - -#include "containers/free_list.hpp" -#include "memory/hip_allocator.hpp" -#include "gda/gda_symm_table.hpp" - -#include - -namespace rocshmem { - -class GDABackend; - -struct user_buf_info_t { - uintptr_t addr; - size_t length; - uint32_t lkey; -}; - -/** - * @brief Scope at which WQEs are issued and completed. This is used to - * determine how to synchronize threads and when to poll the CQ for - * completions. - * thread: Each thread issues WQEs independently - * wave: Thread 0 in each wave issues WQE - * wg: Thread 0 of WAVE 0 issues WQE - */ - -enum class ThreadScope: int { - thread, - wave, - wg -}; - -class ActiveWFInfo { - public: - uint64_t activemask{0}; // Mask of active threads in the wavefront - uint64_t pe_group_mask{0}; // Mask of active threads with the same PE - int pe{-1}; // PE for the threads in pe_group_mask - int num_pe_group_lanes{0}; // Number of active lanes in pe_group_mask - int pe_group_logical_lane_id{0}; // Logical lane id of this thread in pe_group_mask - int pe_group_first_phys_lane_id{0}; // Physical lane id of first thread in pe_group_mask - int pe_group_last_phys_lane_id{0}; // Physical lane id of last thread in pe_group_mask - ThreadScope scope{ThreadScope::thread}; // Threading scope - bool is_pe_group_first{false}; // True if this is the first thread in pe_group_mask - bool is_pe_group_last{false}; // True if this is the last thread in pe_group_mask - - __device__ explicit ActiveWFInfo(int pe, ThreadScope scope = ThreadScope::thread) - : pe(pe), scope(scope) { - // Get active lane mask - activemask = get_active_lane_mask(); - - // Get mask of active lanes with the same PE - switch (scope) { - case ThreadScope::thread: { - pe_group_mask = __match_any_sync(activemask, pe); - num_pe_group_lanes = get_active_lane_count(pe_group_mask); - pe_group_logical_lane_id = get_active_lane_num(pe_group_mask); - pe_group_first_phys_lane_id = get_first_active_lane_id(pe_group_mask); - pe_group_last_phys_lane_id = get_last_active_lane_id(pe_group_mask); - break; - } - // Only thread 0 issues the WQE, so the group is just that thread - case ThreadScope::wave: - case ThreadScope::wg: { - pe_group_mask = 1; - num_pe_group_lanes = 1; - pe_group_logical_lane_id = get_active_lane_num(activemask); - pe_group_first_phys_lane_id = 0; - pe_group_last_phys_lane_id = 0; - } - } - is_pe_group_first = (pe_group_logical_lane_id == 0); - is_pe_group_last = (pe_group_logical_lane_id == num_pe_group_lanes - 1); - } - - // used in CAS based atomic operations at thread scope - __device__ void update(int _pe, ThreadScope _scope = ThreadScope::thread) { - // Get active lane mask - activemask = get_active_lane_mask(); - pe_group_mask = __match_any_sync(activemask, pe); - num_pe_group_lanes = get_active_lane_count(pe_group_mask); - pe_group_logical_lane_id = get_active_lane_num(pe_group_mask); - pe_group_first_phys_lane_id = get_first_active_lane_id(pe_group_mask); - pe_group_last_phys_lane_id = get_last_active_lane_id(pe_group_mask); - is_pe_group_first = (pe_group_logical_lane_id == 0); - is_pe_group_last = (pe_group_logical_lane_id == num_pe_group_lanes - 1); - scope = _scope; - pe = _pe; - } - - __device__ void printInfo() { - printf("PE: %d, Scope: %d, activemask: %llx, " - "pe_group_mask: %llx, num_pe_group_lanes: %d, " - "thread_id: %u, pe_group_logical_lane_id: %d, " - "is_pe_group_first: %d, pe_group_first_phys_lane_id: %d, " - "is_pe_group_last: %d, pe_group_last_phys_lane_id: %d\n", - pe, static_cast(scope), static_cast(activemask), - static_cast(pe_group_mask), num_pe_group_lanes, - threadIdx.x, pe_group_logical_lane_id, - static_cast(is_pe_group_first), pe_group_first_phys_lane_id, - static_cast(is_pe_group_last), pe_group_last_phys_lane_id); - } -}; - -class QueuePair { - public: - friend GDABackend; - - /** - * @brief Constructor. - */ - explicit QueuePair(struct ibv_pd* pd, int gda_provider); - - /** - * @brief Destructor. - */ - virtual ~QueuePair(); - - /** - * @brief Create and enqueue a non-blocking put to a symmetric destination. - * - * The target PE is this QP's connected peer. Resolves the - * remote address and remote key for @p dest and the local key for @p source - * internally (heap or registered buffer), so callers pass symmetric addresses - * only. Small transfers are inlined (no source key). - * - * @param[in] dest Symmetric destination address (local address space). - * @param[in] source Local source address. - * @param[in] length Size in bytes of data transmission. - * @param[in] wf_info Wavefront information. - */ - __device__ void put_nbi(void *dest, const void *source, size_t length, - ActiveWFInfo &wf_info); - - __device__ void put_nbi_single(void *dest, const void *source, size_t length, - bool ring_db); - - __device__ void put_nbi_single(void *raddr, uint32_t rkey, - const void *laddr, uint32_t lkey, - size_t length, bool ring_db = true); - - /** - * @brief Create and enqueue a non-blocking put with explicit rkey/lkey. - * - * Used when each buffer registration has its own keys, distinct from - * the QP's default heap keys. - * - * @param[in] length Size in bytes of data transmission. - * @param[in] raddr Remote destination address. - * @param[in] rkey Remote key for the destination buffer. - * @param[in] laddr Local source address. - * @param[in] lkey Local key for the source buffer. - * @param[in] wf_info Wavefront information. - * @param[in] ring_db Ring doorbell after posting (default true). - */ - __device__ void put_nbi(void *raddr, uint32_t rkey, - const void *laddr, uint32_t lkey, - size_t length, ActiveWFInfo &wf_info, bool ring_db = true); - - /** - * @brief Create and enqueue a non-blocking get from a symmetric source. - * - * The target PE is this QP's connected peer. Resolves the - * remote address and remote key for @p source and the local key for @p dest - * internally (heap or registered buffer), so callers pass symmetric addresses - * only. - * - * @param[in] dest Local destination address. - * @param[in] source Symmetric source address (local address space). - * @param[in] length Size in bytes of data transmission. - * @param[in] wf_info Wavefront information. - */ - __device__ void get_nbi(void *dest, const void *source, size_t length, - ActiveWFInfo &wf_info); - - __device__ void get_nbi_single(void *dest, const void *source, size_t length, - bool ring_db); - - /** - * @brief Empty all completions from the completion queue. - * @param[in] wf_info Wavefront information. - */ - __device__ void quiet(ActiveWFInfo &wf_info); - - __device__ void quiet_single(); - - /** - * @brief Empty all completions from the completion queue. - * @param[in] wf_info Wavefront information. - */ - __device__ void quiet_scope(ActiveWFInfo &wf_info); - - /** - * @brief Create and enqueue an atomic fetch-and-add work queue entry (wqe). - * - * @param[in] dest Destination address for data transmission. - * @param[in] value Data value for the atomic operation. - * @param[in] wf_info Wavefront information. - * - * @return The previous value at dest. - */ - __device__ int64_t atomic_fetch_add(void *dest, int64_t value, - ActiveWFInfo &wf_info); - - /** - * @brief Create and enqueue a non-fetching atomic add work queue entry (wqe). - * - * @param[in] dest Destination address for data transmission. - * @param[in] value Data value for the atomic operation. - * @param[in] wf_info Wavefront information. - */ - __device__ void atomic_add(void *dest, int64_t value, - ActiveWFInfo &wf_info); - - __device__ void atomic_add_single(void *dest, int64_t value); - - /** - * @brief Create and enqueue an atomic cas work queue entry (wqe). - * - * @param[in] dest Destination address for data transmission. - * @param[in] value Data value for the atomic operation. - * @param[in] cond Used in atomic comparisons. - * @param[in] wf_info Wavefront information. - * - * @return An atomic value - */ - __device__ int64_t atomic_cas(void *dest, int64_t atomic_data, - int64_t atomic_cmp, ActiveWFInfo &wf_info); - - /** - * @brief Create and enqueue an atomic cas work queue entry (wqe). - * - * @param[in] dest Destination address for data transmission. - * @param[in] value Data value for the atomic operation. - * @param[in] cond Used in atomic comparisons. - * @param[in] wf_info Wavefront information. - */ - __device__ int64_t atomic_cas_nofetch(void *dest, int64_t atomic_data, - int64_t atomic_cmp, ActiveWFInfo &wf_info); - - /** - * @brief Explicit remote-key atomic variants. - * - * Used when the destination is a symmetrically-registered user buffer whose - * remote key differs from the QP's default heap key. Mirror the default-key - * versions above but forward the caller-supplied @p rkey. - */ - __device__ int64_t atomic_fetch_add(void *dest, uint32_t rkey, int64_t value, - ActiveWFInfo &wf_info); - - __device__ void atomic_add(void *raddr, uint32_t rkey, int64_t value, - ActiveWFInfo &wf_info, bool fence = false); - - __device__ void atomic_add_single(void *raddr, uint32_t rkey, - int64_t value, bool fence = false); - - __device__ int64_t atomic_cas(void *dest, uint32_t rkey, int64_t atomic_data, - int64_t atomic_cmp, ActiveWFInfo &wf_info); - - __device__ int64_t atomic_cas_nofetch(void *dest, uint32_t rkey, - int64_t atomic_data, int64_t atomic_cmp, ActiveWFInfo &wf_info); - - /** - * @brief Explicit-key non-blocking put/get. - * - * get_nbi's default-key form derives the remote key from the QP heap key and - * the local key via get_lkey; this form takes both explicitly for symmetric - * user-buffer transfers. (put_nbi already provides an explicit-key overload.) - */ - __device__ void get_nbi(void *dest, uint32_t lkey, const void *source, - uint32_t rkey, size_t length, ActiveWFInfo &wf_info); - - uintptr_t base_heap = 0; - size_t base_heap_size = 0; - - /** - * @brief Cached remote heap base for this QP's connected peer. - * - * Equals heap_bases[dest_pe], captured once at setup. Lets the (common) heap - * translation run as pure register arithmetic with no memory load. - */ - uintptr_t remote_heap_base{0}; - - /** - * @brief Contiguous slice of registration entries for this QP's fixed - * (dest_pe, nic_idx), indexed by registration slot [0, *symm_count). - * - * Points into the backend's flat entry table; every entry is already - * specialized to this QP, so a match yields the remote address and keys with - * no further dereference. Null when symmetric registration is unavailable. - */ - QpSymmEntry *symm_entries{nullptr}; - - /** - * @brief Shared registration count (number of live entries per slice). - * - * Single device int shared by all QPs; register/unregister publish updates - * here. One load per non-heap lookup (null when registration unavailable). - */ - const int *symm_count{nullptr}; - - /** - * @brief Resolved remote address and remote key for a symmetric target. - */ - struct raddr_info { - uintptr_t raddr; //!< Remote address in the connected peer's address space. - uint32_t rkey; //!< Remote key to use for the transfer. - }; - - /** - * @brief Resolve the remote address and remote key for a symmetric target. - * - * The target PE is this QP's connected peer. Common case is a - * symmetric-heap address, translated from the cached remote heap base using - * this QP's default heap key. Otherwise the address is matched against this - * QP's registration slice, which supplies the peer base and per-NIC key - * directly. - * - * @param[in] sym_addr Symmetric address (valid in the local address space) - * @return Remote address and remote key. Returns {0, 0} for a non-symmetric - * pointer (undefined in SHMEM) so the NIC rejects the transfer, - * rather than fabricating a heap offset that could silently corrupt - * peer memory. - */ - __device__ __forceinline__ raddr_info get_raddr_info(const void *sym_addr) { - uintptr_t addr = reinterpret_cast(sym_addr); - if (is_ptr_in_range(base_heap, base_heap_size, addr)) { - return {remote_heap_base + (addr - base_heap), rkey}; - } - int n = symm_count ? *symm_count : 0; - for (int i = 0; i < n; ++i) { - uintptr_t local_base = symm_entries[i].local_base; - if (is_ptr_in_range(local_base, symm_entries[i].length, addr)) { - return {symm_entries[i].remote_base + (addr - local_base), - symm_entries[i].rkey}; - } - } - return {0, 0}; - } - - /** - * @brief Resolve the local key for a locally-sourced buffer. - * - * Heap buffers use this QP's default heap key; per-QP registered user buffers - * and registered symmetric buffers use their own per-NIC local key. Aborts if - * the address belongs to none of them. - */ - __device__ __forceinline__ uint32_t get_lkey(uintptr_t addr) { - /* Check if in heap */ - if (is_ptr_in_range(base_heap, base_heap_size, addr)) { - return lkey; - } - - /* Get the correct lkey for the user buffer */ - for (size_t i = 0; i < num_user_buffers; i++) { - if (is_ptr_in_range(user_buf_info[i].addr, user_buf_info[i].length, - addr)) { - return user_buf_info[i].lkey; - } - } - - /* Get the correct lkey for a registered symmetric buffer */ - int n = symm_count ? *symm_count : 0; - for (int i = 0; i < n; ++i) { - if (is_ptr_in_range(symm_entries[i].local_base, symm_entries[i].length, - addr)) { - return symm_entries[i].lkey; - } - } - - LOGD_ERROR_ABORT("Valid lkey buffer not found"); - return 0; - } - - private: - /** - * @brief Helper method to build work requests for the send queue. - * - * @param[in] raddr Remote address. - * @param[in] rkey Remote key. - * @param[in] opcode Operation to be performed. - * @param[in] atomic_data An atomic data value to be used. - * @param[in] atomic_cmp An atomic comparison value. - * @param[in] wf_info Wavefront information. - * @param[in] fetching True if the operation returns a value. - * @param[in] fence True to set fence flag on the WQE. - */ - __device__ __attribute__((noinline)) uint64_t - post_wqe_amo(uintptr_t raddr, uint32_t rkey, uint8_t opcode, - int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching = false, bool fence = false); - -#if defined(GDA_IONIC) - __device__ uint64_t ionic_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching = false, bool fence = false); -#endif -#if defined(GDA_BNXT) - __device__ uint64_t bnxt_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching = false, bool fence = false); -#endif -#if defined(GDA_MLX5) - __device__ uint64_t mlx5_post_wqe_amo(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - ActiveWFInfo &wf_info, bool fetching = false, bool fence = false); -#endif - - __device__ __attribute__((noinline)) uint64_t post_wqe_amo_single(uintptr_t raddr, - uint32_t rkey, uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching = false, bool fence = false); - - /** - * @brief Build and post an RMA work queue entry with explicit keys. - * - * @param[in] size Size in bytes of data transmission. - * @param[in] raddr Remote address. - * @param[in] rkey Remote key. - * @param[in] laddr Local address. - * @param[in] lkey Local key. - * @param[in] opcode Operation to be performed. - * @param[in] wf_info Wavefront information. - * @param[in] ring_db Ring doorbell after posting. - */ - __device__ __attribute__((noinline)) void - post_wqe_rma(int32_t length, uintptr_t raddr, uint32_t rkey, - uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db); - - __device__ __attribute__((noinline)) void - post_wqe_rma_single(int32_t length, uintptr_t laddr, uint32_t lkey, - uintptr_t raddr, uint32_t rkey, uint8_t opcode, bool ring_db); - -#if defined(GDA_MLX5) - __device__ uint64_t mlx5_post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetch = false, bool fence = false); - __device__ void mlx5_post_wqe_rma_single(int32_t length, uintptr_t laddr, - uint32_t lkey, uintptr_t raddr, uint32_t rkey, - uint8_t opcode, bool ring_db); - __device__ void mlx5_post_wqe_rma(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db); - __device__ void mlx5_quiet(); - __device__ void mlx5_quiet_single(); -#endif -#if defined(GDA_BNXT) - - __device__ void bnxt_write_rma_wqe(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, uint8_t opcode); - __device__ uint32_t bnxt_write_amo_wqe(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching, bool fence); - - __device__ uint64_t bnxt_post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetching = false, bool fence = false); - __device__ void bnxt_post_wqe_rma_single(int32_t length, uintptr_t laddr, - uint32_t lkey, uintptr_t raddr, uint32_t rkey, - uint8_t opcode, bool ring_db); - __device__ void bnxt_post_wqe_rma(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db); - __device__ void bnxt_quiet(); - __device__ void bnxt_quiet_single(); -#endif -#if defined(GDA_IONIC) - __device__ uint64_t ionic_post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, - uint8_t opcode, int64_t atomic_data, int64_t atomic_cmp, - bool fetch = false, bool fence = false); - __device__ void ionic_post_wqe_rma_single(int32_t length, - uintptr_t laddr, uint32_t lkey, uintptr_t raddr, - uint32_t rkey, uint8_t opcode, bool ring_db); - __device__ void ionic_post_wqe_rma(int32_t length, uintptr_t raddr, - uint32_t rkey, uintptr_t laddr, uint32_t lkey, - uint8_t opcode, ActiveWFInfo &wf_info, bool ring_db); - __device__ void ionic_quiet(ActiveWFInfo &wf_info); - __device__ void ionic_quiet_single(); -#endif - - /** - * @brief Helper method to ring the doorbell - * - * @param[in] db_val Doorbell value is written by method. - */ -#if defined(GDA_MLX5) - __device__ void mlx5_ring_doorbell(uint64_t sq_post, const gda_mlx5_wqe& wqe); -#endif -#if defined(GDA_BNXT) - __device__ void bnxt_ring_doorbell(uint32_t slot_idx); -#endif -#if defined(GDA_IONIC) - __device__ void ionic_ring_doorbell(uint32_t pos); - __device__ void ionic_ring_doorbell_single(uint32_t pos); -#endif - - // TODO: make private once gin_rocshmem_gda_factory uses a proper init API - public: - /* GDAProvider::BNXT START */ - uint64_t *bnxt_dbr; - struct bnxt_device_cq bnxt_cq; - struct bnxt_device_sq bnxt_sq; - - __device__ void bnxt_poll_cq_until(uint32_t requested_available_slots); - [[maybe_unused]] __device__ __attribute__((noinline)) void bnxt_print_cqe_error(uint8_t status); - - /* GDAProvider::BNXT END */ - - /* GDAProvider::MLX5 START */ - - gda_mlx5_device_cq mlx5_cq; - gda_mlx5_device_sq mlx5_sq; - - __device__ void mlx5_poll_cq_until(uint16_t requested_available_slots); - [[maybe_unused]] __device__ __attribute__((noinline)) void mlx5_print_cqe_error(const mlx5_cqe64* cqe, uint8_t opcode); - - /* GDAProvider::MLX5 END */ - - /* GDAProvider::IONIC START */ - - uint64_t *cq_dbreg{nullptr}; - uint64_t cq_dbval{0}; - uint64_t cq_mask{0}; - struct ionic_v1_cqe *ionic_cq_buf{nullptr}; - uint32_t cq_lock{SPIN_LOCK_UNLOCKED}; - uint32_t cq_pos{0}; - uint32_t cq_dbpos{0}; - - uint64_t *sq_dbreg{nullptr}; - uint64_t sq_dbval{0}; - uint64_t sq_mask{0}; - struct ionic_v1_wqe *ionic_sq_buf{nullptr}; - uint32_t sq_lock{SPIN_LOCK_UNLOCKED}; - uint32_t sq_dbprod{0}; - uint32_t sq_prod{0}; - uint32_t sq_msn{0}; - - __device__ uint64_t get_same_qp_lane_mask(); - - /** - * @brief Reserve space in the sq to post this many wqes. - * @param wf_info Wavefront information. - * @param num_wqes number of sq wqes to reserve for this wave. - * @return position of my_tid=0's wqe. - */ - __device__ uint32_t reserve_sq(ActiveWFInfo &wf_info, uint32_t num_wqes); - __device__ uint32_t reserve_sq_single(uint32_t num_wqes); - - /** - * @brief Ring the sq doorbell maintaining order between waves. - * @param wf_info Wavefront information. - * @param my_sq_prod position of my_tid=0's wqe. - * @param num_wqes number of sq wqes posted in this wave. - * @param wqe this thread's wqe. - * @return doorbell producer index. - */ - __device__ uint32_t commit_sq(ActiveWFInfo &wf_info, uint32_t my_sq_prod, - uint32_t my_sq_pos, uint32_t num_wqes); - __device__ uint32_t commit_sq_single(uint32_t my_sq_prod, uint32_t my_sq_pos, - uint32_t num_wqes); - - /** - * @brief Helper method to poll the next completion queue entry. - */ - __device__ __attribute__((noinline)) - void poll_wave_cqes(uint64_t active_lane_mask); - - /** - * @brief Helper method to drain completion queue entries. - * @param wf_info Wavefront information. - * @param cons wait for sq_msn to catch up to this position. - */ - __device__ __attribute__((noinline)) - void ionic_quiet_internal_ccqe(ActiveWFInfo &wf_info, uint32_t cons); - __device__ __attribute__((noinline)) - void ionic_quiet_internal_ccqe_single(uint32_t cons); - - /** - * @brief Helper method to drain completion queue entries. - * @param wf_info Wavefront information. - * @param cons wait for sq_msn to catch up to this position. - */ - __device__ __attribute__((noinline)) - void ionic_quiet_internal(ActiveWFInfo &wf_info, uint32_t cons); - - /* GDAProvider::IONIC END */ - - uint32_t inline_threshold{0}; - - char dev_name[24]; - uint32_t qp_num{0}; - uint32_t rkey{0}; - uint32_t lkey{0}; - - uint64_t* nonfetching_atomic{nullptr}; - uint32_t nonfetching_atomic_lkey{0}; - struct ibv_mr *mr_nonfetching_atomic; - - uint64_t* fetching_atomic{nullptr}; - uint32_t fetching_atomic_lkey{0}; - uint32_t fetching_atomic_idx{0}; - struct ibv_mr *mr_fetching_atomic; - - static constexpr uint32_t FETCHING_ATOMIC_CNT{1024}; - static_assert(FETCHING_ATOMIC_CNT % WF_SIZE == 0); - using FreeListT = FreeList; - FreeListT* fetching_atomic_freelist{nullptr}; - - HIPAllocator allocator{}; - - uint8_t gda_op_rdma_write; - uint8_t gda_op_rdma_read; - uint8_t gda_op_atomic_fa; - uint8_t gda_op_atomic_cs; - - struct ibv_pd* pd_; - std::map user_buffer_mrs; - - struct user_buf_info_t *user_buf_info = nullptr; - size_t num_user_buffers = 0; - int gda_provider_{0}; // host-side only; device dispatch uses constmem.gda_provider - - int buffer_register(uintptr_t addr, size_t length); - int buffer_unregister(uintptr_t addr); - void buffer_unregister_all(); -}; - -} // namespace rocshmem - -#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/CMakeLists.txt b/projects/rocshmem/src/gda/queue_pair/CMakeLists.txt new file mode 100644 index 000000000000..0e3755beef92 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright (c) Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +############################################################################### +# ADD ROCSHMEM TARGET FOR FILES IN CURRENT DIRECTORY +############################################################################### +target_sources( + ${PROJECT_NAME} + PRIVATE + queue_pair_host.cpp +) diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_common.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_common.hpp new file mode 100644 index 000000000000..d729c1a63daf --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_common.hpp @@ -0,0 +1,251 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_COMMON_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_COMMON_HPP_ + +/** + * @file queue_pair_common.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include +#include + +#include + +#include "util.hpp" + +namespace rocshmem { + +struct BufferInfo { + uintptr_t addr; + size_t length; + uint32_t lkey; +}; + +/** + * @brief One symmetric user-buffer registration, pre-specialized to a single + * QueuePair's fixed (dest_pe, nic_idx). + * + * Because every QP talks to exactly one peer over exactly one NIC, all the + * per-PE / per-NIC indexing can be resolved on the host at registration time, + * leaving the device with a self-contained 32-byte record: matching an address + * and producing the remote address + keys needs no pointer chasing. The host + * fills one entry per (pe, nic) per registration (see GDABackend's flat entry + * table), and each QP is handed the contiguous slice for its own (dest_pe, + * nic_idx), scanned by registration slot. + */ +struct SymmBufferInfo { + /** + * @brief This PE's registered alias base (identical across all slices). + * + * Used to recognize whether a symmetric address falls in this registration + * and to compute the intra-registration offset. + */ + uintptr_t local_base; + + /** + * @brief The peer's alias base for this slice's dest_pe. + * + * Remote address for a transfer is remote_base + (sym_addr - local_base). + */ + uintptr_t remote_base; + + /** + * @brief Registered length in bytes. + */ + size_t length; + + /** + * @brief Local key for this slice's nic_idx (locally-sourced buffers). + */ + uint32_t lkey; + + /** + * @brief Remote key for this slice's (dest_pe, nic_idx). + */ + uint32_t rkey; +}; + +/** + * @brief Scope at which WQEs are issued and completed. This is used to + * determine how to synchronize threads and when to poll the CQ for + * completions. + * thread: Each thread issues WQEs independently + * wave: Thread 0 in each wave issues WQE + * wg: Thread 0 of WAVE 0 issues WQE + */ + +enum class ThreadScope: int { + thread, + wave, + wg +}; + +class ActiveWFInfo { + public: + uint64_t activemask{0}; // Mask of active threads in the wavefront + uint64_t pe_group_mask{0}; // Mask of active threads with the same PE + int pe{-1}; // PE for the threads in pe_group_mask + int num_pe_group_lanes{0}; // Number of active lanes in pe_group_mask + int pe_group_logical_lane_id{0}; // Logical lane id of this thread in pe_group_mask + int pe_group_first_phys_lane_id{0}; // Physical lane id of first thread in pe_group_mask + int pe_group_last_phys_lane_id{0}; // Physical lane id of last thread in pe_group_mask + ThreadScope scope{ThreadScope::thread}; // Threading scope + bool is_pe_group_first{false}; // True if this is the first thread in pe_group_mask + bool is_pe_group_last{false}; // True if this is the last thread in pe_group_mask + + __device__ explicit ActiveWFInfo(int pe, ThreadScope scope = ThreadScope::thread) + : pe(pe), scope(scope) { + // Get active lane mask + activemask = get_active_lane_mask(); + + // Get mask of active lanes with the same PE + switch (scope) { + case ThreadScope::thread: { + pe_group_mask = __match_any_sync(activemask, pe); + num_pe_group_lanes = get_active_lane_count(pe_group_mask); + pe_group_logical_lane_id = get_active_lane_num(pe_group_mask); + pe_group_first_phys_lane_id = get_first_active_lane_id(pe_group_mask); + pe_group_last_phys_lane_id = get_last_active_lane_id(pe_group_mask); + break; + } + // Only thread 0 issues the WQE, so the group is just that thread + case ThreadScope::wave: + case ThreadScope::wg: { + pe_group_mask = 1; + num_pe_group_lanes = 1; + pe_group_logical_lane_id = get_active_lane_num(activemask); + pe_group_first_phys_lane_id = 0; + pe_group_last_phys_lane_id = 0; + } + } + is_pe_group_first = (pe_group_logical_lane_id == 0); + is_pe_group_last = (pe_group_logical_lane_id == num_pe_group_lanes - 1); + } + + // used in CAS based atomic operations at thread scope + __device__ void update(int _pe, ThreadScope _scope = ThreadScope::thread) { + // Get active lane mask + activemask = get_active_lane_mask(); + pe_group_mask = __match_any_sync(activemask, pe); + num_pe_group_lanes = get_active_lane_count(pe_group_mask); + pe_group_logical_lane_id = get_active_lane_num(pe_group_mask); + pe_group_first_phys_lane_id = get_first_active_lane_id(pe_group_mask); + pe_group_last_phys_lane_id = get_last_active_lane_id(pe_group_mask); + is_pe_group_first = (pe_group_logical_lane_id == 0); + is_pe_group_last = (pe_group_logical_lane_id == num_pe_group_lanes - 1); + scope = _scope; + pe = _pe; + } + + __device__ void printInfo() { + printf("PE: %d, Scope: %d, activemask: %llx, " + "pe_group_mask: %llx, num_pe_group_lanes: %d, " + "thread_id: %u, pe_group_logical_lane_id: %d, " + "is_pe_group_first: %d, pe_group_first_phys_lane_id: %d, " + "is_pe_group_last: %d, pe_group_last_phys_lane_id: %d\n", + pe, static_cast(scope), static_cast(activemask), + static_cast(pe_group_mask), num_pe_group_lanes, + threadIdx.x, pe_group_logical_lane_id, + static_cast(is_pe_group_first), pe_group_first_phys_lane_id, + static_cast(is_pe_group_last), pe_group_last_phys_lane_id); + } +}; + +/* + * @struct QueuePairTraits + * @brief Defines Provider-specific types and constants. + * + * Each Provider subclass of QueuePairInterface should also define a specialization + * for QueuePairTraits that defines the documented members: + * - QueuePairTraits::OpCode + * - QueuePairTraits::InlineMax + * - QueuePairTraits::InlineThreshold + * + * Additionally, Provider subclasses of QueuePairDevice should also provide the member: + * - QueuePairTraits::Endianness + * + * Sample specialization code for a QueuePairProvider subclass + * of QueuePairDevice: + * @code + * class QueuePairProvider; + * template <> struct QueuePairTraits { + * enum class OpCode : uint8_t { + * RDMA_WRITE = ..., + * RDMA_READ = ..., + * ATOMIC_CS = ..., + * ATOMIC_FA = ..., + * }; + * + * static constexpr endian::Order Endianness = endian::Order::...; + * + * static constexpr size_t InlineMax = ...; + * static constexpr size_t InlineThreshold = ...; + * }; + * @endcode + * + * @tparam Provider Name of Provider class. + */ + +/* + * @enum QueuePairTraits::OpCode + * @brief Enumeration of the Provider-specific opcodes. + * + * @var QueuePairTraits::OpCode::RDMA_WRITE + * Provider-specific RDMA Write opcode. + * + * @var QueuePairTraits::OpCode::RDMA_READ + * Provider-specific RDMA Read opcode. + * + * @var QueuePairTraits::OpCode::ATOMIC_CS + * Provider-specific Compare-and-Swap opcode. + * + * @var QueuePairTraits::OpCode::ATOMIC_FA + * Provider-specific Fetch-Add opcode. + */ + +/* + * @var endian::Order QueuePairTraits::Endianness + * @brief Endianness order of data stored by the hardware for Provider. + * + * @qualifier static + * @qualifier constexpr + */ + +/* + * @var size_t QueuePairTraits::InlineMax + * @brief Maximum number of bytes that can be sent inline in a WQE. + * + * @qualifer static + * @qualifier constexpr + */ + +/* + * @var size_t QueuePairTraits::InlineThreshold + * @brief Preferred maximum number of bytes that should be sent inline in a WQE. + * + * @qualifer static + * @qualifier constexpr + */ + +template +struct QueuePairTraits; + +/* + * @brief Atomic Memory Operation fetching behavior. + */ +enum class AMOFetchType { + Blocking, + NonBlocking, + NonFetching, +}; + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_COMMON_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_device.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_device.hpp new file mode 100644 index 000000000000..b5d9848e3039 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_device.hpp @@ -0,0 +1,388 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_DEVICE_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_DEVICE_HPP_ + +/** + * @file queue_pair_device.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include +#include + +#include +#include + +#include + +#include "constants.hpp" +#include "containers/free_list.hpp" +#include "gda/endian.hpp" +#include "log.hpp" +#include "util.hpp" + +#include "queue_pair_common.hpp" +#include "queue_pair_interface.hpp" +#include "queue_pair_option.hpp" +#include "queue_pair_shmem.hpp" + +namespace rocshmem { + +/* + * @brief Information required for initializing a Queue Pair. + */ +struct QueuePairInitInfo { + uintptr_t heap_laddr; + uintptr_t heap_raddr; + size_t heap_size; + uint32_t heap_lkey; + uint32_t heap_rkey; + + uint64_t *fetching_atomic; + uint64_t *nonfetching_atomic; + uint32_t fetching_atomic_lkey; + uint32_t nonfetching_atomic_lkey; + FreeList *fetching_atomic_freelist; + const BufferInfo *local_buffers; + size_t num_user_buffers; + const SymmBufferInfo *symm_buffers; + const int *symm_count; +}; + +/* + * @brief CRTP base class for Provider-specific Queue Pair implementations. + */ +template +class alignas(ALIGNMENT) QueuePairDevice : public QueuePairSHMEM { +/** + * @name Provider-Defined Members + * + * Members that are defined based on specializations of QueuePairTraits. + * Each Provider must supply their own definitions. + * + * @{ + */ +public: + /** + * @brief Type alias for QueuePairTraits. + */ + using typename QueuePairInterface::Traits; + + /** + * @brief Enumeration of the opcodes for Write, Read, Fetch-Add, and Compare-and-Swap. + * + * A definition must be provided by the QueuePairTraits specialization + * of each Provider subclass. + */ + using typename QueuePairInterface::OpCode; + + /** + * @brief Constant defining the endianness required by the provider. Used for e.g. lkey and rkey. + * + * A definition must be provided by the QueuePairTraits specialization + * of each Provider subclass. + */ + static constexpr endian::Order ProviderEndianness = Traits::Endianness; +/**@}*/ + + + +/** + * @name Non-static Data Members + * + * @{ + */ +protected: + /* + * @brief Cached heap info for this QP. + * + * heap.local_base is heap_bases[my_pe], captured once at setup. + * heap.remote_base is heap_bases[dest_pe], captured once at setup. + * heap.length is the heap size. + * heap.lkey is the LKey for local_base. + * heap.rkey is the RKey for remote_base. + * + * This lets the (common) heap translation run as pure register arithmetic + * with no additional indirection or memory load (besides *this). + */ + SymmBufferInfo heap; + + // Used in most WQEs + uint32_t qp_num; + uint32_t fetching_atomic_idx{0}; + + // Used in atomic WQEs + uint64_t* fetching_atomic; + uint64_t* nonfetching_atomic; + + uint32_t fetching_atomic_lkey; + uint32_t nonfetching_atomic_lkey; + + static constexpr size_t FETCHING_ATOMIC_CNT{1024}; + static_assert(FETCHING_ATOMIC_CNT % WF_SIZE == 0); + using FreeListT = FreeList; + FreeListT* fetching_atomic_freelist; + +private: + // Used by get_laddr + const BufferInfo* local_buffers; + size_t num_user_buffers; + + /** + * @brief Contiguous slice of registration entries for this QP's fixed + * (dest_pe, nic_idx), indexed by registration slot [0, *symm_count). + * + * Points into the backend's flat entry table; every entry is already + * specialized to this QP, so a match yields the remote address and keys with + * no further dereference. Null when symmetric registration is unavailable. + */ + const SymmBufferInfo *symm_buffers; + + /** + * @brief Shared registration count (number of live entries per slice). + * + * Single device int shared by all QPs; register/unregister publish updates + * here. One load per non-heap lookup (null when registration unavailable). + */ + const int *symm_count; +/**@}*/ + + + +/** + * @name Internal + * + * @{ + */ +protected: + /* + * @brief Provider is a friend of QueuePairDevice. + */ + friend Provider; + + template + __device__ __forceinline__ constexpr uint64_t* get_atomic_addr() const { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + if constexpr (Fetch == AMOFetchType::Blocking) { + return fetching_atomic; + } else if constexpr (Fetch == AMOFetchType::NonFetching) { + return nonfetching_atomic; + } + } + + template + __device__ __forceinline__ constexpr uint32_t get_atomic_lkey() const { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + if constexpr (Fetch == AMOFetchType::Blocking) { + return fetching_atomic_lkey; + } else if constexpr (Fetch == AMOFetchType::NonFetching) { + return nonfetching_atomic_lkey; + } + } + +private: + __device__ Provider& provider() { + return static_cast(*this); + } +/**@}*/ + + + +/** + * @name Constructors and Destructors + * + * @{ + */ +protected: + /** + * @brief Constructor. + * + * @param[in] qpn Queue Pair number. + * @param[in] init_info Initialization information for this QP. + */ + __host__ explicit QueuePairDevice(uint32_t qpn, QueuePairInitInfo&& init_info) + : heap{.local_base = init_info.heap_laddr, + .remote_base = init_info.heap_raddr, + .length = init_info.heap_size, + .lkey = to_provider_endianness(init_info.heap_lkey), + .rkey = to_provider_endianness(init_info.heap_rkey)}, + qp_num{qpn}, + fetching_atomic{init_info.fetching_atomic}, + nonfetching_atomic{init_info.nonfetching_atomic}, + fetching_atomic_lkey{to_provider_endianness(init_info.fetching_atomic_lkey)}, + nonfetching_atomic_lkey{to_provider_endianness(init_info.nonfetching_atomic_lkey)}, + fetching_atomic_freelist{init_info.fetching_atomic_freelist}, + local_buffers{init_info.local_buffers}, + num_user_buffers{init_info.num_user_buffers}, + symm_buffers{init_info.symm_buffers}, + symm_count{init_info.symm_count} { } + + /** + * @brief Convenience Constructor for GIN. Unspecified values are nullptr or 0. + * + * A QueuePairDevice constructed in this manner + * must ONLY call explicit-key APIs such as those defined by QueuePairInterface. + * Additionally, only non-fetching atomics may be used: + * fetching (blocking) atomics require that initialization of the array of fetch locations. + * + * @param[in] qpn Queue Pair number. + * @param[in] nonfetching_atomic Single location used for the return values of nonfetching AMOs. + * @param[in] nonfetching_atomic_lkey LKey for nonfetching_atomic. + */ + __host__ explicit QueuePairDevice(uint32_t qpn, + uint64_t *nonfetching_atomic, uint32_t nonfetching_atomic_lkey) + : QueuePairDevice{qpn, QueuePairInitInfo{ + .heap_laddr = 0, .heap_raddr = 0, .heap_size = 0, .heap_lkey = 0, .heap_rkey = 0, + .fetching_atomic = nullptr, .nonfetching_atomic = nonfetching_atomic, + .fetching_atomic_lkey = 0, .nonfetching_atomic_lkey = nonfetching_atomic_lkey, + .fetching_atomic_freelist = nullptr, + .local_buffers = nullptr, .num_user_buffers = 0, + .symm_buffers = nullptr, .symm_count = nullptr}} { } + + __host__ QueuePairDevice(const QueuePairDevice& other) = delete; + __host__ QueuePairDevice& operator=(const QueuePairDevice& other) = delete; + __host__ QueuePairDevice(QueuePairDevice&& other) noexcept = default; + __host__ QueuePairDevice& operator=(QueuePairDevice&& other) = default; + __host__ ~QueuePairDevice() = default; +/**@}*/ + + + +/** + * @name Address Management + * + * @{ + */ +public: + /** + * @brief Resolve the local (origin) virtual address and LKey of a symmetric address. + * + * Attempts to resolve the address in the following order of locations: + * 1. The default symmetric heap. + * 2. All user-registered local buffers. + * 3. All user-registered symmetric buffers. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {laddr, lkey} for addr or {0, std::numeric_limits::max()} if not found. + * Endianness of returned LKey value is ProviderEndianness. + */ + __device__ __forceinline__ + std::tuple get_laddr_info(const void *addr, bool inlined = false) const; + + /** + * @brief Resolve the remote (target) virtual address and RKey of a symmetric address. + * + * Attempts to resolve the address in the following order of locations: + * 1. The default symmetric heap. + * 2. All user-registered symmetric buffers. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {raddr, rkey} for addr or {0, std::numeric_limits::max()} if not found. + * Endianness of returned RKey value is ProviderEndianness. + */ + __device__ __forceinline__ + std::tuple get_raddr_info(const void *addr) const; + + /* + * @brief Query whether data can be inlined into a WQE. + * + * @param[in] size Size of data carried by the WQE. + * @tparam Op OpCode for this WQE. + * + * @return True if size bytes of data can be inlined into a WQE with OpCode Op, else false. + */ + template + static __host__ __device__ __forceinline__ constexpr bool can_inline(size_t size) { + if constexpr (Op == OpCode::RDMA_WRITE) { + return size <= Traits::InlineThreshold; + } else { + return false; + } + } + + /** + * @brief Convert value to ProviderEndianness, byteswapping if necessary. + * + * @param[in] val Value to convert. + * @tparam T Type of val. + * + * @return endian::from_native(val) + */ + template + static __host__ __device__ __forceinline__ constexpr T to_provider_endianness(T val) { + return endian::from_native(val); + } +/**@}*/ +}; + + + +template +__device__ __forceinline__ std::tuple +QueuePairDevice::get_laddr_info(const void *addr, bool inlined) const { + uintptr_t laddr = reinterpret_cast(addr); + /* Check for inlined RDMA_WRITE */ + if (inlined) { + return {laddr, 0}; + } + + /* Check symmetric heap */ + if (is_ptr_in_range(heap.local_base, heap.length, laddr)) [[likely]] { + return {laddr, heap.lkey}; + } + + /* Check user-registered local buffers */ + for (size_t i = 0; i < num_user_buffers; i++) { + if (is_ptr_in_range(local_buffers[i].addr, local_buffers[i].length, laddr)) { + return {laddr, local_buffers[i].lkey}; + } + } + + /* Check user-registered symmetric buffers */ + if (symm_count) { + for (int i = 0; i < *symm_count; i++) { + if (is_ptr_in_range(symm_buffers[i].local_base, symm_buffers[i].length, laddr)) { + return {laddr, symm_buffers[i].lkey}; + } + } + } + + LOGD_ERROR_ABORT("Valid for address %p not found", addr); + return {0, std::numeric_limits::max()}; +} + +template +__device__ __forceinline__ +std::tuple QueuePairDevice::get_raddr_info(const void *addr) const { + uintptr_t laddr = reinterpret_cast(addr); + /* Check symmetric heap */ + if (is_ptr_in_range(heap.local_base, heap.length, laddr)) [[likely]] { + uintptr_t raddr = heap.remote_base + (laddr - heap.local_base); + return {raddr, heap.rkey}; + } + + /* Check user-registered symmetric buffers */ + if (symm_count) { + for (int i = 0; i < *symm_count; i++) { + if (is_ptr_in_range(symm_buffers[i].local_base, symm_buffers[i].length, laddr)) { + uintptr_t raddr = symm_buffers[i].remote_base + (laddr - symm_buffers[i].local_base); + return {raddr, symm_buffers[i].rkey}; + } + } + } + + LOGD_ERROR_ABORT("Valid for address %p not found", addr); + return {0, std::numeric_limits::max()}; +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_DEVICE_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_host.cpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_host.cpp new file mode 100644 index 000000000000..17f15d221da5 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_host.cpp @@ -0,0 +1,280 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include + +#include +#include + +#include + +#include "rocshmem/rocshmem_common.hpp" + +#include "constants.hpp" +#include "containers/free_list_impl.hpp" +#include "log.hpp" +#include "util.hpp" + +#include "queue_pair_common.hpp" +#include "queue_pair_host.hpp" +#include "gda/queue_pair_provider.hpp" + +namespace rocshmem { + +__host__ QueuePairHost::QueuePairHost(struct ibv_pd* pd) : pd{pd} { + int access = IBV_ACCESS_LOCAL_WRITE + | IBV_ACCESS_REMOTE_WRITE + | IBV_ACCESS_REMOTE_READ + | IBV_ACCESS_REMOTE_ATOMIC; + + if (envvar::gda::pcie_relaxed_ordering) { + access |= IBV_ACCESS_RELAXED_ORDERING; + } + + // Allocate and register the fetching and nonfetching atomics arrays + std::tie(nonfetching_atomic, nonfetching_atomic_mr) + = allocate_and_register(1, access); + std::tie(fetching_atomic, fetching_atomic_mr) + = allocate_and_register(FETCHING_ATOMIC_CNT, access); + + allocator.allocate(reinterpret_cast(&fetching_atomic_freelist), sizeof(FreeListT)); + new (fetching_atomic_freelist) FreeListT{allocator}; + + int deviceId; + CHECK_HIP(hipGetDevice(&deviceId)); + int wf_size = get_wf_size(deviceId); + for (size_t i = 0; i < FETCHING_ATOMIC_CNT; i += wf_size) { + fetching_atomic_freelist->push_back(&fetching_atomic[i]); + } + + /* Setup User Buffer Registration Mechanism */ + num_user_buffers = envvar::gda::num_user_buffers; + + CHECK_HIP(hipMalloc(&buffer_info, sizeof(BufferInfo) * num_user_buffers)); + CHECK_HIP(hipMemset(buffer_info, 0, sizeof(BufferInfo) * num_user_buffers)); + + /* Reserve memory to register up to num_user_buffers */ + buffer_info_map.reserve(num_user_buffers); + + /* Create buffer registration info freelist */ + buffer_info_freelist = iterator_freelist{buffer_info, num_user_buffers}; +} + +__host__ QueuePairHost::QueuePairHost(QueuePairHost&& other) noexcept + : buffer_info {std::move(other.buffer_info)}, + num_user_buffers {std::move(other.num_user_buffers)}, + fetching_atomic {std::move(other.fetching_atomic)}, + nonfetching_atomic {std::move(other.nonfetching_atomic)}, + fetching_atomic_freelist{std::move(other.fetching_atomic_freelist)}, + allocator {std::move(other.allocator)}, + buffer_info_map {std::move(other.buffer_info_map)}, + buffer_info_freelist {std::move(other.buffer_info_freelist)}, + pd {std::move(other.pd)}, + fetching_atomic_mr {std::move(other.fetching_atomic_mr)}, + nonfetching_atomic_mr {std::move(other.nonfetching_atomic_mr)} { + other.buffer_info = nullptr; + other.fetching_atomic = nullptr; + other.nonfetching_atomic = nullptr; + other.fetching_atomic_freelist = nullptr; + other.fetching_atomic_mr = nullptr; + other.nonfetching_atomic_mr = nullptr; +} + +__host__ QueuePairHost& QueuePairHost::operator=(QueuePairHost&& other) { + int err = 0; + + /* Step 1: ensure all resources in *this are deallocated */ + if (!buffer_info_map.empty()) { + LOG_WARN("Unmatched buffer_register detected: " + "move assignment operator %s called, but buffer registration map is not empty!", + __PRETTY_FUNCTION__ ); + /* Deregister every memory region registered with this QP */ + for (auto&& [addr, host_info] : buffer_info_map) { + err = ibv.dereg_mr(host_info.mr); + CHECK_ZERO(err, "ibv_dereg_mr (QueuePairHost::operator=(QueuePairHost&&))"); + } + } + + if (buffer_info) { + CHECK_HIP(hipFree(buffer_info)); + } + + if (fetching_atomic_freelist) { + fetching_atomic_freelist->~FreeListT(); + allocator.deallocate(static_cast(fetching_atomic_freelist)); + } + + if (fetching_atomic_mr) { + err = ibv.dereg_mr(fetching_atomic_mr); + CHECK_ZERO(err, "ibv_dereg_mr (fetching_atomic)"); + } + + if (fetching_atomic) { + allocator.deallocate(static_cast(fetching_atomic)); + } + + if (nonfetching_atomic_mr) { + err = ibv.dereg_mr(nonfetching_atomic_mr); + CHECK_ZERO(err, "ibv_dereg_mr (nonfetching_atomic)"); + } + + if (nonfetching_atomic) { + allocator.deallocate(static_cast(nonfetching_atomic)); + } + + /* Step 2: member-wise move of all data members from other to *this */ + buffer_info = std::move(other.buffer_info); + num_user_buffers = std::move(other.num_user_buffers); + fetching_atomic = std::move(other.fetching_atomic); + nonfetching_atomic = std::move(other.nonfetching_atomic); + fetching_atomic_freelist = std::move(other.fetching_atomic_freelist); + allocator = std::move(other.allocator); + buffer_info_map = std::move(other.buffer_info_map); + buffer_info_freelist = std::move(other.buffer_info_freelist); + pd = std::move(other.pd); + fetching_atomic_mr = std::move(other.fetching_atomic_mr); + nonfetching_atomic_mr = std::move(other.nonfetching_atomic_mr); + + /* Step 3: reset allocated data members of other so that it can be safely reused or destroyed */ + other.buffer_info = nullptr; + other.fetching_atomic = nullptr; + other.nonfetching_atomic = nullptr; + other.fetching_atomic_freelist = nullptr; + other.fetching_atomic_mr = nullptr; + other.nonfetching_atomic_mr = nullptr; + + /* Step 4: return *this */ + return *this; +} + +__host__ QueuePairHost::~QueuePairHost() { + int err = 0; + + if (!buffer_info_map.empty()) { + LOG_WARN("Unmatched buffer_register detected: " + "destructor %s called, but buffer registration map is not empty!", + __PRETTY_FUNCTION__ ); + /* Deregister every memory region registered with this QP */ + for (auto&& [addr, host_info] : buffer_info_map) { + err = ibv.dereg_mr(host_info.mr); + CHECK_ZERO(err, "ibv_dereg_mr (QueuePairHost::~QueuePairHost)"); + } + } + + if (buffer_info) { + CHECK_HIP(hipFree(buffer_info)); + } + + if (fetching_atomic_freelist) { + fetching_atomic_freelist->~FreeListT(); + allocator.deallocate(static_cast(fetching_atomic_freelist)); + } + + if (fetching_atomic_mr) { + err = ibv.dereg_mr(fetching_atomic_mr); + CHECK_ZERO(err, "ibv_dereg_mr (fetching_atomic)"); + } + + if (fetching_atomic) { + allocator.deallocate(static_cast(fetching_atomic)); + } + + if (nonfetching_atomic_mr) { + err = ibv.dereg_mr(nonfetching_atomic_mr); + CHECK_ZERO(err, "ibv_dereg_mr (nonfetching_atomic)"); + } + + if (nonfetching_atomic) { + allocator.deallocate(static_cast(nonfetching_atomic)); + } +} + +__host__ int QueuePairHost::buffer_register(void *addr, size_t length) { + if (buffer_info_freelist.empty()) { + assert(buffer_info_map.size() == num_user_buffers); + LOG_WARN("Unable to register user buffer (%p, %zu) with QP. " + "Please increase the value of %s.", + addr, length, envvar::gda::num_user_buffers.get_name().c_str()); + return ROCSHMEM_ERROR; + } + + /* Register addr */ + int access = IBV_ACCESS_LOCAL_WRITE + | IBV_ACCESS_REMOTE_WRITE + | IBV_ACCESS_REMOTE_READ + | IBV_ACCESS_REMOTE_ATOMIC; + + if (envvar::gda::pcie_relaxed_ordering) { + access |= IBV_ACCESS_RELAXED_ORDERING; + } + + struct ibv_mr* mr = ibv.reg_mr(pd, addr, length, access, &allocator); + CHECK_NNULL(mr, "ibv_reg_mr (buffer_register)"); + + /* Get pointer to next free buffer_info entry */ + BufferInfo* info_ptr = buffer_info_freelist.top(); + + /* Try inserting buffer host info into buffer_info_map; detects whether this is a duplicate */ + auto&& [it, inserted] = buffer_info_map.try_emplace(addr, mr, info_ptr); + + if (inserted) [[likely]] { + /* Insertion succceeded, pop the buffer_info entry from the freelist */ + buffer_info_freelist.pop(); + /* Copy buffer info to device */ + BufferInfo info{reinterpret_cast(addr), length, + QueuePair::to_provider_endianness(mr->lkey)}; + CHECK_HIP(hipMemcpy(info_ptr, &info, sizeof(BufferInfo), hipMemcpyHostToDevice)); + return ROCSHMEM_SUCCESS; + } else { + auto&& [key, host_info] = *it; + LOG_WARN("Unable to register user buffer (%p, %zu) with QP: " + "already registered with length=%zu, lkey=%u, rkey=%u.", + addr, length, host_info.mr->length, host_info.mr->lkey, host_info.mr->rkey); + /* Deregister addr */ + int err = ibv.dereg_mr(mr); + CHECK_ZERO(err, "ibv_dereg_mr (buffer_register)"); + return ROCSHMEM_ERROR; + } +} + +__host__ int QueuePairHost::buffer_unregister(void *addr) { + /* Lookup buffer host info */ + auto it = buffer_info_map.find(addr); + + if (it != buffer_info_map.end()) [[likely]] { + auto&& [key, host_info] = *it; + /* Reset buffer_info entry and push to freelist */ + CHECK_HIP(hipMemset(host_info.info_ptr, 0, sizeof(BufferInfo))); + buffer_info_freelist.push(host_info.info_ptr); + /* Deregister addr */ + int err = ibv.dereg_mr(host_info.mr); + CHECK_ZERO(err, "ibv_dereg_mr (buffer_unregister)"); + /* Remove from map */ + buffer_info_map.erase(it); + return ROCSHMEM_SUCCESS; + } else { + LOG_WARN("Unable to unregister user buffer (%p) with this QP: " + "user buffer not registered.", addr); + return ROCSHMEM_ERROR; + } +} + +__host__ int QueuePairHost::buffer_unregister_all() { + int err = 0; + + /* Deregister every memory region registered with this QP */ + for (auto&& [addr, host_info] : buffer_info_map) { + err = ibv.dereg_mr(host_info.mr); + CHECK_ZERO(err, "ibv_dereg_mr (QueuePairHost::buffer_unregister_all)"); + } + buffer_info_map.clear(); + + /* Clear all buffer_info slots */ + CHECK_HIP(hipMemset(buffer_info, 0, sizeof(BufferInfo) * num_user_buffers)); + + /* Reset buffer_info_freelist */ + buffer_info_freelist.reset(buffer_info, num_user_buffers); + return ROCSHMEM_SUCCESS; +} + +} // namespace rocshmem diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_host.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_host.hpp new file mode 100644 index 000000000000..291e1b8f9c6f --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_host.hpp @@ -0,0 +1,275 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_HOST_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_HOST_HPP_ + +/** + * @file queue_pair_host.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "rocshmem/rocshmem_config.h" // NOLINT(build/include_subdir) + +#include "constants.hpp" +#include "containers/free_list.hpp" +#include "memory/hip_allocator.hpp" + +#include "gda/ibv_wrapper.hpp" +#include "queue_pair_common.hpp" + +namespace rocshmem { + +struct BufferHostInfo { + struct ibv_mr* mr; + BufferInfo* info_ptr; + + BufferHostInfo(struct ibv_mr* mr, BufferInfo* info_ptr) : mr{mr}, info_ptr{info_ptr} { } +}; + +template +class iterator_freelist { +private: + std::vector container{}; + + /* Reserve space in container for std::distance(first, last) elements */ + void reserve(Iter first, Iter last) { + container.reserve(std::distance(first, last)); + } + + void init(Iter first, Iter last) { + /* Push back iterators in range [first, last) to container in reverse order + * Sorted so that container.back() always has the smallest iterator */ + while (first != last--) { + container.push_back(last); + } + } + +public: + using value_type = Iter; + using difference_type = typename std::iterator_traits::difference_type; + using size_type = typename decltype(container)::size_type; + + iterator_freelist() = default; + + /* Initialize freelist with iterators in range [first, last) */ + iterator_freelist(Iter first, Iter last) { + reserve(first, last); + init(first, last); + } + + /* Initialize freelist with iterators in range [first, std::next(first, count)) */ + iterator_freelist(Iter first, size_type count) + : iterator_freelist{first, std::next(first, count)} { } + + bool empty() const { + return container.empty(); + } + + size_type size() const { + return container.size(); + } + + Iter top() { + return container.back(); + } + + void push(Iter value) { + std::greater compare{}; + if (container.empty() || compare(top(), value)) { + /* value < top(), so push_back maintains sorted property */ + container.push_back(value); + } else { + /* Use binary search to find insert position for value */ + auto pos = std::upper_bound(container.cbegin(), container.cend(), value, compare); + container.insert(pos, value); + } + } + + void pop() { + container.pop_back(); + } + + void reset(Iter first, Iter last) { + container.clear(); + reserve(first, last); + init(first, last); + } + + void reset(Iter first, size_type count) { + reset(first, std::next(first, count)); + } +}; + + + +/* + * @brief QueuePairHost manages local buffer registration and + * the atomic fetch buffers for a device QueuePair. + */ +class QueuePairHost { +/** + * @name Constructors and Destructors + * + * @{ + */ +public: + /** + * @brief Constructor. + * + * @param[in] pd IBVerbs Protection Domain for registering additional buffers and heaps. + */ + __host__ explicit QueuePairHost(struct ibv_pd* pd); + + /* + * @brief Copy constructor is deleted. + */ + __host__ QueuePairHost(const QueuePairHost& other) = delete; + + /* + * @brief Copy assignment operator is deleted. + */ + __host__ QueuePairHost& operator=(const QueuePairHost& other) = delete; + + /* + * @brief Move constructor. + * + * Performs a member-wise move of all data members from other to *this, + * then resets allocated data members of other so that it can be safely reused or destroyed. + * + * @param[in,out] other QueuePairHost object to move from. + */ + __host__ QueuePairHost(QueuePairHost&& other) noexcept; + + /* + * @brief Move assignment operator. + * + * Cleans up all resources allocated by *this, + * performs a member-wise move of all data members from other to *this, + * then resets allocated data members of other so that it can be safely reused or destroyed. + * + * @param[in,out] other QueuePairHost object to move from. + * @return *this + */ + __host__ QueuePairHost& operator=(QueuePairHost&& other); + + /** + * @brief Destructor. + * + * Cleans up all resources allocated by *this. + */ + __host__ ~QueuePairHost(); +/**@}*/ + + + +/** + * @name Buffer Registration + * + * @{ + */ +public: + /** + * @brief Register buffer for use as local address in rocSHMEM routines + * + * @param[in] addr Base address of buffer. + * @param[in] length Length of buffer. + * + * @retval ROCSHMEM_SUCCESS Buffer registered successfully. + * @retval ROCSHMEM_ERROR Buffer could not be registered. + */ + __host__ int buffer_register(void *addr, size_t length); + + /** + * @brief Unregister buffer. + * Buffer must have previously been registered with buffer_register(void *, size_t). + * + * @param[in] addr Base address of buffer. + * + * @retval ROCSHMEM_SUCCESS Buffer unregistered successfully. + */ + __host__ int buffer_unregister(void *addr); + + /** + * @brief Unregister all registered buffers. + * + * @retval ROCSHMEM_SUCCESS Buffers unregistered successfully. + */ + __host__ int buffer_unregister_all(); +/**@}*/ + + + +/** + * @name Internal + * + * @{ + */ +private: + template + __host__ std::tuple allocate_and_register(size_t count, int access); +/**@}*/ + + + +/** + * @name Non-static Data Members + * + * @{ + */ +public: + // Used by get_laddr + BufferInfo* buffer_info{nullptr}; + size_t num_user_buffers{0}; + + // Used in atomic WQEs + uint64_t* fetching_atomic{nullptr}; + uint64_t* nonfetching_atomic{nullptr}; + + static constexpr size_t FETCHING_ATOMIC_CNT{1024}; + static_assert(FETCHING_ATOMIC_CNT % WF_SIZE == 0); + using FreeListT = FreeList; + FreeListT* fetching_atomic_freelist{nullptr}; + + HIPAllocator allocator{}; + + // Used by host in buffer_register, buffer_unregister + std::unordered_map buffer_info_map{}; + iterator_freelist buffer_info_freelist{}; + struct ibv_pd* pd; + struct ibv_mr* fetching_atomic_mr{nullptr}; + struct ibv_mr* nonfetching_atomic_mr{nullptr}; +/**@}*/ +}; + +template +__host__ std::tuple +QueuePairHost::allocate_and_register(size_t count, int access) { + void* ptr = nullptr; + size_t size = sizeof(T) * count; + allocator.allocate(&ptr, size); + CHECK_HIP(hipMemset(ptr, 0, size)); + struct ibv_mr* mr = ibv.reg_mr(pd, ptr, size, access, &allocator); + CHECK_NNULL(mr, "ibv_reg_mr"); + return {static_cast(ptr), mr}; +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_HOST_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_interface.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_interface.hpp new file mode 100644 index 000000000000..f1d26a6362ea --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_interface.hpp @@ -0,0 +1,364 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_INTERFACE_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_INTERFACE_HPP_ + +/** + * @file queue_pair_interface.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include +#include + +#include + +#include + +#include "queue_pair_common.hpp" +#include "queue_pair_option.hpp" + +namespace rocshmem { + +/* + * @brief CRTP mixin class supplying a common mid-level interface to Queue Pair implementations. + */ +template +class QueuePairInterface { +/** + * @name Provider-Defined Members + * + * Members that are defined based on specializations of QueuePairTraits. + * Each Provider must supply their own definitions. + * + * @{ + */ +public: + /** + * @brief Type alias for QueuePairTraits. + */ + using Traits = QueuePairTraits; + + /* Check that Traits provides a valid inlining threshold */ + static_assert(Traits::InlineThreshold <= Traits::InlineMax, + "Inlining threshold must be less than the inlining maximum."); + + /** + * @brief Enumeration of the opcodes for Write, Read, Fetch-Add, and Compare-and-Swap. + * + * A definition must be provided by the QueuePairTraits specialization + * of each Provider subclass. + */ + using OpCode = typename Traits::OpCode; +/**@}*/ + + + +/** + * @name Member Types + * + * @{ + */ +public: + /** + * @brief Helper alias for defining post_wqe_amo and post_wqe_amo_single in subclasses + */ + template + using amo_ret_t = std::conditional_t; +/**@}*/ + + + +/** + * @name Internal + * + * @{ + */ +protected: + /* + * @brief Provider is a friend of QueuePairInterface. + */ + friend Provider; + + __host__ QueuePairInterface() = default; + __device__ QueuePairInterface() = delete; + +private: + __device__ Provider& provider() { + static_assert(std::is_trivially_destructible_v, + "QueuePair providers should have a trivial destructor: " + "memory is managed externally by QueuePairHost."); + return static_cast(*this); + } +/**@}*/ + + + +/** + * @name Remote Memory Access (RMA) + * + * @{ + */ +public: + /** + * @brief Create and enqueue a non-blocking put work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] source Source address for data transmission. Local virtual address at origin. + * @param[in] source_lkey LKey of source address. + * @param[in] nelems Size in bytes of data transmission. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void put_nbi(uintptr_t dest, uint32_t dest_rkey, + uintptr_t source, uint32_t source_lkey, size_t nelems, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + provider().template post_wqe_rma( + source, source_lkey, dest, dest_rkey, nelems, wf_info, post_options); + } + template + __device__ void put_nbi_single(uintptr_t dest, uint32_t dest_rkey, + uintptr_t source, uint32_t source_lkey, size_t nelems, + PostOpt post_options = {}) { + provider().template post_wqe_rma_single( + source, source_lkey, dest, dest_rkey, nelems, post_options); + } + + /** + * @brief Create and enqueue a non-blocking get work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] dest Destination address for data transmission. Local virtual address at origin. + * @param[in] dest_lkey LKey of destination address. + * @param[in] source Source address for data transmission. Remote virtual address at target. + * @param[in] source_rkey RKey of source address. + * @param[in] nelems Size in bytes of data transmission. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void get_nbi(uintptr_t dest, uint32_t dest_lkey, + uintptr_t source, uint32_t source_rkey, size_t nelems, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + provider().template post_wqe_rma( + dest, dest_lkey, source, source_rkey, nelems, wf_info, post_options); + } + template + __device__ void get_nbi_single(uintptr_t dest, uint32_t dest_lkey, + uintptr_t source, uint32_t source_rkey, size_t nelems, + PostOpt post_options = {}) { + provider().template post_wqe_rma_single( + dest, dest_lkey, source, source_rkey, nelems, post_options); + } +/**@}*/ + + + +/** + * @name Atomic Memory Operations (AMO) + * + * @{ + */ +public: + /** + * @brief Create and enqueue a blocking atomic fetch-and-add work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ uint64_t atomic_fetch_add(uintptr_t dest, uint32_t dest_rkey, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + return provider().template post_wqe_amo( + dest, dest_rkey, value, 0, wf_info, post_options); + } + template + __device__ uint64_t atomic_fetch_add_single(uintptr_t dest, uint32_t dest_rkey, uint64_t value, + PostOpt post_options = {}) { + return provider().template post_wqe_amo_single( + dest, dest_rkey, value, 0, post_options); + } + +#if 0 // NON-BLOCKING AMOs NOT YET IMPLEMENTED + /** + * @brief Create and enqueue a non-blocking atomic fetch-and-add work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] fetch Address for fetched value. Local virtual address at origin. + * @param[in] fetch_lkey LKey of address for fetched value. + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ void atomic_fetch_add_nbi(uintptr_t fetch, uint32_t fetch_lkey, + uintptr_t dest, uint32_t dest_rkey, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + provider().template post_wqe_amo( + fetch, fetch_lkey, dest, dest_rkey, value, 0, wf_info, post_options); + } + template + __device__ void atomic_fetch_add_nbi_single(uintptr_t fetch, uint32_t fetch_lkey, + uintptr_t dest, uint32_t dest_rkey, uint64_t value, + PostOpt post_options = {}) { + provider().template post_wqe_amo_single( + fetch, fetch_lkey, dest, dest_rkey, value, 0, post_options); + } +#endif + + /** + * @brief Create and enqueue a non-fetching atomic add work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void atomic_add(uintptr_t dest, uint32_t dest_rkey, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + provider().template post_wqe_amo( + dest, dest_rkey, value, 0, wf_info, post_options); + } + template + __device__ void atomic_add_single(uintptr_t dest, uint32_t dest_rkey, uint64_t value, + PostOpt post_options = {}) { + provider().template post_wqe_amo_single( + dest, dest_rkey, value, 0, post_options); + } + + /** + * @brief Create and enqueue a blocking atomic compare-and-swap work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] cond Used in atomic comparisons. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ uint64_t atomic_compare_swap(uintptr_t dest, uint32_t dest_rkey, + uint64_t cond, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + return provider().template post_wqe_amo( + dest, dest_rkey, value, cond, wf_info, post_options); + } + template + __device__ uint64_t atomic_compare_swap_single(uintptr_t dest, uint32_t dest_rkey, + uint64_t cond, uint64_t value, + PostOpt post_options = {}) { + return provider().template post_wqe_amo_single( + dest, dest_rkey, value, cond, post_options); + } + +#if 0 // NON-BLOCKING AMOs NOT YET IMPLEMENTED + /** + * @brief Create and enqueue a non-blocking atomic compare-and-swap work queue entry (WQE). + * + * All address arguments are virtual addresses at origin or target, + * as appropriate for the direction of data transfer. + * + * @param[in] fetch Address for fetched value. Local virtual address at origin. + * @param[in] fetch_lkey LKey of address for fetched value. + * @param[in] dest Destination address for data transmission. Remote virtual address at target. + * @param[in] dest_rkey RKey of destination address. + * @param[in] cond Used in atomic comparisons. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ void atomic_compare_swap_nbi(uintptr_t fetch, uint32_t fetch_lkey, + uintptr_t dest, uint32_t dest_rkey, + uint64_t cond, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + provider().template post_wqe_amo( + fetch, fetch_lkey, dest, dest_rkey, value, cond, wf_info, post_options); + } + template + __device__ void atomic_compare_swap_nbi_single(uintptr_t fetch, uint32_t fetch_lkey, + uintptr_t dest, uint32_t dest_rkey, + uint64_t cond, uint64_t value, + PostOpt post_options = {}) { + provider().template post_wqe_amo_single( + fetch, fetch_lkey, dest, dest_rkey, value, cond, post_options); + } +#endif +/**@}*/ + + + +/** + * @name Completion and Ordering. + * + * @{ + */ +public: + /** + * @brief Empty all completions from the completion queue. + * + * @param[in] wf_info Wavefront information. + */ + __device__ void quiet(const ActiveWFInfo& wf_info) { + if (wf_info.is_pe_group_first) { + provider().quiet_single(); + } + } +/**@}*/ +}; + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_INTERFACE_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_mock.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_mock.hpp new file mode 100644 index 000000000000..b71639f42e85 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_mock.hpp @@ -0,0 +1,290 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_MOCK_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_MOCK_HPP_ + +#include + +#include + +#include "gda/endian.hpp" + +#include "queue_pair_common.hpp" +#include "queue_pair_interface.hpp" +#include "queue_pair_option.hpp" +#include "queue_pair_shmem.hpp" + +namespace rocshmem { + +class QueuePairMock; + +template <> struct QueuePairTraits { + /** + * @brief QueuePair mock object OpCode enumeration. + */ + enum class OpCode { + RDMA_WRITE, + RDMA_READ, + ATOMIC_CS, + ATOMIC_FA, + }; + + /** + * @brief QueuePair mock object uses native ordering. + */ + static constexpr endian::Order Endianness = endian::Order::Native; + + /** + * @brief QueuePair mock object inlining maximum is 16 bytes. + */ + static constexpr size_t InlineMax = sizeof(__uint128_t); + + + /* + * @brief QueuePair mock object preferred inlining threshold is 8 bytes. + */ + static constexpr size_t InlineThreshold = sizeof(uint64_t); +}; + +class QueuePairMock : public QueuePairSHMEM { +public: + static __device__ inline size_t rma_count{0}; + static __device__ inline size_t rma_inline_count{0}; + static __device__ inline size_t amo_count{0}; + static __device__ inline size_t quiet_count{0}; + +public: + __host__ QueuePairMock() = default; + __host__ QueuePairMock(const QueuePairMock& other) = delete; + __host__ QueuePairMock& operator=(const QueuePairMock& other) = delete; + __host__ QueuePairMock(QueuePairMock&& other) noexcept = default; + __host__ QueuePairMock& operator=(QueuePairMock&& other) = default; + __host__ ~QueuePairMock() = default; + +public: + template + __device__ __forceinline__ + void post_wqe_rma(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __forceinline__ + void post_wqe_rma_single(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt = {}); + + template + __device__ __forceinline__ + amo_ret_t post_wqe_amo(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __forceinline__ + amo_ret_t post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + PostOpt = {}); + + __device__ __forceinline__ void quiet_single(); + + + /** + * @brief Resolve the local (origin) virtual address and LKey of a symmetric address. + * QueuePair mock object returns the local (origin) virtual address unmodified and LKey = 0. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {reinterpret_cast(addr), 0} + */ + __device__ __forceinline__ + std::tuple get_laddr_info(const void *addr, bool inlined = false) const { + uintptr_t laddr = reinterpret_cast(addr); + return {laddr, 0}; + } + + /** + * @brief Resolve the remote (target) virtual address and RKey of a symmetric address. + * QueuePair mock object returns the remote (target) virtual address unmodified and LKey = 0. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {reinterpret_cast(addr), 0} + */ + __device__ __forceinline__ + std::tuple get_raddr_info(const void *addr) const { + uintptr_t laddr = reinterpret_cast(addr); + return {laddr, 0}; + } + + /* + * @brief Query whether data can be inlined into a WQE. + * + * @param[in] size Size of data carried by the WQE. + * @tparam Op OpCode for this WQE. + * + * @return True if size bytes of data can be inlined into a WQE with OpCode Op, else false. + */ + template + static __host__ __device__ __forceinline__ constexpr bool can_inline(size_t size) { + if constexpr (Op == OpCode::RDMA_WRITE) { + return size <= Traits::InlineThreshold; + } else { + return false; + } + } + + /** + * @brief Convert value to Endianness of provider, byteswapping if necessary. + * QueuePair mock object returns the value unmodified: mock objects use native endian ordering. + * + * @param[in] val Value to convert. + * @tparam T Type of val. + * + * @return val + */ + template + static __host__ __device__ __forceinline__ constexpr T to_provider_endianness(T val) { + return val; + } +}; + + + +template +__device__ __forceinline__ void QueuePairMock::post_wqe_rma( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt options) { + if (!laddr || !raddr) { + return; + } + + if (wf_info.is_pe_group_first) { + __hip_atomic_fetch_add(&rma_count, wf_info.num_pe_group_lanes, + __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } + + if (can_inline(size)) { + __hip_atomic_fetch_add(&rma_inline_count, 1, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } + + if constexpr (Op == OpCode::RDMA_WRITE) { + // RDMA_WRITE: dest is raddr, src is laddr + memcpy(reinterpret_cast(raddr), reinterpret_cast(laddr), size); + } else if constexpr (Op == OpCode::RDMA_READ) { + // RDMA_READ: dest is laddr, src is raddr + memcpy(reinterpret_cast(laddr), reinterpret_cast(raddr), size); + } else { + static_assert(false, "invalid OpCode: post_wqe_rma only supports RDMA_WRITE and RDMA_READ"); + } +} + +template +__device__ __forceinline__ void QueuePairMock::post_wqe_rma_single( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt options) { + if (!laddr || !raddr) { + return; + } + + __hip_atomic_fetch_add(&rma_count, 1, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + + if (can_inline(size)) { + __hip_atomic_fetch_add(&rma_inline_count, 1, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } + + if constexpr (Op == OpCode::RDMA_WRITE) { + // RDMA_WRITE: dest is raddr, src is laddr + memcpy(reinterpret_cast(raddr), reinterpret_cast(laddr), size); + } else if constexpr (Op == OpCode::RDMA_READ) { + // RDMA_READ: dest is laddr, src is raddr + memcpy(reinterpret_cast(laddr), reinterpret_cast(raddr), size); + } else { + static_assert(false, "invalid OpCode: post_wqe_rma_single only supports RDMA_WRITE and RDMA_READ"); + } +} + +template +__device__ __forceinline__ QueuePairMock::amo_ret_t QueuePairMock::post_wqe_amo( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt options) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + if (!raddr) { + // Returns void{}, a prvalue of type void; or uint64_t{}, which performs zero-initialization + return amo_ret_t{}; + } + + if (wf_info.is_pe_group_first) { + __hip_atomic_fetch_add(&amo_count, wf_info.num_pe_group_lanes, + __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } + + if constexpr (Op == OpCode::ATOMIC_CS) { + // ATOMIC_CS: obj is raddr, expected is &compare, desired is swap_add + __hip_atomic_compare_exchange_strong(reinterpret_cast(raddr), &compare, swap_add, + __ATOMIC_RELAXED, __ATOMIC_RELAXED, + __HIP_MEMORY_SCOPE_AGENT); + } else if constexpr (Op == OpCode::ATOMIC_FA) { + // ATOMIC_FA: obj is raddr, arg is swap_add; compare is otherwise unused, use for fetch val + assert(compare == 0); + compare = __hip_atomic_fetch_add(reinterpret_cast(raddr), swap_add, + __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } else { + static_assert(false, "invalid OpCode: post_wqe_amo only supports ATOMIC_CS and ATOMIC_FA"); + } + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet(wf_info); + /* compare either: + * a) unmodified, if ATOMIC_CS and atomic_compare_exchange succeeded (i.e. same as *raddr) + * b) load raddr, if ATOMIC_CS and atomic_compare_exchange failed + * c) prior value at raddr, if ATOMIC_FA */ + return compare; + } +} + +template +__device__ __forceinline__ QueuePairMock::amo_ret_t QueuePairMock::post_wqe_amo_single( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + PostOpt options) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + if (!raddr) { + // Returns void{}, a prvalue of type void; or uint64_t{}, which performs zero-initialization + return amo_ret_t{}; + } + + __hip_atomic_fetch_add(&amo_count, 1, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + + if constexpr (Op == OpCode::ATOMIC_CS) { + // ATOMIC_CS: obj is raddr, expected is &compare, desired is swap_add + __hip_atomic_compare_exchange_strong(reinterpret_cast(raddr), &compare, swap_add, + __ATOMIC_RELAXED, __ATOMIC_RELAXED, + __HIP_MEMORY_SCOPE_AGENT); + } else if constexpr (Op == OpCode::ATOMIC_FA) { + // ATOMIC_FA: obj is raddr, arg is swap_add; compare is otherwise unused, use for fetch val + assert(compare == 0); + compare = __hip_atomic_fetch_add(reinterpret_cast(raddr), swap_add, + __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + } else { + static_assert(false, "invalid OpCode: post_wqe_amo_single only supports ATOMIC_CS and ATOMIC_FA"); + } + + if constexpr (Fetch == AMOFetchType::Blocking) { + quiet_single(); + /* compare either: + * a) unmodified, if ATOMIC_CS and atomic_compare_exchange succeeded (i.e. same as *raddr) + * b) load raddr, if ATOMIC_CS and atomic_compare_exchange failed + * c) prior value at raddr, if ATOMIC_FA */ + return compare; + } +} + +__device__ __forceinline__ void QueuePairMock::quiet_single() { + // Release ensures that prior stores and RMW occur before the increment of quiet_count + __hip_atomic_fetch_add(&quiet_count, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT); +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_MOCK_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_option.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_option.hpp new file mode 100644 index 000000000000..7c0d07b7e6f0 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_option.hpp @@ -0,0 +1,278 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_OPTION_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_OPTION_HPP_ + +/** + * @file queue_pair_option.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include + +#include + +#include "queue_pair_common.hpp" + +namespace rocshmem { + +namespace QueuePairOption { + /* + * @brief Helper alias for option tag types before C++26 std::constant_wrapper. + * + * Options are defined as types derived from constant_t + * so that type-based tag dispatching can be used. + */ + template + using constant_t = std::integral_constant; + + /* + * @brief Type trait for defining default values for options. + */ + template typename option_tag> struct default_option { }; + + /* + * @brief Helper type alias for the default_option type trait. + */ + template typename option_tag> + using default_option_t = typename default_option::type; + + /* + * @brief Helper variable for the default option value. + */ + template typename option_tag> + constexpr inline auto default_option_v = default_option_t::value; + + enum class UpdateThread { + All, + Last, + None, + }; + + /* + * @brief Option: whether the send queue doorbell will be rung. + */ + template + struct ring_db_tag : constant_t { }; + + template <> struct default_option { + /* default: DO ring the doorbell */ + using type = ring_db_tag; + }; + + /* + * @brief Option: whether thread safety will be enforced. + */ + template + struct thread_safe_tag : constant_t { }; + + template <> struct default_option { + /* default: DO use thread safety */ + using type = thread_safe_tag; + }; + + /* + * @brief Option: whether the number of available send queue entries will be checked. + */ + template + struct check_sq_tag : constant_t { }; + + template <> struct default_option { + /* default: DO check the SQ */ + using type = check_sq_tag; + }; + + /* + * @brief Option: which threads' WQEs will generate CQEs. + */ + template + struct update_cq_tag : constant_t { }; + + template <> struct default_option { + /* default: all WQEs update the CQ */ + using type = update_cq_tag; + }; + + /* forward declaration */ + template struct PostOpt; + + /* Clang versions < 22 implicitly treats deduction guides as __host__ functions + * unless marked otherwise by explicit attributes. + * + * Clang versions >= 22 implicitly treats all deduction guides as __host__ __device__ + * and issues a warning when they have explicit attributes. + * See https://clang.llvm.org/docs/HIPSupport.html#deduction-guides for details. + * + * Disable this warning so that Clang >= 22 doesn't cause issues; + * remove the attributes once we no longer support older compiler versions. */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-attributes" + /* deduction guide, required before C++20 */ + template __host__ __device__ PostOpt(Options...) -> PostOpt; +#pragma clang diagnostic pop + + /* Base case with all options defined */ + template + struct PostOpt, + thread_safe_tag, + check_sq_tag, + update_cq_tag> { + /* explicitly-defaulted default constructor */ + __host__ __device__ constexpr PostOpt() = default; + /* constructor for type deduction from tags */ + __host__ __device__ constexpr PostOpt(ring_db_tag, + thread_safe_tag, + check_sq_tag, + update_cq_tag) { } + + /* static constexpr data members to simplify option access */ + static constexpr auto RingDB = ring_db; + static constexpr auto ThreadSafe = thread_safe; + static constexpr auto CheckSQ = check_sq; + static constexpr auto UpdateCQ = update_cq; + + static __device__ constexpr inline bool signal_completion(const ActiveWFInfo& wf_info) { + if constexpr (UpdateCQ == UpdateThread::All) { + // all WQEs update the CQ + return true; + } else if constexpr (UpdateCQ == UpdateThread::Last) { + // only the last WQE in each group updates the CQ + return wf_info.is_pe_group_last; + } else { + // no WQEs update the CQ + return false; + } + } + + static __device__ constexpr inline bool signal_completion_single() { + if constexpr (UpdateCQ == UpdateThread::All) { + // all WQEs update the CQ + return true; + } else if constexpr (UpdateCQ == UpdateThread::Last) { + // singleton groups, so all threads are "last": all WQEs update the CQ + return true; + } else { + // no WQEs update the CQ + return false; + } + } + }; + + /* Extraneous parameters, + * else matches PostOpt */ + template + struct PostOpt, + thread_safe_tag, + check_sq_tag, + update_cq_tag, + Options...> { + static_assert(sizeof...(Options) == 0, "Too many or invalid options"); + }; + + /* Missing update_cq_tag, + * else matches PostOpt */ + template + struct PostOpt, + thread_safe_tag, + check_sq_tag, + Options...> + : PostOpt, + thread_safe_tag, + check_sq_tag, + default_option_t, + Options...> { + __host__ __device__ constexpr PostOpt(ring_db_tag, + thread_safe_tag, + check_sq_tag, + Options...) { } + /* inherit constructor */ + using PostOpt, + thread_safe_tag, + check_sq_tag, + default_option_t, + Options... + >::PostOpt; + }; + + /* Missing check_sq_tag, + * else matches PostOpt */ + template + struct PostOpt, + thread_safe_tag, + Options...> + : PostOpt, + thread_safe_tag, + default_option_t, + Options...> { + __host__ __device__ constexpr PostOpt(ring_db_tag, + thread_safe_tag, + Options...) { } + /* inherit constructor */ + using PostOpt, + thread_safe_tag, + default_option_t, + Options... + >::PostOpt; + }; + + /* Missing thread_safe_tag, + * else matches PostOpt */ + template + struct PostOpt, + Options...> + : PostOpt, + default_option_t, + Options...> { + __host__ __device__ constexpr PostOpt(ring_db_tag, + Options...) { } + /* inherit constructor */ + using PostOpt, + default_option_t, + Options... + >::PostOpt; + }; + + /* Missing ring_db_tag, + * else matches PostOpt */ + template + struct PostOpt + : PostOpt, + Options...> { + __host__ __device__ constexpr PostOpt(Options...) { } + /* inherit constructor */ + using PostOpt, + Options... + >::PostOpt; + }; + + /* ensure default PostOpt<> uses all the default options */ + static_assert(PostOpt<>::RingDB == default_option_v && + PostOpt<>::ThreadSafe == default_option_v && + PostOpt<>::CheckSQ == default_option_v && + PostOpt<>::UpdateCQ == default_option_v); + +} // namespace QueuePairOption + +/* + * @brief Type alias helper for WQE posting options. + */ +using QueuePairOption::PostOpt; + +/* bring QueuePairOption::UpdateThread into scope */ +using QueuePairOption::UpdateThread; + +/* constexpr variable templates to simplify usage */ +template constexpr inline auto RingDB = QueuePairOption::ring_db_tag{}; +template constexpr inline auto ThreadSafe = QueuePairOption::thread_safe_tag{}; +template constexpr inline auto CheckSQ = QueuePairOption::check_sq_tag{}; +template constexpr inline auto UpdateCQ = QueuePairOption::update_cq_tag{}; + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_OPTION_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair/queue_pair_shmem.hpp b/projects/rocshmem/src/gda/queue_pair/queue_pair_shmem.hpp new file mode 100644 index 000000000000..5a62a1f0cb84 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair/queue_pair_shmem.hpp @@ -0,0 +1,351 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_SHMEM_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_SHMEM_HPP_ + +/** + * @file queue_pair_shmem.hpp + * + * @section DESCRIPTION + * An IB QueuePair (SQ and CQ) that the device can use to perform network + * operations. Most important rocSHMEM operations are performed by this + * class. + */ + +#include +#include + +#include + +#include + +#include "queue_pair_common.hpp" +#include "queue_pair_interface.hpp" +#include "queue_pair_option.hpp" + +namespace rocshmem { + +/* + * @brief CRTP mixin class supplying a common SHMEM-like interface to Queue Pair implementations. + */ +template +class QueuePairSHMEM : public QueuePairInterface { +/** + * @name Inherited Members + * + * Members that are inherited from QueuePairInterface. + * + * @{ + */ +public: + using typename QueuePairInterface::Traits; + using typename QueuePairInterface::OpCode; + + using QueuePairInterface::put_nbi; + using QueuePairInterface::put_nbi_single; + using QueuePairInterface::get_nbi; + using QueuePairInterface::get_nbi_single; + using QueuePairInterface::atomic_fetch_add; + using QueuePairInterface::atomic_fetch_add_single; + using QueuePairInterface::atomic_add; + using QueuePairInterface::atomic_add_single; + using QueuePairInterface::atomic_compare_swap; + using QueuePairInterface::atomic_compare_swap_single; + using QueuePairInterface::quiet; +/**@}*/ + + + +/** + * @name Internal + * + * @{ + */ +protected: + /* + * @brief Provider is a friend of QueuePairSHMEM. + */ + friend Provider; + + __host__ QueuePairSHMEM() = default; + __device__ QueuePairSHMEM() = delete; + +private: + __device__ Provider& provider() { + return static_cast(*this); + } +/**@}*/ + + + +/** + * @name Remote Memory Access (RMA) + * + * @{ + */ +public: + /** + * @brief Create and enqueue a non-blocking put work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] source Source symmetric address for data transmission. + * @param[in] nelems Size in bytes of data transmission. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void put_nbi(void *dest, const void *source, size_t nelems, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + bool inlined_wqe = Provider::template can_inline(nelems); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + auto [s_laddr, s_lkey] = provider().get_laddr_info(source, inlined_wqe); + provider().put_nbi(d_raddr, d_rkey, s_laddr, s_lkey, nelems, wf_info, post_options); + } + template + __device__ void put_nbi_single(void *dest, const void *source, size_t nelems, + PostOpt post_options = {}) { + bool inlined_wqe = Provider::template can_inline(nelems); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + auto [s_laddr, s_lkey] = provider().get_laddr_info(source, inlined_wqe); + provider().put_nbi_single(d_raddr, d_rkey, s_laddr, s_lkey, nelems, post_options); + } + + /** + * @brief Create and enqueue a non-blocking get work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] source Source symmetric address for data transmission. + * @param[in] nelems Size in bytes of data transmission. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void get_nbi(void *dest, const void *source, size_t nelems, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [d_laddr, d_lkey] = provider().get_laddr_info(dest); + auto [s_raddr, s_rkey] = provider().get_raddr_info(source); + provider().get_nbi(d_laddr, d_lkey, s_raddr, s_rkey, nelems, wf_info, post_options); + } + template + __device__ void get_nbi_single(void *dest, const void *source, size_t nelems, + PostOpt post_options = {}) { + auto [d_laddr, d_lkey] = provider().get_laddr_info(dest); + auto [s_raddr, s_rkey] = provider().get_raddr_info(source); + provider().get_nbi_single(d_laddr, d_lkey, s_raddr, s_rkey, nelems, post_options); + } +/**@}*/ + + + +/** + * @name Atomic Memory Operations (AMO) + * + * @{ + */ +public: + /** + * @brief Create and enqueue a blocking atomic fetch-and-add work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ uint64_t atomic_fetch_add(void *dest, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + return provider().atomic_fetch_add(d_raddr, d_rkey, value, wf_info, post_options); + } + template + __device__ uint64_t atomic_fetch_add_single(void *dest, uint64_t value, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + return provider().atomic_fetch_add_single(d_raddr, d_rkey, value, post_options); + } + +#if 0 // NON-BLOCKING AMOs NOT YET IMPLEMENTED + /** + * @brief Create and enqueue a non-blocking atomic fetch-and-add work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] fetch Symmetric address for fetched value. + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ void atomic_fetch_add_nbi(uint64_t *fetch, void *dest, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [f_laddr, f_lkey] = provider().get_laddr_info(fetch); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_fetch_add_nbi( + f_laddr, f_lkey, d_raddr, d_rkey, value, wf_info, post_options); + } + template + __device__ void atomic_fetch_add_nbi_single(uint64_t *fetch, void *dest, uint64_t value, + PostOpt post_options = {}) { + auto [f_laddr, f_lkey] = provider().get_laddr_info(fetch); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_fetch_add_nbi_single( + f_laddr, f_lkey, d_raddr, d_rkey, value, post_options); + } +#endif + + /** + * @brief Create and enqueue a non-fetching atomic add work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + */ + template + __device__ void atomic_add(void *dest, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_add(d_raddr, d_rkey, value, wf_info, post_options); + } + template + __device__ void atomic_add_single(void *dest, uint64_t value, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_add_single(d_raddr, d_rkey, value, post_options); + } + + /** + * @brief Create and enqueue a blocking atomic compare-and-swap work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] cond Used in atomic comparisons. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ uint64_t atomic_compare_swap(void *dest, uint64_t cond, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + return provider().atomic_compare_swap(d_raddr, d_rkey, cond, value, wf_info, post_options); + } + template + __device__ uint64_t atomic_compare_swap_single(void *dest, uint64_t cond, uint64_t value, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + return provider().atomic_compare_swap_single(d_raddr, d_rkey, cond, value, post_options); + } + +#if 0 // NON-BLOCKING AMOs NOT YET IMPLEMENTED + /** + * @brief Create and enqueue a non-blocking atomic compare-and-swap work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] fetch Symmetric address for fetched value. + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] cond Used in atomic comparisons. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * @return An atomic value. + */ + template + __device__ void atomic_compare_swap_nbi(uint64_t *fetch, void *dest, + uint64_t cond, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [f_laddr, f_lkey] = provider().get_laddr_info(fetch); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_compare_swap_nbi( + f_laddr, f_lkey, d_raddr, d_rkey, cond, value, wf_info, post_options); + } + template + __device__ void atomic_compare_swap_nbi_single(uint64_t *fetch, void *dest, + uint64_t cond, uint64_t value, + PostOpt post_options = {}) { + auto [f_laddr, f_lkey] = provider().get_laddr_info(fetch); + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + provider().atomic_compare_swap_nbi_single( + f_laddr, f_lkey, d_raddr, d_rkey, cond, value, post_options); + } +#endif + + /** + * @brief Create and enqueue a non-fetching atomic compare-and-swap work queue entry (WQE). + * + * All address arguments are symmetric addresses in the default or user symmetric heaps. + * Virtual addresses, LKeys, and RKeys are resolved internally. + * + * @param[in] dest Destination symmetric address for data transmission. + * @param[in] cond Used in atomic comparisons. + * @param[in] value Data value for the atomic operation. + * @param[in] wf_info Wavefront information. + * + * @tparam PostOptions Options to use when posting these WQEs. + * + * + * @return An atomic value. + */ + template + __device__ void atomic_compare_swap_nofetch(void *dest, uint64_t cond, uint64_t value, + const ActiveWFInfo& wf_info, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + /* QueuePairInterface doesn't provide atomic_compare_swap_nofetch */ + provider().template post_wqe_amo( + d_raddr, d_rkey, value, cond, wf_info, post_options); + } + template + __device__ void atomic_compare_swap_nofetch_single(void *dest, uint64_t cond, uint64_t value, + PostOpt post_options = {}) { + auto [d_raddr, d_rkey] = provider().get_raddr_info(dest); + /* QueuePairInterface doesn't provide atomic_compare_swap_nofetch_single */ + provider().template post_wqe_amo_single( + d_raddr, d_rkey, value, cond, post_options); + } +/**@}*/ +}; + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_SHMEM_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair_mux.hpp b/projects/rocshmem/src/gda/queue_pair_mux.hpp new file mode 100644 index 000000000000..2d016a64af77 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair_mux.hpp @@ -0,0 +1,478 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_MUX_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_MUX_HPP_ + +#include +#include +#include +#include +#include + +#include + +#include "rocshmem/rocshmem_config.h" // NOLINT(build/include_subdir) + +#include "constmem.hpp" +#include "gda_enums.hpp" +#include "log.hpp" +#include "util.hpp" + +#include "gda/queue_pair/queue_pair_interface.hpp" +#include "gda/queue_pair/queue_pair_shmem.hpp" + +#if defined(GDA_IONIC) +#include "gda/ionic/queue_pair_ionic.hpp" +#endif +#if defined(GDA_BNXT) +#include "gda/bnxt/queue_pair_bnxt.hpp" +#endif +#if defined(GDA_MLX5) +#include "gda/mlx5/queue_pair_mlx5.hpp" +#endif + +namespace rocshmem { + +class QueuePairMux; + +template <> struct QueuePairTraits { + /** + * @brief Mux OpCode enumeration + */ + enum class OpCode { + RDMA_WRITE, + RDMA_READ, + ATOMIC_CS, + ATOMIC_FA, + }; + + static constexpr size_t InlineMax = + std::min({ +#if defined(GDA_IONIC) + QueuePairTraits::InlineMax, +#endif +#if defined(GDA_BNXT) + QueuePairTraits::InlineMax, +#endif +#if defined(GDA_MLX5) + QueuePairTraits::InlineMax, +#endif + }); + + static constexpr size_t InlineThreshold = + std::min({ +#if defined(GDA_IONIC) + QueuePairTraits::InlineThreshold, +#endif +#if defined(GDA_BNXT) + QueuePairTraits::InlineThreshold, +#endif +#if defined(GDA_MLX5) + QueuePairTraits::InlineThreshold, +#endif + }); +}; + +class QueuePairMux : public QueuePairSHMEM { +private: + union QueuePairUnion { +#if defined(GDA_IONIC) + QueuePairIONIC ionic; +#endif +#if defined(GDA_BNXT) + QueuePairBNXT bnxt; +#endif +#if defined(GDA_MLX5) + QueuePairMLX5 mlx5; +#endif + }; + static_assert(std::is_aggregate_v, + "QueuePairUnion must be an aggregate type."); + static_assert(std::is_trivially_destructible_v, + "QueuePairUnion must have a trivial destructor."); + + QueuePairUnion qp; + + /** + * @brief Underlying GDA Provider. + * Only used by __host__ code, __device__ code should use constmem.gda_provider instead. + */ + static inline GDAProvider provider{GDAProvider::UNSET}; + +public: +#if defined(GDA_IONIC) + __host__ QueuePairMux(QueuePairIONIC&& ionic) + : qp{.ionic = std::move(ionic)} { provider = GDAProvider::IONIC; } +#endif +#if defined(GDA_BNXT) + __host__ QueuePairMux(QueuePairBNXT&& bnxt) + : qp{.bnxt = std::move(bnxt)} { provider = GDAProvider::BNXT; } +#endif +#if defined(GDA_MLX5) + __host__ QueuePairMux(QueuePairMLX5&& mlx5) + : qp{.mlx5 = std::move(mlx5)} { provider = GDAProvider::MLX5; } +#endif + + __host__ QueuePairMux(const QueuePairMux& other) = delete; + __host__ QueuePairMux& operator=(const QueuePairMux& other) = delete; + __host__ QueuePairMux(QueuePairMux&& other) noexcept = default; + __host__ QueuePairMux& operator=(QueuePairMux&& other) = default; + __host__ ~QueuePairMux() = default; + +public: + template + __device__ __forceinline__ + void post_wqe_rma(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __forceinline__ + void post_wqe_rma_single(uintptr_t laddr, uint32_t lkey, + uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt = {}); + + template + __device__ __forceinline__ + amo_ret_t post_wqe_amo(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt = {}); + + template + __device__ __forceinline__ + amo_ret_t post_wqe_amo_single(uintptr_t raddr, uint32_t rkey, + uint64_t swap_add, uint64_t compare, + PostOpt = {}); + + __device__ __forceinline__ void quiet_single(); + + + /** + * @brief Resolve the local (origin) virtual address and LKey of a symmetric address. + * + * Attempts to resolve the address in the following order of locations: + * 1. The default symmetric heap. + * 2. All user-registered local buffers. + * 3. All user-registered symmetric buffers. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {laddr, lkey} for addr or {0, std::numeric_limits::max()} if not found. + * Endianness of returned LKey value is ProviderEndianness. + */ + __device__ __forceinline__ + std::tuple get_laddr_info(const void *addr, bool inlined = false) const; + + /** + * @brief Resolve the remote (target) virtual address and RKey of a symmetric address. + * + * Attempts to resolve the address in the following order of locations: + * 1. The default symmetric heap. + * 2. All user-registered symmetric buffers. + * + * @param[in] addr Symmetric address to resolve. + * + * @return {raddr, rkey} for addr or {0, std::numeric_limits::max()} if not found. + * Endianness of returned RKey value is ProviderEndianness. + */ + __device__ __forceinline__ + std::tuple get_raddr_info(const void *addr) const; + + /* + * @brief Query whether data can be inlined into a WQE. + * + * @param[in] size Size of data carried by the WQE. + * @tparam Op OpCode for this WQE. + * + * @return True if size bytes of data can be inlined into a WQE with OpCode Op, else false. + */ + template + static __host__ __device__ __forceinline__ constexpr bool can_inline(size_t size); + + /** + * @brief Convert value to Endianness of selected provider, byteswapping if necessary. + * + * @param[in] val Value to convert. + * @tparam T Type of val. + * + * @return QueuePairProvider::to_provider_endianness(val) + */ + template + static __host__ __device__ __forceinline__ T to_provider_endianness(T val); + +private: + static __host__ __forceinline__ GDAProvider get_provider() { return provider; } + static __device__ __forceinline__ GDAProvider get_provider() { return constmem.gda_provider; } + + [[noreturn]] static __host__ __forceinline__ void invalid_provider() { + static_assert(std::is_same_v, int>); + LOG_ERROR_ABORT("Invalid GDAProvider (%d)", static_cast(get_provider())); + } + + [[noreturn]] static __device__ __forceinline__ void invalid_provider() { + assert(false /* invalid GDAProvider */); + __builtin_unreachable(); + } + + /* + * @brief Convert from QueuePairMux::OpCode to the equivalent Provider::OpCode + */ + template + static __host__ __device__ constexpr typename Provider::OpCode provider_op(); +}; + + + +template +__host__ __device__ constexpr typename Provider::OpCode QueuePairMux::provider_op() { + if constexpr (Op == OpCode::RDMA_WRITE) { + return Provider::OpCode::RDMA_WRITE; + } else if constexpr (Op == OpCode::RDMA_READ) { + return Provider::OpCode::RDMA_READ; + } else if constexpr (Op == OpCode::ATOMIC_FA) { + return Provider::OpCode::ATOMIC_FA; + } else if constexpr (Op == OpCode::ATOMIC_CS) { + return Provider::OpCode::ATOMIC_CS; + } else { + // might need some tricks if CWG2518 / P2593R1 isn't implemented by a supported compiler + static_assert(false, "Invalid or unimplemented OpCode"); + } +} + +template +__device__ __forceinline__ void QueuePairMux::post_wqe_rma( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + const ActiveWFInfo& wf_info, PostOpt options) { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.post_wqe_rma()>( + laddr, lkey, raddr, rkey, size, wf_info, options); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.post_wqe_rma()>( + laddr, lkey, raddr, rkey, size, wf_info, options); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.post_wqe_rma()>( + laddr, lkey, raddr, rkey, size, wf_info, options); +#endif + default: + invalid_provider(); + } +} + +template +__device__ __forceinline__ void QueuePairMux::post_wqe_rma_single( + uintptr_t laddr, uint32_t lkey, uintptr_t raddr, uint32_t rkey, size_t size, + PostOpt options) { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.post_wqe_rma_single()>( + laddr, lkey, raddr, rkey, size, options); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.post_wqe_rma_single()>( + laddr, lkey, raddr, rkey, size, options); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.post_wqe_rma_single()>( + laddr, lkey, raddr, rkey, size, options); +#endif + default: + invalid_provider(); + } +} + +template +__device__ __forceinline__ QueuePairMux::amo_ret_t QueuePairMux::post_wqe_amo( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + const ActiveWFInfo& wf_info, PostOpt options) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.post_wqe_amo(), Fetch>( + raddr, rkey, swap_add, compare, wf_info, options); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.post_wqe_amo(), Fetch>( + raddr, rkey, swap_add, compare, wf_info, options); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.post_wqe_amo(), Fetch>( + raddr, rkey, swap_add, compare, wf_info, options); +#endif + default: + invalid_provider(); + } +} + +template +__device__ __forceinline__ QueuePairMux::amo_ret_t QueuePairMux::post_wqe_amo_single( + uintptr_t raddr, uint32_t rkey, uint64_t swap_add, uint64_t compare, + PostOpt options) { + static_assert(Fetch != AMOFetchType::NonBlocking, "non-blocking AMOs not yet implemented"); + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.post_wqe_amo_single(), Fetch>( + raddr, rkey, swap_add, compare, options); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.post_wqe_amo_single(), Fetch>( + raddr, rkey, swap_add, compare, options); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.post_wqe_amo_single(), Fetch>( + raddr, rkey, swap_add, compare, options); +#endif + default: + invalid_provider(); + } +} + +__device__ __forceinline__ void QueuePairMux::quiet_single() { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.quiet_single(); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.quiet_single(); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.quiet_single(); +#endif + default: + invalid_provider(); + } +} + +__device__ __forceinline__ std::tuple +QueuePairMux::get_laddr_info(const void *addr, bool inlined) const { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.get_laddr_info(addr, inlined); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.get_laddr_info(addr, inlined); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.get_laddr_info(addr, inlined); +#endif + default: + invalid_provider(); + } +} + +__device__ __forceinline__ +std::tuple QueuePairMux::get_raddr_info(const void *addr) const { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return qp.ionic.get_raddr_info(addr); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return qp.bnxt.get_raddr_info(addr); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return qp.mlx5.get_raddr_info(addr); +#endif + default: + invalid_provider(); + } +} + +template +__host__ __device__ __forceinline__ constexpr bool QueuePairMux::can_inline(size_t size) { + /* + * We don't know which GDA provider will be selected at runtime, + * so we define QueuePairTraits::InlineThreshold + * as the minimum InlineThreshold of the available GDA providers. + * + * This is a sufficient lower bound that can be used at compile-time + * to e.g. allocate a buffer that can be inlined + * regardless of which provider is selected at runtime. + * + * At runtime, however, we can do better: + * once we know what provider was selected, + * we can query QueuePairProvider::can_inline directly. + * + * To enable this behavior, QueuePairMux::can_inline must detect + * whether it was called from within a manifestly constant-evaluated context. + * This capability is not available in standard C++17. + * In C++20, this can be done using std::is_constant_evaluated. + * In C++23, this can be done using consteval-if. + * + * In C++17, we can backport the C++20 behavior using Clang's __builtin_is_constant_evaluated(). + * Once we enable C++20/C++23, move to standards-compliant behavior. + */ + if (__builtin_is_constant_evaluated()) { + // In a constant-evaluated context use the minimum InlineThreshold. + if constexpr (Op == OpCode::RDMA_WRITE) { + return size <= Traits::InlineThreshold; + } else { + return false; + } + } else { + // At runtime, dispatch to the selected GDA provider. + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return QueuePairIONIC::can_inline()>(size); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return QueuePairBNXT::can_inline()>(size); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return QueuePairMLX5::can_inline()>(size); +#endif + default: + invalid_provider(); + } + } +} + +template +__host__ __device__ __forceinline__ T QueuePairMux::to_provider_endianness(T val) { + switch (get_provider()) { +#if defined(GDA_IONIC) + case GDAProvider::IONIC: + return QueuePairIONIC::to_provider_endianness(val); +#endif +#if defined(GDA_BNXT) + case GDAProvider::BNXT: + return QueuePairBNXT::to_provider_endianness(val); +#endif +#if defined(GDA_MLX5) + case GDAProvider::MLX5: + return QueuePairMLX5::to_provider_endianness(val); +#endif + default: + invalid_provider(); + } +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_MUX_HPP_ diff --git a/projects/rocshmem/src/gda/queue_pair_provider.hpp b/projects/rocshmem/src/gda/queue_pair_provider.hpp new file mode 100644 index 000000000000..e7bb99cbc199 --- /dev/null +++ b/projects/rocshmem/src/gda/queue_pair_provider.hpp @@ -0,0 +1,29 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#ifndef LIBRARY_SRC_GDA_QUEUE_PAIR_PROVIDER_HPP_ +#define LIBRARY_SRC_GDA_QUEUE_PAIR_PROVIDER_HPP_ + +#include "rocshmem/rocshmem_config.h" // NOLINT(build/include_subdir) + +/* define GDA_QUEUEPAIR_MOCK prior to including queue_pair_provider.hpp + * to use mock QueuePair objects */ + +#if defined(GDA_QUEUEPAIR_MOCK) +#include "queue_pair/queue_pair_mock.hpp" +namespace rocshmem { using QueuePair = QueuePairMock; } +#elif defined(GDA_MUX) +#include "queue_pair_mux.hpp" +namespace rocshmem { using QueuePair = QueuePairMux; } +#elif defined(GDA_IONIC) +#include "gda/ionic/queue_pair_ionic.hpp" +namespace rocshmem { using QueuePair = QueuePairIONIC; } +#elif defined(GDA_BNXT) +#include "gda/bnxt/queue_pair_bnxt.hpp" +namespace rocshmem { using QueuePair = QueuePairBNXT; } +#elif defined(GDA_MLX5) +#include "gda/mlx5/queue_pair_mlx5.hpp" +namespace rocshmem { using QueuePair = QueuePairMLX5; } +#endif + +#endif // LIBRARY_SRC_GDA_QUEUE_PAIR_PROVIDER_HPP_ diff --git a/projects/rocshmem/src/gda/topology.cpp b/projects/rocshmem/src/gda/topology.cpp index 864b1f7d61ea..9d63329d4d5d 100644 --- a/projects/rocshmem/src/gda/topology.cpp +++ b/projects/rocshmem/src/gda/topology.cpp @@ -26,11 +26,64 @@ #include "topology.hpp" #include "ibv_wrapper.hpp" +#include #include #include using namespace rocshmem; +#define ERR_CHECK(cmd) \ + do { \ + int error = cmd; \ + if (error != 0) { \ + fprintf(stderr, "error: %d at %s:%d\n", error, __FILE__, __LINE__); \ + exit(EXIT_FAILURE); \ + } \ +} while (0) + +// Helper macros for calling RDMA functions and reporting errors +#ifdef VERBS_DEBUG +#define IBV_CALL(__func__, ...) \ + do { \ + int error = __func__(__VA_ARGS__); \ + if (error != 0) { \ + fprintf(stderr,"Encountered IbVerbs error (%d) at line (%d) " \ + "and function (%s)", (error), __LINE__, #__func__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define IBV_PTR_CALL(__ptr__, __func__, ...) \ + do { \ + __ptr__ = __func__(__VA_ARGS__); \ + if (__ptr__ == nullptr) { \ + fprintf(stderr, "Encountered IbVerbs nullptr error at line (%d) " \ + "and function (%s)", __LINE__, #__func__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) +#else +#define IBV_CALL(__func__, ...) \ + do { \ + int error = __func__(__VA_ARGS__); \ + if (error != 0) { \ + fprintf(stderr, "Encountered IbVerbs error (%d) in func (%s) " \ + , error, #__func__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define IBV_PTR_CALL(__ptr__, __func__, ...) \ + do { \ + __ptr__ = __func__(__VA_ARGS__); \ + if (__ptr__ == nullptr) { \ + fprintf(stderr, "Encountered IbVerbs nullptr error in func (%s) ", \ + #__func__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) +#endif + namespace rocshmem { @@ -78,7 +131,7 @@ namespace rocshmem mistakeCount, numPages, targetId); return -1; } - return ROCSHMEM_SUCCESS; + return 0; } // Allocate memory @@ -124,7 +177,7 @@ namespace rocshmem LOG_ERROR("Unsupported memory type (%d)", memType); return -1; } - return ROCSHMEM_SUCCESS; + return 0; } // Deallocate memory @@ -151,7 +204,7 @@ namespace rocshmem LOG_ERROR("Attempting to deallocate unrecognized memory type (%d)", memType); return -1; } - return ROCSHMEM_SUCCESS; + return 0; } @@ -215,7 +268,7 @@ namespace rocshmem exeDevice.exeType); return -1; } - return ROCSHMEM_SUCCESS; + return 0; } // Get the hsa_agent_t associated with a MemDevice @@ -286,7 +339,7 @@ namespace rocshmem version = 2; } } - return ROCSHMEM_SUCCESS; + return 0; } static bool IsIPv4MappedIPv6(const union ibv_gid &gid) @@ -309,7 +362,7 @@ namespace rocshmem int const& portNum, std::pair& gidInfo) { - if(gidInfo.first >= 0) return ROCSHMEM_SUCCESS; // honor user choice + if(gidInfo.first >= 0) return 0; // honor user choice union ibv_gid gid; GidPriority highestPriority = GidPriority::UNKNOWN; @@ -319,7 +372,7 @@ namespace rocshmem IBV_CALL(ibv.query_gid, context, portNum, i, &gid); if (!IsConfiguredGid(gid)) continue; int gidCurrRoceVersion; - if(GetRoceVersionNumber(context, portNum, i, gidCurrRoceVersion) != ROCSHMEM_SUCCESS) continue; + if(GetRoceVersionNumber(context, portNum, i, gidCurrRoceVersion) != 0) continue; GidPriority currPriority; if (IsIPv4MappedIPv6(gid)) { currPriority = (gidCurrRoceVersion == 2) ? GidPriority::ROCEV2_IPV4 : GidPriority::ROCEV1_IPV4; @@ -341,7 +394,7 @@ namespace rocshmem } gidInfo.first = gidIndex; gidInfo.second = GidPriorityStr[highestPriority]; - return ROCSHMEM_SUCCESS; + return 0; } vector const& GetIbvDeviceList() @@ -387,7 +440,7 @@ namespace rocshmem ibvDevice.isRoce = true; std::pair gidInfo (-1, ""); auto res = GetGidIndex(context, portAttr.gid_tbl_len, activePort, gidInfo); - if (res == ROCSHMEM_SUCCESS) { + if (res == 0) { ibvDevice.gidIndex = gidInfo.first; ibvDevice.gidDescriptor = gidInfo.second; } @@ -587,7 +640,7 @@ namespace rocshmem } currNode->description = description; - return ROCSHMEM_SUCCESS; + return 0; } // Returns root node for PCIe tree. Constructed on first use diff --git a/projects/rocshmem/src/gda/topology.hpp b/projects/rocshmem/src/gda/topology.hpp index 1d842497bb00..0a6b1e9cd01b 100644 --- a/projects/rocshmem/src/gda/topology.hpp +++ b/projects/rocshmem/src/gda/topology.hpp @@ -327,59 +327,3 @@ namespace rocshmem //========================================================================================== // End of rocshmem API //========================================================================================== - -// Error check macros -#define ROCSHMEM_SUCCESS 0 - -#define ERR_CHECK(cmd) \ - do { \ - int error = cmd; \ - if (error != 0) { \ - fprintf(stderr, "error: %d at %s:%d\n", error, __FILE__, __LINE__); \ - exit(EXIT_FAILURE); \ - } \ -} while (0) - -// Helper macros for calling RDMA functions and reporting errors -#ifdef VERBS_DEBUG -#define IBV_CALL(__func__, ...) \ - do { \ - int error = __func__(__VA_ARGS__); \ - if (error != 0) { \ - fprintf(stderr,"Encountered IbVerbs error (%d) at line (%d) " \ - "and function (%s)", (error), __LINE__, #__func__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) - -#define IBV_PTR_CALL(__ptr__, __func__, ...) \ - do { \ - __ptr__ = __func__(__VA_ARGS__); \ - if (__ptr__ == nullptr) { \ - fprintf(stderr, "Encountered IbVerbs nullptr error at line (%d) " \ - "and function (%s)", __LINE__, #__func__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) -#else -#define IBV_CALL(__func__, ...) \ - do { \ - int error = __func__(__VA_ARGS__); \ - if (error != 0) { \ - fprintf(stderr, "Encountered IbVerbs error (%d) in func (%s) " \ - , error, #__func__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) - -#define IBV_PTR_CALL(__ptr__, __func__, ...) \ - do { \ - __ptr__ = __func__(__VA_ARGS__); \ - if (__ptr__ == nullptr) { \ - fprintf(stderr, "Encountered IbVerbs nullptr error in func (%s) ", \ - #__func__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) -#endif - diff --git a/projects/rocshmem/tests/functional_tests/qp_ping_pong_tester.cpp b/projects/rocshmem/tests/functional_tests/qp_ping_pong_tester.cpp index 5e725eb3a360..f01beb0908ce 100644 --- a/projects/rocshmem/tests/functional_tests/qp_ping_pong_tester.cpp +++ b/projects/rocshmem/tests/functional_tests/qp_ping_pong_tester.cpp @@ -26,7 +26,7 @@ #include #include "gda/context_gda_device.hpp" -#include "gda/queue_pair.hpp" +#include "gda/queue_pair_provider.hpp" #include "assembly.hpp" #include "verify_results_kernels.hpp" @@ -60,26 +60,11 @@ __global__ void QpPingPongTest(int loop, int skip, long long int *start_time, QueuePair &qp = gda_ctx->qps[target]; - uintptr_t local_base = - reinterpret_cast(gda_ctx->base_heap[pe]); - uintptr_t remote_base = - reinterpret_cast(gda_ctx->base_heap[target]); - int wg_id = hipBlockIdx_x; - uintptr_t r_buf_offset = - reinterpret_cast(&r_buf[wg_id]) - local_base; - void *remote_r_buf = reinterpret_cast(remote_base + r_buf_offset); - - char *my_data_s = data_s_buf + size * wg_id; - uintptr_t data_r_offset = - reinterpret_cast(data_r_buf + size * wg_id) - local_base; - void *remote_data_r = reinterpret_cast(remote_base + data_r_offset); - - uint64_t *my_sig = &sig_addr[wg_id]; - uintptr_t sig_offset = - reinterpret_cast(my_sig) - local_base; - void *remote_sig = reinterpret_cast(remote_base + sig_offset); + void *data_s = &data_s_buf[wg_id * size]; + void *data_r = &data_r_buf[wg_id * size]; + uint64_t *sig = &sig_addr[wg_id]; // Drain all setup loads from HBM before entering the timed loop. #if defined(__GFX12__) @@ -96,28 +81,27 @@ __global__ void QpPingPongTest(int loop, int skip, long long int *start_time, int val = i + 1; if (op_type <= 1) { - void *src = (op_type == 0) ? static_cast(&val) - : static_cast(my_data_s); + void *src = (op_type == 0) ? static_cast(&val) : data_s; if (op_type == 1) { - *reinterpret_cast(my_data_s) = val; + *reinterpret_cast(data_s) = val; } if (pe == 0) { - qp.put_nbi_single(remote_r_buf, src, sizeof(int), true); + qp.put_nbi_single(&r_buf[wg_id], src, sizeof(int)); while (uncached_load(&r_buf[wg_id]) != val) {} } else { while (uncached_load(&r_buf[wg_id]) != val) {} - qp.put_nbi_single(remote_r_buf, src, sizeof(int), true); + qp.put_nbi_single(&r_buf[wg_id], src, sizeof(int)); } } else { uint64_t expected = static_cast(i + 1); if (pe == 0) { - qp.put_nbi_single(remote_data_r, my_data_s, size, false); - qp.atomic_add_single(remote_sig, 1); - while (uncached_load(my_sig) < expected) {} + qp.put_nbi_single(data_r, data_s, size, PostOpt{RingDB}); + qp.atomic_add_single(sig, 1, PostOpt{RingDB}); + while (uncached_load(sig) < expected) {} } else { - while (uncached_load(my_sig) < expected) {} - qp.put_nbi_single(remote_data_r, my_data_s, size, false); - qp.atomic_add_single(remote_sig, 1); + while (uncached_load(sig) < expected) {} + qp.put_nbi_single(data_r, data_s, size, PostOpt{RingDB}); + qp.atomic_add_single(sig, 1, PostOpt{RingDB}); } } } diff --git a/projects/rocshmem/tests/functional_tests/qp_put_nbi_tester.cpp b/projects/rocshmem/tests/functional_tests/qp_put_nbi_tester.cpp index 1c405411d593..b76d6f05f52c 100644 --- a/projects/rocshmem/tests/functional_tests/qp_put_nbi_tester.cpp +++ b/projects/rocshmem/tests/functional_tests/qp_put_nbi_tester.cpp @@ -26,7 +26,7 @@ #include #include "gda/context_gda_device.hpp" -#include "gda/queue_pair.hpp" +#include "gda/queue_pair_provider.hpp" #include "verify_results_kernels.hpp" using namespace rocshmem; @@ -51,11 +51,6 @@ __global__ void QpPutNbiTest(int loop, int skip, long long int *start_time, QueuePair &qp = gda_ctx->qps[target]; - uintptr_t local_base = - reinterpret_cast(gda_ctx->base_heap[pe]); - uintptr_t remote_base = - reinterpret_cast(gda_ctx->base_heap[target]); - int wg_id = hipBlockIdx_x; int start_slot = (batch - (skip % batch)) % batch; @@ -76,11 +71,7 @@ __global__ void QpPutNbiTest(int loop, int skip, long long int *start_time, } } - uintptr_t d_offset = - reinterpret_cast(dest + size * slot) - local_base; - void *remote_addr = reinterpret_cast(remote_base + d_offset); - - qp.put_nbi_single(remote_addr, source, size, true); + qp.put_nbi_single(&dest[slot * size], source, size); } qp.quiet_single();