From 40b8bdd554fa6d8294789e5c56acfb2a9bcacb42 Mon Sep 17 00:00:00 2001 From: tvegas1 Date: Tue, 6 Oct 2026 21:56:45 +0300 Subject: [PATCH 1/9] UCP/UCT/CUDA: Avoid gdr_copy for non-pinnable memory gdr_copy pinning fails with EINVAL for memory localized to a GPU locality domain and for stream-ordered allocations, because nvidia_p2p_get_pages refuses memory the driver did not place as GDR-capable. Registrability cannot discriminate them, as an exportable memory pool is registrable. Add an orthogonal UCS_MEM_FLAG_PEER_MEM_PINNABLE flag, cleared by cuda_copy for both cases and required by gdr_copy. Select the mem type endpoint RMA lane by the buffer memory flags, instead of always taking the first lane. The datatype iterator already carries the flags, so the proto v2 path adds no memory type query. The remaining callers pass NULL, and the flags are resolved only if a memory domain requires them. --- config/m4/cuda.m4 | 3 + src/ucp/core/ucp_device.c | 8 +- src/ucp/dt/datatype_iter.inl | 8 +- src/ucp/dt/dt.c | 102 ++++++++++++++--- src/ucp/dt/dt.h | 11 +- src/ucp/dt/dt_contig.h | 10 +- src/ucp/dt/dt_iov.c | 10 +- src/ucp/dt/dt_iov.h | 6 +- src/ucp/rma/amo_offload.c | 10 +- src/ucp/rma/amo_sw.c | 9 +- src/ucp/rma/rma_sw.c | 4 +- src/ucp/tag/offload.c | 3 +- src/ucp/tag/tag_match.inl | 3 +- src/ucs/memory/memory_type.h | 9 +- src/uct/cuda/cuda_copy/cuda_copy_md.c | 49 +++++++- src/uct/cuda/gdr_copy/gdr_copy_md.c | 14 ++- test/gtest/ucp/test_ucp_mem_type.cc | 93 +++++++++++++++ test/gtest/uct/cuda/cuda_vmm_mem_buffer.h | 67 +++++++++++ .../gtest/uct/cuda/test_switch_cuda_device.cc | 106 +++++++++++++++++- test/gtest/uct/test_md.cc | 42 ++++++- 20 files changed, 501 insertions(+), 66 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index 553bd212d89..ecaa07c8c5a 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -255,6 +255,9 @@ AS_IF([test "x$cuda_checked" != "xyes"], AC_CHECK_DECLS([CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN], [], [], [[#include ]]) + AC_CHECK_DECLS([CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE], + [], [], [[#include ]]) + CPPFLAGS="$save_CPPFLAGS" LDFLAGS="$save_LDFLAGS" LIBS="$save_LIBS" diff --git a/src/ucp/core/ucp_device.c b/src/ucp/core/ucp_device.c index ec9d086cbc5..e4dbd285a69 100644 --- a/src/ucp/core/ucp_device.c +++ b/src/ucp/core/ucp_device.c @@ -285,7 +285,8 @@ static ucs_status_t ucp_device_mem_list_export_handle( return status; } - ucp_mem_type_unpack(worker, mem->address, handle, handle_size, mem_type); + ucp_mem_type_unpack(worker, mem->address, handle, handle_size, mem_type, + NULL); return UCS_OK; } @@ -901,7 +902,7 @@ ucs_status_t ucp_device_counter_init(ucp_worker_h worker, mem_type = ucp_device_counter_mem_type(worker->context, counter_ptr, params); ucp_dt_contig_unpack(worker, counter_ptr, &counter_value, - sizeof(counter_value), mem_type, + sizeof(counter_value), mem_type, NULL, sizeof(counter_value)); return UCS_OK; } @@ -916,6 +917,7 @@ uint64_t ucp_device_counter_read(ucp_worker_h worker, mem_type = ucp_device_counter_mem_type(worker->context, counter_ptr, params); ucp_dt_contig_pack(worker, &counter_value, counter_ptr, - sizeof(counter_value), mem_type, sizeof(counter_value)); + sizeof(counter_value), mem_type, NULL, + sizeof(counter_value)); return counter_value; } diff --git a/src/ucp/dt/datatype_iter.inl b/src/ucp/dt/datatype_iter.inl index de8746bca48..4a1cc181bde 100644 --- a/src/ucp/dt/datatype_iter.inl +++ b/src/ucp/dt/datatype_iter.inl @@ -415,7 +415,7 @@ ucp_datatype_iter_next_pack(const ucp_datatype_iter_t *dt_iter, dt_iter->offset); ucp_dt_contig_pack(worker, dest, src, length, (ucs_memory_type_t)dt_iter->mem_info.type, - dt_iter->length); + &dt_iter->mem_info, dt_iter->length); break; case UCP_DATATYPE_IOV: ucp_datatype_iter_iov_check(dt_iter); @@ -427,7 +427,7 @@ ucp_datatype_iter_next_pack(const ucp_datatype_iter_t *dt_iter, &next_iter->type.iov.iov_offset, &next_iter->type.iov.iov_index, (ucs_memory_type_t)dt_iter->mem_info.type, - dt_iter->length); + &dt_iter->mem_info, dt_iter->length); break; case UCP_DATATYPE_GENERIC: if (max_length != 0) { @@ -482,7 +482,7 @@ ucp_datatype_iter_unpack(ucp_datatype_iter_t *dt_iter, ucp_worker_h worker, dest = UCS_PTR_BYTE_OFFSET(dt_iter->type.contig.buffer, offset); ucp_dt_contig_unpack(worker, dest, src, length, (ucs_memory_type_t)dt_iter->mem_info.type, - dt_iter->length); + &dt_iter->mem_info, dt_iter->length); status = UCS_OK; break; case UCP_DATATYPE_IOV: @@ -493,7 +493,7 @@ ucp_datatype_iter_unpack(ucp_datatype_iter_t *dt_iter, ucp_worker_h worker, &dt_iter->type.iov.iov_offset, &dt_iter->type.iov.iov_index, (ucs_memory_type_t)dt_iter->mem_info.type, - dt_iter->length); + &dt_iter->mem_info, dt_iter->length); ucs_assert(unpacked_length <= length); dt_iter->offset += unpacked_length; status = UCS_OK; diff --git a/src/ucp/dt/dt.c b/src/ucp/dt/dt.c index 9e19724b32b..9d4d2a80af4 100644 --- a/src/ucp/dt/dt.c +++ b/src/ucp/dt/dt.c @@ -60,14 +60,83 @@ ucs_status_t ucp_dt_mem_info_verify(const char *dt_name, size_t index, } +/* + * Select the first RMA lane of the mem type endpoint whose memory domain + * accepts the buffer. @a mem_info holds the memory flags resolved by the + * caller, or is NULL when they are unknown, in which case they are resolved on + * the first memory domain which actually requires them. Returns UCP_NULL_LANE + * if no lane is compatible. + */ +static ucp_lane_index_t +ucp_mem_type_ep_lane(ucp_ep_h ep, void *address, size_t length, + const ucp_memory_info_t *mem_info) +{ + ucp_context_h context = ep->worker->context; + const ucp_ep_config_key_t *key = &ucp_ep_config(ep)->key; + const uct_md_attr_v2_t *md_attr; + ucp_memory_info_t detected; + ucp_lane_index_t lane; + unsigned i; + + for (i = 0; key->rma_lanes[i] != UCP_NULL_LANE; ++i) { + lane = key->rma_lanes[i]; + md_attr = &context->tl_mds[ucp_ep_md_index(ep, lane)].attr; + if (md_attr->required_mem_flags == 0) { + return lane; + } + + if (mem_info == NULL) { + /* Coverity wrongly resolves the memory domain deregister function + * pointer to 'uct_cuda_ipc_mem_dereg' and considers 'address' as + * freed */ + /* coverity[pass_freed_arg] */ + ucp_memory_detect(context, address, length, &detected); + mem_info = &detected; + } + + if (ucs_test_all_flags(mem_info->flags, + md_attr->required_mem_flags)) { + return lane; + } + } + + return UCP_NULL_LANE; +} + +/* + * Select the mem type endpoint lane which can register the buffer, and register + * it. @a mem_info holds the memory flags resolved by the caller, or is NULL + * when they are unknown. + */ +static ucs_status_t +ucp_mem_type_lane_reg(ucp_worker_h worker, ucp_ep_h ep, void *address, + size_t length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, + ucp_lane_index_t *lane_p, + ucp_mtype_pack_context_t *pack_context) +{ + ucp_lane_index_t lane = ucp_mem_type_ep_lane(ep, address, length, mem_info); + + if (lane == UCP_NULL_LANE) { + ucs_error("no mem type lane can register %s buffer %p length %zu", + ucs_memory_type_names[mem_type], address, length); + return UCS_ERR_UNSUPPORTED; + } + + *lane_p = lane; + return ucp_mem_type_reg_buffers(worker, address, length, mem_type, + ucp_ep_md_index(ep, lane), pack_context); +} + UCS_PROFILE_FUNC_VOID(ucp_mem_type_unpack, - (worker, buffer, recv_data, recv_length, mem_type), + (worker, buffer, recv_data, recv_length, mem_type, + mem_info), ucp_worker_h worker, void *buffer, const void *recv_data, - size_t recv_length, ucs_memory_type_t mem_type) + size_t recv_length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info) { ucp_ep_h ep = worker->mem_type_ep[mem_type]; ucp_lane_index_t lane; - unsigned md_index; ucs_status_t status; ucp_mtype_pack_context_t pack_context; @@ -75,11 +144,8 @@ UCS_PROFILE_FUNC_VOID(ucp_mem_type_unpack, return; } - lane = ucp_ep_config(ep)->key.rma_lanes[0]; - md_index = ucp_ep_md_index(ep, lane); - - status = ucp_mem_type_reg_buffers(worker, buffer, recv_length, mem_type, - md_index, &pack_context); + status = ucp_mem_type_lane_reg(worker, ep, buffer, recv_length, mem_type, + mem_info, &lane, &pack_context); if (status != UCS_OK) { ucs_fatal("failed to register buffer with mem type domain %s", ucs_memory_type_names[mem_type]); @@ -96,13 +162,13 @@ UCS_PROFILE_FUNC_VOID(ucp_mem_type_unpack, } UCS_PROFILE_FUNC_VOID(ucp_mem_type_pack, - (worker, dest, src, length, mem_type), + (worker, dest, src, length, mem_type, mem_info), ucp_worker_h worker, void *dest, const void *src, - size_t length, ucs_memory_type_t mem_type) + size_t length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info) { ucp_ep_h ep = worker->mem_type_ep[mem_type]; ucp_lane_index_t lane; - ucp_md_index_t md_index; ucs_status_t status; ucp_mtype_pack_context_t pack_context; @@ -110,11 +176,8 @@ UCS_PROFILE_FUNC_VOID(ucp_mem_type_pack, return; } - lane = ucp_ep_config(ep)->key.rma_lanes[0]; - md_index = ucp_ep_md_index(ep, lane); - - status = ucp_mem_type_reg_buffers(worker, (void *)src, length, mem_type, - md_index, &pack_context); + status = ucp_mem_type_lane_reg(worker, ep, (void*)src, length, mem_type, + mem_info, &lane, &pack_context); if (status != UCS_OK) { ucs_fatal("failed to register buffer with mem type domain %s", ucs_memory_type_names[mem_type]); @@ -141,18 +204,21 @@ size_t ucp_dt_pack(ucp_worker_h worker, ucp_datatype_t datatype, return length; } + /* This path is driven by 'ucp_dt_state_t', which unlike the datatype + * iterator does not carry the resolved memory flags of the buffer */ switch (datatype & UCP_DATATYPE_CLASS_MASK) { case UCP_DATATYPE_CONTIG: ucp_dt_contig_pack(worker, dest, UCS_PTR_BYTE_OFFSET(src, state->offset), - length, mem_type, length); + length, mem_type, NULL, length); result_len = length; break; case UCP_DATATYPE_IOV: UCS_PROFILE_CALL_VOID(ucp_dt_iov_gather, worker, dest, src, length, &state->dt.iov.iov_offset, - &state->dt.iov.iovcnt_offset, mem_type, length); + &state->dt.iov.iovcnt_offset, mem_type, NULL, + length); result_len = length; break; diff --git a/src/ucp/dt/dt.h b/src/ucp/dt/dt.h index edc38cf9d70..fd999d4bf0e 100644 --- a/src/ucp/dt/dt.h +++ b/src/ucp/dt/dt.h @@ -78,12 +78,19 @@ ucp_memory_info_equal(const ucp_memory_info_t *a, const ucp_memory_info_t *b) (a->flags == b->flags); } +/* + * Copy to/from memory which is not accessible from the CPU, through the memory + * type endpoint. @a mem_info holds the memory flags already resolved for the + * buffer, or is NULL when they are unknown and have to be looked up. + */ void ucp_mem_type_pack(ucp_worker_h worker, void *dest, const void *src, - size_t length, ucs_memory_type_t mem_type); + size_t length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info); void ucp_mem_type_unpack(ucp_worker_h worker, void *buffer, const void *recv_data, size_t recv_length, - ucs_memory_type_t mem_type); + ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info); static UCS_F_ALWAYS_INLINE void diff --git a/src/ucp/dt/dt_contig.h b/src/ucp/dt/dt_contig.h index 6c2f6061816..d7bff9f2921 100644 --- a/src/ucp/dt/dt_contig.h +++ b/src/ucp/dt/dt_contig.h @@ -38,24 +38,26 @@ static UCS_F_ALWAYS_INLINE size_t ucp_contig_dt_length(ucp_datatype_t datatype, static UCS_F_ALWAYS_INLINE void ucp_dt_contig_pack(ucp_worker_h worker, void *dest, const void *src, - size_t length, ucs_memory_type_t mem_type, size_t total_len) + size_t length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len) { if (ucs_likely(UCP_MEM_IS_ACCESSIBLE_FROM_CPU(mem_type))) { ucp_memcpy_pack(dest, src, length, total_len, "memcpy_pack"); } else { - ucp_mem_type_pack(worker, dest, src, length, mem_type); + ucp_mem_type_pack(worker, dest, src, length, mem_type, mem_info); } } static UCS_F_ALWAYS_INLINE void ucp_dt_contig_unpack(ucp_worker_h worker, void *dest, const void *src, - size_t length, ucs_memory_type_t mem_type, size_t total_len) + size_t length, ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len) { if (ucs_likely(UCP_MEM_IS_ACCESSIBLE_FROM_CPU(mem_type))) { ucp_memcpy_unpack(dest, src, length, total_len, "memcpy_unpack"); } else { - ucp_mem_type_unpack(worker, dest, src, length, mem_type); + ucp_mem_type_unpack(worker, dest, src, length, mem_type, mem_info); } } diff --git a/src/ucp/dt/dt_iov.c b/src/ucp/dt/dt_iov.c index b1df342a6b9..67201278f7a 100644 --- a/src/ucp/dt/dt_iov.c +++ b/src/ucp/dt/dt_iov.c @@ -25,7 +25,8 @@ void ucp_dt_iov_gather(ucp_worker_h worker, void *dest, const ucp_dt_iov_t *iov, size_t length, size_t *iov_offset, size_t *iovcnt_offset, - ucs_memory_type_t mem_type, size_t total_len) + ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len) { size_t length_it = 0; size_t item_len, item_reminder, item_len_to_copy; @@ -40,7 +41,7 @@ void ucp_dt_iov_gather(ucp_worker_h worker, void *dest, const ucp_dt_iov_t *iov, ucp_dt_contig_pack(worker, UCS_PTR_BYTE_OFFSET(dest, length_it), UCS_PTR_BYTE_OFFSET(iov[*iovcnt_offset].buffer, *iov_offset), - item_len_to_copy, mem_type, total_len); + item_len_to_copy, mem_type, mem_info, total_len); length_it += item_len_to_copy; ucs_assert(length_it <= length); @@ -56,7 +57,8 @@ void ucp_dt_iov_gather(ucp_worker_h worker, void *dest, const ucp_dt_iov_t *iov, size_t ucp_dt_iov_scatter(ucp_worker_h worker, const ucp_dt_iov_t *iov, size_t iovcnt, const void *src, size_t length, size_t *iov_offset, size_t *iovcnt_offset, - ucs_memory_type_t mem_type, size_t total_len) + ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len) { size_t length_it = 0; size_t item_len, item_len_to_copy; @@ -71,7 +73,7 @@ size_t ucp_dt_iov_scatter(ucp_worker_h worker, const ucp_dt_iov_t *iov, UCS_PTR_BYTE_OFFSET(iov[*iovcnt_offset].buffer, *iov_offset), UCS_PTR_BYTE_OFFSET(src, length_it), - item_len_to_copy, mem_type, total_len); + item_len_to_copy, mem_type, mem_info, total_len); length_it += item_len_to_copy; ucs_assert(length_it <= length); diff --git a/src/ucp/dt/dt_iov.h b/src/ucp/dt/dt_iov.h index 602b44dafa9..9920dd01071 100644 --- a/src/ucp/dt/dt_iov.h +++ b/src/ucp/dt/dt_iov.h @@ -55,7 +55,8 @@ static inline size_t ucp_dt_iov_length(const ucp_dt_iov_t *iov, size_t iovcnt) */ void ucp_dt_iov_gather(ucp_worker_h worker, void *dest, const ucp_dt_iov_t *iov, size_t length, size_t *iov_offset, size_t *iovcnt_offset, - ucs_memory_type_t mem_type, size_t total_len); + ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len); /** @@ -81,7 +82,8 @@ void ucp_dt_iov_gather(ucp_worker_h worker, void *dest, const ucp_dt_iov_t *iov, size_t ucp_dt_iov_scatter(ucp_worker_h worker, const ucp_dt_iov_t *iov, size_t iovcnt, const void *src, size_t length, size_t *iov_offset, size_t *iovcnt_offset, - ucs_memory_type_t mem_type, size_t total_len); + ucs_memory_type_t mem_type, + const ucp_memory_info_t *mem_info, size_t total_len); /** diff --git a/src/ucp/rma/amo_offload.c b/src/ucp/rma/amo_offload.c index e4e12e87268..b2f7af5979e 100644 --- a/src/ucp/rma/amo_offload.c +++ b/src/ucp/rma/amo_offload.c @@ -23,7 +23,7 @@ ucp_amo_memtype_unpack_reply_buffer(ucp_request_t *req) { ucp_dt_contig_unpack(req->send.ep->worker, req->send.amo.reply_buffer, &req->send.amo.result, req->send.state.dt_iter.length, - ucp_amo_request_reply_mem_type(req), + ucp_amo_request_reply_mem_type(req), NULL, req->send.state.dt_iter.length); } @@ -73,7 +73,10 @@ ucp_proto_amo_progress(uct_pending_req_t *self, ucp_operation_id_t op_id, UCS_MEMORY_TYPE_HOST; ucp_dt_contig_pack(ep->worker, &req->send.amo.value, req->send.state.dt_iter.type.contig.buffer, - op_size, mem_type, op_size); + op_size, mem_type, + is_memtype ? &req->send.state.dt_iter.mem_info : + NULL, + op_size); req->flags |= UCP_REQUEST_FLAG_PROTO_AMO_PACKED; } @@ -86,7 +89,8 @@ ucp_proto_amo_progress(uct_pending_req_t *self, ucp_operation_id_t op_id, if (op_id == UCP_OP_ID_AMO_CSWAP) { ucp_dt_contig_pack(ep->worker, &req->send.amo.result, req->send.amo.reply_buffer, op_size, - ucp_amo_request_reply_mem_type(req), op_size); + ucp_amo_request_reply_mem_type(req), NULL, + op_size); } status = ucp_ep_rma_handle_fence(ep, req, UCS_BIT(spriv->super.lane)); diff --git a/src/ucp/rma/amo_sw.c b/src/ucp/rma/amo_sw.c index 7e3b5b00b4a..7846987291d 100644 --- a/src/ucp/rma/amo_sw.c +++ b/src/ucp/rma/amo_sw.c @@ -39,10 +39,11 @@ static size_t ucp_amo_sw_pack(void *dest, ucp_request_t *req, int fetch, if (worker->context->config.ext.proto_enable) { ucp_dt_contig_pack(worker, atomich + 1, &req->send.amo.value, size, - UCS_MEMORY_TYPE_HOST, size); + UCS_MEMORY_TYPE_HOST, NULL, size); if (req->send.amo.uct_op == UCT_ATOMIC_OP_CSWAP) { - ucp_dt_contig_pack(worker, cswaph, req->send.amo.reply_buffer, size, - ucp_amo_request_reply_mem_type(req), size); + ucp_dt_contig_pack(worker, cswaph, req->send.amo.reply_buffer, + size, ucp_amo_request_reply_mem_type(req), NULL, + size); length += size; } } else { @@ -305,7 +306,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_atomic_rep_handler, (arg, data, length, am_fl if (worker->context->config.ext.proto_enable) { ucp_dt_contig_unpack(worker, req->send.amo.reply_buffer, hdr + 1, frag_length, ucp_amo_request_reply_mem_type(req), - frag_length); + NULL, frag_length); } else { memcpy(req->send.buffer, hdr + 1, frag_length); } diff --git a/src/ucp/rma/rma_sw.c b/src/ucp/rma/rma_sw.c index 1862c1ed58c..3eca0315db3 100644 --- a/src/ucp/rma/rma_sw.c +++ b/src/ucp/rma/rma_sw.c @@ -80,7 +80,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_put_handler, (arg, data, length, am_flags), UCP_WORKER_GET_EP_BY_ID(&ep, worker, puth->ep_id, return UCS_OK, "SW PUT request"); ucp_dt_contig_unpack(worker, (void*)puth->address, puth + 1, - length - sizeof(*puth), puth->mem_type, + length - sizeof(*puth), puth->mem_type, NULL, length - sizeof(*puth)); ucp_rma_sw_send_cmpl(ep); return UCS_OK; @@ -115,7 +115,7 @@ static size_t ucp_rma_sw_pack_get_reply(void *dest, void *arg) hdr->offset = req->send.state.dt_iter.offset; ucp_dt_contig_pack(req->send.ep->worker, hdr + 1, (char*)req->send.buffer + hdr->offset, length, - req->send.mem_type, req->send.length); + req->send.mem_type, NULL, req->send.length); return sizeof(*hdr) + length; } diff --git a/src/ucp/tag/offload.c b/src/ucp/tag/offload.c index 9242b9c7ea8..e53952dc2ea 100644 --- a/src/ucp/tag/offload.c +++ b/src/ucp/tag/offload.c @@ -166,7 +166,8 @@ UCS_PROFILE_FUNC_VOID(ucp_tag_offload_rndv_cb, it to the host memory staging buffer for further processing. */ header_host_copy = ucs_alloca(header_length); ucp_mem_type_pack(req->recv.worker, header_host_copy, header, - header_length, req->recv.dt_iter.mem_info.type); + header_length, req->recv.dt_iter.mem_info.type, + &req->recv.dt_iter.mem_info); ucp_tag_rndv_matched(req->recv.worker, req, header_host_copy, header_length); } diff --git a/src/ucp/tag/tag_match.inl b/src/ucp/tag/tag_match.inl index da772a99167..de2c7119f69 100644 --- a/src/ucp/tag/tag_match.inl +++ b/src/ucp/tag/tag_match.inl @@ -275,7 +275,8 @@ ucp_request_recv_offload_data(ucp_request_t *req, const void *data, } ucp_dt_contig_unpack(req->recv.worker, UCS_PTR_BYTE_OFFSET(buffer, offset), - data, length, req->recv.dt_iter.mem_info.type, length); + data, length, req->recv.dt_iter.mem_info.type, + &req->recv.dt_iter.mem_info, length); req->status = UCS_OK; out: diff --git a/src/ucs/memory/memory_type.h b/src/ucs/memory/memory_type.h index f21fb7c91d5..af6868f10d9 100644 --- a/src/ucs/memory/memory_type.h +++ b/src/ucs/memory/memory_type.h @@ -61,7 +61,14 @@ typedef enum ucs_mem_flags { * Memory can be accessed for inter-process memory type copy by a peer on * a different node. */ - UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE = UCS_BIT(1) + UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE = UCS_BIT(1), + + /** + * Memory can be pinned by the GPU peer memory driver. Memory localized + * to a GPU locality domain is never placed as GDR-capable, and + * stream-ordered allocations are only compatible with dma_buf mappings. + */ + UCS_MEM_FLAG_PEER_MEM_PINNABLE = UCS_BIT(2) } ucs_mem_flags_t; diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index 3bcde5a3d10..319f0f7b131 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -453,7 +453,7 @@ uct_cuda_copy_mem_release_fabric(uct_cuda_copy_alloc_handle_t *alloc_handle) static int uct_cuda_copy_detect_vmm(const void *address, ucs_memory_type_t *vmm_mem_type, CUdevice *cuda_device, - int *is_host_located) + int *is_host_located, int *is_localized) { #ifdef HAVE_CUMEMRETAINALLOCATIONHANDLE CUmemGenericAllocationHandle alloc_handle; @@ -471,6 +471,7 @@ static int uct_cuda_copy_detect_vmm(const void *address, *vmm_mem_type = UCS_MEMORY_TYPE_UNKNOWN; *cuda_device = CU_DEVICE_INVALID; *is_host_located = 0; + *is_localized = 0; status = UCT_CUDADRV_FUNC_LOG_DEBUG( cuMemGetAllocationPropertiesFromHandle(&prop, alloc_handle)); @@ -495,6 +496,7 @@ static int uct_cuda_copy_detect_vmm(const void *address, CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN) { *cuda_device = (CUdevice)prop.location.localized.deviceId; *vmm_mem_type = UCS_MEMORY_TYPE_CUDA; + *is_localized = 1; #endif } @@ -628,7 +630,8 @@ static ucs_status_t uct_cuda_copy_md_query_attributes(const uct_cuda_copy_md_t *md, const void *address, size_t length, ucs_memory_info_t *mem_info, - int *is_async_managed, int *is_host_located) + int *is_async_managed, int *is_host_located, + int *is_localized) { #define UCT_CUDA_MEM_QUERY_NUM_ATTRS 4 CUmemorytype cuda_mem_type = CU_MEMORYTYPE_HOST; @@ -644,9 +647,10 @@ uct_cuda_copy_md_query_attributes(const uct_cuda_copy_md_t *md, *is_async_managed = 0; *is_host_located = 0; + *is_localized = 0; is_vmm = uct_cuda_copy_detect_vmm(address, &mem_info->type, &cuda_device, - is_host_located); + is_host_located, is_localized); if (is_vmm) { if (mem_info->type == UCS_MEMORY_TYPE_UNKNOWN) { return UCS_ERR_INVALID_ADDR; @@ -901,10 +905,38 @@ uct_cuda_copy_md_is_registrable(uct_cuda_copy_md_t *md, return dmabuf_fd != UCT_DMABUF_FD_INVALID; } +static int uct_cuda_copy_md_is_mempool(const void *address) +{ +#if HAVE_DECL_CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE + CUmemoryPool mempool = NULL; + + /* Depending on the driver a plain allocation is reported either as a NULL + * handle or as an invalid value, both mean the memory is not pool-backed */ + if (cuPointerGetAttribute(&mempool, CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE, + (CUdeviceptr)address) != CUDA_SUCCESS) { + return 0; + } + + return mempool != NULL; +#else + return 0; +#endif +} + +/* The driver refuses to pin memory it did not place as GDR-capable: localized + * memory is never placed that way, as it belongs to a single GPU locality + * domain, and memory pools are only compatible with dma_buf mappings. */ +static int +uct_cuda_copy_md_is_peer_mem_pinnable(const void *address, int is_localized) +{ + return !is_localized && !uct_cuda_copy_md_is_mempool(address); +} + static uint8_t uct_cuda_copy_md_detect_mem_flags(uct_cuda_copy_md_t *md, const ucs_memory_info_t *mem_info, int is_async_managed, int is_host_located, + int is_localized, const uct_cuda_copy_md_dmabuf_t *dmabuf) { uint8_t mem_flags = 0; @@ -914,6 +946,11 @@ uct_cuda_copy_md_detect_mem_flags(uct_cuda_copy_md_t *md, mem_flags |= UCS_MEM_FLAG_REGISTRABLE; } + if (uct_cuda_copy_md_is_peer_mem_pinnable(mem_info->base_address, + is_localized)) { + mem_flags |= UCS_MEM_FLAG_PEER_MEM_PINNABLE; + } + return mem_flags | uct_cuda_copy_md_detect_memtype_copy_flags(mem_info); } @@ -935,6 +972,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, int dmabuf_queried = 0; int is_async_managed = 0; int is_host_located = 0; + int is_localized = 0; CUdevice cur_cuda_device = CU_DEVICE_INVALID; CUdevice avail_cuda_device = CU_DEVICE_INVALID; ucs_memory_info_t detected_mem_info = {}; @@ -960,7 +998,8 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, status = uct_cuda_copy_md_query_attributes(md, address, length, &addr_mem_info, &is_async_managed, - &is_host_located); + &is_host_located, + &is_localized); if (status != UCS_OK) { return status; } @@ -1023,7 +1062,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, if (address != NULL) { addr_mem_info.mem_flags = uct_cuda_copy_md_detect_mem_flags( md, &detected_mem_info, is_async_managed, is_host_located, - dmabuf_queried ? &dmabuf : NULL); + is_localized, dmabuf_queried ? &dmabuf : NULL); ucs_memtype_cache_update(addr_mem_info.base_address, addr_mem_info.alloc_length, addr_mem_info.type, addr_mem_info.sys_dev, diff --git a/src/uct/cuda/gdr_copy/gdr_copy_md.c b/src/uct/cuda/gdr_copy/gdr_copy_md.c index b2a268fd20e..82723d75f32 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_md.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_md.c @@ -116,11 +116,15 @@ uct_gdr_copy_md_query(uct_md_h uct_md, uct_md_attr_v2_t *md_attr) uct_gdr_copy_md_t *md = ucs_derived_of(uct_md, uct_gdr_copy_md_t); uct_md_base_md_query(md_attr); - md_attr->flags = UCT_MD_FLAG_REG | UCT_MD_FLAG_NEED_RKEY; - md_attr->reg_mem_types = UCS_BIT(UCS_MEMORY_TYPE_CUDA); - md_attr->access_mem_types = UCS_BIT(UCS_MEMORY_TYPE_CUDA); - md_attr->rkey_packed_size = sizeof(uct_gdr_copy_key_t); - md_attr->reg_cost = md->reg_cost; + md_attr->flags = UCT_MD_FLAG_REG | UCT_MD_FLAG_NEED_RKEY; + md_attr->reg_mem_types = UCS_BIT(UCS_MEMORY_TYPE_CUDA); + md_attr->access_mem_types = UCS_BIT(UCS_MEMORY_TYPE_CUDA); + md_attr->rkey_packed_size = sizeof(uct_gdr_copy_key_t); + md_attr->reg_cost = md->reg_cost; + /* Pinning goes through nvidia_p2p_get_pages, which the driver refuses + * for memory it did not place as GDR-capable, such as memory localized to + * a GPU locality domain, or memory pools */ + md_attr->required_mem_flags = UCS_MEM_FLAG_PEER_MEM_PINNABLE; /* In absence of own cache require proper alignment from the global cache */ if (md->rcache == NULL) { diff --git a/test/gtest/ucp/test_ucp_mem_type.cc b/test/gtest/ucp/test_ucp_mem_type.cc index 9181eb48bf4..2d99f3c467b 100644 --- a/test/gtest/ucp/test_ucp_mem_type.cc +++ b/test/gtest/ucp/test_ucp_mem_type.cc @@ -11,6 +11,8 @@ extern "C" { #include #include #include +#include +#include } @@ -181,3 +183,94 @@ UCS_TEST_P(test_ucp_cuda, sparse_regions) { } UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_cuda, all, "all") + +/* + * Stream-ordered CUDA memory is not peer memory pinnable, so gdr_copy cannot + * register it. Pack/unpack through the CUDA memory type endpoint has to fall + * back from the preferred gdr_copy lane to cuda_copy. + */ +class test_ucp_mem_type_non_pinnable : public ucp_test { +public: + static void get_test_variants(std::vector &variants) + { + add_variant(variants, UCP_FEATURE_AM); + } + + virtual void init() override + { + /* Type the stream-ordered allocations as plain CUDA memory, so they + * reach the CUDA memory type endpoint where gdr_copy is selected */ + modify_config("CUDA_COPY_ASYNC_MEM_TYPE", "cuda", SETENV_IF_NOT_EXIST); + ucp_test::init(); + } + +protected: + static constexpr size_t BUF_SIZE = 65536; + static constexpr uint64_t SEED = 0xdeadbeefcafebabeull; + + /* The datatype iterator passes the resolved memory flags, while the + * protocols which do not carry them pass NULL */ + void test_pack_unpack(int pass_mem_info) + { + std::vector host_buf(BUF_SIZE); + ucp_memory_info_t mem_info; + + if (!mem_buffer::is_async_supported(UCS_MEMORY_TYPE_CUDA)) { + UCS_TEST_SKIP_R("CUDA async allocation is not supported"); + } + + if (sender().worker()->mem_type_ep[UCS_MEMORY_TYPE_CUDA] == NULL) { + UCS_TEST_SKIP_R("no CUDA mem type endpoint"); + } + + mem_buffer src(BUF_SIZE, UCS_MEMORY_TYPE_CUDA, + mem_buffer::alloc_mode::ASYNC); + mem_buffer dst(BUF_SIZE, UCS_MEMORY_TYPE_CUDA, + mem_buffer::alloc_mode::ASYNC); + + ucp_memory_detect(sender().ucph(), src.ptr(), BUF_SIZE, &mem_info); + if (mem_info.flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE) { + UCS_TEST_SKIP_R("CUDA async memory is peer memory pinnable"); + } + + ASSERT_EQ(UCS_MEMORY_TYPE_CUDA, mem_info.type); + + const ucp_memory_info_t *mem_info_arg = pass_mem_info ? &mem_info : + NULL; + + mem_buffer::pattern_fill(src.ptr(), BUF_SIZE, SEED, + UCS_MEMORY_TYPE_CUDA); + mem_buffer::pattern_fill(dst.ptr(), BUF_SIZE, 0, UCS_MEMORY_TYPE_CUDA); + + ucp_mem_type_pack(sender().worker(), host_buf.data(), src.ptr(), + BUF_SIZE, UCS_MEMORY_TYPE_CUDA, mem_info_arg); + mem_buffer::pattern_check(host_buf.data(), BUF_SIZE, SEED); + + ucp_mem_type_unpack(sender().worker(), dst.ptr(), host_buf.data(), + BUF_SIZE, UCS_MEMORY_TYPE_CUDA, mem_info_arg); + mem_buffer::pattern_check(dst.ptr(), BUF_SIZE, SEED, + UCS_MEMORY_TYPE_CUDA); + } +}; + +UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_resolved_flags) +{ + test_pack_unpack(1); +} + +/* Without the flags the lane is picked by a memtype cache lookup, a quiet + * registration attempt on the preferred lane, and detection as a last resort */ +UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_unknown_flags) +{ + test_pack_unpack(0); +} + +/* The memtype cache is the first source of flags, make sure the fallback also + * holds when it cannot answer */ +UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_no_memtype_cache, + "MEMTYPE_CACHE=n") +{ + test_pack_unpack(0); +} + +UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_mem_type_non_pinnable, all, "all") diff --git a/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h b/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h index 73a22f93786..d84a204aa85 100644 --- a/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h +++ b/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h @@ -210,4 +210,71 @@ class cuda_host_vmm_mem_buffer : public cuda_vmm_mem_buffer { }; #endif +#if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN +/* Memory localized to a single GPU locality domain, which the driver never + * places as GDR-capable */ +class cuda_localized_mem_buffer : public cuda_vmm_mem_buffer { +public: + cuda_localized_mem_buffer(size_t size, ucs_memory_type_t mem_type) + { + skip_unless_ok( + alloc(size, 0, CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN)); + } +}; +#endif + +#if CUDA_VERSION >= 11020 +/* Device memory from an exportable stream-ordered pool. Exportability makes the + * allocation dmabuf-exportable, and therefore registrable, which plain + * stream-ordered memory from 'mem_buffer' is not guaranteed to be. */ +class cuda_exportable_mem_pool_buffer { +public: + cuda_exportable_mem_pool_buffer(size_t size) : m_size(size) + { + CUmemPoolProps props = {}; + CUdevice device; + + if (cuCtxGetDevice(&device) != CUDA_SUCCESS) { + UCS_TEST_SKIP_R("no CUDA device in the current context"); + } + + props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED; + props.handleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR; + props.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + props.location.id = device; + + if (cuMemPoolCreate(&m_pool, &props) != CUDA_SUCCESS) { + UCS_TEST_SKIP_R("failed to create an exportable CUDA memory pool"); + } + + if ((cuMemAllocFromPoolAsync(&m_ptr, m_size, m_pool, 0) != + CUDA_SUCCESS) || + (cuStreamSynchronize(0) != CUDA_SUCCESS)) { + cuMemPoolDestroy(m_pool); + UCS_TEST_SKIP_R("failed to allocate from a CUDA memory pool"); + } + } + + ~cuda_exportable_mem_pool_buffer() + { + if (m_ptr != 0) { + cuMemFreeAsync(m_ptr, 0); + cuStreamSynchronize(0); + } + + cuMemPoolDestroy(m_pool); + } + + void *ptr() const + { + return (void*)m_ptr; + } + +private: + size_t m_size = 0; + CUmemoryPool m_pool = 0; + CUdeviceptr m_ptr = 0; +}; +#endif + #endif diff --git a/test/gtest/uct/cuda/test_switch_cuda_device.cc b/test/gtest/uct/cuda/test_switch_cuda_device.cc index d1e4690474e..f8c497c08a7 100644 --- a/test/gtest/uct/cuda/test_switch_cuda_device.cc +++ b/test/gtest/uct/cuda/test_switch_cuda_device.cc @@ -353,17 +353,65 @@ class test_mem_alloc_device : public test_switch_cuda_device { EXPECT_NE(nullptr, cuda_mem_ctx); } + void query_mem_attr(void *address, size_t size, + uct_md_mem_attr_v2_t *mem_attr) + { + mem_attr->field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | + UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS; + EXPECT_UCS_OK(uct_md_mem_query_v2(md(), address, size, mem_attr)); + } + void query_managed_registrable(void *address, size_t size) { uct_md_mem_attr_v2_t mem_attr = {}; - mem_attr.field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | - UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS; - EXPECT_UCS_OK(uct_md_mem_query_v2(md(), address, size, &mem_attr)); + query_mem_attr(address, size, &mem_attr); EXPECT_EQ(UCS_MEMORY_TYPE_CUDA_MANAGED, mem_attr.mem_type); EXPECT_TRUE(mem_attr.mem_flags & UCS_MEM_FLAG_REGISTRABLE); } + ucs_mem_flags_t query_mem_flags(void *address, size_t size) + { + uct_md_mem_attr_v2_t mem_attr = {}; + + query_mem_attr(address, size, &mem_attr); + return static_cast(mem_attr.mem_flags); + } + + void query_peer_mem_pinnable(void *address, size_t size, int expected) + { + EXPECT_EQ(expected, !!(query_mem_flags(address, size) & + UCS_MEM_FLAG_PEER_MEM_PINNABLE)); + } + + /* Peer memory pinnability is orthogonal to registrability, so check both */ + void query_registrable_pinnable(void *address, size_t size, + int exp_registrable, + int exp_peer_mem_pinnable) + { + ucs_mem_flags_t mem_flags = query_mem_flags(address, size); + + EXPECT_EQ(exp_registrable, !!(mem_flags & UCS_MEM_FLAG_REGISTRABLE)); + EXPECT_EQ(exp_peer_mem_pinnable, + !!(mem_flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE)); + } + + /* Stream-ordered memory from the default pool, as allocated by + * 'cudaMallocAsync' */ + void test_mem_pool_not_peer_mem_pinnable() + { + constexpr size_t size = 4 * UCS_MBYTE; + + if (!mem_buffer::is_async_supported(UCS_MEMORY_TYPE_CUDA)) { + UCS_TEST_SKIP_R("asynchronous CUDA memory is not supported"); + } + + mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA, + mem_buffer::alloc_mode::ASYNC); + + query_peer_mem_pinnable(buffer.ptr(), size, 0); + } + void test_async_managed_mem_pool_registrable() { constexpr size_t size = 192; @@ -512,6 +560,58 @@ UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_gpu_pref_loc, test_async_managed_mem_pool_registrable(); } +/* Plain device memory has device pages, so the peer memory driver can pin it */ +UCS_TEST_P(test_mem_alloc_device, legacy_mem_peer_mem_pinnable) +{ + constexpr size_t size = 4 * UCS_MBYTE; + mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); + + query_registrable_pinnable(buffer.ptr(), size, 1, 1); +} + +#if CUDA_VERSION >= 11020 +/* Stream-ordered memory is only compatible with dma_buf mappings, so the peer + * memory driver cannot pin it, even when an exportable pool makes it + * registrable. This is the combination gdr_copy has to reject. */ +UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_peer_mem_pinnable, + "CUDA_COPY_ASYNC_MEM_TYPE=cuda", "CUDA_COPY_DMABUF=try") +{ + constexpr size_t size = 4 * UCS_MBYTE; + cuda_exportable_mem_pool_buffer buffer(size); + + query_registrable_pinnable(buffer.ptr(), size, 1, 0); +} +#endif + +/* Pinning does not become possible for the default, non-exportable pool */ +UCS_TEST_P(test_mem_alloc_device, mem_pool_not_peer_mem_pinnable, + "CUDA_COPY_ASYNC_MEM_TYPE=cuda") +{ + test_mem_pool_not_peer_mem_pinnable(); +} + +/* The default typing of stream-ordered memory as managed must not report it as + * pinnable either */ +UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_peer_mem_pinnable, + "CUDA_COPY_ASYNC_MEM_TYPE=cuda-managed") +{ + test_mem_pool_not_peer_mem_pinnable(); +} + +#if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN +/* Localized memory belongs to a single GPU locality domain, which the driver + * never places as GDR-capable, so the peer memory driver cannot pin it even + * though it stays registrable. This is the case reported in #11708. */ +UCS_TEST_P(test_mem_alloc_device, localized_mem_not_peer_mem_pinnable, + "CUDA_COPY_REG_WHOLE_ALLOC=off") +{ + constexpr size_t size = 4 * UCS_MBYTE; + cuda_localized_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); + + query_registrable_pinnable(buffer.ptr(), size, 1, 0); +} +#endif + UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_registrable_cuda_type, "CUDA_COPY_DMABUF=try", "CUDA_COPY_ASYNC_MEM_TYPE=cuda") diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index 9bdfbe12737..92aad90fda7 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -1376,15 +1376,16 @@ UCT_MD_INSTANTIATE_TEST_CASE(test_cuda) class test_gdr_copy : public test_md { protected: + static constexpr size_t REG_SIZE = 65536; + ucs_status_t register_mem() { - constexpr size_t size = 65536; - void *address = NULL; + void *address = NULL; uct_mem_h memh; ucs_status_t status; - alloc_memory(&address, size, NULL, UCS_MEMORY_TYPE_CUDA); - status = reg_mem(UCT_MD_MEM_ACCESS_ALL, address, size, &memh); + alloc_memory(&address, REG_SIZE, NULL, UCS_MEMORY_TYPE_CUDA); + status = reg_mem(UCT_MD_MEM_ACCESS_ALL, address, REG_SIZE, &memh); if (status == UCS_OK) { (void)uct_md_mem_dereg(md(), memh); } @@ -1392,6 +1393,21 @@ class test_gdr_copy : public test_md { free_memory(address, UCS_MEMORY_TYPE_CUDA); return status; } + + ucs_status_t register_async_mem() + { + mem_buffer buffer(REG_SIZE, UCS_MEMORY_TYPE_CUDA, + mem_buffer::alloc_mode::ASYNC); + uct_mem_h memh; + ucs_status_t status; + + status = reg_mem(UCT_MD_MEM_ACCESS_ALL, buffer.ptr(), REG_SIZE, &memh); + if (status == UCS_OK) { + (void)uct_md_mem_dereg(md(), memh); + } + + return status; + } }; UCS_TEST_SKIP_COND_P(test_gdr_copy, gdr_copy_reg_cuda_default_pin, @@ -1423,6 +1439,24 @@ UCS_TEST_SKIP_COND_P(test_gdr_copy, gdr_copy_reg_cuda_try_pcie_pin, ASSERT_UCS_OK(register_mem()); } +/* Pinning goes through nvidia_p2p_get_pages, so memory without device pages + * must never be offered to this memory domain */ +UCS_TEST_P(test_gdr_copy, gdr_copy_requires_peer_mem_pinnable) +{ + EXPECT_TRUE(md_attr().required_mem_flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE); +} + +/* The reason for the requirement above: stream-ordered memory is only + * compatible with dma_buf mappings, so the pin itself fails */ +UCS_TEST_SKIP_COND_P(test_gdr_copy, gdr_copy_reg_async_cuda_fail, + !check_caps(UCT_MD_FLAG_REG) || + !mem_buffer::is_async_supported( + UCS_MEMORY_TYPE_CUDA)) +{ + scoped_log_handler slh(wrap_errors_logger); + ASSERT_UCS_STATUS_EQ(UCS_ERR_IO_ERROR, register_async_mem()); +} + _UCT_MD_INSTANTIATE_TEST_CASE(test_gdr_copy, gdr_copy) #endif /* HAVE_DECL_GDR_PIN_BUFFER_V2 */ From ed89e64ee65a8be47ade8df031f2634d2aebea20 Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Wed, 7 Oct 2026 16:57:10 +0300 Subject: [PATCH 2/9] UCP/UCT/GTEST: Address review comments Drop the registrability assertion from the localized memory test, as exporting a dma_buf fd for such memory is platform dependent, and remove the github reference from its comment. Fix two comments in test_ucp_mem_type which still described selecting the lane by a memtype cache lookup and a quiet registration attempt, instead of resolving the memory flags once on the lane which requires them. Update the copyright year of the modified files. --- src/ucp/dt/dt_contig.h | 2 +- src/ucp/dt/dt_iov.h | 2 +- src/ucp/rma/rma_sw.c | 2 +- src/ucp/tag/tag_match.inl | 2 +- src/uct/cuda/gdr_copy/gdr_copy_md.c | 2 +- test/gtest/ucp/test_ucp_mem_type.cc | 10 +++++----- test/gtest/uct/cuda/test_switch_cuda_device.cc | 7 ++++--- test/gtest/uct/test_md.cc | 2 +- 8 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/ucp/dt/dt_contig.h b/src/ucp/dt/dt_contig.h index d7bff9f2921..8a5d23c92ca 100644 --- a/src/ucp/dt/dt_contig.h +++ b/src/ucp/dt/dt_contig.h @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2015. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2024. ALL RIGHTS RESERVED. * * See file LICENSE for terms. diff --git a/src/ucp/dt/dt_iov.h b/src/ucp/dt/dt_iov.h index 9920dd01071..344ffe6110a 100644 --- a/src/ucp/dt/dt_iov.h +++ b/src/ucp/dt/dt_iov.h @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2015. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ diff --git a/src/ucp/rma/rma_sw.c b/src/ucp/rma/rma_sw.c index 3eca0315db3..ca225dcf402 100644 --- a/src/ucp/rma/rma_sw.c +++ b/src/ucp/rma/rma_sw.c @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2018. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Huawei Technologies Co., Ltd. 2021. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2024. ALL RIGHTS RESERVED. * diff --git a/src/ucp/tag/tag_match.inl b/src/ucp/tag/tag_match.inl index de2c7119f69..2c6100c1126 100644 --- a/src/ucp/tag/tag_match.inl +++ b/src/ucp/tag/tag_match.inl @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2024. ALL RIGHTS RESERVED. * * See file LICENSE for terms. diff --git a/src/uct/cuda/gdr_copy/gdr_copy_md.c b/src/uct/cuda/gdr_copy/gdr_copy_md.c index 82723d75f32..362b27a57b4 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_md.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_md.c @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2017-2019. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2017-2026. ALL RIGHTS RESERVED. * See file LICENSE for terms. */ diff --git a/test/gtest/ucp/test_ucp_mem_type.cc b/test/gtest/ucp/test_ucp_mem_type.cc index 2d99f3c467b..9060f67186a 100644 --- a/test/gtest/ucp/test_ucp_mem_type.cc +++ b/test/gtest/ucp/test_ucp_mem_type.cc @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2017. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -258,15 +258,15 @@ UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_resolved_flags) test_pack_unpack(1); } -/* Without the flags the lane is picked by a memtype cache lookup, a quiet - * registration attempt on the preferred lane, and detection as a last resort */ +/* Without the flags from the caller, they are resolved from the buffer address + * on the first lane whose memory domain requires them */ UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_unknown_flags) { test_pack_unpack(0); } -/* The memtype cache is the first source of flags, make sure the fallback also - * holds when it cannot answer */ +/* Resolving the flags hits the memtype cache first, make sure the slow path + * also reports them when the cache is disabled */ UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_no_memtype_cache, "MEMTYPE_CACHE=n") { diff --git a/test/gtest/uct/cuda/test_switch_cuda_device.cc b/test/gtest/uct/cuda/test_switch_cuda_device.cc index f8c497c08a7..05136308267 100644 --- a/test/gtest/uct/cuda/test_switch_cuda_device.cc +++ b/test/gtest/uct/cuda/test_switch_cuda_device.cc @@ -600,15 +600,16 @@ UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_peer_mem_pinnable, #if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN /* Localized memory belongs to a single GPU locality domain, which the driver - * never places as GDR-capable, so the peer memory driver cannot pin it even - * though it stays registrable. This is the case reported in #11708. */ + * never places as GDR-capable, so the peer memory driver cannot pin it. + * Registrability is not asserted, as it depends on whether the platform can + * export a dma_buf fd for such memory. */ UCS_TEST_P(test_mem_alloc_device, localized_mem_not_peer_mem_pinnable, "CUDA_COPY_REG_WHOLE_ALLOC=off") { constexpr size_t size = 4 * UCS_MBYTE; cuda_localized_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); - query_registrable_pinnable(buffer.ptr(), size, 1, 0); + query_peer_mem_pinnable(buffer.ptr(), size, 0); } #endif diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index 92aad90fda7..fdf9901d079 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2016. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2016 - 2017. ALL RIGHTS RESERVED. * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. * See file LICENSE for terms. From f20c701d742c05018c44fb273e34512386dc8219 Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Wed, 7 Oct 2026 22:32:42 +0300 Subject: [PATCH 3/9] UCP/UCS/GTEST: Address review comments --- src/ucp/dt/dt.c | 13 ++++--- src/ucs/memory/memory_type.h | 6 ++-- src/uct/cuda/cuda_copy/cuda_copy_md.c | 7 ++-- src/uct/cuda/gdr_copy/gdr_copy_md.c | 2 +- test/gtest/ucp/test_ucp_mem_type.cc | 19 ++++++++--- .../gtest/uct/cuda/test_switch_cuda_device.cc | 34 +++++++++---------- test/gtest/uct/test_md.cc | 4 +-- 7 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/ucp/dt/dt.c b/src/ucp/dt/dt.c index 9d4d2a80af4..2af46b7038a 100644 --- a/src/ucp/dt/dt.c +++ b/src/ucp/dt/dt.c @@ -68,8 +68,8 @@ ucs_status_t ucp_dt_mem_info_verify(const char *dt_name, size_t index, * if no lane is compatible. */ static ucp_lane_index_t -ucp_mem_type_ep_lane(ucp_ep_h ep, void *address, size_t length, - const ucp_memory_info_t *mem_info) +ucp_mem_type_ep_find_lane(ucp_ep_h ep, const void *address, size_t length, + const ucp_memory_info_t *mem_info) { ucp_context_h context = ep->worker->context; const ucp_ep_config_key_t *key = &ucp_ep_config(ep)->key; @@ -80,7 +80,7 @@ ucp_mem_type_ep_lane(ucp_ep_h ep, void *address, size_t length, for (i = 0; key->rma_lanes[i] != UCP_NULL_LANE; ++i) { lane = key->rma_lanes[i]; - md_attr = &context->tl_mds[ucp_ep_md_index(ep, lane)].attr; + md_attr = ucp_ep_md_attr(ep, lane); if (md_attr->required_mem_flags == 0) { return lane; } @@ -98,6 +98,10 @@ ucp_mem_type_ep_lane(ucp_ep_h ep, void *address, size_t length, md_attr->required_mem_flags)) { return lane; } + + /* TODO: Also take the memory flags into account in the protocol + * performance estimation, as they are already part of the selection + * key. Performance model here diverges from the lane actually used. */ } return UCP_NULL_LANE; @@ -115,7 +119,8 @@ ucp_mem_type_lane_reg(ucp_worker_h worker, ucp_ep_h ep, void *address, ucp_lane_index_t *lane_p, ucp_mtype_pack_context_t *pack_context) { - ucp_lane_index_t lane = ucp_mem_type_ep_lane(ep, address, length, mem_info); + ucp_lane_index_t lane = ucp_mem_type_ep_find_lane(ep, address, length, + mem_info); if (lane == UCP_NULL_LANE) { ucs_error("no mem type lane can register %s buffer %p length %zu", diff --git a/src/ucs/memory/memory_type.h b/src/ucs/memory/memory_type.h index af6868f10d9..6d3d005f294 100644 --- a/src/ucs/memory/memory_type.h +++ b/src/ucs/memory/memory_type.h @@ -64,11 +64,9 @@ typedef enum ucs_mem_flags { UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE = UCS_BIT(1), /** - * Memory can be pinned by the GPU peer memory driver. Memory localized - * to a GPU locality domain is never placed as GDR-capable, and - * stream-ordered allocations are only compatible with dma_buf mappings. + * Memory pages can be pinned for direct access by another device. */ - UCS_MEM_FLAG_PEER_MEM_PINNABLE = UCS_BIT(2) + UCS_MEM_FLAG_PINNABLE = UCS_BIT(2) } ucs_mem_flags_t; diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index 319f0f7b131..8abf1ed69f8 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -927,7 +927,7 @@ static int uct_cuda_copy_md_is_mempool(const void *address) * memory is never placed that way, as it belongs to a single GPU locality * domain, and memory pools are only compatible with dma_buf mappings. */ static int -uct_cuda_copy_md_is_peer_mem_pinnable(const void *address, int is_localized) +uct_cuda_copy_md_is_pinnable(const void *address, int is_localized) { return !is_localized && !uct_cuda_copy_md_is_mempool(address); } @@ -946,9 +946,8 @@ uct_cuda_copy_md_detect_mem_flags(uct_cuda_copy_md_t *md, mem_flags |= UCS_MEM_FLAG_REGISTRABLE; } - if (uct_cuda_copy_md_is_peer_mem_pinnable(mem_info->base_address, - is_localized)) { - mem_flags |= UCS_MEM_FLAG_PEER_MEM_PINNABLE; + if (uct_cuda_copy_md_is_pinnable(mem_info->base_address, is_localized)) { + mem_flags |= UCS_MEM_FLAG_PINNABLE; } return mem_flags | uct_cuda_copy_md_detect_memtype_copy_flags(mem_info); diff --git a/src/uct/cuda/gdr_copy/gdr_copy_md.c b/src/uct/cuda/gdr_copy/gdr_copy_md.c index 362b27a57b4..fb5f7f61804 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_md.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_md.c @@ -124,7 +124,7 @@ uct_gdr_copy_md_query(uct_md_h uct_md, uct_md_attr_v2_t *md_attr) /* Pinning goes through nvidia_p2p_get_pages, which the driver refuses * for memory it did not place as GDR-capable, such as memory localized to * a GPU locality domain, or memory pools */ - md_attr->required_mem_flags = UCS_MEM_FLAG_PEER_MEM_PINNABLE; + md_attr->required_mem_flags = UCS_MEM_FLAG_PINNABLE; /* In absence of own cache require proper alignment from the global cache */ if (md->rcache == NULL) { diff --git a/test/gtest/ucp/test_ucp_mem_type.cc b/test/gtest/ucp/test_ucp_mem_type.cc index 9060f67186a..bf51ac0b6e6 100644 --- a/test/gtest/ucp/test_ucp_mem_type.cc +++ b/test/gtest/ucp/test_ucp_mem_type.cc @@ -12,6 +12,7 @@ extern "C" { #include #include #include +#include #include } @@ -185,7 +186,7 @@ UCS_TEST_P(test_ucp_cuda, sparse_regions) { UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_cuda, all, "all") /* - * Stream-ordered CUDA memory is not peer memory pinnable, so gdr_copy cannot + * Stream-ordered CUDA memory is not pinnable, so gdr_copy cannot * register it. Pack/unpack through the CUDA memory type endpoint has to fall * back from the preferred gdr_copy lane to cuda_copy. */ @@ -214,23 +215,33 @@ class test_ucp_mem_type_non_pinnable : public ucp_test { { std::vector host_buf(BUF_SIZE); ucp_memory_info_t mem_info; + ucp_ep_h mem_type_ep; if (!mem_buffer::is_async_supported(UCS_MEMORY_TYPE_CUDA)) { UCS_TEST_SKIP_R("CUDA async allocation is not supported"); } - if (sender().worker()->mem_type_ep[UCS_MEMORY_TYPE_CUDA] == NULL) { + mem_type_ep = sender().worker()->mem_type_ep[UCS_MEMORY_TYPE_CUDA]; + if (mem_type_ep == NULL) { UCS_TEST_SKIP_R("no CUDA mem type endpoint"); } + /* Without a preferred lane which demands memory flags, pack/unpack has + * nothing to fall back from */ + ucp_lane_index_t lane = ucp_ep_config(mem_type_ep)->key.rma_lanes[0]; + if ((lane == UCP_NULL_LANE) || + (ucp_ep_md_attr(mem_type_ep, lane)->required_mem_flags == 0)) { + UCS_TEST_SKIP_R("first CUDA mem type lane requires no memory flags"); + } + mem_buffer src(BUF_SIZE, UCS_MEMORY_TYPE_CUDA, mem_buffer::alloc_mode::ASYNC); mem_buffer dst(BUF_SIZE, UCS_MEMORY_TYPE_CUDA, mem_buffer::alloc_mode::ASYNC); ucp_memory_detect(sender().ucph(), src.ptr(), BUF_SIZE, &mem_info); - if (mem_info.flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE) { - UCS_TEST_SKIP_R("CUDA async memory is peer memory pinnable"); + if (mem_info.flags & UCS_MEM_FLAG_PINNABLE) { + UCS_TEST_SKIP_R("CUDA async memory is pinnable"); } ASSERT_EQ(UCS_MEMORY_TYPE_CUDA, mem_info.type); diff --git a/test/gtest/uct/cuda/test_switch_cuda_device.cc b/test/gtest/uct/cuda/test_switch_cuda_device.cc index 05136308267..b61b4d38761 100644 --- a/test/gtest/uct/cuda/test_switch_cuda_device.cc +++ b/test/gtest/uct/cuda/test_switch_cuda_device.cc @@ -378,27 +378,25 @@ class test_mem_alloc_device : public test_switch_cuda_device { return static_cast(mem_attr.mem_flags); } - void query_peer_mem_pinnable(void *address, size_t size, int expected) + void query_pinnable(void *address, size_t size, int expected) { - EXPECT_EQ(expected, !!(query_mem_flags(address, size) & - UCS_MEM_FLAG_PEER_MEM_PINNABLE)); + EXPECT_EQ(expected, + !!(query_mem_flags(address, size) & UCS_MEM_FLAG_PINNABLE)); } - /* Peer memory pinnability is orthogonal to registrability, so check both */ + /* Pinnability is orthogonal to registrability, so check both */ void query_registrable_pinnable(void *address, size_t size, - int exp_registrable, - int exp_peer_mem_pinnable) + int exp_registrable, int exp_pinnable) { ucs_mem_flags_t mem_flags = query_mem_flags(address, size); EXPECT_EQ(exp_registrable, !!(mem_flags & UCS_MEM_FLAG_REGISTRABLE)); - EXPECT_EQ(exp_peer_mem_pinnable, - !!(mem_flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE)); + EXPECT_EQ(exp_pinnable, !!(mem_flags & UCS_MEM_FLAG_PINNABLE)); } /* Stream-ordered memory from the default pool, as allocated by * 'cudaMallocAsync' */ - void test_mem_pool_not_peer_mem_pinnable() + void test_mem_pool_not_pinnable() { constexpr size_t size = 4 * UCS_MBYTE; @@ -409,7 +407,7 @@ class test_mem_alloc_device : public test_switch_cuda_device { mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA, mem_buffer::alloc_mode::ASYNC); - query_peer_mem_pinnable(buffer.ptr(), size, 0); + query_pinnable(buffer.ptr(), size, 0); } void test_async_managed_mem_pool_registrable() @@ -561,7 +559,7 @@ UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_gpu_pref_loc, } /* Plain device memory has device pages, so the peer memory driver can pin it */ -UCS_TEST_P(test_mem_alloc_device, legacy_mem_peer_mem_pinnable) +UCS_TEST_P(test_mem_alloc_device, legacy_mem_pinnable) { constexpr size_t size = 4 * UCS_MBYTE; mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); @@ -573,7 +571,7 @@ UCS_TEST_P(test_mem_alloc_device, legacy_mem_peer_mem_pinnable) /* Stream-ordered memory is only compatible with dma_buf mappings, so the peer * memory driver cannot pin it, even when an exportable pool makes it * registrable. This is the combination gdr_copy has to reject. */ -UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_peer_mem_pinnable, +UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_pinnable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda", "CUDA_COPY_DMABUF=try") { constexpr size_t size = 4 * UCS_MBYTE; @@ -584,18 +582,18 @@ UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_peer_mem_pinnable, #endif /* Pinning does not become possible for the default, non-exportable pool */ -UCS_TEST_P(test_mem_alloc_device, mem_pool_not_peer_mem_pinnable, +UCS_TEST_P(test_mem_alloc_device, mem_pool_not_pinnable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda") { - test_mem_pool_not_peer_mem_pinnable(); + test_mem_pool_not_pinnable(); } /* The default typing of stream-ordered memory as managed must not report it as * pinnable either */ -UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_peer_mem_pinnable, +UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_pinnable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda-managed") { - test_mem_pool_not_peer_mem_pinnable(); + test_mem_pool_not_pinnable(); } #if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN @@ -603,13 +601,13 @@ UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_peer_mem_pinnable, * never places as GDR-capable, so the peer memory driver cannot pin it. * Registrability is not asserted, as it depends on whether the platform can * export a dma_buf fd for such memory. */ -UCS_TEST_P(test_mem_alloc_device, localized_mem_not_peer_mem_pinnable, +UCS_TEST_P(test_mem_alloc_device, localized_mem_not_pinnable, "CUDA_COPY_REG_WHOLE_ALLOC=off") { constexpr size_t size = 4 * UCS_MBYTE; cuda_localized_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); - query_peer_mem_pinnable(buffer.ptr(), size, 0); + query_pinnable(buffer.ptr(), size, 0); } #endif diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index fdf9901d079..86efcfb65ae 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -1441,9 +1441,9 @@ UCS_TEST_SKIP_COND_P(test_gdr_copy, gdr_copy_reg_cuda_try_pcie_pin, /* Pinning goes through nvidia_p2p_get_pages, so memory without device pages * must never be offered to this memory domain */ -UCS_TEST_P(test_gdr_copy, gdr_copy_requires_peer_mem_pinnable) +UCS_TEST_P(test_gdr_copy, gdr_copy_requires_pinnable) { - EXPECT_TRUE(md_attr().required_mem_flags & UCS_MEM_FLAG_PEER_MEM_PINNABLE); + EXPECT_TRUE(md_attr().required_mem_flags & UCS_MEM_FLAG_PINNABLE); } /* The reason for the requirement above: stream-ordered memory is only From 731a0278e195e5749278668da24e170ec50665eb Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Thu, 8 Oct 2026 11:32:34 +0300 Subject: [PATCH 4/9] UCP/UCS/UCT/GTEST: Address review comments --- src/ucp/core/ucp_device.c | 42 +++++++++++++++------------ src/uct/cuda/cuda_copy/cuda_copy_md.c | 15 ++++++---- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/ucp/core/ucp_device.c b/src/ucp/core/ucp_device.c index e4dbd285a69..789215333ec 100644 --- a/src/ucp/core/ucp_device.c +++ b/src/ucp/core/ucp_device.c @@ -873,23 +873,26 @@ void ucp_device_mem_list_release(void *mem_list_h) uct_mem_free(&info.mem); } -static ucs_memory_type_t -ucp_device_counter_mem_type(ucp_context_h context, const void *counter_ptr, - const ucp_device_counter_params_t *params) +static void +ucp_device_counter_mem_info(ucp_context_h context, const void *counter_ptr, + const ucp_device_counter_params_t *params, + ucp_memory_info_t *mem_info) { - ucs_memory_info_t mem_info; - if (params->field_mask & UCP_DEVICE_COUNTER_PARAMS_FIELD_MEMH) { - return params->memh->mem_type; + *mem_info = ucp_memory_info_from_memh(params->memh); + return; } if (params->field_mask & UCP_DEVICE_COUNTER_PARAMS_FIELD_MEM_TYPE) { - return params->mem_type; + /* The memory flags are not part of the caller's hint, so the memory + * type endpoint can only use a lane which does not require any */ + mem_info->type = params->mem_type; + mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; + mem_info->flags = 0; + return; } - ucp_memory_detect_internal(context, counter_ptr, sizeof(uint64_t), - &mem_info); - return mem_info.type; + ucp_memory_detect(context, counter_ptr, sizeof(uint64_t), mem_info); } ucs_status_t ucp_device_counter_init(ucp_worker_h worker, @@ -897,12 +900,13 @@ ucs_status_t ucp_device_counter_init(ucp_worker_h worker, void *counter_ptr) { uint64_t counter_value = 0; - ucs_memory_type_t mem_type; + ucp_memory_info_t mem_info; - mem_type = ucp_device_counter_mem_type(worker->context, counter_ptr, - params); + ucp_device_counter_mem_info(worker->context, counter_ptr, params, + &mem_info); ucp_dt_contig_unpack(worker, counter_ptr, &counter_value, - sizeof(counter_value), mem_type, NULL, + sizeof(counter_value), + (ucs_memory_type_t)mem_info.type, &mem_info, sizeof(counter_value)); return UCS_OK; } @@ -912,12 +916,12 @@ uint64_t ucp_device_counter_read(ucp_worker_h worker, void *counter_ptr) { uint64_t counter_value = 0; - ucs_memory_type_t mem_type; + ucp_memory_info_t mem_info; - mem_type = ucp_device_counter_mem_type(worker->context, counter_ptr, - params); + ucp_device_counter_mem_info(worker->context, counter_ptr, params, + &mem_info); ucp_dt_contig_pack(worker, &counter_value, counter_ptr, - sizeof(counter_value), mem_type, NULL, - sizeof(counter_value)); + sizeof(counter_value), (ucs_memory_type_t)mem_info.type, + &mem_info, sizeof(counter_value)); return counter_value; } diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index 8abf1ed69f8..16ace430325 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -908,12 +908,17 @@ uct_cuda_copy_md_is_registrable(uct_cuda_copy_md_t *md, static int uct_cuda_copy_md_is_mempool(const void *address) { #if HAVE_DECL_CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE - CUmemoryPool mempool = NULL; + CUpointer_attribute attr_type = CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE; + CUmemoryPool mempool = NULL; + void *attr_data = &mempool; + ucs_status_t status; - /* Depending on the driver a plain allocation is reported either as a NULL - * handle or as an invalid value, both mean the memory is not pool-backed */ - if (cuPointerGetAttribute(&mempool, CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE, - (CUdeviceptr)address) != CUDA_SUCCESS) { + /* The attribute only applies to stream-ordered memory, and the plural form + * reports a NULL handle for any other memory instead of failing */ + status = UCT_CUDADRV_FUNC_LOG_DEBUG( + cuPointerGetAttributes(1, &attr_type, &attr_data, + (CUdeviceptr)address)); + if (status != UCS_OK) { return 0; } From ebf6e985490313d0a69cc6bdbc79d4cb098a7db7 Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Thu, 8 Oct 2026 11:02:40 +0200 Subject: [PATCH 5/9] UCP/CORE: Address review comments --- src/ucp/core/ucp_device.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/ucp/core/ucp_device.c b/src/ucp/core/ucp_device.c index 789215333ec..61bebe15778 100644 --- a/src/ucp/core/ucp_device.c +++ b/src/ucp/core/ucp_device.c @@ -873,26 +873,29 @@ void ucp_device_mem_list_release(void *mem_list_h) uct_mem_free(&info.mem); } -static void +/* + * Resolve the memory info of the counter area. Returns NULL when the memory + * flags cannot be resolved, in which case only @a mem_info->type is set and + * the memory type endpoint looks the flags up by itself. + */ +static const ucp_memory_info_t * ucp_device_counter_mem_info(ucp_context_h context, const void *counter_ptr, const ucp_device_counter_params_t *params, ucp_memory_info_t *mem_info) { if (params->field_mask & UCP_DEVICE_COUNTER_PARAMS_FIELD_MEMH) { *mem_info = ucp_memory_info_from_memh(params->memh); - return; + return mem_info; } if (params->field_mask & UCP_DEVICE_COUNTER_PARAMS_FIELD_MEM_TYPE) { - /* The memory flags are not part of the caller's hint, so the memory - * type endpoint can only use a lane which does not require any */ - mem_info->type = params->mem_type; - mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - mem_info->flags = 0; - return; + /* The memory flags are not part of the caller's hint */ + mem_info->type = params->mem_type; + return NULL; } ucp_memory_detect(context, counter_ptr, sizeof(uint64_t), mem_info); + return mem_info; } ucs_status_t ucp_device_counter_init(ucp_worker_h worker, @@ -901,12 +904,13 @@ ucs_status_t ucp_device_counter_init(ucp_worker_h worker, { uint64_t counter_value = 0; ucp_memory_info_t mem_info; + const ucp_memory_info_t *resolved; - ucp_device_counter_mem_info(worker->context, counter_ptr, params, - &mem_info); + resolved = ucp_device_counter_mem_info(worker->context, counter_ptr, params, + &mem_info); ucp_dt_contig_unpack(worker, counter_ptr, &counter_value, sizeof(counter_value), - (ucs_memory_type_t)mem_info.type, &mem_info, + (ucs_memory_type_t)mem_info.type, resolved, sizeof(counter_value)); return UCS_OK; } @@ -917,11 +921,12 @@ uint64_t ucp_device_counter_read(ucp_worker_h worker, { uint64_t counter_value = 0; ucp_memory_info_t mem_info; + const ucp_memory_info_t *resolved; - ucp_device_counter_mem_info(worker->context, counter_ptr, params, - &mem_info); + resolved = ucp_device_counter_mem_info(worker->context, counter_ptr, params, + &mem_info); ucp_dt_contig_pack(worker, &counter_value, counter_ptr, sizeof(counter_value), (ucs_memory_type_t)mem_info.type, - &mem_info, sizeof(counter_value)); + resolved, sizeof(counter_value)); return counter_value; } From 9e779dab30771ff8f287bc7e853e61b00cc433cd Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Thu, 8 Oct 2026 12:46:37 +0200 Subject: [PATCH 6/9] UCS/UCT/UCP: Add default memory flags for undetected memory Several places fill in memory flags for a buffer whose capabilities were never detected, and all of them hardcoded UCS_MEM_FLAG_REGISTRABLE. Now that UCS_MEM_FLAG_PINNABLE gates gdr_copy, such a buffer would never be considered pinnable, silently disabling gdr_copy. Add UCS_MEM_FLAGS_DEFAULT with all capabilities a memory domain can require, and use it as the undetected default. Detection results are left untouched, so a buffer the cuda_copy MD identifies as localized or mempool-backed still reports no PINNABLE flag. UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE is excluded from the default, as it is not part of any required_mem_flags and must only be reported when actually detected. --- src/ucm/cuda/cudamem.c | 6 +++--- src/ucp/core/ucp_context.c | 2 +- src/ucp/core/ucp_context.h | 2 +- src/ucp/core/ucp_mm.c | 4 ++-- src/ucs/memory/memory_type.h | 10 ++++++++++ src/ucs/memory/memtype_cache.c | 6 +++--- src/ucs/memory/memtype_cache.h | 2 +- src/uct/base/uct_md.c | 6 +++--- src/uct/cuda/cuda_copy/cuda_copy_md.c | 2 +- 9 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/ucm/cuda/cudamem.c b/src/ucm/cuda/cudamem.c index a8fd0774d6e..b9341e4d8de 100644 --- a/src/ucm/cuda/cudamem.c +++ b/src/ucm/cuda/cudamem.c @@ -173,7 +173,7 @@ static void ucm_cuda_dispatch_mem_alloc(CUdeviceptr ptr, size_t length) { /* Indicate unknown type and let cuda_md detect attributes. */ ucm_cuda_dispatch_mem_type_alloc(ptr, length, UCS_MEMORY_TYPE_LAST, - UCS_MEM_FLAG_REGISTRABLE); + UCS_MEM_FLAGS_DEFAULT); } static void ucm_cuda_dispatch_mem_free(CUdeviceptr ptr, size_t length, @@ -206,7 +206,7 @@ static void ucm_cuda_dispatch_mem_free(CUdeviceptr ptr, size_t length, event.mem_type.size = length; event.mem_type.mem_type = mem_type; event.mem_type.sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - event.mem_type.mem_flags = UCS_MEM_FLAG_REGISTRABLE; + event.mem_type.mem_flags = UCS_MEM_FLAGS_DEFAULT; ucm_event_dispatch(UCM_EVENT_MEM_TYPE_FREE, &event); } @@ -468,7 +468,7 @@ static int ucm_cudamem_scan_regions_cb(void *arg, void *addr, size_t length, event.mem_type.size = length; event.mem_type.mem_type = UCS_MEMORY_TYPE_LAST; /* unknown memory type */ event.mem_type.sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - event.mem_type.mem_flags = UCS_MEM_FLAG_REGISTRABLE; + event.mem_type.mem_flags = UCS_MEM_FLAGS_DEFAULT; ucm_event_enter(); handler->cb(UCM_EVENT_MEM_TYPE_ALLOC, &event, handler->arg); diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index c29dec49eb1..6214e9fa558 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -2250,7 +2250,7 @@ ucp_fill_resources(ucp_context_h context, const ucp_config_t *config) context->dmabuf_mds[mem_type] = UCP_NULL_RESOURCE; context->alloc_md[mem_type].md_index = UCP_NULL_RESOURCE; context->alloc_md[mem_type].sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - context->alloc_md[mem_type].mem_flags = UCS_MEM_FLAG_REGISTRABLE; + context->alloc_md[mem_type].mem_flags = UCS_MEM_FLAGS_DEFAULT; context->alloc_md[mem_type].initialized = 0; } diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index 49c8a473c71..a43afd08a87 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -720,7 +720,7 @@ ucp_memory_info_set_host(ucp_memory_info_t *mem_info) { mem_info->type = UCS_MEMORY_TYPE_HOST; mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - mem_info->flags = UCS_MEM_FLAG_REGISTRABLE; + mem_info->flags = UCS_MEM_FLAGS_DEFAULT; } static UCS_F_ALWAYS_INLINE void diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index 7e89411843b..f944911a006 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -41,7 +41,7 @@ ucp_mem_dummy_handle_t ucp_mem_dummy_handle = { .parent = &ucp_mem_dummy_handle.memh, .mem_type = UCS_MEMORY_TYPE_HOST, .sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN, - .mem_flags = UCS_MEM_FLAG_REGISTRABLE, + .mem_flags = UCS_MEM_FLAGS_DEFAULT, .md_map = 0, .inv_md_map = 0, .reg_id = 0, @@ -52,7 +52,7 @@ ucp_mem_dummy_handle_t ucp_mem_dummy_handle = { const ucp_memory_info_t ucp_mem_info_unknown = { .type = UCS_MEMORY_TYPE_UNKNOWN, .sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN, - .flags = UCS_MEM_FLAG_REGISTRABLE + .flags = UCS_MEM_FLAGS_DEFAULT }; diff --git a/src/ucs/memory/memory_type.h b/src/ucs/memory/memory_type.h index 6d3d005f294..fad5e3e00cb 100644 --- a/src/ucs/memory/memory_type.h +++ b/src/ucs/memory/memory_type.h @@ -70,6 +70,16 @@ typedef enum ucs_mem_flags { } ucs_mem_flags_t; +/** + * Memory capabilities a memory domain can require from a buffer, used as the + * default when they were not detected for the buffer itself. + * UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE is excluded, as it is not required by + * any memory domain and must only be reported when detected. + */ +#define UCS_MEM_FLAGS_DEFAULT \ + (UCS_MEM_FLAG_REGISTRABLE | UCS_MEM_FLAG_PINNABLE) + + /** * Array of string names for each memory type */ diff --git a/src/ucs/memory/memtype_cache.c b/src/ucs/memory/memtype_cache.c index 337825436ea..40f0a0913e1 100644 --- a/src/ucs/memory/memtype_cache.c +++ b/src/ucs/memory/memtype_cache.c @@ -105,9 +105,9 @@ ucs_memory_info_set_unknown(ucs_memory_info_t *mem_info) mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; mem_info->base_address = NULL; mem_info->alloc_length = -1; - /* Unknown memory: assume registrable. The type is UNKNOWN so consumers - * re-detect via the MD anyway, which overwrites this default. */ - mem_info->mem_flags = UCS_MEM_FLAG_REGISTRABLE; + /* Unknown memory: assume all capabilities. The type is UNKNOWN so + * consumers re-detect via the MD anyway, which overwrites this default. */ + mem_info->mem_flags = UCS_MEM_FLAGS_DEFAULT; } static ucs_pgt_dir_t *ucs_memtype_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) diff --git a/src/ucs/memory/memtype_cache.h b/src/ucs/memory/memtype_cache.h index 7f38ed0ad26..d32bd1040c6 100644 --- a/src/ucs/memory/memtype_cache.h +++ b/src/ucs/memory/memtype_cache.h @@ -128,7 +128,7 @@ ucs_memory_info_set_host(ucs_memory_info_t *mem_info) mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; mem_info->base_address = NULL; mem_info->alloc_length = -1; - mem_info->mem_flags = UCS_MEM_FLAG_REGISTRABLE; + mem_info->mem_flags = UCS_MEM_FLAGS_DEFAULT; } END_C_DECLS diff --git a/src/uct/base/uct_md.c b/src/uct/base/uct_md.c index 02dca7aab5b..c1234380c82 100644 --- a/src/uct/base/uct_md.c +++ b/src/uct/base/uct_md.c @@ -706,10 +706,10 @@ ucs_status_t uct_md_mem_query_v2(uct_md_h md, const void *address, uint64_t field_mask = mem_attr->field_mask; ucs_status_t status; - /* Default to registrable; MDs that detect per-buffer registrability - * (e.g. cuda_copy) override this. */ + /* Default to all capabilities; MDs that detect them per-buffer (e.g. + * cuda_copy) override this. */ if (field_mask & UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS) { - mem_attr->mem_flags = UCS_MEM_FLAG_REGISTRABLE; + mem_attr->mem_flags = UCS_MEM_FLAGS_DEFAULT; } status = md->ops->mem_query(md, address, length, mem_attr); diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index 16ace430325..dddbb421395 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -1075,7 +1075,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, if (mem_attr->field_mask & UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS) { mem_attr->mem_flags = (address != NULL) ? addr_mem_info.mem_flags : - UCS_MEM_FLAG_REGISTRABLE; + UCS_MEM_FLAGS_DEFAULT; } /* If dmabuf_fd was returned to the caller, the caller is responsible to From b7d2590a582161470898c00c58817eadb9cc8888 Mon Sep 17 00:00:00 2001 From: Thomas Vegas <125344591+tvegas1@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:49:26 +0200 Subject: [PATCH 7/9] UCP/UCT/CUDA: Avoid gdr_copy for non-pinnable memory Co-authored-by: tomerg-nvidia --- src/uct/base/uct_md.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uct/base/uct_md.c b/src/uct/base/uct_md.c index c1234380c82..8ddab829bd1 100644 --- a/src/uct/base/uct_md.c +++ b/src/uct/base/uct_md.c @@ -706,7 +706,7 @@ ucs_status_t uct_md_mem_query_v2(uct_md_h md, const void *address, uint64_t field_mask = mem_attr->field_mask; ucs_status_t status; - /* Default to all capabilities; MDs that detect them per-buffer (e.g. + /* Use default flags; MDs that detect them per-buffer (e.g. * cuda_copy) override this. */ if (field_mask & UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS) { mem_attr->mem_flags = UCS_MEM_FLAGS_DEFAULT; From 2b024f1e565cd56c2ce43049ad76c14948cba09d Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Fri, 9 Oct 2026 02:38:09 -0700 Subject: [PATCH 8/9] UCS/UCT/CUDA: Detect GDR capability from the CUDA driver Only gdr_copy requires the pinnable flag, and it only registers CUDA device memory, so rename it to UCS_MEM_FLAG_GDR_CAPABLE and report it only when cuda_copy detects it, instead of assuming it by default for every memory type. All the defaults go back to UCS_MEM_FLAG_REGISTRABLE, so UCS_MEM_FLAGS_DEFAULT is removed. Detect the flag from what the driver reports: the gpuDirectRDMACapable flag of VMM allocations, and the CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE attribute queried together with the other pointer attributes otherwise. Guessing from the allocation kind missed device VMM allocated without gpuDirectRDMACapable, host-located VMM, and all memory allocated under an MPS MLOPart device, where localized memory is reported as plain device memory. The memory pool query and the localized marker are no longer needed. --- config/m4/cuda.m4 | 2 +- src/ucm/cuda/cudamem.c | 6 +- src/ucp/core/ucp_context.c | 2 +- src/ucp/core/ucp_context.h | 2 +- src/ucp/core/ucp_mm.c | 4 +- src/ucp/rndv/proto_rndv.c | 6 +- src/ucs/memory/memory_type.h | 16 +--- src/ucs/memory/memtype_cache.c | 6 +- src/ucs/memory/memtype_cache.h | 2 +- src/uct/base/uct_md.c | 6 +- src/uct/cuda/cuda_copy/cuda_copy_md.c | 73 ++++++--------- src/uct/cuda/gdr_copy/gdr_copy_md.c | 5 +- test/gtest/ucp/test_ucp_mem_type.cc | 16 ++-- test/gtest/uct/cuda/cuda_vmm_mem_buffer.h | 22 ++++- .../gtest/uct/cuda/test_switch_cuda_device.cc | 91 +++++++++++++------ test/gtest/uct/test_md.cc | 8 +- 16 files changed, 144 insertions(+), 123 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index ecaa07c8c5a..36568678f44 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -255,7 +255,7 @@ AS_IF([test "x$cuda_checked" != "xyes"], AC_CHECK_DECLS([CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN], [], [], [[#include ]]) - AC_CHECK_DECLS([CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE], + AC_CHECK_DECLS([CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE], [], [], [[#include ]]) CPPFLAGS="$save_CPPFLAGS" diff --git a/src/ucm/cuda/cudamem.c b/src/ucm/cuda/cudamem.c index b9341e4d8de..a8fd0774d6e 100644 --- a/src/ucm/cuda/cudamem.c +++ b/src/ucm/cuda/cudamem.c @@ -173,7 +173,7 @@ static void ucm_cuda_dispatch_mem_alloc(CUdeviceptr ptr, size_t length) { /* Indicate unknown type and let cuda_md detect attributes. */ ucm_cuda_dispatch_mem_type_alloc(ptr, length, UCS_MEMORY_TYPE_LAST, - UCS_MEM_FLAGS_DEFAULT); + UCS_MEM_FLAG_REGISTRABLE); } static void ucm_cuda_dispatch_mem_free(CUdeviceptr ptr, size_t length, @@ -206,7 +206,7 @@ static void ucm_cuda_dispatch_mem_free(CUdeviceptr ptr, size_t length, event.mem_type.size = length; event.mem_type.mem_type = mem_type; event.mem_type.sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - event.mem_type.mem_flags = UCS_MEM_FLAGS_DEFAULT; + event.mem_type.mem_flags = UCS_MEM_FLAG_REGISTRABLE; ucm_event_dispatch(UCM_EVENT_MEM_TYPE_FREE, &event); } @@ -468,7 +468,7 @@ static int ucm_cudamem_scan_regions_cb(void *arg, void *addr, size_t length, event.mem_type.size = length; event.mem_type.mem_type = UCS_MEMORY_TYPE_LAST; /* unknown memory type */ event.mem_type.sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - event.mem_type.mem_flags = UCS_MEM_FLAGS_DEFAULT; + event.mem_type.mem_flags = UCS_MEM_FLAG_REGISTRABLE; ucm_event_enter(); handler->cb(UCM_EVENT_MEM_TYPE_ALLOC, &event, handler->arg); diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index 6214e9fa558..c29dec49eb1 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -2250,7 +2250,7 @@ ucp_fill_resources(ucp_context_h context, const ucp_config_t *config) context->dmabuf_mds[mem_type] = UCP_NULL_RESOURCE; context->alloc_md[mem_type].md_index = UCP_NULL_RESOURCE; context->alloc_md[mem_type].sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - context->alloc_md[mem_type].mem_flags = UCS_MEM_FLAGS_DEFAULT; + context->alloc_md[mem_type].mem_flags = UCS_MEM_FLAG_REGISTRABLE; context->alloc_md[mem_type].initialized = 0; } diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index a43afd08a87..49c8a473c71 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -720,7 +720,7 @@ ucp_memory_info_set_host(ucp_memory_info_t *mem_info) { mem_info->type = UCS_MEMORY_TYPE_HOST; mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; - mem_info->flags = UCS_MEM_FLAGS_DEFAULT; + mem_info->flags = UCS_MEM_FLAG_REGISTRABLE; } static UCS_F_ALWAYS_INLINE void diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index f944911a006..7e89411843b 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -41,7 +41,7 @@ ucp_mem_dummy_handle_t ucp_mem_dummy_handle = { .parent = &ucp_mem_dummy_handle.memh, .mem_type = UCS_MEMORY_TYPE_HOST, .sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN, - .mem_flags = UCS_MEM_FLAGS_DEFAULT, + .mem_flags = UCS_MEM_FLAG_REGISTRABLE, .md_map = 0, .inv_md_map = 0, .reg_id = 0, @@ -52,7 +52,7 @@ ucp_mem_dummy_handle_t ucp_mem_dummy_handle = { const ucp_memory_info_t ucp_mem_info_unknown = { .type = UCS_MEMORY_TYPE_UNKNOWN, .sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN, - .flags = UCS_MEM_FLAGS_DEFAULT + .flags = UCS_MEM_FLAG_REGISTRABLE }; diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index 12911249b1c..d0dfb98bb05 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -188,11 +188,11 @@ ucp_proto_rndv_rkey_mem_flags_estimate(const ucp_proto_init_params_t *params) } /* - * Derive UCS_MEM_FLAG_REGISTRABLE from matching local MDs which - * require it and whose remote MDs are present in the rkey. + * Derive the memory flags required by matching local MDs whose remote + * MDs are present in the rkey. * * UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE is intentionally not inferred - * here: unlike REGISTRABLE it is not an MD required_mem_flags bit, + * here: it is not an MD required_mem_flags bit, * and is checked only when packing keys in * ucp_proto_rndv_ctrl_get_md_map(). Runtime packing uses the real * local buffer flags; omitting it from this estimate can only make diff --git a/src/ucs/memory/memory_type.h b/src/ucs/memory/memory_type.h index fad5e3e00cb..bdb468c102e 100644 --- a/src/ucs/memory/memory_type.h +++ b/src/ucs/memory/memory_type.h @@ -64,22 +64,14 @@ typedef enum ucs_mem_flags { UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE = UCS_BIT(1), /** - * Memory pages can be pinned for direct access by another device. + * CUDA device memory allocated as GPUDirect RDMA (GDR) capable, so its + * pages can be pinned for peer access, as done by gdr_copy. Reported only + * when detected for the buffer, never assumed by default. */ - UCS_MEM_FLAG_PINNABLE = UCS_BIT(2) + UCS_MEM_FLAG_GDR_CAPABLE = UCS_BIT(2) } ucs_mem_flags_t; -/** - * Memory capabilities a memory domain can require from a buffer, used as the - * default when they were not detected for the buffer itself. - * UCS_MEM_FLAG_MEMTYPE_COPY_INTER_NODE is excluded, as it is not required by - * any memory domain and must only be reported when detected. - */ -#define UCS_MEM_FLAGS_DEFAULT \ - (UCS_MEM_FLAG_REGISTRABLE | UCS_MEM_FLAG_PINNABLE) - - /** * Array of string names for each memory type */ diff --git a/src/ucs/memory/memtype_cache.c b/src/ucs/memory/memtype_cache.c index 40f0a0913e1..337825436ea 100644 --- a/src/ucs/memory/memtype_cache.c +++ b/src/ucs/memory/memtype_cache.c @@ -105,9 +105,9 @@ ucs_memory_info_set_unknown(ucs_memory_info_t *mem_info) mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; mem_info->base_address = NULL; mem_info->alloc_length = -1; - /* Unknown memory: assume all capabilities. The type is UNKNOWN so - * consumers re-detect via the MD anyway, which overwrites this default. */ - mem_info->mem_flags = UCS_MEM_FLAGS_DEFAULT; + /* Unknown memory: assume registrable. The type is UNKNOWN so consumers + * re-detect via the MD anyway, which overwrites this default. */ + mem_info->mem_flags = UCS_MEM_FLAG_REGISTRABLE; } static ucs_pgt_dir_t *ucs_memtype_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) diff --git a/src/ucs/memory/memtype_cache.h b/src/ucs/memory/memtype_cache.h index d32bd1040c6..7f38ed0ad26 100644 --- a/src/ucs/memory/memtype_cache.h +++ b/src/ucs/memory/memtype_cache.h @@ -128,7 +128,7 @@ ucs_memory_info_set_host(ucs_memory_info_t *mem_info) mem_info->sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN; mem_info->base_address = NULL; mem_info->alloc_length = -1; - mem_info->mem_flags = UCS_MEM_FLAGS_DEFAULT; + mem_info->mem_flags = UCS_MEM_FLAG_REGISTRABLE; } END_C_DECLS diff --git a/src/uct/base/uct_md.c b/src/uct/base/uct_md.c index 8ddab829bd1..02dca7aab5b 100644 --- a/src/uct/base/uct_md.c +++ b/src/uct/base/uct_md.c @@ -706,10 +706,10 @@ ucs_status_t uct_md_mem_query_v2(uct_md_h md, const void *address, uint64_t field_mask = mem_attr->field_mask; ucs_status_t status; - /* Use default flags; MDs that detect them per-buffer (e.g. - * cuda_copy) override this. */ + /* Default to registrable; MDs that detect per-buffer registrability + * (e.g. cuda_copy) override this. */ if (field_mask & UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS) { - mem_attr->mem_flags = UCS_MEM_FLAGS_DEFAULT; + mem_attr->mem_flags = UCS_MEM_FLAG_REGISTRABLE; } status = md->ops->mem_query(md, address, length, mem_attr); diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index dddbb421395..cfd3da69fef 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -453,7 +453,7 @@ uct_cuda_copy_mem_release_fabric(uct_cuda_copy_alloc_handle_t *alloc_handle) static int uct_cuda_copy_detect_vmm(const void *address, ucs_memory_type_t *vmm_mem_type, CUdevice *cuda_device, - int *is_host_located, int *is_localized) + int *is_host_located, int *is_gdr_capable) { #ifdef HAVE_CUMEMRETAINALLOCATIONHANDLE CUmemGenericAllocationHandle alloc_handle; @@ -471,7 +471,7 @@ static int uct_cuda_copy_detect_vmm(const void *address, *vmm_mem_type = UCS_MEMORY_TYPE_UNKNOWN; *cuda_device = CU_DEVICE_INVALID; *is_host_located = 0; - *is_localized = 0; + *is_gdr_capable = 0; status = UCT_CUDADRV_FUNC_LOG_DEBUG( cuMemGetAllocationPropertiesFromHandle(&prop, alloc_handle)); @@ -479,7 +479,8 @@ static int uct_cuda_copy_detect_vmm(const void *address, goto out; } - *cuda_device = (CUdevice)prop.location.id; + *cuda_device = (CUdevice)prop.location.id; + *is_gdr_capable = prop.allocFlags.gpuDirectRDMACapable; #if HAVE_DECL_CU_MEM_LOCATION_TYPE_HOST if ((prop.location.type == CU_MEM_LOCATION_TYPE_HOST) || (prop.location.type == CU_MEM_LOCATION_TYPE_HOST_NUMA) || @@ -496,7 +497,6 @@ static int uct_cuda_copy_detect_vmm(const void *address, CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN) { *cuda_device = (CUdevice)prop.location.localized.deviceId; *vmm_mem_type = UCS_MEMORY_TYPE_CUDA; - *is_localized = 1; #endif } @@ -631,9 +631,13 @@ uct_cuda_copy_md_query_attributes(const uct_cuda_copy_md_t *md, const void *address, size_t length, ucs_memory_info_t *mem_info, int *is_async_managed, int *is_host_located, - int *is_localized) + int *is_gdr_capable) { +#if HAVE_DECL_CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE +#define UCT_CUDA_MEM_QUERY_NUM_ATTRS 5 +#else #define UCT_CUDA_MEM_QUERY_NUM_ATTRS 4 +#endif CUmemorytype cuda_mem_type = CU_MEMORYTYPE_HOST; uint32_t is_managed = 0; CUcontext cuda_mem_ctx = NULL; @@ -647,10 +651,10 @@ uct_cuda_copy_md_query_attributes(const uct_cuda_copy_md_t *md, *is_async_managed = 0; *is_host_located = 0; - *is_localized = 0; + *is_gdr_capable = 0; is_vmm = uct_cuda_copy_detect_vmm(address, &mem_info->type, &cuda_device, - is_host_located, is_localized); + is_host_located, is_gdr_capable); if (is_vmm) { if (mem_info->type == UCS_MEMORY_TYPE_UNKNOWN) { return UCS_ERR_INVALID_ADDR; @@ -664,6 +668,12 @@ uct_cuda_copy_md_query_attributes(const uct_cuda_copy_md_t *md, attr_data[2] = &cuda_device; attr_type[3] = CU_POINTER_ATTRIBUTE_CONTEXT; attr_data[3] = &cuda_mem_ctx; +#if HAVE_DECL_CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE + attr_type[4] = CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE; + attr_data[4] = is_gdr_capable; +#else + *is_gdr_capable = 1; +#endif status = UCT_CUDADRV_FUNC_LOG_ERR( cuPointerGetAttributes(ucs_static_array_size(attr_data), @@ -905,43 +915,11 @@ uct_cuda_copy_md_is_registrable(uct_cuda_copy_md_t *md, return dmabuf_fd != UCT_DMABUF_FD_INVALID; } -static int uct_cuda_copy_md_is_mempool(const void *address) -{ -#if HAVE_DECL_CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE - CUpointer_attribute attr_type = CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE; - CUmemoryPool mempool = NULL; - void *attr_data = &mempool; - ucs_status_t status; - - /* The attribute only applies to stream-ordered memory, and the plural form - * reports a NULL handle for any other memory instead of failing */ - status = UCT_CUDADRV_FUNC_LOG_DEBUG( - cuPointerGetAttributes(1, &attr_type, &attr_data, - (CUdeviceptr)address)); - if (status != UCS_OK) { - return 0; - } - - return mempool != NULL; -#else - return 0; -#endif -} - -/* The driver refuses to pin memory it did not place as GDR-capable: localized - * memory is never placed that way, as it belongs to a single GPU locality - * domain, and memory pools are only compatible with dma_buf mappings. */ -static int -uct_cuda_copy_md_is_pinnable(const void *address, int is_localized) -{ - return !is_localized && !uct_cuda_copy_md_is_mempool(address); -} - static uint8_t uct_cuda_copy_md_detect_mem_flags(uct_cuda_copy_md_t *md, const ucs_memory_info_t *mem_info, int is_async_managed, int is_host_located, - int is_localized, + int is_gdr_capable, const uct_cuda_copy_md_dmabuf_t *dmabuf) { uint8_t mem_flags = 0; @@ -951,8 +929,11 @@ uct_cuda_copy_md_detect_mem_flags(uct_cuda_copy_md_t *md, mem_flags |= UCS_MEM_FLAG_REGISTRABLE; } - if (uct_cuda_copy_md_is_pinnable(mem_info->base_address, is_localized)) { - mem_flags |= UCS_MEM_FLAG_PINNABLE; + /* The driver only pins memory it allocated as GPUDirect RDMA capable, + * which excludes memory pools, localized and host-located VMM, and any + * memory allocated under an MPS MLOPart device */ + if ((mem_info->type == UCS_MEMORY_TYPE_CUDA) && is_gdr_capable) { + mem_flags |= UCS_MEM_FLAG_GDR_CAPABLE; } return mem_flags | uct_cuda_copy_md_detect_memtype_copy_flags(mem_info); @@ -976,7 +957,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, int dmabuf_queried = 0; int is_async_managed = 0; int is_host_located = 0; - int is_localized = 0; + int is_gdr_capable = 0; CUdevice cur_cuda_device = CU_DEVICE_INVALID; CUdevice avail_cuda_device = CU_DEVICE_INVALID; ucs_memory_info_t detected_mem_info = {}; @@ -1003,7 +984,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, &addr_mem_info, &is_async_managed, &is_host_located, - &is_localized); + &is_gdr_capable); if (status != UCS_OK) { return status; } @@ -1066,7 +1047,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, if (address != NULL) { addr_mem_info.mem_flags = uct_cuda_copy_md_detect_mem_flags( md, &detected_mem_info, is_async_managed, is_host_located, - is_localized, dmabuf_queried ? &dmabuf : NULL); + is_gdr_capable, dmabuf_queried ? &dmabuf : NULL); ucs_memtype_cache_update(addr_mem_info.base_address, addr_mem_info.alloc_length, addr_mem_info.type, addr_mem_info.sys_dev, @@ -1075,7 +1056,7 @@ ucs_status_t uct_cuda_copy_md_mem_query(uct_md_h tl_md, const void *address, if (mem_attr->field_mask & UCT_MD_MEM_ATTR_V2_FIELD_MEM_FLAGS) { mem_attr->mem_flags = (address != NULL) ? addr_mem_info.mem_flags : - UCS_MEM_FLAGS_DEFAULT; + UCS_MEM_FLAG_REGISTRABLE; } /* If dmabuf_fd was returned to the caller, the caller is responsible to diff --git a/src/uct/cuda/gdr_copy/gdr_copy_md.c b/src/uct/cuda/gdr_copy/gdr_copy_md.c index fb5f7f61804..6df679487ea 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_md.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_md.c @@ -122,9 +122,8 @@ uct_gdr_copy_md_query(uct_md_h uct_md, uct_md_attr_v2_t *md_attr) md_attr->rkey_packed_size = sizeof(uct_gdr_copy_key_t); md_attr->reg_cost = md->reg_cost; /* Pinning goes through nvidia_p2p_get_pages, which the driver refuses - * for memory it did not place as GDR-capable, such as memory localized to - * a GPU locality domain, or memory pools */ - md_attr->required_mem_flags = UCS_MEM_FLAG_PINNABLE; + * for memory it did not allocate as GPUDirect RDMA capable */ + md_attr->required_mem_flags = UCS_MEM_FLAG_GDR_CAPABLE; /* In absence of own cache require proper alignment from the global cache */ if (md->rcache == NULL) { diff --git a/test/gtest/ucp/test_ucp_mem_type.cc b/test/gtest/ucp/test_ucp_mem_type.cc index bf51ac0b6e6..431ac117b23 100644 --- a/test/gtest/ucp/test_ucp_mem_type.cc +++ b/test/gtest/ucp/test_ucp_mem_type.cc @@ -186,11 +186,11 @@ UCS_TEST_P(test_ucp_cuda, sparse_regions) { UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_cuda, all, "all") /* - * Stream-ordered CUDA memory is not pinnable, so gdr_copy cannot + * Stream-ordered CUDA memory is not GDR capable, so gdr_copy cannot * register it. Pack/unpack through the CUDA memory type endpoint has to fall * back from the preferred gdr_copy lane to cuda_copy. */ -class test_ucp_mem_type_non_pinnable : public ucp_test { +class test_ucp_mem_type_non_gdr_capable : public ucp_test { public: static void get_test_variants(std::vector &variants) { @@ -240,8 +240,8 @@ class test_ucp_mem_type_non_pinnable : public ucp_test { mem_buffer::alloc_mode::ASYNC); ucp_memory_detect(sender().ucph(), src.ptr(), BUF_SIZE, &mem_info); - if (mem_info.flags & UCS_MEM_FLAG_PINNABLE) { - UCS_TEST_SKIP_R("CUDA async memory is pinnable"); + if (mem_info.flags & UCS_MEM_FLAG_GDR_CAPABLE) { + UCS_TEST_SKIP_R("CUDA async memory is GDR capable"); } ASSERT_EQ(UCS_MEMORY_TYPE_CUDA, mem_info.type); @@ -264,24 +264,24 @@ class test_ucp_mem_type_non_pinnable : public ucp_test { } }; -UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_resolved_flags) +UCS_TEST_P(test_ucp_mem_type_non_gdr_capable, pack_unpack_resolved_flags) { test_pack_unpack(1); } /* Without the flags from the caller, they are resolved from the buffer address * on the first lane whose memory domain requires them */ -UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_unknown_flags) +UCS_TEST_P(test_ucp_mem_type_non_gdr_capable, pack_unpack_unknown_flags) { test_pack_unpack(0); } /* Resolving the flags hits the memtype cache first, make sure the slow path * also reports them when the cache is disabled */ -UCS_TEST_P(test_ucp_mem_type_non_pinnable, pack_unpack_no_memtype_cache, +UCS_TEST_P(test_ucp_mem_type_non_gdr_capable, pack_unpack_no_memtype_cache, "MEMTYPE_CACHE=n") { test_pack_unpack(0); } -UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_mem_type_non_pinnable, all, "all") +UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_mem_type_non_gdr_capable, all, "all") diff --git a/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h b/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h index d84a204aa85..7d8e15ee67d 100644 --- a/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h +++ b/test/gtest/uct/cuda/cuda_vmm_mem_buffer.h @@ -59,7 +59,8 @@ class cuda_vmm_mem_buffer { CUresult alloc(size_t size, unsigned handle_type, CUmemLocationType location_type = CU_MEM_LOCATION_TYPE_DEVICE, size_t num_chunks = 1, - unsigned char locality_domain_id = 0) + unsigned char locality_domain_id = 0, + bool gdr_capable = false) { size_t granularity = 0; CUmemAllocationProp prop = {}; @@ -73,8 +74,9 @@ class cuda_vmm_mem_buffer { "context"); } - prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; - prop.location.type = location_type; + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; + prop.location.type = location_type; + prop.allocFlags.gpuDirectRDMACapable = gdr_capable; #if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN if (location_type == CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN) { prop.location.localized.deviceId = (unsigned char)device; @@ -200,6 +202,16 @@ class cuda_posix_fd_mem_buffer : public cuda_vmm_mem_buffer { } }; +/* Device memory allocated as GPUDirect RDMA capable */ +class cuda_gdr_vmm_mem_buffer : public cuda_vmm_mem_buffer { +public: + cuda_gdr_vmm_mem_buffer(size_t size, ucs_memory_type_t mem_type) + { + skip_unless_ok( + alloc(size, 0, CU_MEM_LOCATION_TYPE_DEVICE, 1, 0, true)); + } +}; + #if CUDA_VERSION >= 12020 class cuda_host_vmm_mem_buffer : public cuda_vmm_mem_buffer { public: @@ -211,8 +223,8 @@ class cuda_host_vmm_mem_buffer : public cuda_vmm_mem_buffer { #endif #if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN -/* Memory localized to a single GPU locality domain, which the driver never - * places as GDR-capable */ +/* Memory localized to a single GPU locality domain, which the driver cannot + * allocate as GPUDirect RDMA capable */ class cuda_localized_mem_buffer : public cuda_vmm_mem_buffer { public: cuda_localized_mem_buffer(size_t size, ucs_memory_type_t mem_type) diff --git a/test/gtest/uct/cuda/test_switch_cuda_device.cc b/test/gtest/uct/cuda/test_switch_cuda_device.cc index b61b4d38761..c60095570ec 100644 --- a/test/gtest/uct/cuda/test_switch_cuda_device.cc +++ b/test/gtest/uct/cuda/test_switch_cuda_device.cc @@ -378,25 +378,26 @@ class test_mem_alloc_device : public test_switch_cuda_device { return static_cast(mem_attr.mem_flags); } - void query_pinnable(void *address, size_t size, int expected) + void query_gdr_capable(void *address, size_t size, int expected) { - EXPECT_EQ(expected, - !!(query_mem_flags(address, size) & UCS_MEM_FLAG_PINNABLE)); + ucs_mem_flags_t mem_flags = query_mem_flags(address, size); + + EXPECT_EQ(expected, !!(mem_flags & UCS_MEM_FLAG_GDR_CAPABLE)); } - /* Pinnability is orthogonal to registrability, so check both */ - void query_registrable_pinnable(void *address, size_t size, - int exp_registrable, int exp_pinnable) + /* GDR capability is orthogonal to registrability, so check both */ + void query_registrable_gdr_capable(void *address, size_t size, + int exp_registrable, int exp_gdr_capable) { ucs_mem_flags_t mem_flags = query_mem_flags(address, size); EXPECT_EQ(exp_registrable, !!(mem_flags & UCS_MEM_FLAG_REGISTRABLE)); - EXPECT_EQ(exp_pinnable, !!(mem_flags & UCS_MEM_FLAG_PINNABLE)); + EXPECT_EQ(exp_gdr_capable, !!(mem_flags & UCS_MEM_FLAG_GDR_CAPABLE)); } /* Stream-ordered memory from the default pool, as allocated by * 'cudaMallocAsync' */ - void test_mem_pool_not_pinnable() + void test_mem_pool_not_gdr_capable() { constexpr size_t size = 4 * UCS_MBYTE; @@ -407,7 +408,7 @@ class test_mem_alloc_device : public test_switch_cuda_device { mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA, mem_buffer::alloc_mode::ASYNC); - query_pinnable(buffer.ptr(), size, 0); + query_gdr_capable(buffer.ptr(), size, 0); } void test_async_managed_mem_pool_registrable() @@ -558,56 +559,81 @@ UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_gpu_pref_loc, test_async_managed_mem_pool_registrable(); } -/* Plain device memory has device pages, so the peer memory driver can pin it */ -UCS_TEST_P(test_mem_alloc_device, legacy_mem_pinnable) +/* Plain device memory is allocated as GPUDirect RDMA capable */ +UCS_TEST_P(test_mem_alloc_device, legacy_mem_gdr_capable) { constexpr size_t size = 4 * UCS_MBYTE; mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); - query_registrable_pinnable(buffer.ptr(), size, 1, 1); + query_registrable_gdr_capable(buffer.ptr(), size, 1, 1); +} + +/* VMM device memory is GDR capable only when requested at allocation */ +UCS_TEST_P(test_mem_alloc_device, vmm_mem_not_gdr_capable) +{ + constexpr size_t size = 4 * UCS_MBYTE; + cuda_vmm_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); + + query_gdr_capable(buffer.ptr(), size, 0); +} + +UCS_TEST_P(test_mem_alloc_device, gdr_vmm_mem_gdr_capable) +{ + constexpr size_t size = 4 * UCS_MBYTE; + cuda_gdr_vmm_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); + + query_gdr_capable(buffer.ptr(), size, 1); +} + +/* gdr_copy only registers CUDA device memory, so managed memory is never + * reported as GDR capable */ +UCS_TEST_P(test_mem_alloc_device, managed_mem_not_gdr_capable) +{ + constexpr size_t size = 4 * UCS_MBYTE; + mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA_MANAGED); + + query_registrable_gdr_capable(buffer.ptr(), size, 1, 0); } #if CUDA_VERSION >= 11020 -/* Stream-ordered memory is only compatible with dma_buf mappings, so the peer - * memory driver cannot pin it, even when an exportable pool makes it - * registrable. This is the combination gdr_copy has to reject. */ -UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_pinnable, +/* Stream-ordered memory is not GDR capable, even when an exportable pool makes + * it registrable. This is the combination gdr_copy has to reject. */ +UCS_TEST_P(test_mem_alloc_device, exportable_mem_pool_not_gdr_capable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda", "CUDA_COPY_DMABUF=try") { constexpr size_t size = 4 * UCS_MBYTE; cuda_exportable_mem_pool_buffer buffer(size); - query_registrable_pinnable(buffer.ptr(), size, 1, 0); + query_registrable_gdr_capable(buffer.ptr(), size, 1, 0); } #endif -/* Pinning does not become possible for the default, non-exportable pool */ -UCS_TEST_P(test_mem_alloc_device, mem_pool_not_pinnable, +/* The default, non-exportable pool is not GDR capable either */ +UCS_TEST_P(test_mem_alloc_device, mem_pool_not_gdr_capable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda") { - test_mem_pool_not_pinnable(); + test_mem_pool_not_gdr_capable(); } /* The default typing of stream-ordered memory as managed must not report it as - * pinnable either */ -UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_pinnable, + * GDR capable either */ +UCS_TEST_P(test_mem_alloc_device, async_managed_mem_pool_not_gdr_capable, "CUDA_COPY_ASYNC_MEM_TYPE=cuda-managed") { - test_mem_pool_not_pinnable(); + test_mem_pool_not_gdr_capable(); } #if HAVE_DECL_CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN -/* Localized memory belongs to a single GPU locality domain, which the driver - * never places as GDR-capable, so the peer memory driver cannot pin it. +/* Localized memory cannot be allocated as GPUDirect RDMA capable. * Registrability is not asserted, as it depends on whether the platform can * export a dma_buf fd for such memory. */ -UCS_TEST_P(test_mem_alloc_device, localized_mem_not_pinnable, +UCS_TEST_P(test_mem_alloc_device, localized_mem_not_gdr_capable, "CUDA_COPY_REG_WHOLE_ALLOC=off") { constexpr size_t size = 4 * UCS_MBYTE; cuda_localized_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); - query_pinnable(buffer.ptr(), size, 0); + query_gdr_capable(buffer.ptr(), size, 0); } #endif @@ -641,6 +667,17 @@ UCS_TEST_P(test_mem_alloc_device, host_vmm_mem_registrable, query_registrable_no_current_context(buffer.ptr(), size); } + +/* Host-located VMM is typed as CUDA memory, but cannot be allocated as + * GPUDirect RDMA capable */ +UCS_TEST_P(test_mem_alloc_device, host_vmm_mem_not_gdr_capable, + "CUDA_COPY_REG_WHOLE_ALLOC=off") +{ + constexpr size_t size = 4 * UCS_MBYTE; + cuda_host_vmm_mem_buffer buffer(size, UCS_MEMORY_TYPE_CUDA); + + query_registrable_gdr_capable(buffer.ptr(), size, 1, 0); +} #endif UCS_TEST_P(test_mem_alloc_device, legacy_cuda_no_memtype_copy_inter_node) diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index 86efcfb65ae..a05e64d354a 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -1439,11 +1439,11 @@ UCS_TEST_SKIP_COND_P(test_gdr_copy, gdr_copy_reg_cuda_try_pcie_pin, ASSERT_UCS_OK(register_mem()); } -/* Pinning goes through nvidia_p2p_get_pages, so memory without device pages - * must never be offered to this memory domain */ -UCS_TEST_P(test_gdr_copy, gdr_copy_requires_pinnable) +/* Pinning goes through nvidia_p2p_get_pages, so memory which is not GPUDirect + * RDMA capable must never be offered to this memory domain */ +UCS_TEST_P(test_gdr_copy, gdr_copy_requires_gdr_capable) { - EXPECT_TRUE(md_attr().required_mem_flags & UCS_MEM_FLAG_PINNABLE); + EXPECT_TRUE(md_attr().required_mem_flags & UCS_MEM_FLAG_GDR_CAPABLE); } /* The reason for the requirement above: stream-ordered memory is only From 521c60bf2ad2202970be07471bc4306b10604c8f Mon Sep 17 00:00:00 2001 From: Thomas Vegas <125344591+tvegas1@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:51:27 +0200 Subject: [PATCH 9/9] UCP/UCT/CUDA: Avoid gdr_copy for non gdr capable memory Co-authored-by: tomerg-nvidia --- src/ucs/memory/memory_type.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ucs/memory/memory_type.h b/src/ucs/memory/memory_type.h index bdb468c102e..ce7e3994bb2 100644 --- a/src/ucs/memory/memory_type.h +++ b/src/ucs/memory/memory_type.h @@ -65,8 +65,7 @@ typedef enum ucs_mem_flags { /** * CUDA device memory allocated as GPUDirect RDMA (GDR) capable, so its - * pages can be pinned for peer access, as done by gdr_copy. Reported only - * when detected for the buffer, never assumed by default. + * pages can be pinned for peer access. */ UCS_MEM_FLAG_GDR_CAPABLE = UCS_BIT(2) } ucs_mem_flags_t;