From 754dd7f690f7402601b99b29980c434b37425486 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Sun, 19 Jul 2026 12:14:57 -0700 Subject: [PATCH 01/14] UCT/IB/MLX5/RC: Implement put_sgl_zcopy --- src/uct/ib/mlx5/rc/rc_mlx5.h | 10 ++- src/uct/ib/mlx5/rc/rc_mlx5_common.h | 5 +- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 105 +++++++++++++++++++++++++++- src/uct/ib/mlx5/rc/rc_mlx5_iface.c | 30 +++++++- 4 files changed, 145 insertions(+), 5 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.h b/src/uct/ib/mlx5/rc/rc_mlx5.h index d346a44f86d..ffdd34b5373 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5.h @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -123,6 +123,14 @@ ucs_status_t uct_rc_mlx5_base_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, uct_rkey_t rkey, uct_completion_t *comp); +ucs_status_t +uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, + const size_t *lengths, uct_mem_h const *memhs, + const uint64_t *remote_addrs, + uct_rkey_t const *rkeys, const size_t *counts, + const size_t *strides, size_t count, + uct_completion_t *comp); + ucs_status_t uct_rc_mlx5_base_ep_get_bcopy(uct_ep_h tl_ep, uct_unpack_callback_t unpack_cb, void *arg, size_t length, uint64_t remote_addr, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_common.h b/src/uct/ib/mlx5/rc/rc_mlx5_common.h index f5617bca388..31b72b5f738 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_common.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5_common.h @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -134,6 +134,9 @@ enum { UCT_RC_MLX5_CQE_APP_OP_TM_CONSUMED_MSG = 0xA }; +#define UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT 128 + + #define UCT_RC_MLX5_RMA_MAX_IOV(_av_size) \ ((UCT_IB_MLX5_MAX_SEND_WQE_SIZE - ((_av_size) + \ sizeof(struct mlx5_wqe_raddr_seg) + sizeof(struct mlx5_wqe_ctrl_seg))) / \ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 9cf09264368..70897391264 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -175,6 +175,109 @@ ucs_status_t uct_rc_mlx5_base_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, return status; } +ucs_status_t +uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, + const size_t *lengths, uct_mem_h const *memhs, + const uint64_t *remote_addrs, + uct_rkey_t const *rkeys, + const size_t UCS_V_UNUSED *counts, + const size_t UCS_V_UNUSED *strides, + size_t count, uct_completion_t *comp) +{ + UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); + uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; + size_t total = 0; + struct mlx5_wqe_ctrl_seg *ctrl = NULL; + struct mlx5_wqe_raddr_seg *raddr; + struct mlx5_wqe_data_seg *dptr; + size_t wqe_size, i; + uint8_t fm_ce_se, fence_flag; + uint16_t sn, pi, res_count; + uint64_t addr; + uct_rkey_t rkey; + void *curr; + + UCT_CHECK_PARAM(count <= UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, + "put_sgl_zcopy count(%zu) should be limited by %zu", + count, (size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT); + ucs_assert(ep->super.flags & UCT_RC_EP_FLAG_CONNECTED); + ucs_assert(count > 0); + + if (iface->super.tx.cq_available < (int)count) { + UCS_STATS_UPDATE_COUNTER(iface->super.stats, + UCT_RC_IFACE_STAT_NO_CQE, 1); + UCS_STATS_UPDATE_COUNTER(ep->super.super.stats, + UCT_EP_STAT_NO_RES, 1); + return UCS_ERR_NO_RESOURCE; + } + + UCT_RC_CHECK_NUM_RDMA_READ_RET(&iface->super, UCS_ERR_NO_RESOURCE); + UCT_RC_CHECK_TXQP_VALUE_RET(&iface->super, &ep->super, + UCS_ERR_NO_RESOURCE, count - 1); + + wqe_size = sizeof(*ctrl) + sizeof(*raddr) + sizeof(*dptr); + sn = txwq->sw_pi; + + ucs_assert(!(txwq->flags & UCT_IB_MLX5_TXWQ_FLAG_FAILED)); + ucs_assert(ucs_div_round_up(wqe_size, MLX5_SEND_WQE_BB) == 1); + + pi = sn; + curr = txwq->curr; + + fence_flag = uct_rc_ep_fm(&iface->super, &txwq->fi, 1) ? + iface->config.put_fence_flag : 0; + + for (i = 0; i < count; i++) { + UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE, + "put_sgl_zcopy"); + + fm_ce_se = ((i == 0) ? fence_flag : 0) | + ((i == count - 1) ? MLX5_WQE_CTRL_CQ_UPDATE : 0); + ctrl = curr; + + uct_ib_mlx5_set_ctrl_seg(ctrl, pi, MLX5_OPCODE_RDMA_WRITE, 0, + txwq->super.qp_num, fm_ce_se, 0, wqe_size); + + addr = remote_addrs[i]; + if (fence_flag != 0) { + rkey = uct_ib_resolve_atomic_rkey( + rkeys[i], ep->super.atomic_mr_offset, &addr); + } else { + rkey = uct_ib_md_direct_rkey(rkeys[i]); + } + + raddr = uct_ib_mlx5_txwq_wrap_none(txwq, ctrl + 1); + uct_ib_mlx5_ep_set_rdma_seg(raddr, addr, rkey); + + dptr = uct_ib_mlx5_txwq_wrap_none(txwq, raddr + 1); + uct_ib_mlx5_set_data_seg(dptr, buffers[i], lengths[i], + uct_ib_memh_get_lkey(memhs[i])); + + curr = UCS_PTR_BYTE_OFFSET(ctrl, MLX5_SEND_WQE_BB); + curr = uct_ib_mlx5_txwq_wrap_exact(txwq, curr); + pi++; + total += lengths[i]; + } + + res_count = pi - 1 - txwq->prev_sw_pi; + txwq->prev_sw_pi += res_count; + txwq->sw_pi = pi; + txwq->curr = curr; + txwq->sig_pi = txwq->prev_sw_pi; + + uct_rc_txqp_posted(&ep->super.txqp, &iface->super, res_count, 1); + uct_ib_mlx5_txwq_ring_doorbell(txwq, ctrl, txwq->sw_pi, 1); + + uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, + uct_rc_ep_send_op_completion_handler, comp, sn, + UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY, NULL, 0, total); + + UCT_TL_EP_STAT_OP(&ep->super.super, PUT, ZCOPY, total); + uct_rc_ep_enable_flush_remote(&ep->super); + + return UCS_INPROGRESS; +} + ucs_status_t uct_rc_mlx5_base_ep_get_bcopy(uct_ep_h tl_ep, uct_unpack_callback_t unpack_cb, void *arg, size_t length, uint64_t remote_addr, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index 10423703781..268af072ced 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -1011,14 +1011,40 @@ static UCS_CLASS_DEFINE_NEW_FUNC(uct_rc_mlx5_iface_t, uct_iface_t, uct_md_h, static UCS_CLASS_DEFINE_DELETE_FUNC(uct_rc_mlx5_iface_t, uct_iface_t); +static ucs_status_t +uct_rc_mlx5_ep_put_sgl_zcopy(uct_ep_h ep, void * const *buffers, + const size_t *lengths, uct_mem_h const *memhs, + const uint64_t *remote_addrs, + uct_rkey_t const *rkeys, const size_t *counts, + const size_t *strides, size_t count, + uct_completion_t *comp) +{ + ucs_status_t status; + + status = uct_ib_mlx5_ext_ep_put_sgl_zcopy(ep, buffers, lengths, memhs, + remote_addrs, rkeys, counts, + strides, count, comp); + if (status == UCS_ERR_UNSUPPORTED) { + status = uct_rc_mlx5_base_ep_put_sgl_zcopy(ep, buffers, lengths, memhs, + remote_addrs, rkeys, counts, + strides, count, comp); + } + + return status; +} + static ucs_status_t uct_rc_mlx5_iface_query_v2(uct_iface_h UCS_V_UNUSED iface, uct_iface_attr_v2_t *iface_attr) { size_t max_sgl = uct_ib_mlx5_ext_max_put_sgl_zcopy_count(); + if (max_sgl == 0) { + max_sgl = UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT; + } + if (iface_attr->field_mask & UCT_IFACE_ATTR_FIELD_CAP_FLAGS) { - iface_attr->cap.flags = (max_sgl > 0) ? UCT_IFACE_FLAG_V2_PUT_SGL_ZCOPY : 0; + iface_attr->cap.flags = UCT_IFACE_FLAG_V2_PUT_SGL_ZCOPY; } if (iface_attr->field_mask & UCT_IFACE_ATTR_FIELD_MAX_PUT_SGL_ZCOPY_COUNT) { @@ -1040,7 +1066,7 @@ static uct_rc_iface_ops_t uct_rc_mlx5_iface_ops = { .iface_is_reachable_v2 = uct_rc_mlx5_iface_is_reachable_v2, .ep_is_connected = uct_rc_mlx5_base_ep_is_connected, .ep_get_device_ep = (uct_ep_get_device_ep_func_t)ucs_empty_function_return_unsupported, - .ep_put_sgl_zcopy = uct_ib_mlx5_ext_ep_put_sgl_zcopy + .ep_put_sgl_zcopy = uct_rc_mlx5_ep_put_sgl_zcopy }, .create_cq = uct_rc_mlx5_iface_common_create_cq, .destroy_cq = uct_rc_mlx5_iface_common_destroy_cq, From 88dbcff9d180f573f378038e451fec0b583cc178 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 21 Jul 2026 04:20:59 -0700 Subject: [PATCH 02/14] UCT/IB/MLX5/RC: Reject strided put_sgl_zcopy --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 70897391264..45bf608b0ef 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -179,10 +179,9 @@ ucs_status_t uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, const size_t *lengths, uct_mem_h const *memhs, const uint64_t *remote_addrs, - uct_rkey_t const *rkeys, - const size_t UCS_V_UNUSED *counts, - const size_t UCS_V_UNUSED *strides, - size_t count, uct_completion_t *comp) + uct_rkey_t const *rkeys, const size_t *counts, + const size_t *strides, size_t count, + uct_completion_t *comp) { UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; @@ -203,6 +202,12 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, ucs_assert(ep->super.flags & UCT_RC_EP_FLAG_CONNECTED); ucs_assert(count > 0); + /* TODO: add strided elements support */ + if (ucs_unlikely((counts != NULL) || (strides != NULL))) { + ucs_error("put_sgl_zcopy does not support strided elements"); + return UCS_ERR_UNSUPPORTED; + } + if (iface->super.tx.cq_available < (int)count) { UCS_STATS_UPDATE_COUNTER(iface->super.stats, UCT_RC_IFACE_STAT_NO_CQE, 1); From ec2c4b0db5b5a7f72817b846a0484e729440359f Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 21 Jul 2026 11:07:26 -0700 Subject: [PATCH 03/14] UCT/IB/MLX5/RC: Validate lengths before consuming fence state in put_sgl_zcopy --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 45bf608b0ef..fcd375582b7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -220,6 +220,12 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, UCT_RC_CHECK_TXQP_VALUE_RET(&iface->super, &ep->super, UCS_ERR_NO_RESOURCE, count - 1); + /* Validate all lengths before consuming the fence state */ + for (i = 0; i < count; i++) { + UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE, + "put_sgl_zcopy"); + } + wqe_size = sizeof(*ctrl) + sizeof(*raddr) + sizeof(*dptr); sn = txwq->sw_pi; @@ -233,9 +239,6 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, iface->config.put_fence_flag : 0; for (i = 0; i < count; i++) { - UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE, - "put_sgl_zcopy"); - fm_ce_se = ((i == 0) ? fence_flag : 0) | ((i == count - 1) ? MLX5_WQE_CTRL_CQ_UPDATE : 0); ctrl = curr; From 572312d00925070dc8e69b4583250497ee074f25 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 21 Jul 2026 12:50:08 -0700 Subject: [PATCH 04/14] UCT/IB/MLX5/RC: Cap put_sgl_zcopy count --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 8 ++++++++ src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 9 +++++---- src/uct/ib/mlx5/rc/rc_mlx5_iface.c | 7 ++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 9548df6b0cd..1239c5e7cf9 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -24,6 +24,14 @@ uct_rc_mlx5_iface_common_t) +static UCS_F_ALWAYS_INLINE size_t +uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_iface_t *iface) +{ + return ucs_min((size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, + iface->config.tx_qp_len); +} + + static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_ep_fence_put(uct_rc_mlx5_iface_common_t *iface, uct_ib_mlx5_txwq_t *txwq, uct_rkey_t *rkey, uint64_t *addr, uint16_t offset, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index fcd375582b7..ea7457afca8 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -189,16 +189,17 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, struct mlx5_wqe_ctrl_seg *ctrl = NULL; struct mlx5_wqe_raddr_seg *raddr; struct mlx5_wqe_data_seg *dptr; - size_t wqe_size, i; + size_t wqe_size, i, max_count; uint8_t fm_ce_se, fence_flag; uint16_t sn, pi, res_count; uint64_t addr; uct_rkey_t rkey; void *curr; - UCT_CHECK_PARAM(count <= UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, - "put_sgl_zcopy count(%zu) should be limited by %zu", - count, (size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT); + max_count = uct_rc_mlx5_base_put_sgl_zcopy_max_count(&iface->super); + UCT_CHECK_PARAM(count <= max_count, + "put_sgl_zcopy count(%zu) should be limited by %zu", count, + max_count); ucs_assert(ep->super.flags & UCT_RC_EP_FLAG_CONNECTED); ucs_assert(count > 0); diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index 268af072ced..668dd222907 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -1034,13 +1034,14 @@ uct_rc_mlx5_ep_put_sgl_zcopy(uct_ep_h ep, void * const *buffers, } static ucs_status_t -uct_rc_mlx5_iface_query_v2(uct_iface_h UCS_V_UNUSED iface, +uct_rc_mlx5_iface_query_v2(uct_iface_h tl_iface, uct_iface_attr_v2_t *iface_attr) { - size_t max_sgl = uct_ib_mlx5_ext_max_put_sgl_zcopy_count(); + uct_rc_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_iface_t); + size_t max_sgl = uct_ib_mlx5_ext_max_put_sgl_zcopy_count(); if (max_sgl == 0) { - max_sgl = UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT; + max_sgl = uct_rc_mlx5_base_put_sgl_zcopy_max_count(iface); } if (iface_attr->field_mask & UCT_IFACE_ATTR_FIELD_CAP_FLAGS) { From 0e7195a65a70262977b846a102b03e8305cbc269 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 21 Jul 2026 13:19:31 -0700 Subject: [PATCH 05/14] UCT/IB/MLX5/RC: Cap put_sgl_zcopy count by QP and BB limits --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 8 +++++--- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 2 +- src/uct/ib/mlx5/rc/rc_mlx5_iface.c | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 1239c5e7cf9..89b4f3a84c2 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -25,10 +25,12 @@ static UCS_F_ALWAYS_INLINE size_t -uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_iface_t *iface) +uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_mlx5_iface_common_t *iface) { - return ucs_min((size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, - iface->config.tx_qp_len); + size_t max_count = ucs_min((size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, + iface->super.config.tx_qp_len); + + return ucs_min(max_count, iface->tx.bb_max); } diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index ea7457afca8..f8a4ba36cab 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -196,7 +196,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, uct_rkey_t rkey; void *curr; - max_count = uct_rc_mlx5_base_put_sgl_zcopy_max_count(&iface->super); + max_count = uct_rc_mlx5_base_put_sgl_zcopy_max_count(iface); UCT_CHECK_PARAM(count <= max_count, "put_sgl_zcopy count(%zu) should be limited by %zu", count, max_count); diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index 668dd222907..aecf06ea607 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -1037,8 +1037,9 @@ static ucs_status_t uct_rc_mlx5_iface_query_v2(uct_iface_h tl_iface, uct_iface_attr_v2_t *iface_attr) { - uct_rc_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_iface_t); - size_t max_sgl = uct_ib_mlx5_ext_max_put_sgl_zcopy_count(); + uct_rc_mlx5_iface_common_t *iface = ucs_derived_of(tl_iface, + uct_rc_mlx5_iface_common_t); + size_t max_sgl = uct_ib_mlx5_ext_max_put_sgl_zcopy_count(); if (max_sgl == 0) { max_sgl = uct_rc_mlx5_base_put_sgl_zcopy_max_count(iface); From 685abdad12dafc08f8dd959ccd2296d780e7fa61 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Wed, 22 Jul 2026 05:54:16 -0700 Subject: [PATCH 06/14] UCT/IB/MLX5: Fix dead-store warning for max_count in put_sgl_zcopy --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index f8a4ba36cab..67135218cc8 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -189,12 +189,13 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, struct mlx5_wqe_ctrl_seg *ctrl = NULL; struct mlx5_wqe_raddr_seg *raddr; struct mlx5_wqe_data_seg *dptr; - size_t wqe_size, i, max_count; + size_t wqe_size, i; uint8_t fm_ce_se, fence_flag; uint16_t sn, pi, res_count; uint64_t addr; uct_rkey_t rkey; void *curr; + size_t UCS_V_UNUSED max_count; max_count = uct_rc_mlx5_base_put_sgl_zcopy_max_count(iface); UCT_CHECK_PARAM(count <= max_count, From 63756a4bc22afc4f7f33e64af372af96e4600f30 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Fri, 24 Jul 2026 02:16:47 -0700 Subject: [PATCH 07/14] UCT/IB/MLX5: Validate put_sgl_zcopy lengths before resource checks --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 67135218cc8..9e56e52ca86 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -210,6 +210,12 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, return UCS_ERR_UNSUPPORTED; } + /* Validate all lengths before resource checks and fence state */ + for (i = 0; i < count; i++) { + UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE, + "put_sgl_zcopy"); + } + if (iface->super.tx.cq_available < (int)count) { UCS_STATS_UPDATE_COUNTER(iface->super.stats, UCT_RC_IFACE_STAT_NO_CQE, 1); @@ -222,12 +228,6 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, UCT_RC_CHECK_TXQP_VALUE_RET(&iface->super, &ep->super, UCS_ERR_NO_RESOURCE, count - 1); - /* Validate all lengths before consuming the fence state */ - for (i = 0; i < count; i++) { - UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE, - "put_sgl_zcopy"); - } - wqe_size = sizeof(*ctrl) + sizeof(*raddr) + sizeof(*dptr); sn = txwq->sw_pi; From 9128ce39cc7428b2feb974d8ebd3b7a801167c8d Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Fri, 24 Jul 2026 02:40:48 -0700 Subject: [PATCH 08/14] UCT/IB/MLX5: Use fence state for put_sgl_zcopy rkey selection --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 9e56e52ca86..4f7651edd92 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -196,6 +196,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, uct_rkey_t rkey; void *curr; size_t UCS_V_UNUSED max_count; + int fence; max_count = uct_rc_mlx5_base_put_sgl_zcopy_max_count(iface); UCT_CHECK_PARAM(count <= max_count, @@ -237,8 +238,8 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, pi = sn; curr = txwq->curr; - fence_flag = uct_rc_ep_fm(&iface->super, &txwq->fi, 1) ? - iface->config.put_fence_flag : 0; + fence = uct_rc_ep_fm(&iface->super, &txwq->fi, 1); + fence_flag = fence ? iface->config.put_fence_flag : 0; for (i = 0; i < count; i++) { fm_ce_se = ((i == 0) ? fence_flag : 0) | @@ -249,7 +250,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, txwq->super.qp_num, fm_ce_se, 0, wqe_size); addr = remote_addrs[i]; - if (fence_flag != 0) { + if (fence) { rkey = uct_ib_resolve_atomic_rkey( rkeys[i], ep->super.atomic_mr_offset, &addr); } else { From 9538693d0540b23b95658c85e836a9fb34b1d4c7 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Fri, 24 Jul 2026 03:45:50 -0700 Subject: [PATCH 09/14] UCT/IB/MLX5: Limit PUT SGL zcopy to half the TX queue --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 6 ++---- src/uct/ib/mlx5/rc/rc_mlx5_common.h | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 89b4f3a84c2..ff3948b1ef7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -27,10 +27,8 @@ static UCS_F_ALWAYS_INLINE size_t uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_mlx5_iface_common_t *iface) { - size_t max_count = ucs_min((size_t)UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT, - iface->super.config.tx_qp_len); - - return ucs_min(max_count, iface->tx.bb_max); + /* Cap at half the QP so one SGL cannot monopolize the send queue. */ + return ucs_min(iface->super.config.tx_qp_len / 2, iface->tx.bb_max); } diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_common.h b/src/uct/ib/mlx5/rc/rc_mlx5_common.h index 31b72b5f738..f5617bca388 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_common.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5_common.h @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -134,9 +134,6 @@ enum { UCT_RC_MLX5_CQE_APP_OP_TM_CONSUMED_MSG = 0xA }; -#define UCT_RC_MLX5_PUT_SGL_ZCOPY_MAX_COUNT 128 - - #define UCT_RC_MLX5_RMA_MAX_IOV(_av_size) \ ((UCT_IB_MLX5_MAX_SEND_WQE_SIZE - ((_av_size) + \ sizeof(struct mlx5_wqe_raddr_seg) + sizeof(struct mlx5_wqe_ctrl_seg))) / \ From 0c19cfa75963f10854e16286e3710300ffe739e6 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Fri, 24 Jul 2026 04:19:15 -0700 Subject: [PATCH 10/14] UCT/IB/RC: Centralize TX CQ capacity calculation --- src/uct/ib/rc/base/rc_iface.c | 4 ++-- src/uct/ib/rc/base/rc_iface.h | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/uct/ib/rc/base/rc_iface.c b/src/uct/ib/rc/base/rc_iface.c index 82709760b03..a33e622965c 100644 --- a/src/uct/ib/rc/base/rc_iface.c +++ b/src/uct/ib/rc/base/rc_iface.c @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2021. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Huawei Technologies Co., Ltd. 2021. ALL RIGHTS RESERVED. * * See file LICENSE for terms. @@ -587,7 +587,7 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_iface_ops_t *tl_ops, tx_cq_size = uct_ib_cq_size(&self->super, init_attr, UCT_IB_DIR_TX); /* Prevent title CQE overwriting */ - self->tx.cq_available = tx_cq_size - 2; + self->tx.cq_available = uct_rc_iface_tx_cq_capacity(tx_cq_size); self->rx.srq.available = 0; self->rx.srq.quota = 0; self->config.tx_qp_len = config->super.tx.queue_len; diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index aaf58e1ad2d..737fc4de75d 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -22,6 +22,7 @@ #define UCT_RC_QP_TABLE_SIZE UCS_BIT(UCT_RC_QP_TABLE_ORDER) #define UCT_RC_QP_TABLE_MEMB_ORDER (UCT_IB_QPN_ORDER - UCT_RC_QP_TABLE_ORDER) #define UCT_RC_QP_MAX_RETRY_COUNT 7 +#define UCT_RC_TX_CQ_RESERVED 2 #define UCT_RC_CHECK_AM_SHORT(_am_id, _length, _header_t, _max_inline) \ UCT_CHECK_AM_ID(_am_id); \ @@ -487,6 +488,12 @@ uct_rc_iface_have_tx_cqe_avail(uct_rc_iface_t* iface) return iface->tx.cq_available > 0; } +static UCS_F_ALWAYS_INLINE unsigned +uct_rc_iface_tx_cq_capacity(unsigned tx_cq_len) +{ + return tx_cq_len - UCT_RC_TX_CQ_RESERVED; +} + /** * Release RDMA_READ credits back to RC iface. * RDMA_READ credits are freed in completion callbacks, but not released to From e7c4d756ae386d1ec0b2a26c6c33956b050dcf00 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Fri, 24 Jul 2026 04:24:35 -0700 Subject: [PATCH 11/14] UCT/IB/MLX5: Cap PUT SGL zcopy by usable TX CQ credits --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index ff3948b1ef7..d73f4d9f76e 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -27,8 +27,13 @@ static UCS_F_ALWAYS_INLINE size_t uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_mlx5_iface_common_t *iface) { + size_t max_count; + /* Cap at half the QP so one SGL cannot monopolize the send queue. */ - return ucs_min(iface->super.config.tx_qp_len / 2, iface->tx.bb_max); + max_count = ucs_min(iface->super.config.tx_qp_len / 2, iface->tx.bb_max); + + return ucs_min(max_count, + uct_rc_iface_tx_cq_capacity(iface->super.config.tx_cq_len)); } From d05605f3357fe24f836ce42c8f0741e459924771 Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Sun, 26 Jul 2026 05:45:10 -0700 Subject: [PATCH 12/14] UCT/IB/MLX5: Cap PUT SGL count by usable TX CQ credits --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index d73f4d9f76e..74a78633ca4 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2024. ALL RIGHTS RESERVED. * * See file LICENSE for terms. @@ -32,8 +32,9 @@ uct_rc_mlx5_base_put_sgl_zcopy_max_count(uct_rc_mlx5_iface_common_t *iface) /* Cap at half the QP so one SGL cannot monopolize the send queue. */ max_count = ucs_min(iface->super.config.tx_qp_len / 2, iface->tx.bb_max); - return ucs_min(max_count, - uct_rc_iface_tx_cq_capacity(iface->super.config.tx_cq_len)); + max_count = ucs_min(max_count, + uct_rc_iface_tx_cq_capacity(iface->super.config.tx_cq_len)); + return max_count; } From 1af61f3082d0e19c36ca6692fc0c2368f987678b Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Sun, 26 Jul 2026 12:41:20 -0700 Subject: [PATCH 13/14] UCT/RC: Add per-EP CQ reserve for multi-CQE operations --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 9 ++------- src/uct/ib/rc/base/rc_ep.c | 10 +++++++--- src/uct/ib/rc/base/rc_ep.h | 27 +++++++++++++++++++++------ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 4f7651edd92..8094d9aaf1a 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -217,13 +217,8 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, "put_sgl_zcopy"); } - if (iface->super.tx.cq_available < (int)count) { - UCS_STATS_UPDATE_COUNTER(iface->super.stats, - UCT_RC_IFACE_STAT_NO_CQE, 1); - UCS_STATS_UPDATE_COUNTER(ep->super.super.stats, - UCT_EP_STAT_NO_RES, 1); - return UCS_ERR_NO_RESOURCE; - } + UCT_RC_CHECK_CQE_VALUE_RET(&iface->super, &ep->super, + UCS_ERR_NO_RESOURCE, count - 1); UCT_RC_CHECK_NUM_RDMA_READ_RET(&iface->super, UCS_ERR_NO_RESOURCE); UCT_RC_CHECK_TXQP_VALUE_RET(&iface->super, &ep->super, diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index d9bf15b8baf..36177b3b6a7 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * Copyright (c) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED. * Copyright (C) Huawei Technologies Co., Ltd. 2021. ALL RIGHTS RESERVED. * @@ -150,6 +150,7 @@ UCS_CLASS_INIT_FUNC(uct_rc_ep_t, uct_rc_iface_t *iface, uint32_t qp_num, self->path_index = UCT_EP_PARAMS_GET_PATH_INDEX(params); self->flags = 0; self->txqp_reserve = 0; + self->cq_reserve = 0; status = uct_rc_fc_init(&self->fc, iface UCS_STATS_ARG(self->super.stats)); if (status != UCS_OK) { @@ -315,7 +316,8 @@ ucs_status_t uct_rc_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *n, uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_ep_t); - if (uct_rc_ep_has_tx_resources(ep) && + if (uct_rc_ep_have_tx_cqe_avail(ep) && + uct_rc_ep_has_tx_resources(ep) && uct_rc_iface_has_tx_resources(iface)) { return UCS_ERR_BUSY; } @@ -346,10 +348,12 @@ ucs_arbiter_cb_result_t uct_rc_ep_process_pending(ucs_arbiter_t *arbiter, status = uct_rc_iface_invoke_pending_cb(iface, req); if (status == UCS_OK) { ep->txqp_reserve = 0; + ep->cq_reserve = 0; return UCS_ARBITER_CB_RESULT_REMOVE_ELEM; } else if (status == UCS_INPROGRESS) { return UCS_ARBITER_CB_RESULT_NEXT_GROUP; - } else if (!uct_rc_iface_has_tx_resources(iface)) { + } else if (!uct_rc_iface_has_tx_resources(iface) || + !uct_rc_ep_have_tx_cqe_avail(ep)) { /* No iface resources */ return UCS_ARBITER_CB_RESULT_STOP; } diff --git a/src/uct/ib/rc/base/rc_ep.h b/src/uct/ib/rc/base/rc_ep.h index 4db0e5fc8a5..09f296a82be 100644 --- a/src/uct/ib/rc/base/rc_ep.h +++ b/src/uct/ib/rc/base/rc_ep.h @@ -1,5 +1,5 @@ /** -* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED. +* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -97,14 +97,19 @@ enum { /* tx_moderation == 0 for TLs which don't support it */ \ if (ucs_unlikely((_iface)->tx.cq_available <= \ (signed)(_iface)->config.tx_moderation)) { \ - if (!uct_rc_iface_have_tx_cqe_avail(_iface)) { \ - UCS_STATS_UPDATE_COUNTER((_iface)->stats, UCT_RC_IFACE_STAT_NO_CQE, 1); \ - UCS_STATS_UPDATE_COUNTER((_ep)->super.stats, UCT_EP_STAT_NO_RES, 1); \ - return _ret; \ - } \ + UCT_RC_CHECK_CQE_VALUE_RET(_iface, _ep, _ret, 0); \ (_ep)->txqp.unsignaled = RC_UNSIGNALED_INF; \ } +#define UCT_RC_CHECK_CQE_VALUE_RET(_iface, _ep, _ret, _value) \ + if ((_iface)->tx.cq_available <= (signed)(_value)) { \ + (_ep)->cq_reserve = ucs_max((_ep)->cq_reserve, \ + (uint16_t)(_value)); \ + UCS_STATS_UPDATE_COUNTER((_iface)->stats, UCT_RC_IFACE_STAT_NO_CQE, 1); \ + UCS_STATS_UPDATE_COUNTER((_ep)->super.stats, UCT_EP_STAT_NO_RES, 1); \ + return _ret; \ + } + #define UCT_RC_CHECK_TXQP_RET(_iface, _ep, _ret) \ UCT_RC_CHECK_TXQP_VALUE_RET(_iface, _ep, _ret, 0) @@ -225,6 +230,7 @@ struct uct_rc_ep { uct_rc_fc_t fc; uint16_t atomic_mr_offset; uint16_t txqp_reserve; + uint16_t cq_reserve; uint8_t path_index; uint8_t flags; }; @@ -356,6 +362,15 @@ static UCS_F_ALWAYS_INLINE int uct_rc_ep_has_tx_resources(uct_rc_ep_t *ep) uct_rc_fc_has_resources(iface, &ep->fc); } +static UCS_F_ALWAYS_INLINE int +uct_rc_ep_have_tx_cqe_avail(uct_rc_ep_t *ep) +{ + uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.iface, + uct_rc_iface_t); + + return iface->tx.cq_available > (signed)ep->cq_reserve; +} + static UCS_F_ALWAYS_INLINE void uct_rc_txqp_add_send_op(uct_rc_txqp_t *txqp, uct_rc_iface_send_op_t *op) { From 60c8d464f91239edc527f2f4f9b34563caaf6f1c Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Sun, 26 Jul 2026 13:17:59 -0700 Subject: [PATCH 14/14] UCT/IB/MLX5: Handle zero-count PUT SGL operations --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 8094d9aaf1a..88c33f0008c 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -203,7 +203,6 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, "put_sgl_zcopy count(%zu) should be limited by %zu", count, max_count); ucs_assert(ep->super.flags & UCT_RC_EP_FLAG_CONNECTED); - ucs_assert(count > 0); /* TODO: add strided elements support */ if (ucs_unlikely((counts != NULL) || (strides != NULL))) { @@ -211,6 +210,8 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, return UCS_ERR_UNSUPPORTED; } + UCT_SKIP_ZERO_LENGTH(count); + /* Validate all lengths before resource checks and fence state */ for (i = 0; i < count; i++) { UCT_CHECK_LENGTH(lengths[i], 0, UCT_IB_MAX_MESSAGE_SIZE,