From ab74f2b1f9303fab6f5646e82b60e8c769246614 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 24 Aug 2026 17:36:28 +0300 Subject: [PATCH 01/21] UCT/IB/MLX5: Track RC send PSN with path MTU --- src/uct/ib/mlx5/ib_mlx5.c | 12 +++-- src/uct/ib/mlx5/ib_mlx5.h | 3 ++ src/uct/ib/mlx5/rc/rc_mlx5.inl | 63 +++++++++++++++++++++++++ src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 6 ++- test/gtest/uct/ib/test_rc.cc | 83 +++++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 4 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 40cbba23578..4ce9b82cb31 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -693,9 +693,12 @@ void uct_ib_mlx5_devx_uar_cleanup(uct_ib_mlx5_devx_uar_t *uar) void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq) { - txwq->curr = txwq->qstart; - txwq->sw_pi = 0; - txwq->prev_sw_pi = UINT16_MAX; + txwq->curr = txwq->qstart; + txwq->sw_pi = 0; + txwq->prev_sw_pi = UINT16_MAX; + txwq->next_first_psn = 0; + txwq->path_mtu_mask = 0; + txwq->path_mtu_shift = 0; #if UCS_ENABLE_ASSERT txwq->hw_ci = 0xFFFF; txwq->flags = 0; @@ -721,6 +724,9 @@ void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) UCS_VFS_TYPE_U16, "sw_pi"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->prev_sw_pi, UCS_VFS_TYPE_U16, "prev_sw_pi"); + ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, + &txwq->next_first_psn, UCS_VFS_TYPE_U32, + "next_first_psn"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->qstart, UCS_VFS_TYPE_POINTER, "qstart"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->qend, diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 0e013b798ab..0ebc375c253 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -686,6 +686,9 @@ typedef struct uct_ib_mlx5_txwq { uct_ib_mlx5_qp_t super; uint16_t sw_pi; /* PI for next WQE */ uint16_t prev_sw_pi; /* PI where last WQE *started* */ + uint32_t next_first_psn; /* First PSN of next WQE */ + uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ + uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ uct_ib_mlx5_mmio_reg_t *reg; void *curr; volatile uint32_t *dbrec; diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 50ae3e943f8..9f19c49f390 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -452,6 +452,45 @@ uct_rc_mlx5_ep_fm_cq_update(uct_rc_mlx5_iface_common_t *iface, return fm_ce_se; } +static UCS_F_ALWAYS_INLINE void +uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, + enum ibv_mtu path_mtu) +{ + size_t mtu = uct_ib_mtu_value(path_mtu); + + ucs_assert(mtu <= UINT16_MAX); + ucs_assert(ucs_is_pow2(mtu)); + + txwq->path_mtu_mask = mtu - 1; + txwq->path_mtu_shift = ucs_ilog2(mtu); +} + +static UCS_F_ALWAYS_INLINE uint32_t +uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, + size_t message_length) +{ + ucs_assert(txwq->path_mtu_shift > 0); + + return ucs_max((size_t)1, + (message_length + txwq->path_mtu_mask) >> + txwq->path_mtu_shift); +} + +static UCS_F_ALWAYS_INLINE void +uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, uint32_t num_packets) +{ + txwq->next_first_psn = + (txwq->next_first_psn + num_packets) & UCS_MASK(24); +} + +static UCS_F_ALWAYS_INLINE void +uct_rc_mlx5_txwq_update_psn(uct_ib_mlx5_txwq_t *txwq, + size_t message_length) +{ + uct_rc_mlx5_txwq_add_psn(txwq, + uct_rc_mlx5_num_packets(txwq, message_length)); +} + static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_common_post_send(uct_rc_mlx5_iface_common_t *iface, int qp_type, uct_rc_txqp_t *txqp, uct_ib_mlx5_txwq_t *txwq, @@ -532,6 +571,9 @@ static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txqp_inline_iov_post( uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, MLX5_OPCODE_SEND, 0, fm_ce_se, dci_channel, wqe_size, 0, INT_MAX, NULL); + if (qp_type == IBV_QPT_RC) { + uct_rc_mlx5_txwq_add_psn(txwq, 1); + } } /* @@ -627,6 +669,10 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, NULL); + if ((qp_type == IBV_QPT_RC) && (opcode != MLX5_OPCODE_NOP)) { + /* Inline short operations always fit in one RC packet. */ + uct_rc_mlx5_txwq_add_psn(txwq, 1); + } } /* @@ -810,6 +856,9 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, (opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, log_sge); + if (qp_type == IBV_QPT_RC) { + uct_rc_mlx5_txwq_update_psn(txwq, length); + } } static UCS_F_ALWAYS_INLINE @@ -830,6 +879,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty struct mlx5_wqe_inl_data_seg *inl; uct_rc_mlx5_hdr_t *rch; unsigned wqe_size, inl_seg_size, ctrl_av_size; + size_t message_length; void *next_seg; uint8_t opmod; #if HAVE_MLX5_MMO @@ -868,6 +918,8 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty wqe_size = ctrl_av_size + inl_seg_size + uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt); opmod = 0; + message_length = uct_iov_total_length(iov, iovcnt) + sizeof(*rch) + + am_hdr_len; ucs_assert(wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE); break; @@ -883,6 +935,8 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty wqe_size = ctrl_av_size + inl_seg_size + uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt); opmod = 0; + message_length = uct_iov_total_length(iov, iovcnt) + + sizeof(struct ibv_tmh); uct_rc_mlx5_fill_tmh((struct ibv_tmh*)(inl + 1), tag, app_ctx, IBV_TMH_EAGER); @@ -905,6 +959,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty uct_ib_mlx5_set_data_seg_iov(txwq, (void*)(raddr + 1), iov, iovcnt); opmod = 0; + message_length = uct_iov_total_length(iov, iovcnt); break; #if HAVE_MLX5_MMO @@ -940,6 +995,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty wqe_size = sizeof(*ctrl) + sizeof(*dma_seg) + (2 * sizeof(*dptr)); opmod = UCT_IB_MLX5_OPMOD_MMO_DMA; + message_length = 0; break; #endif default: @@ -950,6 +1006,9 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty opcode_flags & UCT_RC_MLX5_OPCODE_MASK, opmod, fm_ce_se, dci_channel, wqe_size, ib_imm_be, max_log_sge, NULL); + if (qp_type == IBV_QPT_RC) { + uct_rc_mlx5_txwq_update_psn(txwq, message_length); + } } /* @@ -1099,6 +1158,10 @@ uct_rc_mlx5_txqp_tag_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se, dci_channel, wqe_size, imm_val_be, INT_MAX, NULL); + if (qp_type == IBV_QPT_RC) { + /* Inline tag operations always fit in one RC packet. */ + uct_rc_mlx5_txwq_add_psn(txwq, 1); + } } static UCS_F_ALWAYS_INLINE void diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 95e48a643ba..e9456ff3a29 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -215,6 +215,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, struct mlx5_wqe_raddr_seg *raddr; struct mlx5_wqe_data_seg *dptr; size_t wqe_size, i; + uint32_t num_packets = 0; uint8_t fm_ce_se, fence_flag; uint16_t sn, pi, res_count; uint64_t addr; @@ -288,7 +289,8 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, curr = UCS_PTR_BYTE_OFFSET(ctrl, MLX5_SEND_WQE_BB); curr = uct_ib_mlx5_txwq_wrap_exact(txwq, curr); pi++; - total += lengths[i]; + total += lengths[i]; + num_packets += uct_rc_mlx5_num_packets(txwq, lengths[i]); } res_count = pi - 1 - txwq->prev_sw_pi; @@ -299,6 +301,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, uct_rc_txqp_posted(&ep->super.txqp, &iface->super, res_count, 1); uct_ib_mlx5_txwq_ring_doorbell(txwq, ctrl, txwq->sw_pi, 1); + uct_rc_mlx5_txwq_add_psn(txwq, num_packets); uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, uct_rc_ep_send_op_completion_handler, comp, sn, @@ -973,6 +976,7 @@ uct_rc_mlx5_ep_connect_to_ep_v2(uct_ep_h tl_ep, if (status != UCS_OK) { return status; } + uct_rc_mlx5_txwq_set_path_mtu(&ep->super.tx.wq, path_mtu); ep->super.super.atomic_mr_offset = uct_ib_md_atomic_offset( rc_addr->atomic_mr_id); diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index e9946df00c3..b0747445353 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -144,6 +144,89 @@ UCS_TEST_P(test_rc, fence_am_short_consumed, "RC_FENCE=weak") UCT_INSTANTIATE_RC_TEST_CASE(test_rc) +#ifdef HAVE_MLX5_DV +class test_rc_mlx5_psn : public test_rc { +public: + void init() override + { + uct_test::init(); + + modify_config("IB_PATH_MTU", "4096"); + m_e1 = uct_test::create_entity(0); + m_entities.push_back(m_e1); + + check_skip_test(); + + modify_config("IB_PATH_MTU", "512"); + m_e2 = uct_test::create_entity(0); + m_entities.push_back(m_e2); + + connect(); + } + +protected: + uct_ib_mlx5_txwq_t *txwq() + { + return &ucs_derived_of(m_e1->ep(0), + uct_rc_mlx5_ep_t)->super.tx.wq; + } +}; + +UCS_TEST_P(test_rc_mlx5_psn, path_mtu_from_rc_qp) +{ + uct_ib_iface_t *local_iface = + ucs_derived_of(m_e1->iface(), uct_ib_iface_t); + uct_ib_iface_t *peer_iface = + ucs_derived_of(m_e2->iface(), uct_ib_iface_t); + + EXPECT_EQ(IBV_MTU_4096, local_iface->config.path_mtu); + EXPECT_EQ(IBV_MTU_512, peer_iface->config.path_mtu); + EXPECT_EQ(511, txwq()->path_mtu_mask); + EXPECT_EQ(9, txwq()->path_mtu_shift); +} + +UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, + !check_caps(UCT_IFACE_FLAG_PUT_ZCOPY)) +{ + static const size_t length = 513; + uct_ib_mlx5_txwq_t *wq = txwq(); + mapped_buffer sendbuf(length, 0ul, *m_e1); + mapped_buffer recvbuf(length, 0ul, *m_e2); + uct_completion_t comp; + ucs_status_t status; + + EXPECT_EQ(0, wq->next_first_psn); + + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); + EXPECT_EQ(1, wq->next_first_psn); + + comp.func = [](uct_completion_t*) {}; + comp.count = 1; + comp.status = UCS_OK; + + UCS_TEST_GET_BUFFER_IOV(iov, iovcnt, sendbuf.ptr(), sendbuf.length(), + sendbuf.memh(), 1); + status = uct_ep_put_zcopy(m_e1->ep(0), iov, iovcnt, recvbuf.addr(), + recvbuf.rkey(), &comp); + ASSERT_UCS_OK_OR_INPROGRESS(status); + + /* The RC QP uses the peer's 512-byte path MTU, so 513 bytes use 2 PSNs. */ + EXPECT_EQ(3, wq->next_first_psn); + + if (status == UCS_INPROGRESS) { + wait_for_value(&comp.count, 0, true); + } + + wq->next_first_psn = UCS_MASK(24); + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); + EXPECT_EQ(0, wq->next_first_psn); + + flush(); +} + +_UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_psn, rc_mlx5) +#endif + class test_rc_max_wr : public test_rc { protected: From 31d155afeb5ce81851148f71fb2503cdb6288caf Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 24 Aug 2026 19:00:27 +0300 Subject: [PATCH 02/21] UCT/IB/MLX5: Avoid duplicate RC IOV traversal Report the payload length while building mlx5 data segments so RC PSN tracking does not walk the zcopy IOV again on the send fast path. --- src/uct/ib/mlx5/ib_mlx5.inl | 31 +++++++++++++++++++++------ src/uct/ib/mlx5/rc/rc_mlx5.inl | 39 ++++++++++++++++------------------ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.inl b/src/uct/ib/mlx5/ib_mlx5.inl index 193ccb80bd6..9bac1c40863 100644 --- a/src/uct/ib/mlx5/ib_mlx5.inl +++ b/src/uct/ib/mlx5/ib_mlx5.inl @@ -507,12 +507,15 @@ uct_ib_mlx5_set_data_seg(struct mlx5_wqe_data_seg *dptr, static UCS_F_ALWAYS_INLINE -size_t uct_ib_mlx5_set_data_seg_iov(uct_ib_mlx5_txwq_t *txwq, - struct mlx5_wqe_data_seg *dptr, - const uct_iov_t *iov, size_t iovcnt) +size_t uct_ib_mlx5_set_data_seg_iov_length(uct_ib_mlx5_txwq_t *txwq, + struct mlx5_wqe_data_seg *dptr, + const uct_iov_t *iov, size_t iovcnt, + size_t *iov_length_p) { - size_t wqe_size = 0; + size_t iov_length = 0; + size_t wqe_size = 0; size_t iov_it; + size_t length; for (iov_it = 0; iov_it < iovcnt; ++iov_it) { if (!iov[iov_it].length) { /* Skip zero length WQE*/ @@ -522,17 +525,31 @@ size_t uct_ib_mlx5_set_data_seg_iov(uct_ib_mlx5_txwq_t *txwq, /* place data into the buffer */ dptr = uct_ib_mlx5_txwq_wrap_any(txwq, dptr); - uct_ib_mlx5_set_data_seg(dptr, iov[iov_it].buffer, - uct_iov_get_length(iov + iov_it), + length = uct_iov_get_length(iov + iov_it); + uct_ib_mlx5_set_data_seg(dptr, iov[iov_it].buffer, length, uct_ib_memh_get_lkey(iov[iov_it].memh)); - wqe_size += sizeof(*dptr); + iov_length += length; + wqe_size += sizeof(*dptr); ++dptr; } + if (iov_length_p != NULL) { + *iov_length_p = iov_length; + } + return wqe_size; } +static UCS_F_ALWAYS_INLINE +size_t uct_ib_mlx5_set_data_seg_iov(uct_ib_mlx5_txwq_t *txwq, + struct mlx5_wqe_data_seg *dptr, + const uct_iov_t *iov, size_t iovcnt) +{ + return uct_ib_mlx5_set_data_seg_iov_length(txwq, dptr, iov, iovcnt, NULL); +} + + static UCS_F_ALWAYS_INLINE void uct_ib_mlx5_bf_copy_bb(void * restrict dst, void * restrict src) { diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 9f19c49f390..8d1919dd716 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -879,7 +879,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty struct mlx5_wqe_inl_data_seg *inl; uct_rc_mlx5_hdr_t *rch; unsigned wqe_size, inl_seg_size, ctrl_av_size; - size_t message_length; + size_t iov_length, message_length; void *next_seg; uint8_t opmod; #if HAVE_MLX5_MMO @@ -902,9 +902,6 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty inl_seg_size = ucs_align_up_pow2(sizeof(*inl) + sizeof(*rch) + am_hdr_len, UCT_IB_MLX5_WQE_SEG_SIZE); - ucs_assert(uct_iov_total_length(iov, iovcnt) + sizeof(*rch) + am_hdr_len <= - iface->super.super.config.seg_size); - /* Inline segment with AM ID and header */ inl = next_seg; inl->byte_count = htonl((sizeof(*rch) + am_hdr_len) | MLX5_INLINE_SEG); @@ -915,12 +912,13 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty /* Data segment with payload */ dptr = (struct mlx5_wqe_data_seg *)((char *)inl + inl_seg_size); - wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt); - opmod = 0; - message_length = uct_iov_total_length(iov, iovcnt) + sizeof(*rch) + - am_hdr_len; + wqe_size = ctrl_av_size + inl_seg_size + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, dptr, iov, iovcnt, &iov_length); + opmod = 0; + message_length = iov_length + sizeof(*rch) + am_hdr_len; + ucs_assert(message_length <= iface->super.super.config.seg_size); ucs_assert(wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE); break; @@ -932,11 +930,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty inl = next_seg; inl->byte_count = htonl(sizeof(struct ibv_tmh) | MLX5_INLINE_SEG); dptr = uct_ib_mlx5_txwq_wrap_exact(txwq, (char *)inl + inl_seg_size); - wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt); - opmod = 0; - message_length = uct_iov_total_length(iov, iovcnt) + - sizeof(struct ibv_tmh); + wqe_size = ctrl_av_size + inl_seg_size + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, dptr, iov, iovcnt, &iov_length); + opmod = 0; + message_length = iov_length + sizeof(struct ibv_tmh); uct_rc_mlx5_fill_tmh((struct ibv_tmh*)(inl + 1), tag, app_ctx, IBV_TMH_EAGER); @@ -949,17 +947,16 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty /* Fall through */ case MLX5_OPCODE_RDMA_WRITE: /* Set RDMA segment */ - ucs_assert(uct_iov_total_length(iov, iovcnt) <= UCT_IB_MAX_MESSAGE_SIZE); - raddr = next_seg; uct_ib_mlx5_ep_set_rdma_seg(raddr, remote_addr, rkey); /* Data segment */ - wqe_size = ctrl_av_size + sizeof(*raddr) + - uct_ib_mlx5_set_data_seg_iov(txwq, (void*)(raddr + 1), - iov, iovcnt); - opmod = 0; - message_length = uct_iov_total_length(iov, iovcnt); + wqe_size = ctrl_av_size + sizeof(*raddr) + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, (void*)(raddr + 1), iov, iovcnt, + &message_length); + opmod = 0; + ucs_assert(message_length <= UCT_IB_MAX_MESSAGE_SIZE); break; #if HAVE_MLX5_MMO From 029080824bfc1f2687c92fbc992d941f1f6e48b1 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 24 Aug 2026 19:12:24 +0300 Subject: [PATCH 03/21] UCT/IB/MLX5: Compact send work queue layout Place the cached path MTU fields after the send indices so they do not introduce padding before the MMIO register pointer. --- src/uct/ib/mlx5/ib_mlx5.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 0ebc375c253..206d9251220 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -687,8 +687,6 @@ typedef struct uct_ib_mlx5_txwq { uint16_t sw_pi; /* PI for next WQE */ uint16_t prev_sw_pi; /* PI where last WQE *started* */ uint32_t next_first_psn; /* First PSN of next WQE */ - uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ - uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ uct_ib_mlx5_mmio_reg_t *reg; void *curr; volatile uint32_t *dbrec; @@ -696,6 +694,8 @@ typedef struct uct_ib_mlx5_txwq { void *qend; uint16_t bb_max; uint16_t sig_pi; /* PI for last signaled WQE */ + uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ + uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ #if UCS_ENABLE_ASSERT uint16_t hw_ci; /* First BB index of last completed WQE */ uint8_t flags; /* Debug flags */ From 119b4a4d16f0407bc7d76a994f4f2a557ee791b6 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 24 Aug 2026 19:26:01 +0300 Subject: [PATCH 04/21] UCT/IB/MLX5: Defer RC PSN masking to readers Keep the next-PSN counter free-running so RC posting only performs an addition. Mask the value when exposing a hardware PSN through the accessor and VFS. --- src/uct/ib/mlx5/ib_mlx5.c | 17 +++++++++++++++-- src/uct/ib/mlx5/ib_mlx5.h | 9 ++++++++- src/uct/ib/mlx5/rc/rc_mlx5.inl | 3 +-- test/gtest/uct/ib/test_rc.cc | 9 +++++---- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 4ce9b82cb31..0323d9edb26 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -715,6 +715,19 @@ void uct_ib_mlx5_init_wq_buf(uct_ib_mlx5_txwq_t *txwq) uct_ib_mlx5_set_ctrl_qpn_ds(uct_ib_mlx5_txwq_get_wqe(txwq, 0xffff), 0, 1); } + +static void +uct_ib_mlx5_txwq_vfs_show_next_first_psn(void *obj, + ucs_string_buffer_t *strb, + void *arg_ptr, uint64_t arg_u64) +{ + uct_ib_mlx5_txwq_t *txwq = arg_ptr; + + ucs_string_buffer_appendf( + strb, "%u\n", uct_ib_mlx5_txwq_get_next_first_psn(txwq)); +} + + void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) { ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, @@ -724,8 +737,8 @@ void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) UCS_VFS_TYPE_U16, "sw_pi"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->prev_sw_pi, UCS_VFS_TYPE_U16, "prev_sw_pi"); - ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, - &txwq->next_first_psn, UCS_VFS_TYPE_U32, + ucs_vfs_obj_add_ro_file(parent_obj, + uct_ib_mlx5_txwq_vfs_show_next_first_psn, txwq, 0, "next_first_psn"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->qstart, UCS_VFS_TYPE_POINTER, "qstart"); diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 206d9251220..34a984fc963 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -686,7 +686,7 @@ typedef struct uct_ib_mlx5_txwq { uct_ib_mlx5_qp_t super; uint16_t sw_pi; /* PI for next WQE */ uint16_t prev_sw_pi; /* PI where last WQE *started* */ - uint32_t next_first_psn; /* First PSN of next WQE */ + uint32_t next_first_psn; /* Free-running PSN counter */ uct_ib_mlx5_mmio_reg_t *reg; void *curr; volatile uint32_t *dbrec; @@ -704,6 +704,13 @@ typedef struct uct_ib_mlx5_txwq { } uct_ib_mlx5_txwq_t; +static UCS_F_ALWAYS_INLINE uint32_t +uct_ib_mlx5_txwq_get_next_first_psn(const uct_ib_mlx5_txwq_t *txwq) +{ + return txwq->next_first_psn & UCS_MASK(24); +} + + /* Receive work-queue */ typedef struct uct_ib_mlx5_rxwq { /* producer index. It updated when new receive wqe is posted */ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 8d1919dd716..fbf6d74344c 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -479,8 +479,7 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, uint32_t num_packets) { - txwq->next_first_psn = - (txwq->next_first_psn + num_packets) & UCS_MASK(24); + txwq->next_first_psn += num_packets; } static UCS_F_ALWAYS_INLINE void diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index b0747445353..3142ce94c88 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -195,10 +195,10 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, uct_completion_t comp; ucs_status_t status; - EXPECT_EQ(0, wq->next_first_psn); + EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_first_psn(wq)); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); - EXPECT_EQ(1, wq->next_first_psn); + EXPECT_EQ(1, uct_ib_mlx5_txwq_get_next_first_psn(wq)); comp.func = [](uct_completion_t*) {}; comp.count = 1; @@ -211,7 +211,7 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, ASSERT_UCS_OK_OR_INPROGRESS(status); /* The RC QP uses the peer's 512-byte path MTU, so 513 bytes use 2 PSNs. */ - EXPECT_EQ(3, wq->next_first_psn); + EXPECT_EQ(3, uct_ib_mlx5_txwq_get_next_first_psn(wq)); if (status == UCS_INPROGRESS) { wait_for_value(&comp.count, 0, true); @@ -219,7 +219,8 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, wq->next_first_psn = UCS_MASK(24); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); - EXPECT_EQ(0, wq->next_first_psn); + EXPECT_EQ(UCS_BIT(24), wq->next_first_psn); + EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_first_psn(wq)); flush(); } From cb95c060ce6681886aff5dbdcb46471ed387bd9d Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 25 Aug 2026 12:21:02 +0300 Subject: [PATCH 05/21] UCT/IB/MLX5: Initialize GGA path MTU for PSN tracking --- src/uct/ib/mlx5/gga/gga_mlx5.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uct/ib/mlx5/gga/gga_mlx5.c b/src/uct/ib/mlx5/gga/gga_mlx5.c index 0528ffe5d95..a804fbab21c 100644 --- a/src/uct/ib/mlx5/gga/gga_mlx5.c +++ b/src/uct/ib/mlx5/gga/gga_mlx5.c @@ -532,6 +532,7 @@ uct_gga_mlx5_ep_connect_to_ep_v2(uct_ep_h tl_ep, return status; } + uct_rc_mlx5_txwq_set_path_mtu(&ep->super.tx.wq, path_mtu); ep->super.super.atomic_mr_offset = 0; ep->super.super.flags |= UCT_RC_EP_FLAG_CONNECTED; ep->super.super.flush_rkey = From 9c3c513ac962c8b4c16d71814efc47c4256b7066 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 25 Aug 2026 14:08:32 +0300 Subject: [PATCH 06/21] UCT/IB/MLX5: Centralize RC PSN update guard --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 37 ++++++++++++++++----------------- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 2 +- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index fbf6d74344c..d3b29e10151 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -477,16 +477,23 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, } static UCS_F_ALWAYS_INLINE void -uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, uint32_t num_packets) +uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, + uint32_t num_packets) { - txwq->next_first_psn += num_packets; + if (qp_type == IBV_QPT_RC) { + txwq->next_first_psn += num_packets; + } } static UCS_F_ALWAYS_INLINE void -uct_rc_mlx5_txwq_update_psn(uct_ib_mlx5_txwq_t *txwq, +uct_rc_mlx5_txwq_update_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, size_t message_length) { - uct_rc_mlx5_txwq_add_psn(txwq, + if (qp_type != IBV_QPT_RC) { + return; + } + + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, uct_rc_mlx5_num_packets(txwq, message_length)); } @@ -570,9 +577,7 @@ static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txqp_inline_iov_post( uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, MLX5_OPCODE_SEND, 0, fm_ce_se, dci_channel, wqe_size, 0, INT_MAX, NULL); - if (qp_type == IBV_QPT_RC) { - uct_rc_mlx5_txwq_add_psn(txwq, 1); - } + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); } /* @@ -668,9 +673,9 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, NULL); - if ((qp_type == IBV_QPT_RC) && (opcode != MLX5_OPCODE_NOP)) { + if (opcode != MLX5_OPCODE_NOP) { /* Inline short operations always fit in one RC packet. */ - uct_rc_mlx5_txwq_add_psn(txwq, 1); + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); } } @@ -855,9 +860,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, (opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, log_sge); - if (qp_type == IBV_QPT_RC) { - uct_rc_mlx5_txwq_update_psn(txwq, length); - } + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); } static UCS_F_ALWAYS_INLINE @@ -1002,9 +1005,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty opcode_flags & UCT_RC_MLX5_OPCODE_MASK, opmod, fm_ce_se, dci_channel, wqe_size, ib_imm_be, max_log_sge, NULL); - if (qp_type == IBV_QPT_RC) { - uct_rc_mlx5_txwq_update_psn(txwq, message_length); - } + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); } /* @@ -1154,10 +1155,8 @@ uct_rc_mlx5_txqp_tag_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se, dci_channel, wqe_size, imm_val_be, INT_MAX, NULL); - if (qp_type == IBV_QPT_RC) { - /* Inline tag operations always fit in one RC packet. */ - uct_rc_mlx5_txwq_add_psn(txwq, 1); - } + /* Inline tag operations always fit in one RC packet. */ + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); } static UCS_F_ALWAYS_INLINE void diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index e9456ff3a29..d77d199dc00 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -301,7 +301,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, uct_rc_txqp_posted(&ep->super.txqp, &iface->super, res_count, 1); uct_ib_mlx5_txwq_ring_doorbell(txwq, ctrl, txwq->sw_pi, 1); - uct_rc_mlx5_txwq_add_psn(txwq, num_packets); + uct_rc_mlx5_txwq_add_psn(txwq, IBV_QPT_RC, num_packets); uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, uct_rc_ep_send_op_completion_handler, comp, sn, From e76b940cec70e3c47ab676209fe4891c408f7a8a Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:51:22 +0300 Subject: [PATCH 07/21] UCT/IB/MLX5: Cache only path MTU shift Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.c | 1 - src/uct/ib/mlx5/ib_mlx5.h | 1 - src/uct/ib/mlx5/rc/rc_mlx5.inl | 10 ++++++---- test/gtest/uct/ib/test_rc.cc | 1 - 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 0323d9edb26..852015363d1 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -697,7 +697,6 @@ void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq) txwq->sw_pi = 0; txwq->prev_sw_pi = UINT16_MAX; txwq->next_first_psn = 0; - txwq->path_mtu_mask = 0; txwq->path_mtu_shift = 0; #if UCS_ENABLE_ASSERT txwq->hw_ci = 0xFFFF; diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 34a984fc963..c6e3cdaf959 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -694,7 +694,6 @@ typedef struct uct_ib_mlx5_txwq { void *qend; uint16_t bb_max; uint16_t sig_pi; /* PI for last signaled WQE */ - uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ #if UCS_ENABLE_ASSERT uint16_t hw_ci; /* First BB index of last completed WQE */ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index d3b29e10151..e0d824ef462 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -461,7 +461,6 @@ uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, ucs_assert(mtu <= UINT16_MAX); ucs_assert(ucs_is_pow2(mtu)); - txwq->path_mtu_mask = mtu - 1; txwq->path_mtu_shift = ucs_ilog2(mtu); } @@ -469,11 +468,14 @@ static UCS_F_ALWAYS_INLINE uint32_t uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, size_t message_length) { + uint32_t num_packets; + ucs_assert(txwq->path_mtu_shift > 0); - return ucs_max((size_t)1, - (message_length + txwq->path_mtu_mask) >> - txwq->path_mtu_shift); + num_packets = (message_length + UCS_MASK(txwq->path_mtu_shift)) >> + txwq->path_mtu_shift; + ucs_assert(num_packets > 0); + return num_packets; } static UCS_F_ALWAYS_INLINE void diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 3142ce94c88..97f9aca9d13 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -181,7 +181,6 @@ UCS_TEST_P(test_rc_mlx5_psn, path_mtu_from_rc_qp) EXPECT_EQ(IBV_MTU_4096, local_iface->config.path_mtu); EXPECT_EQ(IBV_MTU_512, peer_iface->config.path_mtu); - EXPECT_EQ(511, txwq()->path_mtu_mask); EXPECT_EQ(9, txwq()->path_mtu_shift); } From a15642c7c78533dab498604e5c109fe607bc0c9a Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:53:12 +0300 Subject: [PATCH 08/21] UCT/IB/MLX5: Move path MTU setup to control path Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.h | 3 +++ src/uct/ib/mlx5/rc/rc_mlx5.inl | 12 ------------ src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 11 +++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.h b/src/uct/ib/mlx5/rc/rc_mlx5.h index f40be42c0c5..269c4c0fcb3 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5.h @@ -213,6 +213,9 @@ ucs_status_t uct_rc_mlx5_iface_create_qp(uct_rc_mlx5_iface_common_t *iface, uct_ib_mlx5_txwq_t *txwq, uct_ib_mlx5_qp_attr_t *attr); +void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, + enum ibv_mtu path_mtu); + ucs_status_t uct_rc_mlx5_ep_connect_qp(uct_rc_mlx5_iface_common_t *iface, uct_ib_mlx5_qp_t *qp, uint32_t qp_num, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index e0d824ef462..56e846463d8 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -452,18 +452,6 @@ uct_rc_mlx5_ep_fm_cq_update(uct_rc_mlx5_iface_common_t *iface, return fm_ce_se; } -static UCS_F_ALWAYS_INLINE void -uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, - enum ibv_mtu path_mtu) -{ - size_t mtu = uct_ib_mtu_value(path_mtu); - - ucs_assert(mtu <= UINT16_MAX); - ucs_assert(ucs_is_pow2(mtu)); - - txwq->path_mtu_shift = ucs_ilog2(mtu); -} - static UCS_F_ALWAYS_INLINE uint32_t uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, size_t message_length) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index d77d199dc00..ded91922ff2 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -880,6 +880,17 @@ void uct_rc_mlx5_common_packet_dump(uct_base_iface_t *iface, uct_am_trace_type_t valid_length, buffer, max); } +void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, + enum ibv_mtu path_mtu) +{ + size_t mtu = uct_ib_mtu_value(path_mtu); + + ucs_assert(mtu <= UINT16_MAX); + ucs_assert(ucs_is_pow2(mtu)); + + txwq->path_mtu_shift = ucs_ilog2(mtu); +} + ucs_status_t uct_rc_mlx5_ep_connect_qp(uct_rc_mlx5_iface_common_t *iface, uct_ib_mlx5_qp_t *qp, uint32_t qp_num, From 2727b52c8a19c991c70130ad8715ef167026eea4 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:53:58 +0300 Subject: [PATCH 09/21] UCT/IB/MLX5: Align TXWQ reset assignments Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 852015363d1..f8b78934974 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -693,14 +693,14 @@ void uct_ib_mlx5_devx_uar_cleanup(uct_ib_mlx5_devx_uar_t *uar) void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq) { - txwq->curr = txwq->qstart; - txwq->sw_pi = 0; - txwq->prev_sw_pi = UINT16_MAX; - txwq->next_first_psn = 0; - txwq->path_mtu_shift = 0; + txwq->curr = txwq->qstart; + txwq->sw_pi = 0; + txwq->prev_sw_pi = UINT16_MAX; + txwq->next_first_psn = 0; + txwq->path_mtu_shift = 0; #if UCS_ENABLE_ASSERT - txwq->hw_ci = 0xFFFF; - txwq->flags = 0; + txwq->hw_ci = 0xFFFF; + txwq->flags = 0; #endif uct_ib_fence_info_init(&txwq->fi); } From 7d290d1b49a421a3c585eb9d7cc1e5bbba4505ab Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:54:14 +0300 Subject: [PATCH 10/21] UCT/IB/MLX5: Order initialized send variables Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index ded91922ff2..f85150339f7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -212,10 +212,10 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; size_t total = 0; struct mlx5_wqe_ctrl_seg *ctrl = NULL; + uint32_t num_packets = 0; struct mlx5_wqe_raddr_seg *raddr; struct mlx5_wqe_data_seg *dptr; size_t wqe_size, i; - uint32_t num_packets = 0; uint8_t fm_ce_se, fence_flag; uint16_t sn, pi, res_count; uint64_t addr; From b917dc0aae56e01920f390a803abf9dcab7475d3 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:54:41 +0300 Subject: [PATCH 11/21] UCT/IB/MLX5: Use consistent RC QP checks Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 56e846463d8..61cc6c6e1f6 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -479,12 +479,11 @@ static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txwq_update_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, size_t message_length) { - if (qp_type != IBV_QPT_RC) { - return; + if (qp_type == IBV_QPT_RC) { + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, + uct_rc_mlx5_num_packets(txwq, + message_length)); } - - uct_rc_mlx5_txwq_add_psn(txwq, qp_type, - uct_rc_mlx5_num_packets(txwq, message_length)); } static UCS_F_ALWAYS_INLINE void From 34e3fa129c35455694481a7f06769a378761f019 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:55:51 +0300 Subject: [PATCH 12/21] UCT/IB/MLX5: Align RC send assignments Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 61cc6c6e1f6..74ae9cc93a5 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -903,11 +903,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty /* Data segment with payload */ dptr = (struct mlx5_wqe_data_seg *)((char *)inl + inl_seg_size); - wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov_length( - txwq, dptr, iov, iovcnt, &iov_length); - opmod = 0; - message_length = iov_length + sizeof(*rch) + am_hdr_len; + wqe_size = ctrl_av_size + inl_seg_size + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, dptr, iov, iovcnt, &iov_length); + opmod = 0; + message_length = iov_length + sizeof(*rch) + am_hdr_len; ucs_assert(message_length <= iface->super.super.config.seg_size); ucs_assert(wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE); @@ -921,11 +921,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty inl = next_seg; inl->byte_count = htonl(sizeof(struct ibv_tmh) | MLX5_INLINE_SEG); dptr = uct_ib_mlx5_txwq_wrap_exact(txwq, (char *)inl + inl_seg_size); - wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov_length( - txwq, dptr, iov, iovcnt, &iov_length); - opmod = 0; - message_length = iov_length + sizeof(struct ibv_tmh); + wqe_size = ctrl_av_size + inl_seg_size + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, dptr, iov, iovcnt, &iov_length); + opmod = 0; + message_length = iov_length + sizeof(struct ibv_tmh); uct_rc_mlx5_fill_tmh((struct ibv_tmh*)(inl + 1), tag, app_ctx, IBV_TMH_EAGER); @@ -942,11 +942,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty uct_ib_mlx5_ep_set_rdma_seg(raddr, remote_addr, rkey); /* Data segment */ - wqe_size = ctrl_av_size + sizeof(*raddr) + - uct_ib_mlx5_set_data_seg_iov_length( - txwq, (void*)(raddr + 1), iov, iovcnt, - &message_length); - opmod = 0; + wqe_size = ctrl_av_size + sizeof(*raddr) + + uct_ib_mlx5_set_data_seg_iov_length( + txwq, (void*)(raddr + 1), iov, iovcnt, + &message_length); + opmod = 0; ucs_assert(message_length <= UCT_IB_MAX_MESSAGE_SIZE); break; @@ -981,8 +981,9 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty uct_ib_mlx5_set_data_seg(dptr /* scatter segment */, dma_dst_buf, iov[0].length, dma_dst_key); - wqe_size = sizeof(*ctrl) + sizeof(*dma_seg) + (2 * sizeof(*dptr)); - opmod = UCT_IB_MLX5_OPMOD_MMO_DMA; + wqe_size = sizeof(*ctrl) + sizeof(*dma_seg) + + (2 * sizeof(*dptr)); + opmod = UCT_IB_MLX5_OPMOD_MMO_DMA; message_length = 0; break; #endif From 7f706432fe8a2de35332d2484f0cf216e208c4d0 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 13:56:07 +0300 Subject: [PATCH 13/21] UCT/IB/MLX5: Guard MMO opcode handling Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 74ae9cc93a5..3d9c194637e 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -995,6 +995,12 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty opcode_flags & UCT_RC_MLX5_OPCODE_MASK, opmod, fm_ce_se, dci_channel, wqe_size, ib_imm_be, max_log_sge, NULL); +#if HAVE_MLX5_MMO + if ((opcode_flags & UCT_RC_MLX5_OPCODE_MASK) == MLX5_OPCODE_MMO) { + return; + } +#endif + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); } From 368df8e42de75ae867f60ce9a18520af3c6a4e35 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 14:39:52 +0300 Subject: [PATCH 14/21] UCT/IB/MLX5: Restore cached path MTU mask Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.c | 1 + src/uct/ib/mlx5/ib_mlx5.h | 1 + src/uct/ib/mlx5/rc/rc_mlx5.inl | 2 +- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 1 + test/gtest/uct/ib/test_rc.cc | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index f8b78934974..0c7b5ee7d69 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -697,6 +697,7 @@ void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq) txwq->sw_pi = 0; txwq->prev_sw_pi = UINT16_MAX; txwq->next_first_psn = 0; + txwq->path_mtu_mask = 0; txwq->path_mtu_shift = 0; #if UCS_ENABLE_ASSERT txwq->hw_ci = 0xFFFF; diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index c6e3cdaf959..34a984fc963 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -694,6 +694,7 @@ typedef struct uct_ib_mlx5_txwq { void *qend; uint16_t bb_max; uint16_t sig_pi; /* PI for last signaled WQE */ + uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ #if UCS_ENABLE_ASSERT uint16_t hw_ci; /* First BB index of last completed WQE */ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 3d9c194637e..5384d0618b3 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -460,7 +460,7 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, ucs_assert(txwq->path_mtu_shift > 0); - num_packets = (message_length + UCS_MASK(txwq->path_mtu_shift)) >> + num_packets = (message_length + txwq->path_mtu_mask) >> txwq->path_mtu_shift; ucs_assert(num_packets > 0); return num_packets; diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index f85150339f7..2f602422749 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -888,6 +888,7 @@ void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, ucs_assert(mtu <= UINT16_MAX); ucs_assert(ucs_is_pow2(mtu)); + txwq->path_mtu_mask = mtu - 1; txwq->path_mtu_shift = ucs_ilog2(mtu); } diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 97f9aca9d13..3142ce94c88 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -181,6 +181,7 @@ UCS_TEST_P(test_rc_mlx5_psn, path_mtu_from_rc_qp) EXPECT_EQ(IBV_MTU_4096, local_iface->config.path_mtu); EXPECT_EQ(IBV_MTU_512, peer_iface->config.path_mtu); + EXPECT_EQ(511, txwq()->path_mtu_mask); EXPECT_EQ(9, txwq()->path_mtu_shift); } From 12d49839184b7a8569403b18a64efc8d05d55997 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 14:42:26 +0300 Subject: [PATCH 15/21] UCT/IB/MLX5: Count zero-length sends as one packet Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 5384d0618b3..d6d2a8f2e18 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -456,14 +456,10 @@ static UCS_F_ALWAYS_INLINE uint32_t uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, size_t message_length) { - uint32_t num_packets; - ucs_assert(txwq->path_mtu_shift > 0); - num_packets = (message_length + txwq->path_mtu_mask) >> - txwq->path_mtu_shift; - ucs_assert(num_packets > 0); - return num_packets; + return ucs_max(1u, (message_length + txwq->path_mtu_mask) >> + txwq->path_mtu_shift); } static UCS_F_ALWAYS_INLINE void From c1d7701d79f1f0aea6acab22d1c710d8bb2f5449 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 14:45:12 +0300 Subject: [PATCH 16/21] UCT/IB/MLX5: Fix PSN tracking formatting Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.c | 2 -- src/uct/ib/mlx5/rc/rc_mlx5.h | 2 +- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 0c7b5ee7d69..ea53f02467c 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -715,7 +715,6 @@ void uct_ib_mlx5_init_wq_buf(uct_ib_mlx5_txwq_t *txwq) uct_ib_mlx5_set_ctrl_qpn_ds(uct_ib_mlx5_txwq_get_wqe(txwq, 0xffff), 0, 1); } - static void uct_ib_mlx5_txwq_vfs_show_next_first_psn(void *obj, ucs_string_buffer_t *strb, @@ -727,7 +726,6 @@ uct_ib_mlx5_txwq_vfs_show_next_first_psn(void *obj, strb, "%u\n", uct_ib_mlx5_txwq_get_next_first_psn(txwq)); } - void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) { ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.h b/src/uct/ib/mlx5/rc/rc_mlx5.h index 269c4c0fcb3..04f2fc39038 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5.h @@ -214,7 +214,7 @@ ucs_status_t uct_rc_mlx5_iface_create_qp(uct_rc_mlx5_iface_common_t *iface, uct_ib_mlx5_qp_attr_t *attr); void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, - enum ibv_mtu path_mtu); + enum ibv_mtu path_mtu); ucs_status_t uct_rc_mlx5_ep_connect_qp(uct_rc_mlx5_iface_common_t *iface, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 2f602422749..61fa31b7cfe 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -881,7 +881,7 @@ void uct_rc_mlx5_common_packet_dump(uct_base_iface_t *iface, uct_am_trace_type_t } void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, - enum ibv_mtu path_mtu) + enum ibv_mtu path_mtu) { size_t mtu = uct_ib_mtu_value(path_mtu); From e2eb81a14fde0653d45fa7d9d9bd00a7544a1359 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 16:57:18 +0300 Subject: [PATCH 17/21] UCT/IB/MLX5: Address PSN tracking review comments Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.c | 13 ++++++------- src/uct/ib/mlx5/ib_mlx5.h | 7 ++++--- src/uct/ib/mlx5/rc/rc_mlx5.inl | 2 +- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 1 + test/gtest/uct/ib/test_rc.cc | 14 +++++++------- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index ea53f02467c..07bf2544307 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -696,7 +696,7 @@ void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq) txwq->curr = txwq->qstart; txwq->sw_pi = 0; txwq->prev_sw_pi = UINT16_MAX; - txwq->next_first_psn = 0; + txwq->next_wqe_psn = 0; txwq->path_mtu_mask = 0; txwq->path_mtu_shift = 0; #if UCS_ENABLE_ASSERT @@ -716,14 +716,13 @@ void uct_ib_mlx5_init_wq_buf(uct_ib_mlx5_txwq_t *txwq) } static void -uct_ib_mlx5_txwq_vfs_show_next_first_psn(void *obj, - ucs_string_buffer_t *strb, - void *arg_ptr, uint64_t arg_u64) +uct_ib_mlx5_txwq_vfs_show_next_wqe_psn(void *obj, ucs_string_buffer_t *strb, + void *arg_ptr, uint64_t arg_u64) { uct_ib_mlx5_txwq_t *txwq = arg_ptr; ucs_string_buffer_appendf( - strb, "%u\n", uct_ib_mlx5_txwq_get_next_first_psn(txwq)); + strb, "%u\n", uct_ib_mlx5_txwq_get_next_wqe_psn(txwq)); } void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) @@ -736,8 +735,8 @@ void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj) ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->prev_sw_pi, UCS_VFS_TYPE_U16, "prev_sw_pi"); ucs_vfs_obj_add_ro_file(parent_obj, - uct_ib_mlx5_txwq_vfs_show_next_first_psn, txwq, 0, - "next_first_psn"); + uct_ib_mlx5_txwq_vfs_show_next_wqe_psn, txwq, 0, + "next_wqe_psn"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->qstart, UCS_VFS_TYPE_POINTER, "qstart"); ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->qend, diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 34a984fc963..aa245ef7f34 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -686,7 +686,8 @@ typedef struct uct_ib_mlx5_txwq { uct_ib_mlx5_qp_t super; uint16_t sw_pi; /* PI for next WQE */ uint16_t prev_sw_pi; /* PI where last WQE *started* */ - uint32_t next_first_psn; /* Free-running PSN counter */ + uint32_t next_wqe_psn; /* 1st PSN of the next WQE to be + posted */ uct_ib_mlx5_mmio_reg_t *reg; void *curr; volatile uint32_t *dbrec; @@ -705,9 +706,9 @@ typedef struct uct_ib_mlx5_txwq { static UCS_F_ALWAYS_INLINE uint32_t -uct_ib_mlx5_txwq_get_next_first_psn(const uct_ib_mlx5_txwq_t *txwq) +uct_ib_mlx5_txwq_get_next_wqe_psn(const uct_ib_mlx5_txwq_t *txwq) { - return txwq->next_first_psn & UCS_MASK(24); + return txwq->next_wqe_psn & UCS_MASK(24); } diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index d6d2a8f2e18..31d182e9c0c 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -467,7 +467,7 @@ uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, uint32_t num_packets) { if (qp_type == IBV_QPT_RC) { - txwq->next_first_psn += num_packets; + txwq->next_wqe_psn += num_packets; } } diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 61fa31b7cfe..e1cded70c09 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -988,6 +988,7 @@ uct_rc_mlx5_ep_connect_to_ep_v2(uct_ep_h tl_ep, if (status != UCS_OK) { return status; } + uct_rc_mlx5_txwq_set_path_mtu(&ep->super.tx.wq, path_mtu); ep->super.super.atomic_mr_offset = uct_ib_md_atomic_offset( diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 3142ce94c88..a946583d10c 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -185,7 +185,7 @@ UCS_TEST_P(test_rc_mlx5_psn, path_mtu_from_rc_qp) EXPECT_EQ(9, txwq()->path_mtu_shift); } -UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, +UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_wqe_psn, !check_caps(UCT_IFACE_FLAG_PUT_ZCOPY)) { static const size_t length = 513; @@ -195,10 +195,10 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, uct_completion_t comp; ucs_status_t status; - EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_first_psn(wq)); + EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_wqe_psn(wq)); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); - EXPECT_EQ(1, uct_ib_mlx5_txwq_get_next_first_psn(wq)); + EXPECT_EQ(1, uct_ib_mlx5_txwq_get_next_wqe_psn(wq)); comp.func = [](uct_completion_t*) {}; comp.count = 1; @@ -211,16 +211,16 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_first_psn, ASSERT_UCS_OK_OR_INPROGRESS(status); /* The RC QP uses the peer's 512-byte path MTU, so 513 bytes use 2 PSNs. */ - EXPECT_EQ(3, uct_ib_mlx5_txwq_get_next_first_psn(wq)); + EXPECT_EQ(3, uct_ib_mlx5_txwq_get_next_wqe_psn(wq)); if (status == UCS_INPROGRESS) { wait_for_value(&comp.count, 0, true); } - wq->next_first_psn = UCS_MASK(24); + wq->next_wqe_psn = UCS_MASK(24); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); - EXPECT_EQ(UCS_BIT(24), wq->next_first_psn); - EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_first_psn(wq)); + EXPECT_EQ(UCS_BIT(24), wq->next_wqe_psn); + EXPECT_EQ(0, uct_ib_mlx5_txwq_get_next_wqe_psn(wq)); flush(); } From a4e584dd6631b48f8f4820b12b194c5a4d4168fd Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 17:19:40 +0300 Subject: [PATCH 18/21] UCT/IB/MLX5: Remove redundant IOV helper Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.inl | 9 --------- src/uct/ib/mlx5/ud/ud_mlx5.c | 6 +++--- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.inl b/src/uct/ib/mlx5/ib_mlx5.inl index 9bac1c40863..178f3ca1e61 100644 --- a/src/uct/ib/mlx5/ib_mlx5.inl +++ b/src/uct/ib/mlx5/ib_mlx5.inl @@ -541,15 +541,6 @@ size_t uct_ib_mlx5_set_data_seg_iov_length(uct_ib_mlx5_txwq_t *txwq, } -static UCS_F_ALWAYS_INLINE -size_t uct_ib_mlx5_set_data_seg_iov(uct_ib_mlx5_txwq_t *txwq, - struct mlx5_wqe_data_seg *dptr, - const uct_iov_t *iov, size_t iovcnt) -{ - return uct_ib_mlx5_set_data_seg_iov_length(txwq, dptr, iov, iovcnt, NULL); -} - - static UCS_F_ALWAYS_INLINE void uct_ib_mlx5_bf_copy_bb(void * restrict dst, void * restrict src) { diff --git a/src/uct/ib/mlx5/ud/ud_mlx5.c b/src/uct/ib/mlx5/ud/ud_mlx5.c index e4df3a927e9..5e91fb7c7d1 100644 --- a/src/uct/ib/mlx5/ud/ud_mlx5.c +++ b/src/uct/ib/mlx5/ud/ud_mlx5.c @@ -320,9 +320,9 @@ static UCS_F_ALWAYS_INLINE ucs_status_t uct_ud_mlx5_ep_inline_iov_post( /* set iov to dptr */ if (iovcnt > 0) { wqe_size = ucs_align_up_pow2(wqe_size, UCT_IB_MLX5_WQE_SEG_SIZE); - wqe_size += uct_ib_mlx5_set_data_seg_iov(&iface->tx.wq, - UCS_PTR_BYTE_OFFSET(ctrl, wqe_size), - iov, iovcnt); + wqe_size += uct_ib_mlx5_set_data_seg_iov_length( + &iface->tx.wq, UCS_PTR_BYTE_OFFSET(ctrl, wqe_size), iov, + iovcnt, NULL); } uct_ud_mlx5_post_send(iface, ep, 0, ctrl, wqe_size, neth, From 8f07fb4ff39f72f84eaaf8e7781faeb0e2bccdc7 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 26 Aug 2026 21:49:07 +0300 Subject: [PATCH 19/21] UCT/IB/MLX5: Simplify data segment IOV helper Signed-off-by: Roie Danino --- src/uct/ib/mlx5/ib_mlx5.inl | 8 ++++---- src/uct/ib/mlx5/rc/rc_mlx5.inl | 6 +++--- src/uct/ib/mlx5/ud/ud_mlx5.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.inl b/src/uct/ib/mlx5/ib_mlx5.inl index 178f3ca1e61..7b5916e6ec1 100644 --- a/src/uct/ib/mlx5/ib_mlx5.inl +++ b/src/uct/ib/mlx5/ib_mlx5.inl @@ -507,10 +507,10 @@ uct_ib_mlx5_set_data_seg(struct mlx5_wqe_data_seg *dptr, static UCS_F_ALWAYS_INLINE -size_t uct_ib_mlx5_set_data_seg_iov_length(uct_ib_mlx5_txwq_t *txwq, - struct mlx5_wqe_data_seg *dptr, - const uct_iov_t *iov, size_t iovcnt, - size_t *iov_length_p) +size_t uct_ib_mlx5_set_data_seg_iov(uct_ib_mlx5_txwq_t *txwq, + struct mlx5_wqe_data_seg *dptr, + const uct_iov_t *iov, size_t iovcnt, + size_t *iov_length_p) { size_t iov_length = 0; size_t wqe_size = 0; diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 31d182e9c0c..074886b93ed 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -900,7 +900,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty /* Data segment with payload */ dptr = (struct mlx5_wqe_data_seg *)((char *)inl + inl_seg_size); wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov_length( + uct_ib_mlx5_set_data_seg_iov( txwq, dptr, iov, iovcnt, &iov_length); opmod = 0; message_length = iov_length + sizeof(*rch) + am_hdr_len; @@ -918,7 +918,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty inl->byte_count = htonl(sizeof(struct ibv_tmh) | MLX5_INLINE_SEG); dptr = uct_ib_mlx5_txwq_wrap_exact(txwq, (char *)inl + inl_seg_size); wqe_size = ctrl_av_size + inl_seg_size + - uct_ib_mlx5_set_data_seg_iov_length( + uct_ib_mlx5_set_data_seg_iov( txwq, dptr, iov, iovcnt, &iov_length); opmod = 0; message_length = iov_length + sizeof(struct ibv_tmh); @@ -939,7 +939,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty /* Data segment */ wqe_size = ctrl_av_size + sizeof(*raddr) + - uct_ib_mlx5_set_data_seg_iov_length( + uct_ib_mlx5_set_data_seg_iov( txwq, (void*)(raddr + 1), iov, iovcnt, &message_length); opmod = 0; diff --git a/src/uct/ib/mlx5/ud/ud_mlx5.c b/src/uct/ib/mlx5/ud/ud_mlx5.c index 5e91fb7c7d1..89b0453877e 100644 --- a/src/uct/ib/mlx5/ud/ud_mlx5.c +++ b/src/uct/ib/mlx5/ud/ud_mlx5.c @@ -320,7 +320,7 @@ static UCS_F_ALWAYS_INLINE ucs_status_t uct_ud_mlx5_ep_inline_iov_post( /* set iov to dptr */ if (iovcnt > 0) { wqe_size = ucs_align_up_pow2(wqe_size, UCT_IB_MLX5_WQE_SEG_SIZE); - wqe_size += uct_ib_mlx5_set_data_seg_iov_length( + wqe_size += uct_ib_mlx5_set_data_seg_iov( &iface->tx.wq, UCS_PTR_BYTE_OFFSET(ctrl, wqe_size), iov, iovcnt, NULL); } From d7d6abc1bff475519fb94ddb71b05e0f4481a1a8 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 27 Aug 2026 10:53:32 +0300 Subject: [PATCH 20/21] UCT/IB/MLX5: Handle zero-length PSN update explicitly Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 15 +++++++++++---- test/gtest/uct/ib/test_rc.cc | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 074886b93ed..834f0b540c9 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -458,8 +458,7 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, { ucs_assert(txwq->path_mtu_shift > 0); - return ucs_max(1u, (message_length + txwq->path_mtu_mask) >> - txwq->path_mtu_shift); + return (message_length + txwq->path_mtu_mask) >> txwq->path_mtu_shift; } static UCS_F_ALWAYS_INLINE void @@ -845,7 +844,11 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, (opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, log_sge); - uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); + if (length == 0) { + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); + } else { + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); + } } static UCS_F_ALWAYS_INLINE @@ -997,7 +1000,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty } #endif - uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); + if (message_length == 0) { + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); + } else { + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); + } } /* diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index a946583d10c..56935f9efaa 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -192,6 +192,7 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_wqe_psn, uct_ib_mlx5_txwq_t *wq = txwq(); mapped_buffer sendbuf(length, 0ul, *m_e1); mapped_buffer recvbuf(length, 0ul, *m_e2); + uct_iov_t zero_iov = {}; uct_completion_t comp; ucs_status_t status; @@ -217,6 +218,23 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_wqe_psn, wait_for_value(&comp.count, 0, true); } + zero_iov.buffer = sendbuf.ptr(); + zero_iov.length = 0; + zero_iov.memh = sendbuf.memh(); + zero_iov.stride = 0; + zero_iov.count = 1; + comp.count = 1; + comp.status = UCS_OK; + + status = uct_ep_put_zcopy(m_e1->ep(0), &zero_iov, 1, recvbuf.addr(), + recvbuf.rkey(), &comp); + ASSERT_UCS_OK_OR_INPROGRESS(status); + EXPECT_EQ(4, uct_ib_mlx5_txwq_get_next_wqe_psn(wq)); + + if (status == UCS_INPROGRESS) { + wait_for_value(&comp.count, 0, true); + } + wq->next_wqe_psn = UCS_MASK(24); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0)); EXPECT_EQ(UCS_BIT(24), wq->next_wqe_psn); From d4eca2561a0c8e12ec1a4724970466a8dfe76cd5 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 27 Aug 2026 13:23:05 +0300 Subject: [PATCH 21/21] UCT/IB/MLX5: Avoid zero-length PSN branch Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index 834f0b540c9..302b9197086 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -741,6 +741,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, /* Data segment */ if (length == 0) { wqe_size = ctrl_av_size + sizeof(*raddr); + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); } else { /* dptr cannot wrap, because ctrl+av could be either 2 or 4 segs */ dptr = uct_ib_mlx5_txwq_wrap_none(txwq, raddr + 1); @@ -844,11 +845,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, (opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod, fm_ce_se, dci_channel, wqe_size, imm_val_be, max_log_sge, log_sge); - if (length == 0) { - uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); - } else { - uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); - } + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); } static UCS_F_ALWAYS_INLINE @@ -1000,10 +997,11 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty } #endif - if (message_length == 0) { - uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); - } else { - uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); + uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); + if (opcode_flags == MLX5_OPCODE_RDMA_WRITE) { + /* opcode_flags is constant after inlining, so only zero-length + * PUT_ZCOPY gets the branchless one-PSN adjustment. */ + uct_rc_mlx5_txwq_add_psn(txwq, qp_type, !message_length); } }