From 65b8b97cb2e7e126bf76bbb799262612bf6e0b58 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Sun, 30 Aug 2026 13:50:52 +0300 Subject: [PATCH 01/26] UCT/IB/MLX5: Purge outstanding AM short operations --- src/uct/ib/mlx5/ib_mlx5.c | 40 ++++- src/uct/ib/mlx5/ib_mlx5.h | 12 +- src/uct/ib/mlx5/rc/rc_mlx5.h | 3 + src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 235 ++++++++++++++++++++++++++ src/uct/ib/mlx5/rc/rc_mlx5_iface.c | 2 +- test/gtest/uct/ib/test_ib_mlx5_ext.cc | 4 +- test/gtest/uct/ib/test_rc.cc | 105 ++++++++++++ 7 files changed, 395 insertions(+), 6 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index e9d9255fad8..101c10fc65a 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -858,6 +858,42 @@ void *uct_ib_mlx5_txwq_get_wqe(const uct_ib_mlx5_txwq_t *txwq, uint16_t pi) return UCS_PTR_BYTE_OFFSET(txwq->qstart, (pi % num_bb) * MLX5_SEND_WQE_BB); } +size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl) +{ + uint8_t ds = ntohl(ctrl->qpn_ds) & UINT8_MAX; + + ucs_assert(ds > 0); + ucs_assert((ds * UCT_IB_MLX5_WQE_SEG_SIZE) <= + UCT_IB_MLX5_MAX_SEND_WQE_SIZE); + return ds * UCT_IB_MLX5_WQE_SEG_SIZE; +} + +uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size) +{ + return ci + ucs_div_round_up(wqe_size, MLX5_SEND_WQE_BB); +} + +ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, + uint32_t receiver_next_psn, + uint32_t num_packets) +{ + const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); + uint32_t diff; + + ucs_assert(num_packets > 0); + + diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; + if (diff == psn_half) { + return UCS_ERR_INVALID_PARAM; + } + + if ((diff > 0) && (diff < psn_half) && (diff >= num_packets)) { + return UCS_OK; + } + + return UCS_INPROGRESS; +} + uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, uint16_t outstanding) { @@ -873,8 +909,8 @@ uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, ucs_assert(pi == txwq->hw_ci); do { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, pi); - wqe_size = (ctrl->qpn_ds >> 24) * UCT_IB_MLX5_WQE_SEG_SIZE; - pi += (wqe_size + MLX5_SEND_WQE_BB - 1) / MLX5_SEND_WQE_BB; + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + pi = uct_ib_mlx5_txwq_next_ci(pi, wqe_size); ++count; } while (pi != txwq->sw_pi); diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 3732e228aa9..42465a08692 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -64,6 +64,8 @@ #define UCT_IB_MLX5_CQ_SET_CI 0 #define UCT_IB_MLX5_CQ_ARM_DB 1 #define UCT_IB_MLX5_LOG_MAX_MSG_SIZE 30 +#define UCT_IB_MLX5_PSN_BITS 24 +#define UCT_IB_MLX5_PSN_MASK UCS_MASK(UCT_IB_MLX5_PSN_BITS) #define UCT_IB_MLX5_ATOMIC_MODE_COMP 1 #define UCT_IB_MLX5_ATOMIC_MODE_EXT 3 #define UCT_IB_MLX5_CQE_FLAG_L3_IN_DATA UCS_BIT(28) /* GRH/IP in the receive buffer */ @@ -709,7 +711,7 @@ typedef struct uct_ib_mlx5_txwq { static UCS_F_ALWAYS_INLINE uint32_t uct_ib_mlx5_txwq_get_next_wqe_psn(const uct_ib_mlx5_txwq_t *txwq) { - return txwq->next_wqe_psn & UCS_MASK(24); + return txwq->next_wqe_psn & UCT_IB_MLX5_PSN_MASK; } @@ -917,6 +919,14 @@ ucs_status_t uct_ib_mlx5_txwq_init(uct_priv_worker_t *worker, /* Get pointer to a WQE by producer index */ void *uct_ib_mlx5_txwq_get_wqe(const uct_ib_mlx5_txwq_t *txwq, uint16_t pi); +size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl); + +uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size); + +ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, + uint32_t receiver_next_psn, + uint32_t num_packets); + /* Count how many WQEs are currently posted */ uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, uint16_t outstanding); diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.h b/src/uct/ib/mlx5/rc/rc_mlx5.h index c20045fd2d1..03bb325eb16 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5.h @@ -209,6 +209,9 @@ ucs_status_t uct_rc_mlx5_base_ep_invalidate(uct_ep_h tl_ep, const uct_ep_invalidate_params_t *params); +ucs_status_t uct_rc_mlx5_ep_outstanding_purge( + uct_ep_h tl_ep, const uct_ep_outstanding_purge_params_t *params); + ucs_status_t uct_rc_mlx5_base_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_rc_pending_req_t *req); diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 6b809f8fc6d..c236202c2d6 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -817,6 +817,241 @@ ucs_status_t uct_rc_mlx5_base_ep_invalidate(uct_ep_h tl_ep, IBV_QPS_ERR); } +static uint8_t +uct_ib_mlx5_wqe_opcode(const struct mlx5_wqe_ctrl_seg *ctrl) +{ + return ctrl->opmod_idx_opcode >> 24; +} + +static void +uct_ib_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, + void *dst, size_t length) +{ + size_t copy_len = ucs_min(length, UCS_PTR_BYTE_DIFF(src, txwq->qend)); + + memcpy(dst, src, copy_len); + if (copy_len < length) { + memcpy(UCS_PTR_BYTE_OFFSET(dst, copy_len), txwq->qstart, + length - copy_len); + } +} + +static ucs_status_t +uct_rc_mlx5_fill_am_short_info(const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_inl_data_seg *inl, + size_t wqe_size, uct_ep_op_info_t *info, + int *skip_p, void *callback_data) +{ + size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + uct_rc_mlx5_am_short_hdr_t *am; + + ucs_assert(inline_length >= sizeof(uct_rc_mlx5_hdr_t)); + ucs_assert(wqe_size == + sizeof(struct mlx5_wqe_ctrl_seg) + + ucs_align_up_pow2(sizeof(*inl) + inline_length, + UCT_IB_MLX5_WQE_SEG_SIZE)); + + uct_ib_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); + + am = callback_data; + if ((am->rc_hdr.rc_hdr.am_id & UCT_RC_EP_FC_MASK) == + UCT_RC_EP_FC_PURE_GRANT) { + *skip_p = 1; + return UCS_OK; + } + + if (inline_length < sizeof(*am)) { + return UCS_ERR_UNSUPPORTED; + } + + info->field_mask = UCT_EP_OP_INFO_FIELD_OPERATION | + UCT_EP_OP_INFO_FIELD_AM; + info->operation = UCT_EP_OP_AM_SHORT; + info->am.field_mask = UCT_EP_OP_INFO_AM_FIELD_AM_ID | + UCT_EP_OP_INFO_AM_FIELD_HEADER_VALUE | + UCT_EP_OP_INFO_AM_FIELD_PAYLOAD_DATA; + info->am.am_id = am->rc_hdr.rc_hdr.am_id & ~UCT_RC_EP_FC_MASK; + info->am.header.value = am->am_hdr; + info->am.payload.data.buffer = am + 1; + info->am.payload.data.length = inline_length - sizeof(*am); + return UCS_OK; +} + +static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( + const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size, int *skip_p, + uct_ep_op_info_t *info, void *callback_data) +{ + const struct mlx5_wqe_inl_data_seg *inl; + size_t inline_length, inline_wqe_size; + + memset(info, 0, sizeof(*info)); + *skip_p = 0; + ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); + + inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, + (void*)(ctrl + 1)); + if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { + inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + inline_wqe_size = sizeof(*ctrl) + + ucs_align_up_pow2(sizeof(*inl) + inline_length, + UCT_IB_MLX5_WQE_SEG_SIZE); + if (inline_wqe_size != wqe_size) { + return UCS_ERR_UNSUPPORTED; + } + + return uct_rc_mlx5_fill_am_short_info(txwq, inl, wqe_size, info, + skip_p, callback_data); + } + + return UCS_ERR_UNSUPPORTED; +} + +static size_t uct_rc_mlx5_ep_am_short_wqe_length( + uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl) +{ + const struct mlx5_wqe_inl_data_seg *inl; + + inl = uct_ib_mlx5_txwq_wrap_any(txwq, (void*)(ctrl + 1)); + ucs_assert(inl->byte_count & htonl(MLX5_INLINE_SEG)); + return ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; +} + +static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( + const uct_ep_outstanding_purge_params_t *params) +{ + if (params == NULL) { + ucs_error("rc mlx5: outstanding purge parameters are NULL"); + return UCS_ERR_INVALID_PARAM; + } + + if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_RX_TOKEN) || + (params->rx_token == NULL)) { + ucs_error("rc mlx5: rx token is not set or is NULL"); + return UCS_ERR_INVALID_PARAM; + } + + if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_CB) || + (params->cb == NULL)) { + ucs_error("rc mlx5: callback is not set or is NULL"); + return UCS_ERR_INVALID_PARAM; + } + + return UCS_OK; +} + +ucs_status_t uct_rc_mlx5_ep_outstanding_purge( + uct_ep_h tl_ep, const uct_ep_outstanding_purge_params_t *params) +{ + uct_rc_mlx5_base_ep_t *ep = ucs_derived_of(tl_ep, + uct_rc_mlx5_base_ep_t); + uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; + uct_rc_txqp_t *txqp = &ep->super.txqp; + const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); + const uct_rc_mlx5_rx_token_t *rx_token; + const struct mlx5_wqe_ctrl_seg *ctrl; + uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; + uct_ep_op_info_t info; + uint16_t ci, end_ci, last_completed_ci, start_ci; + uint32_t first_psn, receiver_next_psn, num_packets, psn_diff; + uint32_t total_packets; + uint8_t opcode; + size_t payload_length, wqe_size; + int skip; + void *callback_arg; + ucs_status_t delivery_status, status; + + status = uct_rc_mlx5_ep_outstanding_purge_check_params(params); + if (status != UCS_OK) { + return status; + } + + callback_arg = (params->field_mask & UCT_EP_OUTSTANDING_FIELD_ARG) ? + params->arg : NULL; + + end_ci = txwq->sw_pi; + last_completed_ci = txwq->prev_sw_pi - + (txwq->bb_max - uct_rc_txqp_available(txqp)); + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, last_completed_ci); + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + start_ci = uct_ib_mlx5_txwq_next_ci(last_completed_ci, wqe_size); + if (start_ci == end_ci) { + return UCS_OK; + } + + /* + * Prevalidate all WQE layouts before invoking callbacks so a later + * unsupported WQE cannot cause a partial purge. Recomputing WQE metadata + * keeps purge allocation-free and every pass linear. + */ + total_packets = 0; + for (ci = start_ci; ci != end_ci; + ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + opcode = uct_ib_mlx5_wqe_opcode(ctrl); + if (opcode == MLX5_OPCODE_NOP) { + continue; + } + + if (opcode != MLX5_OPCODE_SEND) { + return UCS_ERR_UNSUPPORTED; + } + + status = uct_rc_mlx5_fill_am_short_op_info( + txwq, ctrl, wqe_size, &skip, &info, callback_data); + if (status != UCS_OK) { + return status; + } + + payload_length = uct_rc_mlx5_ep_am_short_wqe_length(txwq, ctrl); + total_packets += uct_rc_mlx5_num_packets(txwq, payload_length); + } + + rx_token = params->rx_token; + receiver_next_psn = ntohl(*rx_token) & UCT_IB_MLX5_PSN_MASK; + first_psn = (uct_ib_mlx5_txwq_get_next_wqe_psn(txwq) - + total_packets) & UCT_IB_MLX5_PSN_MASK; + ucs_assert(total_packets < psn_half); + /* Reject ambiguity at any point in the outstanding PSN range. */ + psn_diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; + if ((psn_diff >= psn_half) && + ((psn_diff - psn_half) < total_packets)) { + return UCS_ERR_INVALID_PARAM; + } + + for (ci = start_ci; ci != end_ci; + ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + opcode = uct_ib_mlx5_wqe_opcode(ctrl); + if (opcode == MLX5_OPCODE_NOP) { + continue; + } + + payload_length = uct_rc_mlx5_ep_am_short_wqe_length(txwq, ctrl); + num_packets = uct_rc_mlx5_num_packets(txwq, payload_length); + delivery_status = uct_ib_mlx5_psn_delivery_status( + first_psn, receiver_next_psn, num_packets); + ucs_assert(delivery_status != UCS_ERR_INVALID_PARAM); + first_psn = (first_psn + num_packets) & UCT_IB_MLX5_PSN_MASK; + + skip = 1; + if (delivery_status != UCS_OK) { + status = uct_rc_mlx5_fill_am_short_op_info( + txwq, ctrl, wqe_size, &skip, &info, callback_data); + ucs_assert(status == UCS_OK); + } + + if ((delivery_status != UCS_OK) && !skip) { + params->cb(&info, callback_arg); + } + } + + return UCS_OK; +} + ucs_status_t uct_rc_mlx5_base_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_rc_pending_req_t *req) { diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index b611da3cd3d..f452e079a61 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -1239,7 +1239,7 @@ static uct_rc_iface_ops_t uct_rc_mlx5_iface_ops = { .ep_is_connected = uct_rc_mlx5_base_ep_is_connected, .ep_get_device_ep = (uct_ep_get_device_ep_func_t)ucs_empty_function_return_unsupported, .ep_put_sgl_zcopy = uct_rc_mlx5_ep_put_sgl_zcopy, - .ep_outstanding_purge = uct_ib_mlx5_ext_ep_outstanding_purge + .ep_outstanding_purge = uct_rc_mlx5_ep_outstanding_purge }, .create_cq = uct_rc_mlx5_iface_common_create_cq, .destroy_cq = uct_rc_mlx5_iface_common_destroy_cq, diff --git a/test/gtest/uct/ib/test_ib_mlx5_ext.cc b/test/gtest/uct/ib/test_ib_mlx5_ext.cc index 0f747b40b45..a5a4669140e 100644 --- a/test/gtest/uct/ib/test_ib_mlx5_ext.cc +++ b/test/gtest/uct/ib/test_ib_mlx5_ext.cc @@ -104,7 +104,7 @@ UCS_TEST_P(test_uct_ib_mlx5_ext_rc, ep_outstanding_purge) { scoped_log_handler wrap_err(wrap_errors_logger); EXPECT_EQ(UCS_ERR_INVALID_PARAM, - uct_ep_outstanding_purge(m_e1->ep(0), ¶ms)); + uct_ib_mlx5_ext_ep_outstanding_purge(m_e1->ep(0), ¶ms)); } register_plugin("stub"); @@ -118,7 +118,7 @@ UCS_TEST_P(test_uct_ib_mlx5_ext_rc, ep_outstanding_purge) params.cb = purge_cb; params.arg = &callback_invoked; - ASSERT_UCS_OK(uct_ep_outstanding_purge(m_e1->ep(0), ¶ms)); + ASSERT_UCS_OK(uct_ib_mlx5_ext_ep_outstanding_purge(m_e1->ep(0), ¶ms)); EXPECT_TRUE(callback_invoked); } diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index f6478bb6019..4e41db105f2 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -278,6 +278,19 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_wqe_psn, flush(); } +UCS_TEST_P(test_rc_mlx5_psn, delivery_status) +{ + EXPECT_EQ(UCS_INPROGRESS, uct_ib_mlx5_psn_delivery_status(10, 10, 1)); + EXPECT_EQ(UCS_OK, uct_ib_mlx5_psn_delivery_status(10, 11, 1)); + EXPECT_EQ(UCS_INPROGRESS, + uct_ib_mlx5_psn_delivery_status(10, 11, 2)); + EXPECT_EQ(UCS_OK, + uct_ib_mlx5_psn_delivery_status(UCT_IB_MLX5_PSN_MASK, 0, 1)); + EXPECT_EQ(UCS_ERR_INVALID_PARAM, + uct_ib_mlx5_psn_delivery_status( + 0, UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1), 1)); +} + _UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_psn, rc_mlx5) #endif @@ -1533,6 +1546,98 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_token_query, am_short, _UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_token_query, rc_mlx5) +class test_rc_mlx5_outstanding_purge : public test_rc { +public: + void init() override + { + uct_test::init(); + + m_e1 = uct_test::create_entity(0, err_handler); + m_entities.push_back(m_e1); + + check_skip_test(); + + m_e2 = uct_test::create_entity(0, err_handler); + m_entities.push_back(m_e2); + connect(); + } + +protected: + struct operation { + uint8_t am_id; + uint64_t header; + std::vector payload; + }; + + static ucs_status_t err_handler(void*, uct_ep_h, ucs_status_t) + { + return UCS_OK; + } + + static void purge_cb(const uct_ep_op_info_t *info, void *arg) + { + test_rc_mlx5_outstanding_purge *self = + static_cast(arg); + operation op; + const uint8_t *payload = + static_cast(info->am.payload.data.buffer); + + EXPECT_EQ(UCT_EP_OP_AM_SHORT, info->operation); + op.am_id = info->am.am_id; + op.header = info->am.header.value; + op.payload.assign(payload, + payload + info->am.payload.data.length); + self->m_operations.push_back(op); + } + + std::vector m_operations; +}; + +UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + static const uint8_t am_id = 3; + static const uint64_t header = 0x0123456789abcdefull; + const std::vector delivered_payload = {0}; + const std::vector short_payload = {1, 2, 3, 4}; + uct_rc_mlx5_base_ep_t *ep = + ucs_derived_of(m_e1->ep(0), uct_rc_mlx5_base_ep_t); + uct_ib_mlx5_md_t *md = uct_ib_mlx5_iface_md( + ucs_derived_of(m_e1->iface(), uct_ib_iface_t)); + uct_ep_outstanding_purge_params_t purge_params = {}; + uct_ep_invalidate_params_t invalidate_params = {}; + uct_rc_mlx5_rx_token_t rx_token; + + if (!(md->flags & UCT_IB_MLX5_MD_FLAG_DEVX)) { + UCS_TEST_SKIP_R("DEVX is not supported"); + } + + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, + delivered_payload.data(), + delivered_payload.size())); + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, + short_payload.data(), short_payload.size())); + + rx_token = htobe32((uct_ib_mlx5_txwq_get_next_wqe_psn(&ep->tx.wq) - 1) & + UCT_IB_MLX5_PSN_MASK); + ASSERT_UCS_OK(uct_ep_invalidate(m_e1->ep(0), &invalidate_params)); + + purge_params.field_mask = UCT_EP_OUTSTANDING_FIELD_RX_TOKEN | + UCT_EP_OUTSTANDING_FIELD_CB | + UCT_EP_OUTSTANDING_FIELD_ARG; + purge_params.rx_token = &rx_token; + purge_params.cb = purge_cb; + purge_params.arg = this; + ASSERT_UCS_OK(uct_ep_outstanding_purge(m_e1->ep(0), &purge_params)); + + ASSERT_EQ(1, m_operations.size()); + EXPECT_EQ(am_id, m_operations[0].am_id); + EXPECT_EQ(header, m_operations[0].header); + EXPECT_EQ(short_payload, m_operations[0].payload); +} + +_UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_outstanding_purge, rc_mlx5) + class test_rc_srq : public test_rc { public: test_rc_srq() : m_buf8b(NULL), m_buf8k(NULL) From 2e3e70d8cbd913763b30996a328f7cbc08e2d8bb Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 1 Sep 2026 12:54:46 +0300 Subject: [PATCH 02/26] UCT/IB/MLX5: Refine AM short outstanding purge --- src/uct/ib/mlx5/ib_mlx5_ext.c | 55 +---------- src/uct/ib/mlx5/ib_mlx5_ext.h | 4 - src/uct/ib/mlx5/rc/rc_mlx5.inl | 13 +++ src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 134 +++++++++++++------------- test/gtest/Makefile.am | 1 - test/gtest/uct/ib/test_ib_mlx5_ext.cc | 125 ------------------------ 6 files changed, 79 insertions(+), 253 deletions(-) delete mode 100644 test/gtest/uct/ib/test_ib_mlx5_ext.cc diff --git a/src/uct/ib/mlx5/ib_mlx5_ext.c b/src/uct/ib/mlx5/ib_mlx5_ext.c index 643e0dbc21c..ede800d8bb5 100644 --- a/src/uct/ib/mlx5/ib_mlx5_ext.c +++ b/src/uct/ib/mlx5/ib_mlx5_ext.c @@ -111,55 +111,6 @@ uct_ib_mlx5_ext_ep_put_sgl_zcopy(uct_ep_h ep, void * const *buffers, return UCS_ERR_UNSUPPORTED; } -static ucs_status_t uct_ib_mlx5_ext_ep_outstanding_purge_check_param( - const uct_ep_outstanding_purge_params_t *params) -{ - if (params == NULL) { - ucs_error("ib mlx5 ext: outstanding purge parameters are NULL"); - return UCS_ERR_INVALID_PARAM; - } - - if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_RX_TOKEN) || - !(params->field_mask & UCT_EP_OUTSTANDING_FIELD_CB) || - (params->rx_token == NULL) || (params->cb == NULL)) { - ucs_error("ib mlx5 ext: rx token or callback is not set or is NULL"); - return UCS_ERR_INVALID_PARAM; - } - - return UCS_OK; -} - -ucs_status_t uct_ib_mlx5_ext_ep_outstanding_purge( - uct_ep_h ep, const uct_ep_outstanding_purge_params_t *params) -{ - ucs_status_t status; - uct_ib_mlx5_ext_plugin_t *plugin; - - status = uct_ib_mlx5_ext_ep_outstanding_purge_check_param(params); - if (status != UCS_OK) { - return status; - } - - ucs_list_for_each(plugin, &uct_ib_mlx5_ext_plugins, list) { - if (ucs_unlikely(uct_ib_mlx5_ext_is_unsupported_op( - (const void*)plugin->ops.ep_outstanding_purge))) { - continue; - } - - /* The provider validates parameters and token signatures itself, and - * returns an error if it should not handle this request. */ - status = plugin->ops.ep_outstanding_purge(ep, params); - if (status != UCS_OK) { - continue; - } - - return UCS_OK; - } - - ucs_error("ib mlx5 ext: ep outstanding purge is not supported"); - return UCS_ERR_UNSUPPORTED; -} - void uct_ib_mlx5_ext_cleanup(void) { uct_ib_mlx5_ext_plugin_t *plugin, *tmp; @@ -211,7 +162,7 @@ ucs_status_t uct_ib_mlx5_ext_register(const uct_ib_mlx5_ext_ops_t *ops) num_plugins = ucs_list_length(&uct_ib_mlx5_ext_plugins); ucs_debug("ib mlx5 ext: registered plugin name=%s iface_query=%s " - "put_sgl_zcopy=%s outstanding_purge=%s (total=%u)", + "put_sgl_zcopy=%s (total=%u)", plugin->ops.name, uct_ib_mlx5_ext_is_unsupported_op( (const void*)plugin->ops.iface_query) ? @@ -221,10 +172,6 @@ ucs_status_t uct_ib_mlx5_ext_register(const uct_ib_mlx5_ext_ops_t *ops) (const void*)plugin->ops.ep_put_sgl_zcopy) ? "unsupported" : "supported", - uct_ib_mlx5_ext_is_unsupported_op( - (const void*)plugin->ops.ep_outstanding_purge) ? - "unsupported" : - "supported", num_plugins); return UCS_OK; } diff --git a/src/uct/ib/mlx5/ib_mlx5_ext.h b/src/uct/ib/mlx5/ib_mlx5_ext.h index 04a9d2a44b1..af8ad012b3e 100644 --- a/src/uct/ib/mlx5/ib_mlx5_ext.h +++ b/src/uct/ib/mlx5/ib_mlx5_ext.h @@ -77,7 +77,6 @@ typedef struct uct_ib_mlx5_ext_ops { uct_ib_mlx5_ext_iface_query_func_t iface_query; /**< Iface query callback */ uct_ib_mlx5_ext_max_put_sgl_zcopy_count_func_t max_put_sgl_zcopy_count; /**< Maximum PUT SGL zero-copy entry count callback */ uct_ep_put_sgl_zcopy_func_t ep_put_sgl_zcopy; /**< PUT SGL zero-copy callback */ - uct_ep_outstanding_purge_func_t ep_outstanding_purge; /**< Outstanding operation purge callback */ } uct_ib_mlx5_ext_ops_t; /** @@ -118,9 +117,6 @@ ucs_status_t uct_ib_mlx5_ext_ep_put_sgl_zcopy(uct_ep_h ep, size_t count, uct_completion_t *comp); -ucs_status_t uct_ib_mlx5_ext_ep_outstanding_purge( - uct_ep_h ep, const uct_ep_outstanding_purge_params_t *params); - END_C_DECLS #endif /* UCT_IB_MLX5_EXT_H_ */ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index f28d4c9865c..bf25065c429 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -461,6 +461,19 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, return (message_length + txwq->path_mtu_mask) >> txwq->path_mtu_shift; } +static UCS_F_ALWAYS_INLINE uint16_t +uct_rc_mlx5_txwq_first_outstanding_ci(const uct_ib_mlx5_txwq_t *txwq, + const uct_rc_txqp_t *txqp) +{ + uint16_t last_completed_ci = txwq->prev_sw_pi - + (txwq->bb_max - txqp->available); + const struct mlx5_wqe_ctrl_seg *ctrl; + + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, last_completed_ci); + return uct_ib_mlx5_txwq_next_ci(last_completed_ci, + uct_ib_mlx5_wqe_size(ctrl)); +} + static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, uint32_t num_packets) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index c236202c2d6..b9035103bb1 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -842,14 +842,16 @@ uct_rc_mlx5_fill_am_short_info(const uct_ib_mlx5_txwq_t *txwq, size_t wqe_size, uct_ep_op_info_t *info, int *skip_p, void *callback_data) { - size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + size_t inline_wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + + ucs_align_up_pow2(sizeof(*inl) + inline_length, + UCT_IB_MLX5_WQE_SEG_SIZE); uct_rc_mlx5_am_short_hdr_t *am; - ucs_assert(inline_length >= sizeof(uct_rc_mlx5_hdr_t)); - ucs_assert(wqe_size == - sizeof(struct mlx5_wqe_ctrl_seg) + - ucs_align_up_pow2(sizeof(*inl) + inline_length, - UCT_IB_MLX5_WQE_SEG_SIZE)); + if ((inline_length < sizeof(uct_rc_mlx5_hdr_t)) || + (wqe_size != inline_wqe_size)) { + return UCS_ERR_UNSUPPORTED; + } uct_ib_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); @@ -883,23 +885,19 @@ static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( uct_ep_op_info_t *info, void *callback_data) { const struct mlx5_wqe_inl_data_seg *inl; - size_t inline_length, inline_wqe_size; memset(info, 0, sizeof(*info)); *skip_p = 0; ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); + if ((wqe_size < (sizeof(*ctrl) + sizeof(*inl))) || + (wqe_size > UCT_IB_MLX5_MAX_SEND_WQE_SIZE)) { + return UCS_ERR_UNSUPPORTED; + } + inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, (void*)(ctrl + 1)); if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { - inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; - inline_wqe_size = sizeof(*ctrl) + - ucs_align_up_pow2(sizeof(*inl) + inline_length, - UCT_IB_MLX5_WQE_SEG_SIZE); - if (inline_wqe_size != wqe_size) { - return UCS_ERR_UNSUPPORTED; - } - return uct_rc_mlx5_fill_am_short_info(txwq, inl, wqe_size, info, skip_p, callback_data); } @@ -907,15 +905,38 @@ static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( return UCS_ERR_UNSUPPORTED; } -static size_t uct_rc_mlx5_ep_am_short_wqe_length( - uct_ib_mlx5_txwq_t *txwq, - const struct mlx5_wqe_ctrl_seg *ctrl) +static uint32_t uct_rc_mlx5_ep_am_short_num_packets( + uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, uint16_t end_ci) { - const struct mlx5_wqe_inl_data_seg *inl; + const struct mlx5_wqe_ctrl_seg *ctrl; + uint32_t total_packets = 0; + uint16_t ci; + uint8_t opcode; + size_t wqe_size; + + for (ci = start_ci; ci != end_ci; + ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + if ((wqe_size == 0) || + (wqe_size > UCT_IB_MLX5_MAX_SEND_WQE_SIZE)) { + ucs_fatal("rc mlx5: invalid outstanding WQE size %zu", wqe_size); + } + + opcode = uct_ib_mlx5_wqe_opcode(ctrl); + if (opcode == MLX5_OPCODE_NOP) { + continue; + } + + if (opcode != MLX5_OPCODE_SEND) { + ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x", + opcode); + } - inl = uct_ib_mlx5_txwq_wrap_any(txwq, (void*)(ctrl + 1)); - ucs_assert(inl->byte_count & htonl(MLX5_INLINE_SEG)); - return ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + ++total_packets; + } + + return total_packets; } static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( @@ -948,16 +969,15 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_rc_mlx5_base_ep_t); uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; uct_rc_txqp_t *txqp = &ep->super.txqp; - const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; uct_ep_op_info_t info; - uint16_t ci, end_ci, last_completed_ci, start_ci; - uint32_t first_psn, receiver_next_psn, num_packets, psn_diff; + uint16_t ci, end_ci, start_ci; + uint32_t first_psn, receiver_next_psn, psn_diff; uint32_t total_packets; uint8_t opcode; - size_t payload_length, wqe_size; + size_t wqe_size; int skip; void *callback_arg; ucs_status_t delivery_status, status; @@ -970,54 +990,29 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( callback_arg = (params->field_mask & UCT_EP_OUTSTANDING_FIELD_ARG) ? params->arg : NULL; - end_ci = txwq->sw_pi; - last_completed_ci = txwq->prev_sw_pi - - (txwq->bb_max - uct_rc_txqp_available(txqp)); - ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, last_completed_ci); - wqe_size = uct_ib_mlx5_wqe_size(ctrl); - start_ci = uct_ib_mlx5_txwq_next_ci(last_completed_ci, wqe_size); + end_ci = txwq->sw_pi; + if (ep->err_handler_inprogress) { + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, txwq->ft_ci); + start_ci = uct_ib_mlx5_txwq_next_ci(txwq->ft_ci, + uct_ib_mlx5_wqe_size(ctrl)); + } else { + start_ci = uct_rc_mlx5_txwq_first_outstanding_ci(txwq, txqp); + } + if (start_ci == end_ci) { return UCS_OK; } - /* - * Prevalidate all WQE layouts before invoking callbacks so a later - * unsupported WQE cannot cause a partial purge. Recomputing WQE metadata - * keeps purge allocation-free and every pass linear. - */ - total_packets = 0; - for (ci = start_ci; ci != end_ci; - ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { - ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); - wqe_size = uct_ib_mlx5_wqe_size(ctrl); - opcode = uct_ib_mlx5_wqe_opcode(ctrl); - if (opcode == MLX5_OPCODE_NOP) { - continue; - } - - if (opcode != MLX5_OPCODE_SEND) { - return UCS_ERR_UNSUPPORTED; - } - - status = uct_rc_mlx5_fill_am_short_op_info( - txwq, ctrl, wqe_size, &skip, &info, callback_data); - if (status != UCS_OK) { - return status; - } - - payload_length = uct_rc_mlx5_ep_am_short_wqe_length(txwq, ctrl); - total_packets += uct_rc_mlx5_num_packets(txwq, payload_length); - } + total_packets = uct_rc_mlx5_ep_am_short_num_packets(txwq, start_ci, + end_ci); rx_token = params->rx_token; receiver_next_psn = ntohl(*rx_token) & UCT_IB_MLX5_PSN_MASK; first_psn = (uct_ib_mlx5_txwq_get_next_wqe_psn(txwq) - total_packets) & UCT_IB_MLX5_PSN_MASK; - ucs_assert(total_packets < psn_half); - /* Reject ambiguity at any point in the outstanding PSN range. */ + ucs_assert(total_packets < UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1)); psn_diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; - if ((psn_diff >= psn_half) && - ((psn_diff - psn_half) < total_packets)) { + if (psn_diff > total_packets) { return UCS_ERR_INVALID_PARAM; } @@ -1030,18 +1025,19 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( continue; } - payload_length = uct_rc_mlx5_ep_am_short_wqe_length(txwq, ctrl); - num_packets = uct_rc_mlx5_num_packets(txwq, payload_length); delivery_status = uct_ib_mlx5_psn_delivery_status( - first_psn, receiver_next_psn, num_packets); + first_psn, receiver_next_psn, 1); ucs_assert(delivery_status != UCS_ERR_INVALID_PARAM); - first_psn = (first_psn + num_packets) & UCT_IB_MLX5_PSN_MASK; + first_psn = (first_psn + 1) & UCT_IB_MLX5_PSN_MASK; skip = 1; if (delivery_status != UCS_OK) { status = uct_rc_mlx5_fill_am_short_op_info( txwq, ctrl, wqe_size, &skip, &info, callback_data); - ucs_assert(status == UCS_OK); + if (status != UCS_OK) { + ucs_fatal("rc mlx5: failed to parse outstanding WQE at ci %u", + ci); + } } if ((delivery_status != UCS_OK) && !skip) { diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 2638992b103..c9ed084dfc1 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -262,7 +262,6 @@ gtest_SOURCES += \ endif if HAVE_MLX5_DV gtest_SOURCES += \ - uct/ib/test_ib_mlx5_ext.cc \ uct/ib/test_cqe_zipping.cc endif if HAVE_EFA diff --git a/test/gtest/uct/ib/test_ib_mlx5_ext.cc b/test/gtest/uct/ib/test_ib_mlx5_ext.cc deleted file mode 100644 index a5a4669140e..00000000000 --- a/test/gtest/uct/ib/test_ib_mlx5_ext.cc +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2026. ALL RIGHTS RESERVED. - * - * See file LICENSE for terms. - */ - -#include "test_rc.h" - -extern "C" { -#include - -extern ucs_list_link_t uct_ib_mlx5_ext_plugins; -} - - -class test_uct_ib_mlx5_ext_rc : public test_rc { -public: - void init() override - { - save_plugins(); - test_rc::init(); - } - - void cleanup() override - { - uct_ib_mlx5_ext_cleanup(); - uct_test::cleanup(); - restore_plugins(); - } - -protected: - static uint64_t tx_token() - { - return 0x1234; - } - - static uint64_t rx_token() - { - return tx_token() + 1; - } - - void save_plugins() - { - if (ucs_list_is_empty(&uct_ib_mlx5_ext_plugins)) { - ucs_list_head_init(&m_saved_plugins); - } else { - ucs_list_replace(&uct_ib_mlx5_ext_plugins, &m_saved_plugins); - ucs_list_head_init(&uct_ib_mlx5_ext_plugins); - } - } - - void restore_plugins() - { - if (!ucs_list_is_empty(&m_saved_plugins)) { - ucs_list_replace(&m_saved_plugins, &uct_ib_mlx5_ext_plugins); - ucs_list_head_init(&m_saved_plugins); - } - } - - static ucs_status_t - purge_fail(uct_ep_h, const uct_ep_outstanding_purge_params_t*) - { - return UCS_ERR_UNSUPPORTED; - } - - static ucs_status_t - purge(uct_ep_h, const uct_ep_outstanding_purge_params_t *params) - { - uct_ep_op_info_t op_info = {}; - - EXPECT_EQ(rx_token(), *static_cast(params->rx_token)); - params->cb(&op_info, params->arg); - return UCS_OK; - } - - static void purge_cb(const uct_ep_op_info_t*, void *arg) - { - *static_cast(arg) = true; - } - - static void - register_plugin(const char *name, - uct_ep_outstanding_purge_func_t purge_cb = NULL) - { - uct_ib_mlx5_ext_ops_t ops = {}; - - ucs_strncpy_zero(ops.name, name, sizeof(ops.name)); - ops.ep_outstanding_purge = purge_cb; - ASSERT_UCS_OK(uct_ib_mlx5_ext_register(&ops)); - } - -private: - ucs_list_link_t m_saved_plugins = UCS_LIST_INITIALIZER(&m_saved_plugins, - &m_saved_plugins); -}; - - -UCS_TEST_P(test_uct_ib_mlx5_ext_rc, ep_outstanding_purge) -{ - uint64_t rx_token_value = rx_token(); - bool callback_invoked = false; - uct_ep_outstanding_purge_params_t params = {}; - - { - scoped_log_handler wrap_err(wrap_errors_logger); - EXPECT_EQ(UCS_ERR_INVALID_PARAM, - uct_ib_mlx5_ext_ep_outstanding_purge(m_e1->ep(0), ¶ms)); - } - - register_plugin("stub"); - register_plugin("fail", purge_fail); - register_plugin("token", purge); - - params.field_mask = UCT_EP_OUTSTANDING_FIELD_RX_TOKEN | - UCT_EP_OUTSTANDING_FIELD_CB | - UCT_EP_OUTSTANDING_FIELD_ARG; - params.rx_token = &rx_token_value; - params.cb = purge_cb; - params.arg = &callback_invoked; - - ASSERT_UCS_OK(uct_ib_mlx5_ext_ep_outstanding_purge(m_e1->ep(0), ¶ms)); - EXPECT_TRUE(callback_invoked); -} - -_UCT_INSTANTIATE_TEST_CASE(test_uct_ib_mlx5_ext_rc, rc_mlx5) From 1a39e5f351bc862fd4f6966944ea503497a9a637 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 1 Sep 2026 14:25:22 +0300 Subject: [PATCH 03/26] UCT/IB/MLX5: Localize purge PSN helper --- src/uct/ib/mlx5/ib_mlx5.c | 21 --------------------- src/uct/ib/mlx5/ib_mlx5.h | 4 ---- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 21 +++++++++++++++++++++ test/gtest/uct/ib/test_rc.cc | 13 ------------- 4 files changed, 21 insertions(+), 38 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 101c10fc65a..812d6b451bb 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -873,27 +873,6 @@ uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size) return ci + ucs_div_round_up(wqe_size, MLX5_SEND_WQE_BB); } -ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, - uint32_t receiver_next_psn, - uint32_t num_packets) -{ - const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); - uint32_t diff; - - ucs_assert(num_packets > 0); - - diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; - if (diff == psn_half) { - return UCS_ERR_INVALID_PARAM; - } - - if ((diff > 0) && (diff < psn_half) && (diff >= num_packets)) { - return UCS_OK; - } - - return UCS_INPROGRESS; -} - uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, uint16_t outstanding) { diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 42465a08692..3f8656d8b3c 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -923,10 +923,6 @@ size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl); uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size); -ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, - uint32_t receiver_next_psn, - uint32_t num_packets); - /* Count how many WQEs are currently posted */ uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, uint16_t outstanding); diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index b9035103bb1..aa95ce89b8a 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -905,6 +905,27 @@ static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( return UCS_ERR_UNSUPPORTED; } +static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, + uint32_t receiver_next_psn, + uint32_t num_packets) +{ + const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); + uint32_t diff; + + ucs_assert(num_packets > 0); + + diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; + if (diff == psn_half) { + return UCS_ERR_INVALID_PARAM; + } + + if ((diff > 0) && (diff < psn_half) && (diff >= num_packets)) { + return UCS_OK; + } + + return UCS_INPROGRESS; +} + static uint32_t uct_rc_mlx5_ep_am_short_num_packets( uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, uint16_t end_ci) { diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 4e41db105f2..60968078b10 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -278,19 +278,6 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_psn, next_wqe_psn, flush(); } -UCS_TEST_P(test_rc_mlx5_psn, delivery_status) -{ - EXPECT_EQ(UCS_INPROGRESS, uct_ib_mlx5_psn_delivery_status(10, 10, 1)); - EXPECT_EQ(UCS_OK, uct_ib_mlx5_psn_delivery_status(10, 11, 1)); - EXPECT_EQ(UCS_INPROGRESS, - uct_ib_mlx5_psn_delivery_status(10, 11, 2)); - EXPECT_EQ(UCS_OK, - uct_ib_mlx5_psn_delivery_status(UCT_IB_MLX5_PSN_MASK, 0, 1)); - EXPECT_EQ(UCS_ERR_INVALID_PARAM, - uct_ib_mlx5_psn_delivery_status( - 0, UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1), 1)); -} - _UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_psn, rc_mlx5) #endif From 337510a58ee59c1d449d72b6346148d19f02d272 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 2 Sep 2026 12:59:11 +0300 Subject: [PATCH 04/26] UCT/IB/MLX5: Complete AM short purge cleanup Signed-off-by: Roie Danino --- src/uct/ib/mlx5/rc/rc_mlx5.inl | 13 -------- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 55 ++++++++++++++++----------------- test/gtest/uct/ib/test_rc.cc | 13 +++++++- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5.inl b/src/uct/ib/mlx5/rc/rc_mlx5.inl index bf25065c429..f28d4c9865c 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5.inl +++ b/src/uct/ib/mlx5/rc/rc_mlx5.inl @@ -461,19 +461,6 @@ uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, return (message_length + txwq->path_mtu_mask) >> txwq->path_mtu_shift; } -static UCS_F_ALWAYS_INLINE uint16_t -uct_rc_mlx5_txwq_first_outstanding_ci(const uct_ib_mlx5_txwq_t *txwq, - const uct_rc_txqp_t *txqp) -{ - uint16_t last_completed_ci = txwq->prev_sw_pi - - (txwq->bb_max - txqp->available); - const struct mlx5_wqe_ctrl_seg *ctrl; - - ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, last_completed_ci); - return uct_ib_mlx5_txwq_next_ci(last_completed_ci, - uct_ib_mlx5_wqe_size(ctrl)); -} - static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, uint32_t num_packets) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index aa95ce89b8a..3f60e6674c7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -837,10 +837,10 @@ uct_ib_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, } static ucs_status_t -uct_rc_mlx5_fill_am_short_info(const uct_ib_mlx5_txwq_t *txwq, - const struct mlx5_wqe_inl_data_seg *inl, - size_t wqe_size, uct_ep_op_info_t *info, - int *skip_p, void *callback_data) +uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_inl_data_seg *inl, + size_t wqe_size, void *callback_data, + int *skip_p, uct_ep_op_info_t *info) { size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; size_t inline_wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + @@ -879,10 +879,10 @@ uct_rc_mlx5_fill_am_short_info(const uct_ib_mlx5_txwq_t *txwq, return UCS_OK; } -static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( +static ucs_status_t uct_rc_mlx5_op_info_fill_am( const uct_ib_mlx5_txwq_t *txwq, - const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size, int *skip_p, - uct_ep_op_info_t *info, void *callback_data) + const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size, + void *callback_data, int *skip_p, uct_ep_op_info_t *info) { const struct mlx5_wqe_inl_data_seg *inl; @@ -898,8 +898,8 @@ static ucs_status_t uct_rc_mlx5_fill_am_short_op_info( inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, (void*)(ctrl + 1)); if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { - return uct_rc_mlx5_fill_am_short_info(txwq, inl, wqe_size, info, - skip_p, callback_data); + return uct_rc_mlx5_op_info_fill_am_short( + txwq, inl, wqe_size, callback_data, skip_p, info); } return UCS_ERR_UNSUPPORTED; @@ -986,10 +986,8 @@ static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_ep_h tl_ep, const uct_ep_outstanding_purge_params_t *params) { - uct_rc_mlx5_base_ep_t *ep = ucs_derived_of(tl_ep, - uct_rc_mlx5_base_ep_t); - uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; - uct_rc_txqp_t *txqp = &ep->super.txqp; + UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); + uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; @@ -1011,17 +1009,15 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( callback_arg = (params->field_mask & UCT_EP_OUTSTANDING_FIELD_ARG) ? params->arg : NULL; - end_ci = txwq->sw_pi; - if (ep->err_handler_inprogress) { - ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, txwq->ft_ci); - start_ci = uct_ib_mlx5_txwq_next_ci(txwq->ft_ci, - uct_ib_mlx5_wqe_size(ctrl)); - } else { - start_ci = uct_rc_mlx5_txwq_first_outstanding_ci(txwq, txqp); - } + ucs_assert(ep->err_handler_inprogress); + + end_ci = txwq->sw_pi; + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, txwq->ft_ci); + start_ci = uct_ib_mlx5_txwq_next_ci(txwq->ft_ci, + uct_ib_mlx5_wqe_size(ctrl)); if (start_ci == end_ci) { - return UCS_OK; + goto out_purge; } total_packets = uct_rc_mlx5_ep_am_short_num_packets(txwq, start_ci, @@ -1051,21 +1047,24 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( ucs_assert(delivery_status != UCS_ERR_INVALID_PARAM); first_psn = (first_psn + 1) & UCT_IB_MLX5_PSN_MASK; - skip = 1; if (delivery_status != UCS_OK) { - status = uct_rc_mlx5_fill_am_short_op_info( - txwq, ctrl, wqe_size, &skip, &info, callback_data); + status = uct_rc_mlx5_op_info_fill_am( + txwq, ctrl, wqe_size, callback_data, &skip, &info); if (status != UCS_OK) { ucs_fatal("rc mlx5: failed to parse outstanding WQE at ci %u", ci); } - } - if ((delivery_status != UCS_OK) && !skip) { - params->cb(&info, callback_arg); + if (!skip) { + params->cb(&info, callback_arg); + } } } +out_purge: + uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, + UCS_ERR_CANCELED, txwq->prev_sw_pi, 0); + uct_rc_mlx5_ep_update_tx_qp_res(ep, txwq->prev_sw_pi); return UCS_OK; } diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 60968078b10..9c7d794d686 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -1558,7 +1558,7 @@ class test_rc_mlx5_outstanding_purge : public test_rc { static ucs_status_t err_handler(void*, uct_ep_h, ucs_status_t) { - return UCS_OK; + return UCS_INPROGRESS; } static void purge_cb(const uct_ep_op_info_t *info, void *arg) @@ -1593,21 +1593,29 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, ucs_derived_of(m_e1->iface(), uct_ib_iface_t)); uct_ep_outstanding_purge_params_t purge_params = {}; uct_ep_invalidate_params_t invalidate_params = {}; + uct_completion_t flush_comp; uct_rc_mlx5_rx_token_t rx_token; if (!(md->flags & UCT_IB_MLX5_MD_FLAG_DEVX)) { UCS_TEST_SKIP_R("DEVX is not supported"); } + flush_comp.func = [](uct_completion_t*) {}; + flush_comp.count = 1; + flush_comp.status = UCS_OK; + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, delivered_payload.data(), delivered_payload.size())); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, short_payload.data(), short_payload.size())); + EXPECT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_e1->ep(0), 0, &flush_comp)); rx_token = htobe32((uct_ib_mlx5_txwq_get_next_wqe_psn(&ep->tx.wq) - 1) & UCT_IB_MLX5_PSN_MASK); ASSERT_UCS_OK(uct_ep_invalidate(m_e1->ep(0), &invalidate_params)); + wait_for_flag(&ep->err_handler_inprogress); purge_params.field_mask = UCT_EP_OUTSTANDING_FIELD_RX_TOKEN | UCT_EP_OUTSTANDING_FIELD_CB | @@ -1621,6 +1629,9 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, EXPECT_EQ(am_id, m_operations[0].am_id); EXPECT_EQ(header, m_operations[0].header); EXPECT_EQ(short_payload, m_operations[0].payload); + EXPECT_EQ(0, flush_comp.count); + EXPECT_EQ(UCS_ERR_CANCELED, flush_comp.status); + EXPECT_EQ(ep->tx.wq.bb_max, uct_rc_txqp_available(&ep->super.txqp)); } _UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_outstanding_purge, rc_mlx5) From 369e583868018104af7041c72afd52e4b96cc22a Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 2 Sep 2026 16:30:52 +0300 Subject: [PATCH 05/26] UCT/IB/MLX5: Fix AM short purge validation and test --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 21 +++++++++++---------- test/gtest/uct/ib/test_rc.cc | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 3f60e6674c7..b06318e1c0f 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -890,10 +890,10 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( *skip_p = 0; ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); - if ((wqe_size < (sizeof(*ctrl) + sizeof(*inl))) || - (wqe_size > UCT_IB_MLX5_MAX_SEND_WQE_SIZE)) { - return UCS_ERR_UNSUPPORTED; - } + ucs_assertv_always( + (wqe_size >= (sizeof(*ctrl) + sizeof(*inl))) && + (wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE), + "wqe_size=%zu", wqe_size); inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, (void*)(ctrl + 1)); @@ -919,11 +919,11 @@ static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, return UCS_ERR_INVALID_PARAM; } - if ((diff > 0) && (diff < psn_half) && (diff >= num_packets)) { - return UCS_OK; + if ((diff == 0) || (diff > psn_half) || (diff < num_packets)) { + return UCS_INPROGRESS; } - return UCS_INPROGRESS; + return UCS_OK; } static uint32_t uct_rc_mlx5_ep_am_short_num_packets( @@ -968,12 +968,13 @@ static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( return UCS_ERR_INVALID_PARAM; } - if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_RX_TOKEN) || - (params->rx_token == NULL)) { - ucs_error("rc mlx5: rx token is not set or is NULL"); + if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_RX_TOKEN)) { + ucs_error("rc mlx5: rx token is not set"); return UCS_ERR_INVALID_PARAM; } + ucs_assert(params->rx_token != NULL); + if (!(params->field_mask & UCT_EP_OUTSTANDING_FIELD_CB) || (params->cb == NULL)) { ucs_error("rc mlx5: callback is not set or is NULL"); diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 9c7d794d686..e4fd40bf885 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -1591,8 +1591,9 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, ucs_derived_of(m_e1->ep(0), uct_rc_mlx5_base_ep_t); uct_ib_mlx5_md_t *md = uct_ib_mlx5_iface_md( ucs_derived_of(m_e1->iface(), uct_ib_iface_t)); + uct_iface_attr_v2_t iface_attr = {}; uct_ep_outstanding_purge_params_t purge_params = {}; - uct_ep_invalidate_params_t invalidate_params = {}; + uct_ep_invalidate_params_t invalidate_params = {}; uct_completion_t flush_comp; uct_rc_mlx5_rx_token_t rx_token; @@ -1600,22 +1601,32 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, UCS_TEST_SKIP_R("DEVX is not supported"); } + iface_attr.field_mask = UCT_IFACE_ATTR_FIELD_CAP_FLAGS; + ASSERT_UCS_OK(uct_iface_query_v2(m_e1->iface(), &iface_attr)); + if (!(iface_attr.cap.flags & UCT_IFACE_FLAG_V2_QUERY_TOKEN)) { + UCS_TEST_SKIP_R("UCT endpoint outstanding purge is not supported"); + } + flush_comp.func = [](uct_completion_t*) {}; flush_comp.count = 1; flush_comp.status = UCS_OK; + ASSERT_UCS_OK(uct_iface_set_am_handler(m_e2->iface(), am_id, + am_dummy_handler, NULL, 0)); + ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, delivered_payload.data(), delivered_payload.size())); ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, short_payload.data(), short_payload.size())); - EXPECT_EQ(UCS_INPROGRESS, - uct_ep_flush(m_e1->ep(0), 0, &flush_comp)); rx_token = htobe32((uct_ib_mlx5_txwq_get_next_wqe_psn(&ep->tx.wq) - 1) & UCT_IB_MLX5_PSN_MASK); ASSERT_UCS_OK(uct_ep_invalidate(m_e1->ep(0), &invalidate_params)); + EXPECT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_e1->ep(0), 0, &flush_comp)); wait_for_flag(&ep->err_handler_inprogress); + ASSERT_TRUE(ep->err_handler_inprogress); purge_params.field_mask = UCT_EP_OUTSTANDING_FIELD_RX_TOKEN | UCT_EP_OUTSTANDING_FIELD_CB | From 751da50f575f8c513b2f344cfa992d1c8be332d2 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 3 Sep 2026 10:23:27 +0300 Subject: [PATCH 06/26] UCT/IB/MLX5: Remove redundant DEVX test check --- test/gtest/uct/ib/test_rc.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index e4fd40bf885..04e2bcda9d2 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -1589,18 +1589,12 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, const std::vector short_payload = {1, 2, 3, 4}; uct_rc_mlx5_base_ep_t *ep = ucs_derived_of(m_e1->ep(0), uct_rc_mlx5_base_ep_t); - uct_ib_mlx5_md_t *md = uct_ib_mlx5_iface_md( - ucs_derived_of(m_e1->iface(), uct_ib_iface_t)); uct_iface_attr_v2_t iface_attr = {}; uct_ep_outstanding_purge_params_t purge_params = {}; uct_ep_invalidate_params_t invalidate_params = {}; uct_completion_t flush_comp; uct_rc_mlx5_rx_token_t rx_token; - if (!(md->flags & UCT_IB_MLX5_MD_FLAG_DEVX)) { - UCS_TEST_SKIP_R("DEVX is not supported"); - } - iface_attr.field_mask = UCT_IFACE_ATTR_FIELD_CAP_FLAGS; ASSERT_UCS_OK(uct_iface_query_v2(m_e1->iface(), &iface_attr)); if (!(iface_attr.cap.flags & UCT_IFACE_FLAG_V2_QUERY_TOKEN)) { From 30c4fa2430a99bda64547e05001cf32eef5f0aef Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 3 Sep 2026 14:11:24 +0300 Subject: [PATCH 07/26] UCT/IB/MLX5: Refine outstanding purge packet tracking --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 93 ++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index b06318e1c0f..7c96e2a5fa2 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -905,7 +905,7 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( return UCS_ERR_UNSUPPORTED; } -static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, +static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t wqe_first_psn, uint32_t receiver_next_psn, uint32_t num_packets) { @@ -914,7 +914,7 @@ static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, ucs_assert(num_packets > 0); - diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; + diff = (receiver_next_psn - wqe_first_psn) & UCT_IB_MLX5_PSN_MASK; if (diff == psn_half) { return UCS_ERR_INVALID_PARAM; } @@ -926,13 +926,38 @@ static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t first_psn, return UCS_OK; } -static uint32_t uct_rc_mlx5_ep_am_short_num_packets( +static uint32_t uct_rc_mlx5_op_num_packets( + const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) +{ + const struct mlx5_wqe_inl_data_seg *inl; + uint8_t opcode = uct_ib_mlx5_wqe_opcode(ctrl); + + switch (opcode) { + case MLX5_OPCODE_NOP: + return 0; + case MLX5_OPCODE_SEND: + ucs_assertv_always( + wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), + "wqe_size=%zu", wqe_size); + inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, + (void*)(ctrl + 1)); + if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { + return 1; + } + + ucs_fatal("rc mlx5: unsupported outstanding SEND WQE"); + default: + ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x", opcode); + } +} + +static uint32_t uct_rc_mlx5_ep_outstanding_num_packets( uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, uint16_t end_ci) { const struct mlx5_wqe_ctrl_seg *ctrl; - uint32_t total_packets = 0; + uint32_t num_packets = 0; uint16_t ci; - uint8_t opcode; size_t wqe_size; for (ci = start_ci; ci != end_ci; @@ -944,20 +969,10 @@ static uint32_t uct_rc_mlx5_ep_am_short_num_packets( ucs_fatal("rc mlx5: invalid outstanding WQE size %zu", wqe_size); } - opcode = uct_ib_mlx5_wqe_opcode(ctrl); - if (opcode == MLX5_OPCODE_NOP) { - continue; - } - - if (opcode != MLX5_OPCODE_SEND) { - ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x", - opcode); - } - - ++total_packets; + num_packets += uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); } - return total_packets; + return num_packets; } static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( @@ -994,9 +1009,8 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; uct_ep_op_info_t info; uint16_t ci, end_ci, start_ci; - uint32_t first_psn, receiver_next_psn, psn_diff; - uint32_t total_packets; - uint8_t opcode; + uint32_t first_failed_psn, wqe_first_psn, receiver_next_psn, psn_diff; + uint32_t num_outstanding_packets, num_packets; size_t wqe_size; int skip; void *callback_arg; @@ -1021,34 +1035,41 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( goto out_purge; } - total_packets = uct_rc_mlx5_ep_am_short_num_packets(txwq, start_ci, - end_ci); + num_outstanding_packets = uct_rc_mlx5_ep_outstanding_num_packets( + txwq, start_ci, end_ci); rx_token = params->rx_token; receiver_next_psn = ntohl(*rx_token) & UCT_IB_MLX5_PSN_MASK; - first_psn = (uct_ib_mlx5_txwq_get_next_wqe_psn(txwq) - - total_packets) & UCT_IB_MLX5_PSN_MASK; - ucs_assert(total_packets < UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1)); - psn_diff = (receiver_next_psn - first_psn) & UCT_IB_MLX5_PSN_MASK; - if (psn_diff > total_packets) { + first_failed_psn = (uct_ib_mlx5_txwq_get_next_wqe_psn(txwq) - + num_outstanding_packets) & UCT_IB_MLX5_PSN_MASK; + ucs_assert(num_outstanding_packets < + UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1)); + psn_diff = (receiver_next_psn - first_failed_psn) & + UCT_IB_MLX5_PSN_MASK; + if (psn_diff > num_outstanding_packets) { return UCS_ERR_INVALID_PARAM; } + wqe_first_psn = first_failed_psn; for (ci = start_ci; ci != end_ci; ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { - ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); - wqe_size = uct_ib_mlx5_wqe_size(ctrl); - opcode = uct_ib_mlx5_wqe_opcode(ctrl); - if (opcode == MLX5_OPCODE_NOP) { + ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); + wqe_size = uct_ib_mlx5_wqe_size(ctrl); + num_packets = uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); + if (num_packets == 0) { continue; } delivery_status = uct_ib_mlx5_psn_delivery_status( - first_psn, receiver_next_psn, 1); - ucs_assert(delivery_status != UCS_ERR_INVALID_PARAM); - first_psn = (first_psn + 1) & UCT_IB_MLX5_PSN_MASK; - - if (delivery_status != UCS_OK) { + wqe_first_psn, receiver_next_psn, num_packets); + ucs_assertv_always((delivery_status == UCS_OK) || + (delivery_status == UCS_INPROGRESS), + "delivery_status=%s", + ucs_status_string(delivery_status)); + wqe_first_psn = (wqe_first_psn + num_packets) & + UCT_IB_MLX5_PSN_MASK; + + if (delivery_status == UCS_INPROGRESS) { status = uct_rc_mlx5_op_info_fill_am( txwq, ctrl, wqe_size, callback_data, &skip, &info); if (status != UCS_OK) { From 485eb51c29203a8a39956cd5b9ec462ef7a4ab37 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 7 Sep 2026 15:33:15 +0300 Subject: [PATCH 08/26] GTEST/UCT/IB: Defer AM short purge test --- test/gtest/uct/ib/test_rc.cc | 108 ----------------------------------- 1 file changed, 108 deletions(-) diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index 04e2bcda9d2..f6478bb6019 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -1533,114 +1533,6 @@ UCS_TEST_SKIP_COND_P(test_rc_mlx5_token_query, am_short, _UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_token_query, rc_mlx5) -class test_rc_mlx5_outstanding_purge : public test_rc { -public: - void init() override - { - uct_test::init(); - - m_e1 = uct_test::create_entity(0, err_handler); - m_entities.push_back(m_e1); - - check_skip_test(); - - m_e2 = uct_test::create_entity(0, err_handler); - m_entities.push_back(m_e2); - connect(); - } - -protected: - struct operation { - uint8_t am_id; - uint64_t header; - std::vector payload; - }; - - static ucs_status_t err_handler(void*, uct_ep_h, ucs_status_t) - { - return UCS_INPROGRESS; - } - - static void purge_cb(const uct_ep_op_info_t *info, void *arg) - { - test_rc_mlx5_outstanding_purge *self = - static_cast(arg); - operation op; - const uint8_t *payload = - static_cast(info->am.payload.data.buffer); - - EXPECT_EQ(UCT_EP_OP_AM_SHORT, info->operation); - op.am_id = info->am.am_id; - op.header = info->am.header.value; - op.payload.assign(payload, - payload + info->am.payload.data.length); - self->m_operations.push_back(op); - } - - std::vector m_operations; -}; - -UCS_TEST_SKIP_COND_P(test_rc_mlx5_outstanding_purge, am_short, - !check_caps(UCT_IFACE_FLAG_AM_SHORT)) -{ - static const uint8_t am_id = 3; - static const uint64_t header = 0x0123456789abcdefull; - const std::vector delivered_payload = {0}; - const std::vector short_payload = {1, 2, 3, 4}; - uct_rc_mlx5_base_ep_t *ep = - ucs_derived_of(m_e1->ep(0), uct_rc_mlx5_base_ep_t); - uct_iface_attr_v2_t iface_attr = {}; - uct_ep_outstanding_purge_params_t purge_params = {}; - uct_ep_invalidate_params_t invalidate_params = {}; - uct_completion_t flush_comp; - uct_rc_mlx5_rx_token_t rx_token; - - iface_attr.field_mask = UCT_IFACE_ATTR_FIELD_CAP_FLAGS; - ASSERT_UCS_OK(uct_iface_query_v2(m_e1->iface(), &iface_attr)); - if (!(iface_attr.cap.flags & UCT_IFACE_FLAG_V2_QUERY_TOKEN)) { - UCS_TEST_SKIP_R("UCT endpoint outstanding purge is not supported"); - } - - flush_comp.func = [](uct_completion_t*) {}; - flush_comp.count = 1; - flush_comp.status = UCS_OK; - - ASSERT_UCS_OK(uct_iface_set_am_handler(m_e2->iface(), am_id, - am_dummy_handler, NULL, 0)); - - ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, - delivered_payload.data(), - delivered_payload.size())); - ASSERT_UCS_OK(uct_ep_am_short(m_e1->ep(0), am_id, header, - short_payload.data(), short_payload.size())); - - rx_token = htobe32((uct_ib_mlx5_txwq_get_next_wqe_psn(&ep->tx.wq) - 1) & - UCT_IB_MLX5_PSN_MASK); - ASSERT_UCS_OK(uct_ep_invalidate(m_e1->ep(0), &invalidate_params)); - EXPECT_EQ(UCS_INPROGRESS, - uct_ep_flush(m_e1->ep(0), 0, &flush_comp)); - wait_for_flag(&ep->err_handler_inprogress); - ASSERT_TRUE(ep->err_handler_inprogress); - - purge_params.field_mask = UCT_EP_OUTSTANDING_FIELD_RX_TOKEN | - UCT_EP_OUTSTANDING_FIELD_CB | - UCT_EP_OUTSTANDING_FIELD_ARG; - purge_params.rx_token = &rx_token; - purge_params.cb = purge_cb; - purge_params.arg = this; - ASSERT_UCS_OK(uct_ep_outstanding_purge(m_e1->ep(0), &purge_params)); - - ASSERT_EQ(1, m_operations.size()); - EXPECT_EQ(am_id, m_operations[0].am_id); - EXPECT_EQ(header, m_operations[0].header); - EXPECT_EQ(short_payload, m_operations[0].payload); - EXPECT_EQ(0, flush_comp.count); - EXPECT_EQ(UCS_ERR_CANCELED, flush_comp.status); - EXPECT_EQ(ep->tx.wq.bb_max, uct_rc_txqp_available(&ep->super.txqp)); -} - -_UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_outstanding_purge, rc_mlx5) - class test_rc_srq : public test_rc { public: test_rc_srq() : m_buf8b(NULL), m_buf8k(NULL) From f9a2484c1df69b3318fc691de4b46c5f4ff1524a Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 8 Sep 2026 15:40:51 +0300 Subject: [PATCH 09/26] UCT/IB/MLX5: Explain circular PSN comparison --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 7c96e2a5fa2..0f99b787d89 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -914,6 +914,13 @@ static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t wqe_first_psn, ucs_assert(num_packets > 0); + /* + * PSNs wrap in a 24-bit sequence space. Since the outstanding window is + * smaller than half of that space, the masked forward distance determines + * their order: 0 < diff < psn_half means receiver_next_psn is ahead of + * wqe_first_psn, while diff > psn_half means it is behind and the + * subtraction wrapped. Exactly psn_half is ambiguous. + */ diff = (receiver_next_psn - wqe_first_psn) & UCT_IB_MLX5_PSN_MASK; if (diff == psn_half) { return UCS_ERR_INVALID_PARAM; From e9f208db6fed6817ce98f25e7f68454026fde4ae Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 9 Sep 2026 10:48:31 +0300 Subject: [PATCH 10/26] GTEST/UCT: Test outstanding AM short purge --- test/gtest/uct/test_peer_failure.cc | 69 +++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index 75fb0eeaf39..c1e97314817 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -376,6 +376,7 @@ class test_uct_purge_outstanding : public uct_test { struct purge_ctx { test_uct_purge_outstanding *self; uct_completion_t op_comp; + uct_completion_t flush_comp; uint32_t num_ops_purged; uint32_t num_flush_purged; }; @@ -407,19 +408,18 @@ class test_uct_purge_outstanding : public uct_test { return status; } - static bool post_flush(uct_ep_h ep, uct_completion_t *comp) + static void post_flush(uct_ep_h ep, uct_completion_t *comp) { ucs_status_t status; ++comp->count; status = uct_ep_flush(ep, 0, comp); if (status == UCS_INPROGRESS) { - return true; + return; } --comp->count; ASSERT_UCS_OK(status); - return false; } static uint32_t post_until_error(uct_ep_h ep, uct_completion_t *comp, @@ -446,6 +446,9 @@ class test_uct_purge_outstanding : public uct_test { void validate_op(const uct_ep_op_info_t *info, purge_ctx *ctx) { switch (info->operation) { + case UCT_EP_OP_AM_SHORT: + validate_am_short(info, ctx); + return; case UCT_EP_OP_FLUSH: validate_flush(info, ctx); return; @@ -456,6 +459,26 @@ class test_uct_purge_outstanding : public uct_test { } } + void validate_am_short(const uct_ep_op_info_t *info, purge_ctx *ctx) + { + const uint16_t required_fields = + UCT_EP_OP_INFO_AM_FIELD_AM_ID | + UCT_EP_OP_INFO_AM_FIELD_HEADER_VALUE | + UCT_EP_OP_INFO_AM_FIELD_PAYLOAD_DATA; + + ASSERT_TRUE(info->field_mask & UCT_EP_OP_INFO_FIELD_AM); + ASSERT_TRUE(ucs_test_all_flags(info->am.field_mask, required_fields)); + EXPECT_FALSE(info->field_mask & UCT_EP_OP_INFO_FIELD_COMP); + EXPECT_EQ(m_am_short_id, info->am.am_id); + EXPECT_EQ(m_am_short_header, info->am.header.value); + ASSERT_EQ(m_am_short_payload.size(), info->am.payload.data.length); + ASSERT_NE(nullptr, info->am.payload.data.buffer); + EXPECT_EQ(0, memcmp(m_am_short_payload.data(), + info->am.payload.data.buffer, + m_am_short_payload.size())); + ++ctx->num_ops_purged; + } + static void validate_flush(const uct_ep_op_info_t *info, purge_ctx *ctx) { const uint64_t required_fields = UCT_EP_OP_INFO_FIELD_COMP | @@ -465,7 +488,7 @@ class test_uct_purge_outstanding : public uct_test { ASSERT_TRUE(info->flush.field_mask & UCT_EP_OP_INFO_FLUSH_FIELD_FLAGS); EXPECT_EQ(0u, info->flush.flags); - EXPECT_EQ(&ctx->op_comp, info->comp); + EXPECT_EQ(&ctx->flush_comp, info->comp); ASSERT_EQ(0u, ctx->num_flush_purged); ++ctx->num_flush_purged; } @@ -497,6 +520,11 @@ class test_uct_purge_outstanding : public uct_test { { } + static ucs_status_t am_handler(void*, void*, size_t, unsigned) + { + return UCS_OK; + } + void purge_outstanding(purge_ctx *ctx) { uct_ep_outstanding_purge_params_t purge_params = {}; @@ -537,7 +565,8 @@ class test_uct_purge_outstanding : public uct_test { { uct_ep_invalidate_params_t invalidate_params = {}; purge_ctx ctx = { - this, {completion_cb, 0, UCS_OK}, 0, 0}; + this, {completion_cb, 0, UCS_OK}, + {completion_cb, 0, UCS_OK}, 0, 0}; uint32_t num_posted; bool flush_outstanding; ucs_status_t status; @@ -554,34 +583,58 @@ class test_uct_purge_outstanding : public uct_test { } ++num_posted; - flush_outstanding = post_flush(m_sender->ep(0), &ctx.op_comp); ASSERT_UCS_OK(uct_ep_invalidate(m_sender->ep(0), &invalidate_params)); + /* Post after invalidation so flush exercises local cancellation. */ + post_flush(m_sender->ep(0), &ctx.flush_comp); num_posted += post_until_error(m_sender->ep(0), &ctx.op_comp, send_func); wait_for_flag(&m_err_count); ASSERT_EQ(1u, m_err_count); + flush_outstanding = ctx.flush_comp.count != 0; purge_outstanding(&ctx); EXPECT_GT(ctx.num_ops_purged, 0u); EXPECT_LT(ctx.num_ops_purged, num_posted); - EXPECT_EQ(unsigned(flush_outstanding), ctx.num_flush_purged); + /* A pending flush may be completed locally without a purge callback. */ + EXPECT_LE(ctx.num_flush_purged, unsigned(flush_outstanding)); wait_for_value(&ctx.op_comp.count, 0, true); + wait_for_value(&ctx.flush_comp.count, 0, true); if (flush_outstanding) { - EXPECT_EQ(UCS_ERR_CANCELED, ctx.op_comp.status); + EXPECT_EQ(UCS_ERR_CANCELED, ctx.flush_comp.status); } flush(); EXPECT_EQ(0, ctx.op_comp.count); + EXPECT_EQ(0, ctx.flush_comp.count); } entity *m_sender; entity *m_receiver; unsigned m_err_count = 0; + + const uint8_t m_am_short_id = 3; + const uint64_t m_am_short_header = 0x0123456789abcdefull; + const std::vector m_am_short_payload = {1, 2, 3, 4}; }; +UCS_TEST_SKIP_COND_P(test_uct_purge_outstanding, am_short, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + ASSERT_UCS_OK(uct_iface_set_am_handler(m_receiver->iface(), m_am_short_id, + am_handler, NULL, 0)); + + test_purge_outstanding([this](uct_ep_h ep, uct_completion_t*) { + return uct_ep_am_short(ep, m_am_short_id, m_am_short_header, + m_am_short_payload.data(), + m_am_short_payload.size()); + }); +} + +UCT_INSTANTIATE_TEST_CASE(test_uct_purge_outstanding) + class test_uct_peer_failure_multiple : public test_uct_peer_failure { public: From 81c63d1543acdb44adcbe09a6435fa04620d9a62 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 9 Sep 2026 12:52:57 +0300 Subject: [PATCH 11/26] UCT/IB/MLX5: Report outstanding flush operations during purge --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 51 +++++++++++++++++++++++++++-- src/uct/ib/rc/base/rc_iface.h | 6 +++- test/gtest/uct/test_peer_failure.cc | 28 ++++++++++++---- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 0f99b787d89..a50a01a08f7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -762,6 +762,7 @@ ucs_status_t uct_rc_mlx5_base_ep_flush(uct_ep_h tl_ep, unsigned flags, { UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); int already_canceled = ep->super.flags & UCT_RC_EP_FLAG_FLUSH_CANCEL; + uct_rc_iface_send_op_t *op; ucs_status_t status; UCT_CHECK_PARAM(!ucs_test_all_flags(flags, UCT_FLUSH_FLAG_CANCEL | @@ -804,8 +805,17 @@ ucs_status_t uct_rc_mlx5_base_ep_flush(uct_ep_h tl_ep, unsigned flags, 0); } - return uct_rc_txqp_add_flush_comp(&iface->super, &ep->super.super, - &ep->super.txqp, comp, ep->tx.wq.sig_pi); + status = uct_rc_txqp_add_flush_comp(&iface->super, &ep->super.super, + &ep->super.txqp, comp, + ep->tx.wq.sig_pi); + if ((status == UCS_INPROGRESS) && (comp != NULL)) { + op = ucs_queue_tail_elem_non_empty(&ep->super.txqp.outstanding, + uct_rc_iface_send_op_t, queue); + op->flags |= UCT_RC_IFACE_SEND_OP_FLAG_FLUSH; + op->flush_flags = flags; + } + + return status; } ucs_status_t uct_rc_mlx5_base_ep_invalidate(uct_ep_h tl_ep, @@ -982,6 +992,37 @@ static uint32_t uct_rc_mlx5_ep_outstanding_num_packets( return num_packets; } +static void uct_rc_mlx5_ep_purge_flush( + uct_rc_txqp_t *txqp, uint16_t sn, + const uct_ep_outstanding_purge_params_t *params, void *callback_arg, + ucs_queue_iter_t *iter_p) +{ + uct_ep_op_info_t info; + uct_rc_iface_send_op_t *op; + + while (*iter_p != txqp->outstanding.ptail) { + op = ucs_container_of(**iter_p, uct_rc_iface_send_op_t, queue); + if (UCS_CIRCULAR_COMPARE16(op->sn, >, sn)) { + break; + } + + *iter_p = &op->queue.next; + if (!(op->flags & UCT_RC_IFACE_SEND_OP_FLAG_FLUSH)) { + continue; + } + + memset(&info, 0, sizeof(info)); + info.field_mask = UCT_EP_OP_INFO_FIELD_OPERATION | + UCT_EP_OP_INFO_FIELD_COMP | + UCT_EP_OP_INFO_FIELD_FLUSH; + info.operation = UCT_EP_OP_FLUSH; + info.comp = op->user_comp; + info.flush.field_mask = UCT_EP_OP_INFO_FLUSH_FIELD_FLAGS; + info.flush.flags = op->flush_flags; + params->cb(&info, callback_arg); + } +} + static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( const uct_ep_outstanding_purge_params_t *params) { @@ -1011,6 +1052,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( { UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; + ucs_queue_iter_t op_iter = &ep->super.txqp.outstanding.head; const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; @@ -1060,6 +1102,9 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( wqe_first_psn = first_failed_psn; for (ci = start_ci; ci != end_ci; ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + /* A flush follows its signaled WQE, which need not be a NOP. */ + uct_rc_mlx5_ep_purge_flush(&ep->super.txqp, ci - 1, params, + callback_arg, &op_iter); ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); num_packets = uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); @@ -1091,6 +1136,8 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( } out_purge: + uct_rc_mlx5_ep_purge_flush(&ep->super.txqp, txwq->prev_sw_pi, params, + callback_arg, &op_iter); uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, UCS_ERR_CANCELED, txwq->prev_sw_pi, 0); uct_rc_mlx5_ep_update_tx_qp_res(ep, txwq->prev_sw_pi); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 737fc4de75d..2809c50a038 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -107,6 +107,7 @@ enum { /* flags for uct_rc_iface_send_op_t */ enum { + UCT_RC_IFACE_SEND_OP_FLAG_FLUSH = UCS_BIT(10), /* flush_flags field is valid */ UCT_RC_IFACE_SEND_OP_STATUS = UCS_BIT(11), /* status field is valid */ #ifdef NVALGRIND UCT_RC_IFACE_SEND_OP_FLAG_IOV = 0, @@ -340,7 +341,10 @@ struct uct_rc_iface_send_op { uct_rc_send_handler_t handler; uint16_t sn; uint16_t flags; - unsigned length; + union { + unsigned length; + unsigned flush_flags; + }; union { void *buffer; /* atomics / desc / FC_PURE_GRANT request */ diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index c1e97314817..61cd7b0c026 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -379,6 +379,7 @@ class test_uct_purge_outstanding : public uct_test { uct_completion_t flush_comp; uint32_t num_ops_purged; uint32_t num_flush_purged; + unsigned num_flush_completed; }; using send_func_t = @@ -489,7 +490,9 @@ class test_uct_purge_outstanding : public uct_test { UCT_EP_OP_INFO_FLUSH_FIELD_FLAGS); EXPECT_EQ(0u, info->flush.flags); EXPECT_EQ(&ctx->flush_comp, info->comp); - ASSERT_EQ(0u, ctx->num_flush_purged); + ASSERT_LT(ctx->num_flush_purged, 2u); + /* Only the second operation can precede the flushes in the purge. */ + EXPECT_LE(ctx->num_ops_purged, 1u); ++ctx->num_flush_purged; } @@ -520,6 +523,13 @@ class test_uct_purge_outstanding : public uct_test { { } + static void flush_completion_cb(uct_completion_t *comp) + { + purge_ctx *ctx = ucs_container_of(comp, purge_ctx, flush_comp); + + ++ctx->num_flush_completed; + } + static ucs_status_t am_handler(void*, void*, size_t, unsigned) { return UCS_OK; @@ -566,9 +576,9 @@ class test_uct_purge_outstanding : public uct_test { uct_ep_invalidate_params_t invalidate_params = {}; purge_ctx ctx = { this, {completion_cb, 0, UCS_OK}, - {completion_cb, 0, UCS_OK}, 0, 0}; + {flush_completion_cb, 0, UCS_OK}, 0, 0, 0}; uint32_t num_posted; - bool flush_outstanding; + unsigned num_flush_outstanding, num_flush_completed; ucs_status_t status; status = post_op(m_sender->ep(0), &ctx.op_comp, send_func); @@ -586,26 +596,30 @@ class test_uct_purge_outstanding : public uct_test { ASSERT_UCS_OK(uct_ep_invalidate(m_sender->ep(0), &invalidate_params)); /* Post after invalidation so flush exercises local cancellation. */ post_flush(m_sender->ep(0), &ctx.flush_comp); + post_flush(m_sender->ep(0), &ctx.flush_comp); num_posted += post_until_error(m_sender->ep(0), &ctx.op_comp, send_func); wait_for_flag(&m_err_count); ASSERT_EQ(1u, m_err_count); - flush_outstanding = ctx.flush_comp.count != 0; + num_flush_outstanding = ctx.flush_comp.count; + num_flush_completed = ctx.num_flush_completed; purge_outstanding(&ctx); EXPECT_GT(ctx.num_ops_purged, 0u); EXPECT_LT(ctx.num_ops_purged, num_posted); - /* A pending flush may be completed locally without a purge callback. */ - EXPECT_LE(ctx.num_flush_purged, unsigned(flush_outstanding)); + EXPECT_EQ(num_flush_outstanding, ctx.num_flush_purged); wait_for_value(&ctx.op_comp.count, 0, true); wait_for_value(&ctx.flush_comp.count, 0, true); - if (flush_outstanding) { + if (num_flush_outstanding != 0) { EXPECT_EQ(UCS_ERR_CANCELED, ctx.flush_comp.status); + ++num_flush_completed; } + EXPECT_EQ(num_flush_completed, ctx.num_flush_completed); + flush(); EXPECT_EQ(0, ctx.op_comp.count); EXPECT_EQ(0, ctx.flush_comp.count); From 981d6ad0ae76e6cfc079e2f01df95dc8f3163978 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 9 Sep 2026 14:38:58 +0300 Subject: [PATCH 12/26] UCT/IB/MLX5: Keep purge invariant checks always enabled --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index a50a01a08f7..28c8a445c71 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -981,10 +981,10 @@ static uint32_t uct_rc_mlx5_ep_outstanding_num_packets( ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - if ((wqe_size == 0) || - (wqe_size > UCT_IB_MLX5_MAX_SEND_WQE_SIZE)) { - ucs_fatal("rc mlx5: invalid outstanding WQE size %zu", wqe_size); - } + ucs_assertv_always( + (wqe_size > 0) && + (wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE), + "wqe_size=%zu", wqe_size); num_packets += uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); } @@ -1073,7 +1073,8 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( callback_arg = (params->field_mask & UCT_EP_OUTSTANDING_FIELD_ARG) ? params->arg : NULL; - ucs_assert(ep->err_handler_inprogress); + ucs_assertv_always(ep->err_handler_inprogress, + "ep %p is not in deferred error handling", ep); end_ci = txwq->sw_pi; ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, txwq->ft_ci); From 23e6ddc7e9e0951f6c039cce3ebe7b9c1d99c0cb Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 9 Sep 2026 14:38:58 +0300 Subject: [PATCH 13/26] GTEST/UCT: Pass operation data through purge context --- test/gtest/uct/test_peer_failure.cc | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index 61cd7b0c026..695ed0e3918 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -380,6 +380,9 @@ class test_uct_purge_outstanding : public uct_test { uint32_t num_ops_purged; uint32_t num_flush_purged; unsigned num_flush_completed; + uint8_t am_short_id; + uint64_t am_short_header; + std::vector am_short_payload; }; using send_func_t = @@ -470,13 +473,13 @@ class test_uct_purge_outstanding : public uct_test { ASSERT_TRUE(info->field_mask & UCT_EP_OP_INFO_FIELD_AM); ASSERT_TRUE(ucs_test_all_flags(info->am.field_mask, required_fields)); EXPECT_FALSE(info->field_mask & UCT_EP_OP_INFO_FIELD_COMP); - EXPECT_EQ(m_am_short_id, info->am.am_id); - EXPECT_EQ(m_am_short_header, info->am.header.value); - ASSERT_EQ(m_am_short_payload.size(), info->am.payload.data.length); + EXPECT_EQ(ctx->am_short_id, info->am.am_id); + EXPECT_EQ(ctx->am_short_header, info->am.header.value); + ASSERT_EQ(ctx->am_short_payload.size(), info->am.payload.data.length); ASSERT_NE(nullptr, info->am.payload.data.buffer); - EXPECT_EQ(0, memcmp(m_am_short_payload.data(), + EXPECT_EQ(0, memcmp(ctx->am_short_payload.data(), info->am.payload.data.buffer, - m_am_short_payload.size())); + ctx->am_short_payload.size())); ++ctx->num_ops_purged; } @@ -571,12 +574,10 @@ class test_uct_purge_outstanding : public uct_test { &purge_params)); } - void test_purge_outstanding(const send_func_t &send_func) + void test_purge_outstanding(const send_func_t &send_func, + purge_ctx &ctx) { uct_ep_invalidate_params_t invalidate_params = {}; - purge_ctx ctx = { - this, {completion_cb, 0, UCS_OK}, - {flush_completion_cb, 0, UCS_OK}, 0, 0, 0}; uint32_t num_posted; unsigned num_flush_outstanding, num_flush_completed; ucs_status_t status; @@ -628,23 +629,23 @@ class test_uct_purge_outstanding : public uct_test { entity *m_sender; entity *m_receiver; unsigned m_err_count = 0; - - const uint8_t m_am_short_id = 3; - const uint64_t m_am_short_header = 0x0123456789abcdefull; - const std::vector m_am_short_payload = {1, 2, 3, 4}; }; UCS_TEST_SKIP_COND_P(test_uct_purge_outstanding, am_short, !check_caps(UCT_IFACE_FLAG_AM_SHORT)) { - ASSERT_UCS_OK(uct_iface_set_am_handler(m_receiver->iface(), m_am_short_id, + purge_ctx ctx = {this, {completion_cb, 0, UCS_OK}, + {flush_completion_cb, 0, UCS_OK}, 0, 0, 0, + 3, 0x0123456789abcdefull, {1, 2, 3, 4}}; + + ASSERT_UCS_OK(uct_iface_set_am_handler(m_receiver->iface(), ctx.am_short_id, am_handler, NULL, 0)); - test_purge_outstanding([this](uct_ep_h ep, uct_completion_t*) { - return uct_ep_am_short(ep, m_am_short_id, m_am_short_header, - m_am_short_payload.data(), - m_am_short_payload.size()); - }); + test_purge_outstanding([&ctx](uct_ep_h ep, uct_completion_t*) { + return uct_ep_am_short(ep, ctx.am_short_id, ctx.am_short_header, + ctx.am_short_payload.data(), + ctx.am_short_payload.size()); + }, ctx); } UCT_INSTANTIATE_TEST_CASE(test_uct_purge_outstanding) From c72dbd0203656420d7232ed3c8d3d0ebc0d73364 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Wed, 9 Sep 2026 17:57:34 +0300 Subject: [PATCH 14/26] UCT/IB/MLX5: Simplify AM short outstanding purge Keep flush reporting separate from AM short purge, simplify delivery and flow-control handling, and share the test completion counter. --- src/uct/ib/mlx5/ib_mlx5.h | 2 + src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 117 +++++++--------------------- src/uct/ib/rc/base/rc_iface.h | 6 +- test/gtest/uct/test_peer_failure.cc | 55 ++++++------- 4 files changed, 53 insertions(+), 127 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 3f8656d8b3c..f4c138763dd 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -919,8 +919,10 @@ ucs_status_t uct_ib_mlx5_txwq_init(uct_priv_worker_t *worker, /* Get pointer to a WQE by producer index */ void *uct_ib_mlx5_txwq_get_wqe(const uct_ib_mlx5_txwq_t *txwq, uint16_t pi); +/* Get the WQE size in bytes from its control segment */ size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl); +/* Get the index of the WQE that follows a WQE of the given size */ uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size); /* Count how many WQEs are currently posted */ diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 28c8a445c71..fe693deff2e 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -762,7 +762,6 @@ ucs_status_t uct_rc_mlx5_base_ep_flush(uct_ep_h tl_ep, unsigned flags, { UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); int already_canceled = ep->super.flags & UCT_RC_EP_FLAG_FLUSH_CANCEL; - uct_rc_iface_send_op_t *op; ucs_status_t status; UCT_CHECK_PARAM(!ucs_test_all_flags(flags, UCT_FLUSH_FLAG_CANCEL | @@ -805,17 +804,8 @@ ucs_status_t uct_rc_mlx5_base_ep_flush(uct_ep_h tl_ep, unsigned flags, 0); } - status = uct_rc_txqp_add_flush_comp(&iface->super, &ep->super.super, - &ep->super.txqp, comp, - ep->tx.wq.sig_pi); - if ((status == UCS_INPROGRESS) && (comp != NULL)) { - op = ucs_queue_tail_elem_non_empty(&ep->super.txqp.outstanding, - uct_rc_iface_send_op_t, queue); - op->flags |= UCT_RC_IFACE_SEND_OP_FLAG_FLUSH; - op->flush_flags = flags; - } - - return status; + return uct_rc_txqp_add_flush_comp(&iface->super, &ep->super.super, + &ep->super.txqp, comp, ep->tx.wq.sig_pi); } ucs_status_t uct_rc_mlx5_base_ep_invalidate(uct_ep_h tl_ep, @@ -850,7 +840,7 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_inl_data_seg *inl, size_t wqe_size, void *callback_data, - int *skip_p, uct_ep_op_info_t *info) + uct_ep_op_info_t *info) { size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; size_t inline_wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + @@ -858,8 +848,7 @@ uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, UCT_IB_MLX5_WQE_SEG_SIZE); uct_rc_mlx5_am_short_hdr_t *am; - if ((inline_length < sizeof(uct_rc_mlx5_hdr_t)) || - (wqe_size != inline_wqe_size)) { + if (wqe_size != inline_wqe_size) { return UCS_ERR_UNSUPPORTED; } @@ -868,8 +857,7 @@ uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, am = callback_data; if ((am->rc_hdr.rc_hdr.am_id & UCT_RC_EP_FC_MASK) == UCT_RC_EP_FC_PURE_GRANT) { - *skip_p = 1; - return UCS_OK; + return UCS_ERR_NO_ELEM; } if (inline_length < sizeof(*am)) { @@ -892,12 +880,11 @@ uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, static ucs_status_t uct_rc_mlx5_op_info_fill_am( const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size, - void *callback_data, int *skip_p, uct_ep_op_info_t *info) + void *callback_data, uct_ep_op_info_t *info) { const struct mlx5_wqe_inl_data_seg *inl; memset(info, 0, sizeof(*info)); - *skip_p = 0; ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); ucs_assertv_always( @@ -909,15 +896,15 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( (void*)(ctrl + 1)); if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { return uct_rc_mlx5_op_info_fill_am_short( - txwq, inl, wqe_size, callback_data, skip_p, info); + txwq, inl, wqe_size, callback_data, info); } return UCS_ERR_UNSUPPORTED; } -static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t wqe_first_psn, - uint32_t receiver_next_psn, - uint32_t num_packets) +static int uct_ib_mlx5_wqe_is_delivered(uint32_t wqe_first_psn, + uint32_t receiver_next_psn, + uint32_t num_packets) { const uint32_t psn_half = UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1); uint32_t diff; @@ -929,21 +916,14 @@ static ucs_status_t uct_ib_mlx5_psn_delivery_status(uint32_t wqe_first_psn, * smaller than half of that space, the masked forward distance determines * their order: 0 < diff < psn_half means receiver_next_psn is ahead of * wqe_first_psn, while diff > psn_half means it is behind and the - * subtraction wrapped. Exactly psn_half is ambiguous. + * subtraction wrapped. The validated outstanding window excludes the + * ambiguous distance of exactly psn_half. */ diff = (receiver_next_psn - wqe_first_psn) & UCT_IB_MLX5_PSN_MASK; - if (diff == psn_half) { - return UCS_ERR_INVALID_PARAM; - } - - if ((diff == 0) || (diff > psn_half) || (diff < num_packets)) { - return UCS_INPROGRESS; - } - - return UCS_OK; + return (diff < psn_half) && (diff >= num_packets); } -static uint32_t uct_rc_mlx5_op_num_packets( +static uint32_t uct_ib_mlx5_wqe_num_packets( const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) { @@ -969,7 +949,7 @@ static uint32_t uct_rc_mlx5_op_num_packets( } } -static uint32_t uct_rc_mlx5_ep_outstanding_num_packets( +static uint32_t uct_rc_mlx5_txwq_outstanding_num_packets( uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, uint16_t end_ci) { const struct mlx5_wqe_ctrl_seg *ctrl; @@ -986,43 +966,12 @@ static uint32_t uct_rc_mlx5_ep_outstanding_num_packets( (wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE), "wqe_size=%zu", wqe_size); - num_packets += uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); + num_packets += uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); } return num_packets; } -static void uct_rc_mlx5_ep_purge_flush( - uct_rc_txqp_t *txqp, uint16_t sn, - const uct_ep_outstanding_purge_params_t *params, void *callback_arg, - ucs_queue_iter_t *iter_p) -{ - uct_ep_op_info_t info; - uct_rc_iface_send_op_t *op; - - while (*iter_p != txqp->outstanding.ptail) { - op = ucs_container_of(**iter_p, uct_rc_iface_send_op_t, queue); - if (UCS_CIRCULAR_COMPARE16(op->sn, >, sn)) { - break; - } - - *iter_p = &op->queue.next; - if (!(op->flags & UCT_RC_IFACE_SEND_OP_FLAG_FLUSH)) { - continue; - } - - memset(&info, 0, sizeof(info)); - info.field_mask = UCT_EP_OP_INFO_FIELD_OPERATION | - UCT_EP_OP_INFO_FIELD_COMP | - UCT_EP_OP_INFO_FIELD_FLUSH; - info.operation = UCT_EP_OP_FLUSH; - info.comp = op->user_comp; - info.flush.field_mask = UCT_EP_OP_INFO_FLUSH_FIELD_FLAGS; - info.flush.flags = op->flush_flags; - params->cb(&info, callback_arg); - } -} - static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( const uct_ep_outstanding_purge_params_t *params) { @@ -1052,7 +1001,6 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( { UCT_RC_MLX5_BASE_EP_DECL(tl_ep, iface, ep); uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; - ucs_queue_iter_t op_iter = &ep->super.txqp.outstanding.head; const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; @@ -1061,9 +1009,9 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uint32_t first_failed_psn, wqe_first_psn, receiver_next_psn, psn_diff; uint32_t num_outstanding_packets, num_packets; size_t wqe_size; - int skip; void *callback_arg; - ucs_status_t delivery_status, status; + int is_delivered; + ucs_status_t status; status = uct_rc_mlx5_ep_outstanding_purge_check_params(params); if (status != UCS_OK) { @@ -1085,15 +1033,13 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( goto out_purge; } - num_outstanding_packets = uct_rc_mlx5_ep_outstanding_num_packets( + num_outstanding_packets = uct_rc_mlx5_txwq_outstanding_num_packets( txwq, start_ci, end_ci); rx_token = params->rx_token; receiver_next_psn = ntohl(*rx_token) & UCT_IB_MLX5_PSN_MASK; first_failed_psn = (uct_ib_mlx5_txwq_get_next_wqe_psn(txwq) - num_outstanding_packets) & UCT_IB_MLX5_PSN_MASK; - ucs_assert(num_outstanding_packets < - UCS_BIT(UCT_IB_MLX5_PSN_BITS - 1)); psn_diff = (receiver_next_psn - first_failed_psn) & UCT_IB_MLX5_PSN_MASK; if (psn_diff > num_outstanding_packets) { @@ -1103,42 +1049,35 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( wqe_first_psn = first_failed_psn; for (ci = start_ci; ci != end_ci; ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { - /* A flush follows its signaled WQE, which need not be a NOP. */ - uct_rc_mlx5_ep_purge_flush(&ep->super.txqp, ci - 1, params, - callback_arg, &op_iter); ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - num_packets = uct_rc_mlx5_op_num_packets(txwq, ctrl, wqe_size); + num_packets = uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); if (num_packets == 0) { continue; } - delivery_status = uct_ib_mlx5_psn_delivery_status( + is_delivered = uct_ib_mlx5_wqe_is_delivered( wqe_first_psn, receiver_next_psn, num_packets); - ucs_assertv_always((delivery_status == UCS_OK) || - (delivery_status == UCS_INPROGRESS), - "delivery_status=%s", - ucs_status_string(delivery_status)); wqe_first_psn = (wqe_first_psn + num_packets) & UCT_IB_MLX5_PSN_MASK; - if (delivery_status == UCS_INPROGRESS) { + if (!is_delivered) { status = uct_rc_mlx5_op_info_fill_am( - txwq, ctrl, wqe_size, callback_data, &skip, &info); + txwq, ctrl, wqe_size, callback_data, &info); + if (status == UCS_ERR_NO_ELEM) { + continue; + } + if (status != UCS_OK) { ucs_fatal("rc mlx5: failed to parse outstanding WQE at ci %u", ci); } - if (!skip) { - params->cb(&info, callback_arg); - } + params->cb(&info, callback_arg); } } out_purge: - uct_rc_mlx5_ep_purge_flush(&ep->super.txqp, txwq->prev_sw_pi, params, - callback_arg, &op_iter); uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, UCS_ERR_CANCELED, txwq->prev_sw_pi, 0); uct_rc_mlx5_ep_update_tx_qp_res(ep, txwq->prev_sw_pi); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 2809c50a038..737fc4de75d 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -107,7 +107,6 @@ enum { /* flags for uct_rc_iface_send_op_t */ enum { - UCT_RC_IFACE_SEND_OP_FLAG_FLUSH = UCS_BIT(10), /* flush_flags field is valid */ UCT_RC_IFACE_SEND_OP_STATUS = UCS_BIT(11), /* status field is valid */ #ifdef NVALGRIND UCT_RC_IFACE_SEND_OP_FLAG_IOV = 0, @@ -341,10 +340,7 @@ struct uct_rc_iface_send_op { uct_rc_send_handler_t handler; uint16_t sn; uint16_t flags; - union { - unsigned length; - unsigned flush_flags; - }; + unsigned length; union { void *buffer; /* atomics / desc / FC_PURE_GRANT request */ diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index 695ed0e3918..45c693cbec4 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -375,11 +375,10 @@ class test_uct_purge_outstanding : public uct_test { protected: struct purge_ctx { test_uct_purge_outstanding *self; - uct_completion_t op_comp; - uct_completion_t flush_comp; + uct_completion_t comp; uint32_t num_ops_purged; uint32_t num_flush_purged; - unsigned num_flush_completed; + unsigned num_completions; uint8_t am_short_id; uint64_t am_short_header; std::vector am_short_payload; @@ -492,10 +491,8 @@ class test_uct_purge_outstanding : public uct_test { ASSERT_TRUE(info->flush.field_mask & UCT_EP_OP_INFO_FLUSH_FIELD_FLAGS); EXPECT_EQ(0u, info->flush.flags); - EXPECT_EQ(&ctx->flush_comp, info->comp); - ASSERT_LT(ctx->num_flush_purged, 2u); - /* Only the second operation can precede the flushes in the purge. */ - EXPECT_LE(ctx->num_ops_purged, 1u); + EXPECT_EQ(&ctx->comp, info->comp); + ASSERT_EQ(0u, ctx->num_flush_purged); ++ctx->num_flush_purged; } @@ -522,15 +519,11 @@ class test_uct_purge_outstanding : public uct_test { ctx->self->validate_op(info, ctx); } - static void completion_cb(uct_completion_t*) + static void completion_cb(uct_completion_t *comp) { - } - - static void flush_completion_cb(uct_completion_t *comp) - { - purge_ctx *ctx = ucs_container_of(comp, purge_ctx, flush_comp); + purge_ctx *ctx = ucs_container_of(comp, purge_ctx, comp); - ++ctx->num_flush_completed; + ++ctx->num_completions; } static ucs_status_t am_handler(void*, void*, size_t, unsigned) @@ -579,15 +572,15 @@ class test_uct_purge_outstanding : public uct_test { { uct_ep_invalidate_params_t invalidate_params = {}; uint32_t num_posted; - unsigned num_flush_outstanding, num_flush_completed; + unsigned num_outstanding, num_completions; ucs_status_t status; - status = post_op(m_sender->ep(0), &ctx.op_comp, send_func); + status = post_op(m_sender->ep(0), &ctx.comp, send_func); ASSERT_UCS_OK_OR_INPROGRESS(status); num_posted = 1; flush(); - status = post_op(m_sender->ep(0), &ctx.op_comp, send_func); + status = post_op(m_sender->ep(0), &ctx.comp, send_func); if (UCS_STATUS_IS_ERR(status)) { UCS_TEST_ABORT("failed to post operation before invalidation: " << ucs_status_string(status)); @@ -596,34 +589,31 @@ class test_uct_purge_outstanding : public uct_test { ++num_posted; ASSERT_UCS_OK(uct_ep_invalidate(m_sender->ep(0), &invalidate_params)); /* Post after invalidation so flush exercises local cancellation. */ - post_flush(m_sender->ep(0), &ctx.flush_comp); - post_flush(m_sender->ep(0), &ctx.flush_comp); - num_posted += post_until_error(m_sender->ep(0), &ctx.op_comp, + post_flush(m_sender->ep(0), &ctx.comp); + num_posted += post_until_error(m_sender->ep(0), &ctx.comp, send_func); wait_for_flag(&m_err_count); ASSERT_EQ(1u, m_err_count); - num_flush_outstanding = ctx.flush_comp.count; - num_flush_completed = ctx.num_flush_completed; + num_outstanding = ctx.comp.count; + num_completions = ctx.num_completions; purge_outstanding(&ctx); EXPECT_GT(ctx.num_ops_purged, 0u); EXPECT_LT(ctx.num_ops_purged, num_posted); - EXPECT_EQ(num_flush_outstanding, ctx.num_flush_purged); + EXPECT_EQ(0u, ctx.num_flush_purged); - wait_for_value(&ctx.op_comp.count, 0, true); - wait_for_value(&ctx.flush_comp.count, 0, true); - if (num_flush_outstanding != 0) { - EXPECT_EQ(UCS_ERR_CANCELED, ctx.flush_comp.status); - ++num_flush_completed; + wait_for_value(&ctx.comp.count, 0, true); + if (num_outstanding != 0) { + EXPECT_EQ(UCS_ERR_CANCELED, ctx.comp.status); + ++num_completions; } - EXPECT_EQ(num_flush_completed, ctx.num_flush_completed); + EXPECT_EQ(num_completions, ctx.num_completions); flush(); - EXPECT_EQ(0, ctx.op_comp.count); - EXPECT_EQ(0, ctx.flush_comp.count); + EXPECT_EQ(0, ctx.comp.count); } entity *m_sender; @@ -634,8 +624,7 @@ class test_uct_purge_outstanding : public uct_test { UCS_TEST_SKIP_COND_P(test_uct_purge_outstanding, am_short, !check_caps(UCT_IFACE_FLAG_AM_SHORT)) { - purge_ctx ctx = {this, {completion_cb, 0, UCS_OK}, - {flush_completion_cb, 0, UCS_OK}, 0, 0, 0, + purge_ctx ctx = {this, {completion_cb, 0, UCS_OK}, 0, 0, 0, 3, 0x0123456789abcdefull, {1, 2, 3, 4}}; ASSERT_UCS_OK(uct_iface_set_am_handler(m_receiver->iface(), ctx.am_short_id, From 3f6c0a58402e366df44330723a81ebba4d021f23 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 10 Sep 2026 11:08:46 +0300 Subject: [PATCH 15/26] GTEST/UCT: Simplify outstanding purge test posting --- test/gtest/uct/test_peer_failure.cc | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index 45c693cbec4..b98f732ab1e 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -570,23 +570,17 @@ class test_uct_purge_outstanding : public uct_test { void test_purge_outstanding(const send_func_t &send_func, purge_ctx &ctx) { + static constexpr uint32_t NUM_MSG_BEFORE_INVALIDATE = 2; uct_ep_invalidate_params_t invalidate_params = {}; - uint32_t num_posted; + uint32_t num_posted = 0; unsigned num_outstanding, num_completions; ucs_status_t status; - status = post_op(m_sender->ep(0), &ctx.comp, send_func); - ASSERT_UCS_OK_OR_INPROGRESS(status); - num_posted = 1; - flush(); - - status = post_op(m_sender->ep(0), &ctx.comp, send_func); - if (UCS_STATUS_IS_ERR(status)) { - UCS_TEST_ABORT("failed to post operation before invalidation: " - << ucs_status_string(status)); + for (; num_posted < NUM_MSG_BEFORE_INVALIDATE; ++num_posted) { + status = post_op(m_sender->ep(0), &ctx.comp, send_func); + ASSERT_UCS_OK_OR_INPROGRESS(status); } - ++num_posted; ASSERT_UCS_OK(uct_ep_invalidate(m_sender->ep(0), &invalidate_params)); /* Post after invalidation so flush exercises local cancellation. */ post_flush(m_sender->ep(0), &ctx.comp); @@ -601,7 +595,7 @@ class test_uct_purge_outstanding : public uct_test { purge_outstanding(&ctx); EXPECT_GT(ctx.num_ops_purged, 0u); - EXPECT_LT(ctx.num_ops_purged, num_posted); + EXPECT_LE(ctx.num_ops_purged, num_posted); EXPECT_EQ(0u, ctx.num_flush_purged); wait_for_value(&ctx.comp.count, 0, true); From b76e1d68903d8544e37786be21443e39a12eaa69 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 10 Sep 2026 13:10:24 +0300 Subject: [PATCH 16/26] UCT/IB/MLX5: Advertise native token query support --- src/uct/ib/mlx5/rc/rc_mlx5_iface.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index f452e079a61..8f2c00d3466 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -1164,6 +1164,10 @@ uct_rc_mlx5_iface_query_v2(uct_iface_h tl_iface, md = uct_ib_mlx5_iface_md(ucs_derived_of(tl_iface, uct_ib_iface_t)); if (md->flags & UCT_IB_MLX5_MD_FLAG_DEVX) { + if (iface_attr->field_mask & UCT_IFACE_ATTR_FIELD_CAP_FLAGS) { + iface_attr->cap.flags |= UCT_IFACE_FLAG_V2_QUERY_TOKEN; + } + if (iface_attr->field_mask & UCT_IFACE_ATTR_FIELD_RX_TOKEN) { status = uct_rc_mlx5_iface_query_rx_token(tl_iface, iface_attr); if (status != UCS_OK) { From a44bce3b69b3c639ac4cc8ae02fb00c0b0e79d86 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 10 Sep 2026 13:10:25 +0300 Subject: [PATCH 17/26] UCT/IB/MLX5: Simplify purge validation and diagnostics --- src/uct/ib/mlx5/ib_mlx5.c | 14 +++++++----- src/uct/ib/mlx5/ib_mlx5.h | 2 +- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 40 ++++++++++++++------------------- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 812d6b451bb..711eb47cb6e 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -862,15 +862,17 @@ size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl) { uint8_t ds = ntohl(ctrl->qpn_ds) & UINT8_MAX; - ucs_assert(ds > 0); - ucs_assert((ds * UCT_IB_MLX5_WQE_SEG_SIZE) <= - UCT_IB_MLX5_MAX_SEND_WQE_SIZE); + ucs_assertv_always( + (ds > 0) && + ((ds * UCT_IB_MLX5_WQE_SEG_SIZE) <= + UCT_IB_MLX5_MAX_SEND_WQE_SIZE), + "ds=%u", ds); return ds * UCT_IB_MLX5_WQE_SEG_SIZE; } -uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size) +uint16_t uct_ib_mlx5_txwq_next_wqe_index(uint16_t index, size_t wqe_size) { - return ci + ucs_div_round_up(wqe_size, MLX5_SEND_WQE_BB); + return index + ucs_div_round_up(wqe_size, MLX5_SEND_WQE_BB); } uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, @@ -889,7 +891,7 @@ uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, do { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, pi); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - pi = uct_ib_mlx5_txwq_next_ci(pi, wqe_size); + pi = uct_ib_mlx5_txwq_next_wqe_index(pi, wqe_size); ++count; } while (pi != txwq->sw_pi); diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index f4c138763dd..c261dbcd724 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -923,7 +923,7 @@ void *uct_ib_mlx5_txwq_get_wqe(const uct_ib_mlx5_txwq_t *txwq, uint16_t pi); size_t uct_ib_mlx5_wqe_size(const struct mlx5_wqe_ctrl_seg *ctrl); /* Get the index of the WQE that follows a WQE of the given size */ -uint16_t uct_ib_mlx5_txwq_next_ci(uint16_t ci, size_t wqe_size); +uint16_t uct_ib_mlx5_txwq_next_wqe_index(uint16_t index, size_t wqe_size); /* Count how many WQEs are currently posted */ uint16_t uct_ib_mlx5_txwq_num_posted_wqes(const uct_ib_mlx5_txwq_t *txwq, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index fe693deff2e..9b3b2bfbb30 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include /* For htonl */ @@ -839,19 +840,11 @@ uct_ib_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, static ucs_status_t uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_inl_data_seg *inl, - size_t wqe_size, void *callback_data, - uct_ep_op_info_t *info) + void *callback_data, uct_ep_op_info_t *info) { - size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; - size_t inline_wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + - ucs_align_up_pow2(sizeof(*inl) + inline_length, - UCT_IB_MLX5_WQE_SEG_SIZE); + size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; uct_rc_mlx5_am_short_hdr_t *am; - if (wqe_size != inline_wqe_size) { - return UCS_ERR_UNSUPPORTED; - } - uct_ib_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); am = callback_data; @@ -888,15 +881,14 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); ucs_assertv_always( - (wqe_size >= (sizeof(*ctrl) + sizeof(*inl))) && - (wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE), + wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), "wqe_size=%zu", wqe_size); inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, (void*)(ctrl + 1)); if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { return uct_rc_mlx5_op_info_fill_am_short( - txwq, inl, wqe_size, callback_data, info); + txwq, inl, callback_data, info); } return UCS_ERR_UNSUPPORTED; @@ -929,6 +921,8 @@ static uint32_t uct_ib_mlx5_wqe_num_packets( { const struct mlx5_wqe_inl_data_seg *inl; uint8_t opcode = uct_ib_mlx5_wqe_opcode(ctrl); + uint8_t wqe[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; + char wqe_dump[3 * sizeof(wqe)]; switch (opcode) { case MLX5_OPCODE_NOP: @@ -943,9 +937,13 @@ static uint32_t uct_ib_mlx5_wqe_num_packets( return 1; } - ucs_fatal("rc mlx5: unsupported outstanding SEND WQE"); + /* Fall through */ default: - ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x", opcode); + uct_ib_mlx5_txwq_copy_segs(txwq, ctrl, wqe, wqe_size); + ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x size %zu: %s", + opcode, wqe_size, + ucs_str_dump_hex(wqe, wqe_size, wqe_dump, sizeof(wqe_dump), + SIZE_MAX)); } } @@ -958,13 +956,9 @@ static uint32_t uct_rc_mlx5_txwq_outstanding_num_packets( size_t wqe_size; for (ci = start_ci; ci != end_ci; - ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + ci = uct_ib_mlx5_txwq_next_wqe_index(ci, wqe_size)) { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - ucs_assertv_always( - (wqe_size > 0) && - (wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE), - "wqe_size=%zu", wqe_size); num_packets += uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); } @@ -1026,8 +1020,8 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( end_ci = txwq->sw_pi; ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, txwq->ft_ci); - start_ci = uct_ib_mlx5_txwq_next_ci(txwq->ft_ci, - uct_ib_mlx5_wqe_size(ctrl)); + start_ci = uct_ib_mlx5_txwq_next_wqe_index( + txwq->ft_ci, uct_ib_mlx5_wqe_size(ctrl)); if (start_ci == end_ci) { goto out_purge; @@ -1048,7 +1042,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( wqe_first_psn = first_failed_psn; for (ci = start_ci; ci != end_ci; - ci = uct_ib_mlx5_txwq_next_ci(ci, wqe_size)) { + ci = uct_ib_mlx5_txwq_next_wqe_index(ci, wqe_size)) { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); num_packets = uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); From 26af207a881d87fbddd1b2409cdbca6ec753c1ea Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 10 Sep 2026 13:47:28 +0300 Subject: [PATCH 18/26] UCT/IB/MLX5: Reject non-inline AM purge layouts --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 44 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 9b3b2bfbb30..7535edc3d3d 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -819,13 +819,13 @@ ucs_status_t uct_rc_mlx5_base_ep_invalidate(uct_ep_h tl_ep, } static uint8_t -uct_ib_mlx5_wqe_opcode(const struct mlx5_wqe_ctrl_seg *ctrl) +uct_rc_mlx5_wqe_opcode(const struct mlx5_wqe_ctrl_seg *ctrl) { return ctrl->opmod_idx_opcode >> 24; } static void -uct_ib_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, +uct_rc_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, void *dst, size_t length) { size_t copy_len = ucs_min(length, UCS_PTR_BYTE_DIFF(src, txwq->qend)); @@ -845,7 +845,7 @@ uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; uct_rc_mlx5_am_short_hdr_t *am; - uct_ib_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); + uct_rc_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); am = callback_data; if ((am->rc_hdr.rc_hdr.am_id & UCT_RC_EP_FC_MASK) == @@ -878,7 +878,7 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( const struct mlx5_wqe_inl_data_seg *inl; memset(info, 0, sizeof(*info)); - ucs_assert(uct_ib_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); + ucs_assert(uct_rc_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); ucs_assertv_always( wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), @@ -915,14 +915,28 @@ static int uct_ib_mlx5_wqe_is_delivered(uint32_t wqe_first_psn, return (diff < psn_half) && (diff >= num_packets); } +static UCS_F_NOINLINE UCS_F_NORETURN void +uct_rc_mlx5_wqe_unsupported(const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl, + size_t wqe_size) +{ + uint8_t wqe[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; + char wqe_dump[3 * sizeof(wqe)]; + + uct_rc_mlx5_txwq_copy_segs(txwq, ctrl, wqe, wqe_size); + ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x size %zu: %s", + uct_rc_mlx5_wqe_opcode(ctrl), wqe_size, + ucs_str_dump_hex(wqe, wqe_size, wqe_dump, sizeof(wqe_dump), + SIZE_MAX)); +} + static uint32_t uct_ib_mlx5_wqe_num_packets( const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) { const struct mlx5_wqe_inl_data_seg *inl; - uint8_t opcode = uct_ib_mlx5_wqe_opcode(ctrl); - uint8_t wqe[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; - char wqe_dump[3 * sizeof(wqe)]; + uint8_t opcode = uct_rc_mlx5_wqe_opcode(ctrl); + size_t inline_length, inline_wqe_size; switch (opcode) { case MLX5_OPCODE_NOP: @@ -934,16 +948,20 @@ static uint32_t uct_ib_mlx5_wqe_num_packets( inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, (void*)(ctrl + 1)); if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { - return 1; + inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + inline_wqe_size = sizeof(*ctrl) + + ucs_align_up_pow2(sizeof(*inl) + inline_length, + UCT_IB_MLX5_WQE_SEG_SIZE); + /* AM short is fully inline. IOV-backed AM zcopy also starts + * inline, but has trailing data segments we do not support. */ + if (wqe_size == inline_wqe_size) { + return 1; + } } /* Fall through */ default: - uct_ib_mlx5_txwq_copy_segs(txwq, ctrl, wqe, wqe_size); - ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x size %zu: %s", - opcode, wqe_size, - ucs_str_dump_hex(wqe, wqe_size, wqe_dump, sizeof(wqe_dump), - SIZE_MAX)); + uct_rc_mlx5_wqe_unsupported(txwq, ctrl, wqe_size); } } From aec2bcbb889f944627a22fe744bac5a7b659d63c Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Thu, 10 Sep 2026 15:03:19 +0300 Subject: [PATCH 19/26] UCT/IB/MLX5: Advance outstanding purge boundary --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 7535edc3d3d..908e4d679a9 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -1093,6 +1093,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, UCS_ERR_CANCELED, txwq->prev_sw_pi, 0); uct_rc_mlx5_ep_update_tx_qp_res(ep, txwq->prev_sw_pi); + txwq->ft_ci = txwq->prev_sw_pi; return UCS_OK; } From c5ed0022a6b5ad570948976015eeb93f3ac25e94 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 14 Sep 2026 11:22:49 +0300 Subject: [PATCH 20/26] UCT/IB/MLX5: Reuse WQE dump for unsupported purge operations --- src/uct/ib/mlx5/ib_mlx5_log.c | 21 ++++++++++++--------- src/uct/ib/mlx5/ib_mlx5_log.h | 6 ++++++ src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 29 +++++++++++++++-------------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5_log.c b/src/uct/ib/mlx5/ib_mlx5_log.c index 8e1bc9fbfda..86994209920 100644 --- a/src/uct/ib/mlx5/ib_mlx5_log.c +++ b/src/uct/ib/mlx5/ib_mlx5_log.c @@ -16,11 +16,6 @@ #include -static void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, - void *qend, int max_sge, int dump_qp, - uct_log_data_dump_func_t packet_dump_cb, - char *buffer, size_t max, uct_ib_log_sge_t *log_sge); - static void uct_ib_mlx5_resp_error_dump(const uct_ib_mlx5_srq_seg_t *seg, unsigned max_strides, char *buffer, size_t max); @@ -281,7 +276,7 @@ static size_t uct_ib_mlx5_dump_dgram(char *buf, size_t max, void *seg, int is_et UCT_IB_MLX5_AV_FULL_SIZE : UCT_IB_MLX5_AV_BASE_SIZE; } -static void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, +void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, void *qend, int max_sge, int dump_qp, uct_log_data_dump_func_t packet_dump_cb, char *buffer, size_t max, uct_ib_log_sge_t *log_sge) @@ -309,17 +304,25 @@ static void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, uint8_t opmod = ctrl->opmod_idx_opcode & 0xff; uint32_t qp_num = ntohl(ctrl->qpn_ds) >> 8; int ds = ctrl->qpn_ds >> 24; - uct_ib_opcode_t *op = &opcodes[opcode]; + uct_ib_opcode_t *op; char *s = buffer; char *ends = buffer + max; - const char* sg_prefix_arr = (op->flags & UCT_IB_OPCODE_FLAG_HAS_DMA) ? - "GS" : NULL; + const char *sg_prefix_arr; struct ibv_sge sg_list[16]; uint64_t inline_bitmap; int i, is_inline, is_eth; size_t dg_size; void *seg; + if ((opcode >= ucs_static_array_size(opcodes)) || + (opcodes[opcode].name == NULL)) { + snprintf(buffer, max, "unknown opcode 0x%x", opcode); + return; + } + + op = &opcodes[opcode]; + sg_prefix_arr = (op->flags & UCT_IB_OPCODE_FLAG_HAS_DMA) ? "GS" : NULL; + /* QP and WQE index */ if (dump_qp) { snprintf(s, ends - s, "QP 0x%x [%03ld] ", qp_num, diff --git a/src/uct/ib/mlx5/ib_mlx5_log.h b/src/uct/ib/mlx5/ib_mlx5_log.h index 882dab56b96..233c940f843 100644 --- a/src/uct/ib/mlx5/ib_mlx5_log.h +++ b/src/uct/ib/mlx5/ib_mlx5_log.h @@ -24,6 +24,12 @@ ucs_status_t uct_ib_mlx5_completion_with_err(uct_ib_iface_t *iface, ucs_log_level_t log_level); +void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, + void *qend, int max_sge, int dump_qp, + uct_log_data_dump_func_t packet_dump_cb, + char *buffer, size_t max, uct_ib_log_sge_t *log_sge); + + void __uct_ib_mlx5_log_tx(const char *file, int line, const char *function, uct_ib_iface_t *iface, void *wqe, void *qstart, void *qend, int max_sge, uct_ib_log_sge_t *log_sge, diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 908e4d679a9..77fe1d10fc9 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include /* For htonl */ @@ -916,22 +915,22 @@ static int uct_ib_mlx5_wqe_is_delivered(uint32_t wqe_first_psn, } static UCS_F_NOINLINE UCS_F_NORETURN void -uct_rc_mlx5_wqe_unsupported(const uct_ib_mlx5_txwq_t *txwq, +uct_rc_mlx5_wqe_unsupported(uct_ib_iface_t *iface, + const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) { - uint8_t wqe[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; - char wqe_dump[3 * sizeof(wqe)]; + char wqe_dump[256] = {0}; - uct_rc_mlx5_txwq_copy_segs(txwq, ctrl, wqe, wqe_size); + uct_ib_mlx5_wqe_dump(iface, (void*)ctrl, txwq->qstart, txwq->qend, + INT_MAX, 0, NULL, wqe_dump, sizeof(wqe_dump) - 1, + NULL); ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x size %zu: %s", - uct_rc_mlx5_wqe_opcode(ctrl), wqe_size, - ucs_str_dump_hex(wqe, wqe_size, wqe_dump, sizeof(wqe_dump), - SIZE_MAX)); + uct_rc_mlx5_wqe_opcode(ctrl), wqe_size, wqe_dump); } static uint32_t uct_ib_mlx5_wqe_num_packets( - const uct_ib_mlx5_txwq_t *txwq, + uct_ib_iface_t *iface, const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) { const struct mlx5_wqe_inl_data_seg *inl; @@ -961,12 +960,13 @@ static uint32_t uct_ib_mlx5_wqe_num_packets( /* Fall through */ default: - uct_rc_mlx5_wqe_unsupported(txwq, ctrl, wqe_size); + uct_rc_mlx5_wqe_unsupported(iface, txwq, ctrl, wqe_size); } } static uint32_t uct_rc_mlx5_txwq_outstanding_num_packets( - uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, uint16_t end_ci) + uct_ib_iface_t *iface, uct_ib_mlx5_txwq_t *txwq, uint16_t start_ci, + uint16_t end_ci) { const struct mlx5_wqe_ctrl_seg *ctrl; uint32_t num_packets = 0; @@ -978,7 +978,7 @@ static uint32_t uct_rc_mlx5_txwq_outstanding_num_packets( ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - num_packets += uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); + num_packets += uct_ib_mlx5_wqe_num_packets(iface, txwq, ctrl, wqe_size); } return num_packets; @@ -1046,7 +1046,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( } num_outstanding_packets = uct_rc_mlx5_txwq_outstanding_num_packets( - txwq, start_ci, end_ci); + &iface->super.super, txwq, start_ci, end_ci); rx_token = params->rx_token; receiver_next_psn = ntohl(*rx_token) & UCT_IB_MLX5_PSN_MASK; @@ -1063,7 +1063,8 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( ci = uct_ib_mlx5_txwq_next_wqe_index(ci, wqe_size)) { ctrl = uct_ib_mlx5_txwq_get_wqe(txwq, ci); wqe_size = uct_ib_mlx5_wqe_size(ctrl); - num_packets = uct_ib_mlx5_wqe_num_packets(txwq, ctrl, wqe_size); + num_packets = uct_ib_mlx5_wqe_num_packets(&iface->super.super, txwq, + ctrl, wqe_size); if (num_packets == 0) { continue; } From bd56d22da78a930ff57b5f47827427e8fe85272f Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Mon, 14 Sep 2026 13:06:31 +0300 Subject: [PATCH 21/26] UCT/IB/MLX5: Fix WQE dump alignment and DS byte order --- src/uct/ib/mlx5/ib_mlx5_log.c | 8 ++++---- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5_log.c b/src/uct/ib/mlx5/ib_mlx5_log.c index 86994209920..e629a23a5df 100644 --- a/src/uct/ib/mlx5/ib_mlx5_log.c +++ b/src/uct/ib/mlx5/ib_mlx5_log.c @@ -277,9 +277,9 @@ static size_t uct_ib_mlx5_dump_dgram(char *buf, size_t max, void *seg, int is_et } void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, - void *qend, int max_sge, int dump_qp, - uct_log_data_dump_func_t packet_dump_cb, - char *buffer, size_t max, uct_ib_log_sge_t *log_sge) + void *qend, int max_sge, int dump_qp, + uct_log_data_dump_func_t packet_dump_cb, + char *buffer, size_t max, uct_ib_log_sge_t *log_sge) { static uct_ib_opcode_t opcodes[] = { [MLX5_OPCODE_NOP] = { "NOP", 0 }, @@ -303,7 +303,7 @@ void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, uint8_t opcode = ctrl->opmod_idx_opcode >> 24; uint8_t opmod = ctrl->opmod_idx_opcode & 0xff; uint32_t qp_num = ntohl(ctrl->qpn_ds) >> 8; - int ds = ctrl->qpn_ds >> 24; + int ds = ntohl(ctrl->qpn_ds) & UINT8_MAX; uct_ib_opcode_t *op; char *s = buffer; char *ends = buffer + max; diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 77fe1d10fc9..5f5d3fb23c7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -916,15 +916,15 @@ static int uct_ib_mlx5_wqe_is_delivered(uint32_t wqe_first_psn, static UCS_F_NOINLINE UCS_F_NORETURN void uct_rc_mlx5_wqe_unsupported(uct_ib_iface_t *iface, - const uct_ib_mlx5_txwq_t *txwq, - const struct mlx5_wqe_ctrl_seg *ctrl, - size_t wqe_size) + const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl, + size_t wqe_size) { char wqe_dump[256] = {0}; uct_ib_mlx5_wqe_dump(iface, (void*)ctrl, txwq->qstart, txwq->qend, - INT_MAX, 0, NULL, wqe_dump, sizeof(wqe_dump) - 1, - NULL); + INT_MAX, 0, NULL, wqe_dump, sizeof(wqe_dump) - 1, + NULL); ucs_fatal("rc mlx5: unsupported outstanding WQE opcode 0x%x size %zu: %s", uct_rc_mlx5_wqe_opcode(ctrl), wqe_size, wqe_dump); } From 10bd703222c3c783cf04624299c6ec5fa9f1fb74 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 15 Sep 2026 11:28:14 +0300 Subject: [PATCH 22/26] UCT/IB/MLX5: Declare initialized WQE dump variables first --- src/uct/ib/mlx5/ib_mlx5_log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5_log.c b/src/uct/ib/mlx5/ib_mlx5_log.c index e629a23a5df..9277e1a6986 100644 --- a/src/uct/ib/mlx5/ib_mlx5_log.c +++ b/src/uct/ib/mlx5/ib_mlx5_log.c @@ -304,9 +304,9 @@ void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, uint8_t opmod = ctrl->opmod_idx_opcode & 0xff; uint32_t qp_num = ntohl(ctrl->qpn_ds) >> 8; int ds = ntohl(ctrl->qpn_ds) & UINT8_MAX; - uct_ib_opcode_t *op; char *s = buffer; char *ends = buffer + max; + uct_ib_opcode_t *op; const char *sg_prefix_arr; struct ibv_sge sg_list[16]; uint64_t inline_bitmap; @@ -390,8 +390,8 @@ void uct_ib_mlx5_wqe_dump(uct_ib_iface_t *iface, void *wqe, void *qstart, /* Extended atomic segment */ if (op->flags & UCT_IB_OPCODE_FLAG_HAS_EXT_ATOMIC) { - uint64_t add, boundary, compare, swap, compare_mask, swap_mask; int size = 1 << ((opmod & 7) + 2); + uint64_t add, boundary, compare, swap, compare_mask, swap_mask; if (opcode == MLX5_OPCODE_ATOMIC_MASKED_FA) { add = network_to_host(seg, size); From c2942cd5d00822b691153e1333b340b7cd711df6 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 15 Sep 2026 13:29:14 +0300 Subject: [PATCH 23/26] UCT/IB/MLX5: Share inline segment decoding in purge --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 68 ++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 5f5d3fb23c7..2fa33f9616f 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -823,6 +823,29 @@ uct_rc_mlx5_wqe_opcode(const struct mlx5_wqe_ctrl_seg *ctrl) return ctrl->opmod_idx_opcode >> 24; } +static const struct mlx5_wqe_inl_data_seg * +uct_rc_mlx5_wqe_get_inline_seg( + const uct_ib_mlx5_txwq_t *txwq, + const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size, + size_t *inline_length_p) +{ + const struct mlx5_wqe_inl_data_seg *inl; + uint32_t byte_count; + + ucs_assertv_always(wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), + "wqe_size=%zu", wqe_size); + + inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, + (void*)(ctrl + 1)); + byte_count = ntohl(inl->byte_count); + if (!(byte_count & MLX5_INLINE_SEG)) { + return NULL; + } + + *inline_length_p = byte_count & ~MLX5_INLINE_SEG; + return inl; +} + static void uct_rc_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, void *dst, size_t length) @@ -839,9 +862,9 @@ uct_rc_mlx5_txwq_copy_segs(const uct_ib_mlx5_txwq_t *txwq, const void *src, static ucs_status_t uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_inl_data_seg *inl, - void *callback_data, uct_ep_op_info_t *info) + size_t inline_length, void *callback_data, + uct_ep_op_info_t *info) { - size_t inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; uct_rc_mlx5_am_short_hdr_t *am; uct_rc_mlx5_txwq_copy_segs(txwq, inl + 1, callback_data, inline_length); @@ -856,14 +879,14 @@ uct_rc_mlx5_op_info_fill_am_short(const uct_ib_mlx5_txwq_t *txwq, return UCS_ERR_UNSUPPORTED; } - info->field_mask = UCT_EP_OP_INFO_FIELD_OPERATION | - UCT_EP_OP_INFO_FIELD_AM; - info->operation = UCT_EP_OP_AM_SHORT; - info->am.field_mask = UCT_EP_OP_INFO_AM_FIELD_AM_ID | - UCT_EP_OP_INFO_AM_FIELD_HEADER_VALUE | - UCT_EP_OP_INFO_AM_FIELD_PAYLOAD_DATA; - info->am.am_id = am->rc_hdr.rc_hdr.am_id & ~UCT_RC_EP_FC_MASK; - info->am.header.value = am->am_hdr; + info->field_mask = UCT_EP_OP_INFO_FIELD_OPERATION | + UCT_EP_OP_INFO_FIELD_AM; + info->operation = UCT_EP_OP_AM_SHORT; + info->am.field_mask = UCT_EP_OP_INFO_AM_FIELD_AM_ID | + UCT_EP_OP_INFO_AM_FIELD_HEADER_VALUE | + UCT_EP_OP_INFO_AM_FIELD_PAYLOAD_DATA; + info->am.am_id = am->rc_hdr.rc_hdr.am_id & ~UCT_RC_EP_FC_MASK; + info->am.header.value = am->am_hdr; info->am.payload.data.buffer = am + 1; info->am.payload.data.length = inline_length - sizeof(*am); return UCS_OK; @@ -875,19 +898,16 @@ static ucs_status_t uct_rc_mlx5_op_info_fill_am( void *callback_data, uct_ep_op_info_t *info) { const struct mlx5_wqe_inl_data_seg *inl; + size_t inline_length; memset(info, 0, sizeof(*info)); ucs_assert(uct_rc_mlx5_wqe_opcode(ctrl) == MLX5_OPCODE_SEND); - ucs_assertv_always( - wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), - "wqe_size=%zu", wqe_size); - - inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, - (void*)(ctrl + 1)); - if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { + inl = uct_rc_mlx5_wqe_get_inline_seg(txwq, ctrl, wqe_size, + &inline_length); + if (inl != NULL) { return uct_rc_mlx5_op_info_fill_am_short( - txwq, inl, callback_data, info); + txwq, inl, inline_length, callback_data, info); } return UCS_ERR_UNSUPPORTED; @@ -933,21 +953,17 @@ static uint32_t uct_ib_mlx5_wqe_num_packets( uct_ib_iface_t *iface, const uct_ib_mlx5_txwq_t *txwq, const struct mlx5_wqe_ctrl_seg *ctrl, size_t wqe_size) { - const struct mlx5_wqe_inl_data_seg *inl; uint8_t opcode = uct_rc_mlx5_wqe_opcode(ctrl); + const struct mlx5_wqe_inl_data_seg *inl; size_t inline_length, inline_wqe_size; switch (opcode) { case MLX5_OPCODE_NOP: return 0; case MLX5_OPCODE_SEND: - ucs_assertv_always( - wqe_size >= (sizeof(*ctrl) + sizeof(*inl)), - "wqe_size=%zu", wqe_size); - inl = uct_ib_mlx5_txwq_wrap_any((uct_ib_mlx5_txwq_t*)txwq, - (void*)(ctrl + 1)); - if (inl->byte_count & htonl(MLX5_INLINE_SEG)) { - inline_length = ntohl(inl->byte_count) & ~MLX5_INLINE_SEG; + inl = uct_rc_mlx5_wqe_get_inline_seg(txwq, ctrl, wqe_size, + &inline_length); + if (inl != NULL) { inline_wqe_size = sizeof(*ctrl) + ucs_align_up_pow2(sizeof(*inl) + inline_length, UCT_IB_MLX5_WQE_SEG_SIZE); From 75fc3a4b7c24b8cf655d2cc6fd9122db6d7e98ab Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 15 Sep 2026 15:29:57 +0300 Subject: [PATCH 24/26] UCT/IB/MLX5: Complete flushes in WQE purge order --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 15 +++++++++------ test/gtest/uct/test_peer_failure.cc | 27 +++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 2fa33f9616f..58f402cab5c 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -1058,7 +1058,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( txwq->ft_ci, uct_ib_mlx5_wqe_size(ctrl)); if (start_ci == end_ci) { - goto out_purge; + goto out; } num_outstanding_packets = uct_rc_mlx5_txwq_outstanding_num_packets( @@ -1082,7 +1082,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( num_packets = uct_ib_mlx5_wqe_num_packets(&iface->super.super, txwq, ctrl, wqe_size); if (num_packets == 0) { - continue; + goto purge_completions; } is_delivered = uct_ib_mlx5_wqe_is_delivered( @@ -1094,7 +1094,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( status = uct_rc_mlx5_op_info_fill_am( txwq, ctrl, wqe_size, callback_data, &info); if (status == UCS_ERR_NO_ELEM) { - continue; + goto purge_completions; } if (status != UCS_OK) { @@ -1104,11 +1104,14 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( params->cb(&info, callback_arg); } + +purge_completions: + /* Complete flushes after their WQE, before later AM purge callbacks. */ + uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, + UCS_ERR_CANCELED, ci, 0); } -out_purge: - uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, - UCS_ERR_CANCELED, txwq->prev_sw_pi, 0); +out: uct_rc_mlx5_ep_update_tx_qp_res(ep, txwq->prev_sw_pi); txwq->ft_ci = txwq->prev_sw_pi; return UCS_OK; diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index b98f732ab1e..397f97dfd4d 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -377,6 +377,8 @@ class test_uct_purge_outstanding : public uct_test { test_uct_purge_outstanding *self; uct_completion_t comp; uint32_t num_ops_purged; + uint32_t num_ops_purged_at_completion; + uint32_t num_ops_posted_after_flush; uint32_t num_flush_purged; unsigned num_completions; uint8_t am_short_id; @@ -523,6 +525,7 @@ class test_uct_purge_outstanding : public uct_test { { purge_ctx *ctx = ucs_container_of(comp, purge_ctx, comp); + ctx->num_ops_purged_at_completion = ctx->num_ops_purged; ++ctx->num_completions; } @@ -573,6 +576,7 @@ class test_uct_purge_outstanding : public uct_test { static constexpr uint32_t NUM_MSG_BEFORE_INVALIDATE = 2; uct_ep_invalidate_params_t invalidate_params = {}; uint32_t num_posted = 0; + uint32_t num_ops_purged; unsigned num_outstanding, num_completions; ucs_status_t status; @@ -582,10 +586,15 @@ class test_uct_purge_outstanding : public uct_test { } ASSERT_UCS_OK(uct_ep_invalidate(m_sender->ep(0), &invalidate_params)); - /* Post after invalidation so flush exercises local cancellation. */ + /* Make flush wait for failed work even if it reuses a signaled WQE. */ + status = post_op(m_sender->ep(0), &ctx.comp, send_func); + ASSERT_UCS_OK_OR_INPROGRESS(status); + ++num_posted; + post_flush(m_sender->ep(0), &ctx.comp); - num_posted += post_until_error(m_sender->ep(0), &ctx.comp, - send_func); + ctx.num_ops_posted_after_flush = post_until_error( + m_sender->ep(0), &ctx.comp, send_func); + num_posted += ctx.num_ops_posted_after_flush; wait_for_flag(&m_err_count); ASSERT_EQ(1u, m_err_count); @@ -606,6 +615,11 @@ class test_uct_purge_outstanding : public uct_test { EXPECT_EQ(num_completions, ctx.num_completions); + num_ops_purged = ctx.num_ops_purged; + purge_outstanding(&ctx); + EXPECT_EQ(num_ops_purged, ctx.num_ops_purged); + EXPECT_EQ(num_completions, ctx.num_completions); + flush(); EXPECT_EQ(0, ctx.comp.count); } @@ -618,7 +632,7 @@ class test_uct_purge_outstanding : public uct_test { UCS_TEST_SKIP_COND_P(test_uct_purge_outstanding, am_short, !check_caps(UCT_IFACE_FLAG_AM_SHORT)) { - purge_ctx ctx = {this, {completion_cb, 0, UCS_OK}, 0, 0, 0, + purge_ctx ctx = {this, {completion_cb, 0, UCS_OK}, 0, 0, 0, 0, 0, 3, 0x0123456789abcdefull, {1, 2, 3, 4}}; ASSERT_UCS_OK(uct_iface_set_am_handler(m_receiver->iface(), ctx.am_short_id, @@ -629,6 +643,11 @@ UCS_TEST_SKIP_COND_P(test_uct_purge_outstanding, am_short, ctx.am_short_payload.data(), ctx.am_short_payload.size()); }, ctx); + + EXPECT_GT(ctx.num_ops_purged_at_completion, 0u); + EXPECT_GT(ctx.num_ops_posted_after_flush, 0u); + EXPECT_EQ(ctx.num_ops_purged, ctx.num_ops_purged_at_completion + + ctx.num_ops_posted_after_flush); } UCT_INSTANTIATE_TEST_CASE(test_uct_purge_outstanding) From d54595f52d36052c3552cc3ab2eedbd7582fe129 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 15 Sep 2026 15:54:42 +0300 Subject: [PATCH 25/26] UCT/IB/MLX5: Invoke only flush completions during purge --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index 58f402cab5c..be0dd083575 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -1031,6 +1031,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; + uct_rc_iface_send_op_t *op; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; uct_ep_op_info_t info; uint16_t ci, end_ci, start_ci; @@ -1082,7 +1083,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( num_packets = uct_ib_mlx5_wqe_num_packets(&iface->super.super, txwq, ctrl, wqe_size); if (num_packets == 0) { - goto purge_completions; + goto complete_flushes; } is_delivered = uct_ib_mlx5_wqe_is_delivered( @@ -1094,7 +1095,7 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( status = uct_rc_mlx5_op_info_fill_am( txwq, ctrl, wqe_size, callback_data, &info); if (status == UCS_ERR_NO_ELEM) { - goto purge_completions; + goto complete_flushes; } if (status != UCS_OK) { @@ -1105,10 +1106,16 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( params->cb(&info, callback_arg); } -purge_completions: +complete_flushes: /* Complete flushes after their WQE, before later AM purge callbacks. */ - uct_rc_txqp_purge_outstanding(&iface->super, &ep->super.txqp, - UCS_ERR_CANCELED, ci, 0); + ucs_queue_for_each_extract( + op, &ep->super.txqp.outstanding, queue, + (op->handler == uct_rc_ep_flush_op_completion_handler) && + UCS_CIRCULAR_COMPARE16(op->sn, <=, ci)) { + op->flags &= ~UCT_RC_IFACE_SEND_OP_FLAG_INUSE; + uct_invoke_completion(op->user_comp, UCS_ERR_CANCELED); + ucs_mpool_put(op); + } } out: From 1f518a074aa148483dbbd93ec17bd012a048bad8 Mon Sep 17 00:00:00 2001 From: Roie Danino Date: Tue, 15 Sep 2026 16:39:36 +0300 Subject: [PATCH 26/26] UCT/IB/MLX5: Simplify outstanding purge loop --- src/uct/ib/mlx5/rc/rc_mlx5_ep.c | 53 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c index be0dd083575..56aac5aaec7 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_ep.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_ep.c @@ -1024,6 +1024,21 @@ static ucs_status_t uct_rc_mlx5_ep_outstanding_purge_check_params( return UCS_OK; } +static void +uct_rc_mlx5_ep_purge_flushes(uct_rc_mlx5_base_ep_t *ep, uint16_t ci) +{ + uct_rc_iface_send_op_t *op; + + ucs_queue_for_each_extract( + op, &ep->super.txqp.outstanding, queue, + (op->handler == uct_rc_ep_flush_op_completion_handler) && + UCS_CIRCULAR_COMPARE16(op->sn, <=, ci)) { + op->flags &= ~UCT_RC_IFACE_SEND_OP_FLAG_INUSE; + uct_invoke_completion(op->user_comp, UCS_ERR_CANCELED); + ucs_mpool_put(op); + } +} + ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_ep_h tl_ep, const uct_ep_outstanding_purge_params_t *params) { @@ -1031,7 +1046,6 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; const uct_rc_mlx5_rx_token_t *rx_token; const struct mlx5_wqe_ctrl_seg *ctrl; - uct_rc_iface_send_op_t *op; uint8_t callback_data[UCT_IB_MLX5_MAX_SEND_WQE_SIZE]; uct_ep_op_info_t info; uint16_t ci, end_ci, start_ci; @@ -1039,7 +1053,6 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( uint32_t num_outstanding_packets, num_packets; size_t wqe_size; void *callback_arg; - int is_delivered; ucs_status_t status; status = uct_rc_mlx5_ep_outstanding_purge_check_params(params); @@ -1082,40 +1095,24 @@ ucs_status_t uct_rc_mlx5_ep_outstanding_purge( wqe_size = uct_ib_mlx5_wqe_size(ctrl); num_packets = uct_ib_mlx5_wqe_num_packets(&iface->super.super, txwq, ctrl, wqe_size); - if (num_packets == 0) { - goto complete_flushes; - } - - is_delivered = uct_ib_mlx5_wqe_is_delivered( - wqe_first_psn, receiver_next_psn, num_packets); - wqe_first_psn = (wqe_first_psn + num_packets) & - UCT_IB_MLX5_PSN_MASK; - - if (!is_delivered) { + if ((num_packets != 0) && + !uct_ib_mlx5_wqe_is_delivered(wqe_first_psn, receiver_next_psn, + num_packets)) { status = uct_rc_mlx5_op_info_fill_am( txwq, ctrl, wqe_size, callback_data, &info); - if (status == UCS_ERR_NO_ELEM) { - goto complete_flushes; - } - - if (status != UCS_OK) { + if (status == UCS_OK) { + params->cb(&info, callback_arg); + } else if (status != UCS_ERR_NO_ELEM) { ucs_fatal("rc mlx5: failed to parse outstanding WQE at ci %u", ci); } - - params->cb(&info, callback_arg); } -complete_flushes: + wqe_first_psn = (wqe_first_psn + num_packets) & + UCT_IB_MLX5_PSN_MASK; + /* Complete flushes after their WQE, before later AM purge callbacks. */ - ucs_queue_for_each_extract( - op, &ep->super.txqp.outstanding, queue, - (op->handler == uct_rc_ep_flush_op_completion_handler) && - UCS_CIRCULAR_COMPARE16(op->sn, <=, ci)) { - op->flags &= ~UCT_RC_IFACE_SEND_OP_FLAG_INUSE; - uct_invoke_completion(op->user_comp, UCS_ERR_CANCELED); - ucs_mpool_put(op); - } + uct_rc_mlx5_ep_purge_flushes(ep, ci); } out: