Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
f44378a
UCT/RC: Preserve pending order for multi-credit SGL puts
michal-shalev Aug 26, 2026
dc05f89
GTEST/UCP: Scope SGL AM+MT worker to the put-AM pending test
michal-shalev Aug 26, 2026
c5fcd20
GTEST: Replace UCP SGL pending test with UCT RC test
michal-shalev Aug 27, 2026
fdbc016
GTEST/UCT/RC: Cover pending check in put_sgl_zcopy
michal-shalev Aug 27, 2026
b8b3964
UCT/RC: Block sends only when pending is queued
michal-shalev Aug 27, 2026
5b20bda
GTEST/UCT/RC: Dispatch pending to send FC grant
michal-shalev Aug 27, 2026
2f34f63
GTEST/UCT: Drop redundant uct_v2.h include from test_rc
michal-shalev Aug 27, 2026
795119c
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Aug 27, 2026
c586b96
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Aug 27, 2026
e59eb71
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Sep 3, 2026
435a117
UCT/RC: Reserve TX credits instead of blocking sends on pending
michal-shalev Sep 4, 2026
165dea4
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Sep 4, 2026
07800d4
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Sep 7, 2026
f009c9c
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Sep 13, 2026
67dfdf8
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Oct 19, 2026
64fb775
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Oct 19, 2026
377863a
UCT/RC: Reset reserve on ep failure
michal-shalev Oct 19, 2026
2ea9451
GTEST/UCT/RC: Fix DC pending dispatch in pending grant test
michal-shalev Oct 19, 2026
4b9f93e
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Oct 22, 2026
256ff2f
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Oct 29, 2026
035bc2e
GTEST/UCT/RC: Check TX reserve in SGL pending tests
michal-shalev Oct 29, 2026
af88ade
UCT/RC: Tidy reserve and schedule_pending helper signatures
michal-shalev Oct 30, 2026
b2aa6c3
UCT/RC: Explain why pending success keeps the reserve
michal-shalev Oct 30, 2026
72f38c8
Merge remote-tracking branch 'origin/uct-rc-pending-sgl-order' into u…
michal-shalev Oct 30, 2026
4dd65ae
GTEST/UCT/RC: Address review comments in pending grant test
michal-shalev Nov 1, 2026
030c504
Merge branch 'master' into uct-rc-pending-sgl-order
michal-shalev Nov 1, 2026
bfa0c78
UCT/RC: Keep reserve on failure while requests are queued
michal-shalev Nov 1, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/uct/ib/mlx5/rc/rc_mlx5_ep.c
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers,
"put_sgl_zcopy");
}

UCT_RC_CHECK_PENDING(&ep->super);
UCT_RC_CHECK_CQE_VALUE_RET(&iface->super, &ep->super,
UCS_ERR_NO_RESOURCE, count - 1);

Expand Down
20 changes: 12 additions & 8 deletions src/uct/ib/rc/base/rc_ep.c
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,10 @@ UCS_CLASS_INIT_FUNC(uct_rc_ep_t, uct_rc_iface_t *iface, uint32_t qp_num,
return status;
}

self->path_index = UCT_EP_PARAMS_GET_PATH_INDEX(params);
self->flags = 0;
self->txqp_reserve = 0;
self->cq_reserve = 0;
self->path_index = UCT_EP_PARAMS_GET_PATH_INDEX(params);
self->flags = 0;
self->in_pending = 0;
uct_rc_ep_reset_reserve(self);

status = uct_rc_fc_init(&self->fc, iface UCS_STATS_ARG(self->super.stats));
if (status != UCS_OK) {
Expand Down Expand Up @@ -316,7 +316,8 @@ ucs_status_t uct_rc_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *n,
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
uct_rc_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_ep_t);

if (uct_rc_ep_have_tx_cqe_avail(ep) &&
if (!UCT_RC_EP_MUST_BLOCK_SEND(ep) &&
uct_rc_ep_have_tx_cqe_avail(ep) &&
uct_rc_ep_has_tx_resources(ep) &&
uct_rc_iface_has_tx_resources(iface)) {
return UCS_ERR_BUSY;
Expand Down Expand Up @@ -345,10 +346,11 @@ ucs_arbiter_cb_result_t uct_rc_ep_process_pending(ucs_arbiter_t *arbiter,
uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_rc_iface_t);
ucs_status_t status;

status = uct_rc_iface_invoke_pending_cb(iface, req);
ep->in_pending = 1;
Comment thread
michal-shalev marked this conversation as resolved.
Resolved
status = uct_rc_iface_invoke_pending_cb(iface, req);
ep->in_pending = 0;
if (status == UCS_OK) {
ep->txqp_reserve = 0;
ep->cq_reserve = 0;
Comment thread
michal-shalev marked this conversation as resolved.
Resolved
uct_rc_ep_reset_reserve(ep);
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
} else if (status == UCS_INPROGRESS) {
return UCS_ARBITER_CB_RESULT_NEXT_GROUP;
Expand Down Expand Up @@ -434,6 +436,8 @@ void uct_rc_ep_pending_purge(uct_ep_h tl_ep, uct_pending_purge_callback_t cb,

ucs_arbiter_group_purge(&iface->tx.arbiter, &ep->arb_group,
uct_rc_ep_arbiter_purge_cb, &args);

uct_rc_ep_reset_reserve(ep);
}

ucs_status_t uct_rc_ep_fc_grant(uct_pending_req_t *self)
Expand Down
25 changes: 23 additions & 2 deletions src/uct/ib/rc/base/rc_ep.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,25 @@ enum {
#define UCT_RC_DEFINE_ATOMIC_HANDLER_FUNC_NAME(_num_bits, _is_be) \
uct_rc_ep_atomic_handler_##_num_bits##_be##_is_be

#define UCT_RC_EP_MUST_BLOCK_SEND(_ep) \
(!(_ep)->in_pending && \
(!ucs_arbiter_group_is_empty(&(_ep)->arb_group) || \
(((_ep)->cq_reserve | (_ep)->txqp_reserve) != 0)))

#define UCT_RC_CHECK_PENDING_RET(_ep, _ret) \
if (ucs_unlikely(UCT_RC_EP_MUST_BLOCK_SEND(_ep))) { \
UCS_STATS_UPDATE_COUNTER((_ep)->super.stats, UCT_EP_STAT_NO_RES, 1); \
return _ret; \
}

#define UCT_RC_CHECK_PENDING(_ep) \
UCT_RC_CHECK_PENDING_RET(_ep, UCS_ERR_NO_RESOURCE)

/*
* Check for send resources
*/
#define UCT_RC_CHECK_CQE_RET(_iface, _ep, _ret) \
UCT_RC_CHECK_PENDING_RET(_ep, _ret) \
/* tx_moderation == 0 for TLs which don't support it */ \
if (ucs_unlikely((_iface)->tx.cq_available <= \
(signed)(_iface)->config.tx_moderation)) { \
Expand Down Expand Up @@ -201,8 +216,7 @@ enum {

#define UCT_RC_CHECK_RES_AND_FC(_iface, _ep, _am_id) \
UCT_RC_CHECK_RES(_iface, _ep) \
UCT_RC_CHECK_FC(_iface, _ep, _am_id) \
uct_rc_iface_check_pending(_iface, &(_ep)->arb_group);
UCT_RC_CHECK_FC(_iface, _ep, _am_id)

/* this is a common type for all rc and dc transports */
struct uct_rc_txqp {
Expand Down Expand Up @@ -233,6 +247,7 @@ struct uct_rc_ep {
uint16_t cq_reserve;
uint8_t path_index;
uint8_t flags;
uint8_t in_pending;
Comment thread
michal-shalev marked this conversation as resolved.
Resolved
Comment thread
iyastreb marked this conversation as resolved.
Resolved
};


Expand Down Expand Up @@ -354,6 +369,12 @@ static UCS_F_ALWAYS_INLINE void uct_rc_fc_restore_wnd(uct_rc_iface_t *iface,
UCS_STATS_SET_COUNTER(fc->stats, UCT_RC_FC_STAT_FC_WND, fc->fc_wnd);
}

static UCS_F_ALWAYS_INLINE void uct_rc_ep_reset_reserve(uct_rc_ep_t *ep)
{
ep->txqp_reserve = 0;
ep->cq_reserve = 0;
}

static UCS_F_ALWAYS_INLINE int uct_rc_ep_has_tx_resources(uct_rc_ep_t *ep)
{
uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_rc_iface_t);
Expand Down
127 changes: 127 additions & 0 deletions test/gtest/ucp/test_ucp_rma.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@

#include "test_ucp_memheap.h"

#include <atomic>
#include <thread>

extern "C" {
#include <ucp/core/ucp_context.h>
#include <ucp/core/ucp_mm.h> /* for UCP_MEM_IS_ACCESSIBLE_FROM_CPU */
Expand Down Expand Up @@ -1658,4 +1661,128 @@ UCS_TEST_SKIP_COND_P(test_ucp_rma_sgl, put_without_proto,
REMOTE_MASK_DEFAULT, 2, UCS_ERR_UNSUPPORTED);
}

class test_ucp_rma_sgl_am_pending : public test_ucp_rma_sgl {
public:
static void get_base_variants(std::vector<ucp_test_variant>& variants) {
add_variant(variants, UCP_FEATURE_RMA | UCP_FEATURE_AM,
MULTI_THREAD_WORKER);
}

static void get_test_variants(std::vector<ucp_test_variant>& variants) {
add_variant_memtypes(variants, get_base_variants,
UCS_BIT(UCS_MEMORY_TYPE_CUDA) |
UCS_BIT(UCS_MEMORY_TYPE_HOST));
}
};

UCS_TEST_P(test_ucp_rma_sgl_am_pending, put)
{
/* size, count, repeat, num_threads */
static const size_t cases[][4] = {
{4096, 8, 3, 1},
{32768, 64, 3, 2},
{1000000, 100, 3, 4},
{40, 1000, 1, 4},
{40, 1000, 1, 8}
};
static const char am_msg[] = "am";
static const unsigned am_id = 1;

auto am_cb = [](void *, const void *, size_t, void *, size_t,
const ucp_am_recv_param_t *) -> ucs_status_t {
return UCS_OK;
};
ucp_am_handler_param_t am_handler = {};
am_handler.field_mask = UCP_AM_HANDLER_PARAM_FIELD_ID |
UCP_AM_HANDLER_PARAM_FIELD_CB;
am_handler.id = am_id;
am_handler.cb = am_cb;
ASSERT_UCS_OK(ucp_worker_set_am_recv_handler(receiver().worker(),
&am_handler));

std::atomic<bool> stop_progress{false};
Comment thread
michal-shalev marked this conversation as resolved.
Resolved
Comment thread
michal-shalev marked this conversation as resolved.
Resolved
std::thread progress_thread([&]() {
while (!stop_progress.load(std::memory_order_relaxed)) {
ucp_worker_progress(sender().worker());
ucp_worker_progress(receiver().worker());
}
});

auto stop_progress_thread = [&]() {
stop_progress = true;
progress_thread.join();
};

auto wait_req = [&](void *req) {
if (req == NULL) {
return;
}

if (!UCS_PTR_IS_PTR(req)) {
EXPECT_UCS_OK(UCS_PTR_STATUS(req));
return;
}

while (ucp_request_check_status(req) == UCS_INPROGRESS) {
ucp_worker_progress(sender().worker());
ucp_worker_progress(receiver().worker());
}

EXPECT_UCS_OK(ucp_request_check_status(req));
ucp_request_free(req);
};

for (size_t c = 0; c < ucs_static_array_size(cases); c++) {
const size_t size = cases[c][0];
const size_t count = cases[c][1];
const size_t repeat = cases[c][2];
const size_t num_threads = cases[c][3];

sgl_ctx ctx;
init_sgl_ctx(ctx, count, size);

ucp_dt_local_sgl_t local = make_local_sgl(
ctx, LOCAL_MASK_DEFAULT | UCP_DT_LOCAL_SGL_FIELD_MEMHS);
ucp_dt_remote_sgl_t remote = make_remote_sgl(ctx, REMOTE_MASK_DEFAULT);
ucp_request_param_t param = make_sgl_param(&remote, count);

for (size_t r = 0; r < repeat; r++) {
std::vector<std::thread> threads;
threads.reserve(num_threads);

for (size_t t = 0; t < num_threads; t++) {
threads.emplace_back([&]() {
void *put_req = sgl_op_nbx(SGL_OP_PUT, &local, count,
UCP_REMOTE_ADDR_INVALID,
UCP_RKEY_INVALID, &param);
void *flush_req = sender().flush_ep_nb();

ucp_request_param_t am_param = {};
am_param.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS;
am_param.flags = UCP_AM_SEND_FLAG_EAGER;
void *am_req = ucp_am_send_nbx(sender().ep(), am_id, NULL,
0, am_msg, sizeof(am_msg),
&am_param);

wait_req(put_req);
wait_req(flush_req);
wait_req(am_req);
});
}

for (auto &thread : threads) {
thread.join();
}

if (HasFailure() || (num_errors() > 0)) {
stop_progress_thread();
return;
}
}
}

stop_progress_thread();
}

UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_rma_sgl, all, "all")
UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_rma_sgl_am_pending, all, "all")
Loading