Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/ucp/core/ucp_ep.c
Original file line number Diff line number Diff line change
Expand Up @@ -2128,6 +2128,7 @@ static void ucp_ep_recovery_arg_free(ucp_ep_h ep)

seen_peer_id = arg->seen_peer_id;
ucp_ep_refcount_assert(ep, probe, ==, 0);
ucp_wireup_tf_reply_discard(ep);
ucp_ep_tf_cleanup(ep);
ucs_free(arg);
ep->ext->recovery_arg = NULL;
Expand Down Expand Up @@ -2506,8 +2507,14 @@ int ucp_ep_recovery_progress(ucp_ep_h ep)

failed = ucp_ep_get_failed_lanes(ep);
if (failed == 0) {
/* Recovery completed between rounds, the ep operates normally. */
ucs_assert(ucp_ep_recovery_arg(ep) == NULL);
/* A peer request moves the queue pair to error and waits for that
* completion before this lane is failed. Recovery stays idle until
* the completion holds it. */
if ((ucp_ep_recovery_arg(ep) == NULL) ||

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pls write this as one assert, the if + assert form is hard to read:

Suggested change
if ((ucp_ep_recovery_arg(ep) == NULL) ||
/* A peer request moves the queue pair to error and waits for that
* completion before this lane is failed, so a parked reply can keep
* the recovery state alive with no failed lane. */
ucs_assert((ucp_ep_recovery_arg(ep) == NULL) ||
(ucp_ep_recovery_arg(ep)->tf_reply != NULL));
goto done;

(ucp_ep_recovery_arg(ep)->tf_reply == NULL)) {
ucs_assert(ucp_ep_recovery_arg(ep) == NULL);
}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion:

        /* A peer request moves the queue pair to error and waits for that
         * completion before this lane is failed. Recovery stays idle until
         * the completion holds it. */
        ucs_assert((ucp_ep_recovery_arg(ep) == NULL) ||
                   (ucp_ep_recovery_arg(ep)->tf_reply != NULL));

The if (...) { ucs_assert(...) } form is equivalent but much harder to read.


goto done;
}

Expand Down
3 changes: 3 additions & 0 deletions src/ucp/core/ucp_ep.h
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,9 @@ typedef struct ucp_ep_recovery_arg {
* outstanding operations are resolved,
* outstanding re-posts also go here */
ucs_queue_head_t tf_pending_q;
/* LANES_ADDR reply waiting for error completions. Owned by wireup.
* NULL when no reply is waiting. */
void *tf_reply;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO pls forward-declare ucp_wireup_tf_reply_t and use it here instead of void*, so this pointer keeps type checking.

} ucp_ep_recovery_arg_t;


Expand Down
14 changes: 11 additions & 3 deletions src/ucp/core/ucp_worker.c
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <ucp/core/ucp_context.h>
#include <ucp/proto/proto_common.inl>
#include <ucp/wireup/address.h>
#include <ucp/wireup/wireup.h>
#include <ucp/wireup/wireup_cm.h>
#include <ucp/wireup/wireup_ep.h>
#include <ucp/wireup/wireup_lane_info.h>
Expand Down Expand Up @@ -482,18 +483,22 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane,
if (wireup_ep == NULL) {
/* This endpoint is already held. Keep it until its completion. */
if (ucp_ep_tf_is_held(ucp_ep, lane, uct_ep)) {
ucp_wireup_tf_lane_errored(ucp_ep, lane);
return UCS_INPROGRESS;
}

if (ucp_ep_tf_hold(ucp_ep, lane, uct_ep) == UCS_OK) {
/* Purge after return, once the outstanding range is
* recorded. */
* recorded. The hold is the error this request was waiting
* for. A deferred reply rebuilds the lane only after it. */
ucp_ep_tf_schedule_token_purge(ucp_ep, lane);
ucp_wireup_tf_lane_errored(ucp_ep, lane);
return UCS_INPROGRESS;
}
}

ucp_ep_set_lanes_failed(ucp_ep, UCS_BIT(lane), status);
ucp_wireup_tf_lane_errored(ucp_ep, lane);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

blocker: ucp_ep_set_lanes_failed() can fail the whole ep and destroy it (ucp_ep_set_failed() -> ucp_ep_disconnected() -> ucp_ep_destroy_internal()), so ucp_ep may already be freed here.

Suggested change
ucp_wireup_tf_lane_errored(ucp_ep, lane);
ucp_wireup_tf_lane_errored(ucp_ep, lane);
ucp_ep_set_lanes_failed(ucp_ep, UCS_BIT(lane), status);
return UCS_OK;

return UCS_OK;
}

Expand Down Expand Up @@ -565,8 +570,8 @@ ucp_worker_iface_error_handler(void *arg, uct_ep_h uct_ep, ucs_status_t status)

UCS_ASYNC_BLOCK(&worker->async);

ucs_debug("worker %p: error handler called for UCT EP %p: %s",
worker, uct_ep, ucs_status_string(status));
ucs_debug("worker %p: error handler called for UCT EP %p: %s", worker,
uct_ep, ucs_status_string(status));

if (ucp_worker_is_uct_ep_discarding(worker, uct_ep)) {
ucs_debug("UCT EP %p is being discarded on UCP Worker %p",
Expand All @@ -588,6 +593,9 @@ ucp_worker_iface_error_handler(void *arg, uct_ep_h uct_ep, ucs_status_t status)
}
}

ucs_debug("worker %p: uct_ep %p is ep %p lane %d slot %p", worker, uct_ep,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: this looks like leftover debugging and mostly repeats the debug message at the top of the handler, pls drop it.

ucp_ep, lane, ucp_ep_get_lane(ucp_ep, lane));

status = ucp_worker_iface_handle_uct_ep_failure(ucp_ep, lane, uct_ep,
status);

Expand Down
Loading
Loading